Feat[mc_downloader]: native library downloading and extraction

Currently implemented only for JNA

app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/Tools.java

@@ -325,6 +325,14 @@ public final class Tools {
             }
             javaArgList.add("-Dlog4j.configurationFile=" + configFile);
         }
+
+        File versionSpecificNativesDir = new File(Tools.DIR_CACHE, "natives/"+versionId);
+        if(versionSpecificNativesDir.exists()) {
+            String dirPath = versionSpecificNativesDir.getAbsolutePath();
+            javaArgList.add("-Djava.library.path="+dirPath+":"+Tools.NATIVE_LIB_DIR);
+            javaArgList.add("-Djna.boot.library.path="+dirPath);
+        }
+
         javaArgList.addAll(Arrays.asList(getMinecraftJVMArgs(versionId, gamedir)));
         javaArgList.add("-cp");
         javaArgList.add(launchClassPath + ":" + getLWJGL3ClassPath());

app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/MinecraftDownloader.java

@@ -39,8 +39,10 @@ import java.util.concurrent.atomic.AtomicReference;
 public class MinecraftDownloader {
     private static final double ONE_MEGABYTE = (1024d * 1024d);
     public static final String MINECRAFT_RES = "https://resources.download.minecraft.net/";
+    private static final String MAVEN_CENTRAL_REPO1 = "https://repo1.maven.org/maven2/";
     private AtomicReference<Exception> mDownloaderThreadException;
     private ArrayList<DownloaderTask> mScheduledDownloadTasks;
+    private ArrayList<File> mDeclaredNatives;
     private AtomicLong mProcessedFileCounter;
     private AtomicLong mProcessedSizeCounter; // Total bytes of processed files (passed SHA1 or downloaded)
     private AtomicLong mInternetUsageCounter; // How many bytes downloaded over Internet
@@ -88,6 +90,7 @@ public class MinecraftDownloader {
 
         mTargetJarFile = createGameJarPath(versionName);
         mScheduledDownloadTasks = new ArrayList<>();
+        mDeclaredNatives = new ArrayList<>();
         mProcessedFileCounter = new AtomicLong(0);
         mProcessedSizeCounter = new AtomicLong(0);
         mInternetUsageCounter = new AtomicLong(0);
@@ -120,6 +123,7 @@ public class MinecraftDownloader {
                 throw thrownException;
             } else {
                 ensureJarFileCopy();
+                extractNatives(versionName);
             }
         }catch (InterruptedException e) {
             // Interrupted while waiting, which means that the download was cancelled.
@@ -167,6 +171,25 @@ public class MinecraftDownloader {
         org.apache.commons.io.FileUtils.copyFile(mSourceJarFile, mTargetJarFile, false);
     }
 
+    private void extractNatives(String versionName) throws IOException {
+        if(mDeclaredNatives.isEmpty()) return;
+        int totalCount = mDeclaredNatives.size();
+
+        ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0,
+                R.string.newdl_extracting_native_libraries, 0, totalCount);
+
+        File targetDirectory = new File(Tools.DIR_CACHE, "natives/"+versionName);
+        FileUtils.ensureDirectory(targetDirectory);
+        NativesExtractor nativesExtractor = new NativesExtractor(targetDirectory);
+        int extractedCount = 0;
+        for(File source : mDeclaredNatives) {
+            nativesExtractor.extractFromAar(source);
+            extractedCount++;
+            ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, extractedCount * 100 / totalCount,
+                    R.string.newdl_extracting_native_libraries, extractedCount, totalCount);
+        }
+    }
+
     private File downloadGameJson(JMinecraftVersionList.Version verInfo) throws IOException, MirrorTamperedException {
         File targetFile = createGameJsonPath(verInfo.id);
         if(verInfo.sha1 == null && targetFile.canRead() && targetFile.isFile())
@@ -274,13 +297,31 @@ public class MinecraftDownloader {
         );
     }
 
+    /**
+     * Schedule the download of an AAR library containing the required natives, for later extraction
+     * and adding to the library path.
+     * @param baseRepository the source Maven repository to download from.
+     * @param dependentLibrary the DependentLibrary to get the path from
+     * @throws IOException in case if download scheduling fails.
+     */
+    private void scheduleNativeLibraryDownload(String baseRepository, DependentLibrary dependentLibrary) throws IOException {
+        String path = FileUtils.removeExtension(Tools.artifactToPath(dependentLibrary)) + ".aar";
+        String downloadUrl = baseRepository + path;
+        File targetPath = new File(Tools.DIR_HOME_LIBRARY, path);
+        mDeclaredNatives.add(targetPath);
+        scheduleDownload(targetPath, DownloadMirror.DOWNLOAD_CLASS_LIBRARIES, downloadUrl, null, 0, true);
+    }
+
     private void scheduleLibraryDownloads(DependentLibrary[] dependentLibraries) throws IOException {
         Tools.preProcessLibraries(dependentLibraries);
         growDownloadList(dependentLibraries.length);
         for(DependentLibrary dependentLibrary : dependentLibraries) {
             // Don't download lwjgl, we have our own bundled in.
             if(dependentLibrary.name.startsWith("org.lwjgl")) continue;
-
+            // Special handling for JNA Android natives
+            if(dependentLibrary.name.startsWith("net.java.dev.jna:jna:")) {
+                scheduleNativeLibraryDownload(MAVEN_CENTRAL_REPO1, dependentLibrary);
+            }
             String libArtifactPath = Tools.artifactToPath(dependentLibrary);
             String sha1 = null, url = null;
             long size = 0;

app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/NativesExtractor.java

@@ -0,0 +1,119 @@
+package net.kdt.pojavlaunch.tasks;
+
+import net.kdt.pojavlaunch.Architecture;
+import net.kdt.pojavlaunch.Tools;
+import net.kdt.pojavlaunch.utils.FileUtils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.zip.CRC32;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+public class NativesExtractor {
+    private static final ArrayList<String> LIBRARY_BLACKLIST = createLibraryBlacklist();
+    private final File mDestinationDir;
+    private final String mLibraryLocation;
+
+    public NativesExtractor(File mDestinationDir) {
+        this.mDestinationDir = mDestinationDir;
+        this.mLibraryLocation = "jni/"+getAarArchitectureName()+"/";
+    }
+
+    /**
+     * Create a library blacklist so that downloaded natives are not able to
+     * override built-in libraries.
+     * @return the resulting blacklist of library file names
+     */
+    private static ArrayList<String> createLibraryBlacklist() {
+        String[] includedLibraryNames = new File(Tools.NATIVE_LIB_DIR).list();
+        ArrayList<String> blacklist = new ArrayList<>(includedLibraryNames.length);
+        for(String libraryName : includedLibraryNames) {
+            // allow overriding jnidispatch (as the integrated version may be too old)
+            if(libraryName.equals("libjnidispatch.so")) continue;
+            blacklist.add(libraryName);
+        }
+        blacklist.trimToSize();
+        return blacklist;
+    }
+
+    private static String getAarArchitectureName() {
+        switch (Architecture.getDeviceArchitecture()) {
+            case Architecture.ARCH_ARM:
+                return "armeabi-v7a";
+            case Architecture.ARCH_ARM64:
+                return "arm64-v8a";
+            case Architecture.ARCH_X86:
+                return "x86";
+            case Architecture.ARCH_X86_64:
+                return "x86_64";
+        }
+        throw new RuntimeException("Unknown CPU architecture!");
+    }
+
+    public void extractFromAar(File source) throws IOException {
+        try (FileInputStream fileInputStream = new FileInputStream(source)) {
+            try(ZipInputStream zipInputStream = new ZipInputStream(fileInputStream)) {
+                // Wrap the ZIP input stream into a non-closeable stream to
+                // avoid it being closed by processEntry()
+                NonCloseableInputStream entryCopyStream = new NonCloseableInputStream(zipInputStream);
+
+                while(true) {
+                    ZipEntry entry = zipInputStream.getNextEntry();
+                    if(entry == null) break;
+
+                    String entryName = entry.getName();
+                    if(!entryName.startsWith(mLibraryLocation) || entry.isDirectory()) continue;
+                    // Entry name is actually the full path, so we need to strip the path before extraction
+                    entryName = FileUtils.getFileName(entryName);
+                    // getFileName may make the file name null, avoid that case.
+                    if(entryName == null || LIBRARY_BLACKLIST.contains(entryName)) continue;
+
+                    processEntry(entryCopyStream, entry, new File(mDestinationDir, entryName));
+                }
+            }
+        }
+    }
+
+    private static long fileCrc32(File target) throws IOException {
+        try(FileInputStream fileInputStream = new FileInputStream(target)) {
+            CRC32 crc32 = new CRC32();
+            byte[] buffer = new byte[1024];
+            int len;
+            while((len = fileInputStream.read(buffer)) != -1) {
+                crc32.update(buffer, 0, len);
+            }
+            return crc32.getValue();
+        }
+    }
+
+    private void processEntry(InputStream sourceStream, ZipEntry zipEntry, File entryDestination) throws IOException {
+        if(entryDestination.exists()) {
+            long expectedSize = zipEntry.getSize();
+            long expectedCrc32 = zipEntry.getCrc();
+            long realSize = entryDestination.length();
+            long realCrc32 = fileCrc32(entryDestination);
+            // File in archive is the same as the local one, don't extract
+            if(realSize == expectedSize && realCrc32 == expectedCrc32) return;
+        }
+        // copyInputStreamToFile copies the stream to a file and then closes it.
+        org.apache.commons.io.FileUtils.copyInputStreamToFile(sourceStream, entryDestination);
+    }
+
+
+    private static class NonCloseableInputStream extends FilterInputStream {
+
+        protected NonCloseableInputStream(InputStream in) {
+            super(in);
+        }
+
+        @Override
+        public void close() {
+            // Do nothing (the point of this class)
+        }
+    }
+}

app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/utils/FileUtils.java

@@ -24,6 +24,17 @@ public class FileUtils {
         return pathOrUrl.substring(lastSlashIndex);
     }
 
+    /**
+     * Remove the extension (all text after the last dot) from a path/URL string.
+     * @param pathOrUrl the path or the URL of the file
+     * @return the input with the extension removed
+     */
+    public static String removeExtension(String pathOrUrl) {
+        int lastDotIndex = pathOrUrl.lastIndexOf('.');
+        if(lastDotIndex == -1) return pathOrUrl;
+        return pathOrUrl.substring(0, lastDotIndex);
+    }
+
     /**
      * Ensure that a directory exists, is a directory and is writable.
      * @param targetFile the directory to check

app_pojavlauncher/src/main/res/values/strings.xml

@@ -429,4 +429,5 @@
     <string name="bta_installer_available_versions">Supported BTA versions</string>
     <string name="bta_installer_untested_versions">Untested BTA versions</string>
     <string name="bta_installer_nightly_versions">Nightly BTA versions</string>
+    <string name="newdl_extracting_native_libraries">Extracting native libraries (%d/%d)</string>
 </resources>