Feat[mcdl]: better Minecraft downloader

Now it does not look like dog poo
This commit is contained in:
artdeell 2023-12-17 13:14:29 +03:00 committed by Maksim Belov
parent 87ec536112
commit dec824af97
7 changed files with 406 additions and 413 deletions

View File

@ -41,6 +41,7 @@ import net.kdt.pojavlaunch.services.ProgressServiceKeeper;
import net.kdt.pojavlaunch.tasks.AsyncMinecraftDownloader;
import net.kdt.pojavlaunch.tasks.AsyncVersionList;
import net.kdt.pojavlaunch.lifecycle.ContextAwareDoneListener;
import net.kdt.pojavlaunch.tasks.NewMinecraftDownloader;
import net.kdt.pojavlaunch.utils.NotificationUtils;
import net.kdt.pojavlaunch.value.launcherprofiles.LauncherProfiles;
import net.kdt.pojavlaunch.value.launcherprofiles.MinecraftProfile;
@ -132,7 +133,7 @@ public class LauncherActivity extends BaseActivity {
}
String normalizedVersionId = AsyncMinecraftDownloader.normalizeVersionId(prof.lastVersionId);
JMinecraftVersionList.Version mcVersion = AsyncMinecraftDownloader.getListedVersion(normalizedVersionId);
new AsyncMinecraftDownloader().start(
new NewMinecraftDownloader().start(
this,
mcVersion,
normalizedVersionId,

View File

@ -161,7 +161,7 @@ public final class Tools {
DIR_HOME_LIBRARY = DIR_GAME_NEW + "/libraries";
DIR_HOME_CRASH = DIR_GAME_NEW + "/crash-reports";
ASSETS_PATH = DIR_GAME_NEW + "/assets";
OBSOLETE_RESOURCES_PATH= DIR_GAME_NEW + "/resources";
OBSOLETE_RESOURCES_PATH = DIR_GAME_NEW + "/resources";
CTRLMAP_PATH = DIR_GAME_HOME + "/controlmap";
CTRLDEF_FILE = DIR_GAME_HOME + "/controlmap/default.json";
NATIVE_LIB_DIR = ctx.getApplicationInfo().nativeLibraryDir;
@ -676,7 +676,7 @@ public final class Tools {
return true; // allow if none match
}
private static void preProcessLibraries(DependentLibrary[] libraries) {
public static void preProcessLibraries(DependentLibrary[] libraries) {
for (int i = 0; i < libraries.length; i++) {
DependentLibrary libItem = libraries[i];
String[] version = libItem.name.split(":")[2].split("\\.");
@ -856,6 +856,10 @@ public final class Tools {
return read(new FileInputStream(path));
}
public static String read(File path) throws IOException {
return read(new FileInputStream(path));
}
public static void write(String path, String content) throws IOException {
File file = new File(path);
FileUtils.ensureParentDirectory(file);

View File

@ -53,6 +53,26 @@ public class DownloadMirror {
DownloadUtils.downloadFileMonitored(urlInput, outputFile, buffer, monitor);
}
/**
* Download a file with the current mirror. If the file is missing on the mirror,
* fall back to the official source.
* @param downloadClass Class of the download. Can either be DOWNLOAD_CLASS_LIBRARIES,
* DOWNLOAD_CLASS_METADATA or DOWNLOAD_CLASS_ASSETS
* @param urlInput The original (Mojang) URL for the download
* @param outputFile The output file for the download
*/
public static void downloadFileMirrored(int downloadClass, String urlInput, File outputFile) throws IOException {
try {
DownloadUtils.downloadFile(getMirrorMapping(downloadClass, urlInput),
outputFile);
return;
}catch (FileNotFoundException e) {
Log.w("DownloadMirror", "Cannot find the file on the mirror", e);
Log.i("DownloadMirror", "Failling back to default source");
}
DownloadUtils.downloadFile(urlInput, outputFile);
}
/**
* Check if the current download source is a mirror and not an official source.
* @return true if the source is a mirror, false otherwise

View File

@ -7,6 +7,7 @@ import net.kdt.pojavlaunch.R;
import net.kdt.pojavlaunch.Tools;
import net.kdt.pojavlaunch.progresskeeper.ProgressKeeper;
import net.kdt.pojavlaunch.tasks.AsyncMinecraftDownloader;
import net.kdt.pojavlaunch.tasks.NewMinecraftDownloader;
import net.kdt.pojavlaunch.utils.DownloadUtils;
import java.io.File;
@ -88,7 +89,7 @@ public class OptiFineDownloadTask implements Runnable, Tools.DownloaderFeedback,
if(minecraftJsonVersion == null) return false;
try {
synchronized (mMinecraftDownloadLock) {
new AsyncMinecraftDownloader().start(null, minecraftJsonVersion, minecraftVersion, this);
new NewMinecraftDownloader().start(null, minecraftJsonVersion, minecraftVersion, this);
mMinecraftDownloadLock.wait();
}
}catch (InterruptedException e) {

View File

@ -1,396 +1,10 @@
package net.kdt.pojavlaunch.tasks;
import static net.kdt.pojavlaunch.PojavApplication.sExecutorService;
import static net.kdt.pojavlaunch.Tools.BYTE_TO_MB;
import static net.kdt.pojavlaunch.mirrors.DownloadMirror.downloadFileMirrored;
import android.app.Activity;
import android.util.Log;
import androidx.annotation.NonNull;
import com.kdt.mcgui.ProgressLayout;
import net.kdt.pojavlaunch.JAssetInfo;
import net.kdt.pojavlaunch.JAssets;
import net.kdt.pojavlaunch.JMinecraftVersionList;
import net.kdt.pojavlaunch.JRE17Util;
import net.kdt.pojavlaunch.R;
import net.kdt.pojavlaunch.Tools;
import net.kdt.pojavlaunch.extra.ExtraConstants;
import net.kdt.pojavlaunch.extra.ExtraCore;
import net.kdt.pojavlaunch.mirrors.DownloadMirror;
import net.kdt.pojavlaunch.mirrors.MirrorTamperedException;
import net.kdt.pojavlaunch.prefs.LauncherPreferences;
import net.kdt.pojavlaunch.progresskeeper.ProgressKeeper;
import net.kdt.pojavlaunch.value.DependentLibrary;
import net.kdt.pojavlaunch.value.MinecraftClientInfo;
import net.kdt.pojavlaunch.value.MinecraftLibraryArtifact;
import org.apache.commons.io.IOUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
public class AsyncMinecraftDownloader {
public static final String MINECRAFT_RES = "https://resources.download.minecraft.net/";
/* Allows each downloading thread to have its own RECYCLED buffer */
private final ConcurrentHashMap<Thread, byte[]> mThreadBuffers = new ConcurrentHashMap<>(5);
public void start(Activity activity, JMinecraftVersionList.Version version,
String realVersion, // this was there for a reason
@NonNull DoneListener listener) {
sExecutorService.execute(() -> {
try {
downloadGame(activity, version, realVersion);
listener.onDownloadDone();
}catch (DownloaderException e) {
listener.onDownloadFailed(e.getCause());
}
});
}
/* we do the throws DownloaderException thing to avoid blanket-catching Exception as a form of anti-lazy-developer protection */
private void downloadGame(Activity activity, JMinecraftVersionList.Version verInfo, String versionName) throws DownloaderException {
final String downVName = "/" + versionName + "/" + versionName;
//Downloading libraries
String minecraftMainJar = Tools.DIR_HOME_VERSION + downVName + ".jar";
JAssets assets = null;
try {
File verJsonDir = new File(Tools.DIR_HOME_VERSION + downVName + ".json");
if (verInfo != null && verInfo.url != null) {
downloadVersionJson(versionName, verJsonDir, verInfo);
}
JMinecraftVersionList.Version originalVersion = Tools.getVersionInfo(versionName, true);
if(Tools.isValidString(originalVersion.inheritsFrom)) {
Log.i("Downloader", "probe: inheritsFrom="+originalVersion.inheritsFrom);
String version = originalVersion.inheritsFrom;
String downName = Tools.DIR_HOME_VERSION+"/"+version+"/"+version+".json";
JMinecraftVersionList.Version listedVersion = getListedVersion(originalVersion.inheritsFrom);
if(listedVersion != null) {
Log.i("Downloader", "probe: verifying "+version);
downloadVersionJson(version, new File(downName), listedVersion);
}else{
Log.i("Downloader", "failed to test source version before downloading.");
Log.i("Downloader", "Inheriting from versions not in the Mojang list?");
Log.i("Downloader", "If so, feel free to open a PR at our GitHub repository to add this feature!");
}
}
verInfo = Tools.getVersionInfo(versionName);
// THIS one function need the activity in the case of an error
if(activity != null && !JRE17Util.installNewJreIfNeeded(activity, verInfo)){
ProgressLayout.clearProgress(ProgressLayout.DOWNLOAD_MINECRAFT);
throw new DownloaderException(new Exception(activity.getString(R.string.exception_failed_to_unpack_jre17)));
}
try {
if(verInfo.assets != null)
assets = downloadIndex(verInfo, new File(Tools.ASSETS_PATH, "indexes/" + verInfo.assets + ".json"));
} catch (IOException e) {
Log.e("AsyncMcDownloader", e.toString(), e);
throw new DownloaderException(e);
}
File outLib;
// Patch the Log4J RCE (CVE-2021-44228)
if (verInfo.logging != null) {
outLib = new File(Tools.DIR_DATA + "/security", verInfo.logging.client.file.id.replace("client", "log4j-rce-patch"));
boolean useLocal = outLib.exists();
if (!useLocal) {
outLib = new File(Tools.DIR_GAME_NEW, verInfo.logging.client.file.id);
}
if (outLib.exists() && !useLocal) {
if(LauncherPreferences.PREF_CHECK_LIBRARY_SHA) {
if(!Tools.compareSHA1(outLib,verInfo.logging.client.file.sha1)) {
outLib.delete();
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.dl_library_sha_fail,verInfo.logging.client.file.id);
}else{
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.dl_library_sha_pass,verInfo.logging.client.file.id);
}
} else if (outLib.length() != verInfo.logging.client.file.size) {
// force updating anyways
outLib.delete();
}
}
if (!outLib.exists()) {
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.mcl_launch_downloading, verInfo.logging.client.file.id);
JMinecraftVersionList.Version finalVerInfo = verInfo;
downloadFileMirrored(
DownloadMirror.DOWNLOAD_CLASS_METADATA,
verInfo.logging.client.file.url, outLib, getByteBuffer(),
(curr, max) -> ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT,
(int) Math.max((float)curr/max*100,0), R.string.mcl_launch_downloading_progress, finalVerInfo.logging.client.file.id, curr/BYTE_TO_MB, max/BYTE_TO_MB)
);
}
}
for (final DependentLibrary libItem : verInfo.libraries) {
if(libItem.name.startsWith("org.lwjgl")){ //Black list
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, "Ignored " + libItem.name);
continue;
}
String libArtifact = Tools.artifactToPath(libItem);
outLib = new File(Tools.DIR_HOME_LIBRARY + "/" + libArtifact);
outLib.getParentFile().mkdirs();
if (!outLib.exists()) {
downloadLibrary(libItem,libArtifact,outLib);
}else{
if(libItem.downloads != null && libItem.downloads.artifact != null && libItem.downloads.artifact.sha1 != null && !libItem.downloads.artifact.sha1.isEmpty()) {
if(!Tools.compareSHA1(outLib,libItem.downloads.artifact.sha1)) {
outLib.delete();
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.dl_library_sha_fail,libItem.name);
downloadLibrary(libItem,libArtifact,outLib);
}else{
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.dl_library_sha_pass,libItem.name);
}
}else{
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.dl_library_sha_unknown,libItem.name);
}
}
}
File minecraftMainFile = new File(minecraftMainJar);
Log.i("Downloader", "originalVersion.inheritsFrom="+originalVersion.inheritsFrom);
Log.i("Downloader", "originalVersion.downloads="+originalVersion.downloads);
MinecraftClientInfo originalClientInfo;
if(originalVersion.inheritsFrom == null) {
if (originalVersion.downloads != null && (originalClientInfo = originalVersion.downloads.get("client")) != null) {
verifyAndDownloadMainJar(originalClientInfo.url, originalClientInfo.sha1, minecraftMainFile);
}
}else if(!minecraftMainFile.exists() || minecraftMainFile.length() == 0) {
File minecraftSourceFile = new File(Tools.DIR_HOME_VERSION + "/" + verInfo.id + "/" + verInfo.id + ".jar");
MinecraftClientInfo inheritedClientInfo;
if(verInfo.downloads != null && (inheritedClientInfo = verInfo.downloads.get("client")) != null) {
verifyAndDownloadMainJar(inheritedClientInfo.url, inheritedClientInfo.sha1, minecraftSourceFile);
}
if(minecraftSourceFile.exists()) {
FileInputStream is = new FileInputStream(minecraftSourceFile);
FileOutputStream os = new FileOutputStream(minecraftMainFile);
IOUtils.copy(is, os);
is.close();
os.close();
}
}
} catch (DownloaderException e) {
ProgressLayout.clearProgress(ProgressLayout.DOWNLOAD_MINECRAFT);
throw e;
} catch (Throwable e) {
Log.e("AsyncMcDownloader", e.toString(),e );
ProgressLayout.clearProgress(ProgressLayout.DOWNLOAD_MINECRAFT);
throw new DownloaderException(e);
}
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.mcl_launch_cleancache);
new File(Tools.DIR_HOME_VERSION).mkdir();
for (File f : new File(Tools.DIR_HOME_VERSION).listFiles()) {
if(f.getName().endsWith(".part")) {
Log.d(Tools.APP_NAME, "Cleaning cache: " + f);
f.delete();
}
}
try {
if(assets != null)
downloadAssets(assets, assets.mapToResources ? new File(Tools.OBSOLETE_RESOURCES_PATH) : new File(Tools.ASSETS_PATH));
} catch (Exception e) {
Log.e("AsyncMcDownloader", e.toString(), e);
ProgressLayout.clearProgress(ProgressLayout.DOWNLOAD_MINECRAFT);
throw new DownloaderException(e);
}
ProgressLayout.clearProgress(ProgressLayout.DOWNLOAD_MINECRAFT);
}
public void verifyAndDownloadMainJar(String url, String sha1, File destination) throws Exception{
while(!destination.exists() || (destination.exists() && !Tools.compareSHA1(destination, sha1))) downloadFileMirrored(DownloadMirror.DOWNLOAD_CLASS_LIBRARIES,
url,
destination, getByteBuffer(),
(curr, max) -> ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT,
(int) Math.max((float)curr/max*100,0), R.string.mcl_launch_downloading_progress, destination.getName(), curr/BYTE_TO_MB, max/BYTE_TO_MB));
}
public void downloadAssets(final JAssets assets, final File outputDir) {
LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>();
final ThreadPoolExecutor executor = new ThreadPoolExecutor(5, 5, 500, TimeUnit.MILLISECONDS, workQueue);
Log.i("AsyncMcDownloader","Assets begin time: " + System.currentTimeMillis());
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.mcl_launch_download_assets);
Map<String, JAssetInfo> assetsObjects = assets.objects;
int assetsSizeBytes = 0;
AtomicInteger downloadedSize = new AtomicInteger(0);
AtomicBoolean localInterrupt = new AtomicBoolean(false);
File objectsDir = new File(outputDir, "objects");
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.mcl_launch_downloading, "assets");
for(String assetKey : assetsObjects.keySet()) {
JAssetInfo asset = assetsObjects.get(assetKey);
assetsSizeBytes += asset.size;
String assetPath = asset.hash.substring(0, 2) + "/" + asset.hash;
File outFile = (assets.mapToResources || assets.virtual) ? new File(outputDir,"/"+assetKey) : new File(objectsDir , assetPath);
boolean skip = outFile.exists();// skip if the file exists
if(LauncherPreferences.PREF_CHECK_LIBRARY_SHA && skip)
skip = Tools.compareSHA1(outFile, asset.hash);
if(skip) {
downloadedSize.addAndGet(asset.size);
continue;
}
if(outFile.exists())
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.dl_library_sha_fail, assetKey);
executor.execute(()->{
try {
downloadAssetFile(outFile, assetPath, downloadedSize);
}catch (IOException e) {
Log.e("AsyncMcManager", e.toString());
localInterrupt.set(true);
}
});
}
executor.shutdown();
try {
Log.i("AsyncMcDownloader","Queue size: " + workQueue.size());
while ((!executor.awaitTermination(1000, TimeUnit.MILLISECONDS))&&(!localInterrupt.get()) /*&&mActivity.mIsAssetsProcessing*/) {
int DLSize = downloadedSize.get();
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT,(int) Math.max((float) DLSize/assetsSizeBytes*100, 0),
R.string.mcl_launch_downloading_progress, "assets", (float)DLSize/BYTE_TO_MB, (float)assetsSizeBytes/BYTE_TO_MB);
}
executor.shutdownNow();
while (!executor.awaitTermination(250, TimeUnit.MILLISECONDS)) {}
Log.i("AsyncMcDownloader","Fully shut down!");
}catch(InterruptedException e) {
Log.e("AsyncMcDownloader", e.toString());
}
Log.i("AsyncMcDownloader", "Assets end time: " + System.currentTimeMillis());
}
public void downloadAssetFile(File outFile, String assetPath, AtomicInteger downloadCounter) throws IOException {
downloadFileMirrored(DownloadMirror.DOWNLOAD_CLASS_ASSETS, MINECRAFT_RES + assetPath, outFile, getByteBuffer(),
new Tools.DownloaderFeedback() {
int prevCurr;
@Override
public void updateProgress(int curr, int max) {
downloadCounter.addAndGet(curr - prevCurr);
prevCurr = curr;
}
}
);
}
protected void downloadLibrary(DependentLibrary libItem, String libArtifact, File outLib) throws Throwable{
String libPathURL;
boolean skipIfFailed = false;
if (libItem.downloads == null || libItem.downloads.artifact == null) {
MinecraftLibraryArtifact artifact = new MinecraftLibraryArtifact();
artifact.url = (libItem.url == null
? "https://libraries.minecraft.net/"
: libItem.url.replace("http://","https://")) + libArtifact;
libItem.downloads = new DependentLibrary.LibraryDownloads(artifact);
skipIfFailed = true;
}
try {
libPathURL = libItem.downloads.artifact.url;
boolean isFileGood = false;
byte timesChecked=0;
while(!isFileGood) {
timesChecked++;
if(timesChecked > 5) throw new RuntimeException("Library download failed after 5 retries");
downloadFileMirrored(DownloadMirror.DOWNLOAD_CLASS_LIBRARIES, libPathURL, outLib, getByteBuffer(),
(curr, max) -> ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT,
(int) Math.max((float)curr/max*100,0), R.string.mcl_launch_downloading_progress, outLib.getName(), curr/BYTE_TO_MB, max/BYTE_TO_MB)
);
isFileGood = (libItem.downloads.artifact.sha1 == null
|| LauncherPreferences.PREF_CHECK_LIBRARY_SHA)
|| Tools.compareSHA1(outLib,libItem.downloads.artifact.sha1);
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0,
isFileGood ? R.string.dl_library_sha_pass : R.string.dl_library_sha_unknown
,outLib.getName());
}
} catch (Throwable th) {
Log.e("AsyncMcDownloader", th.toString(), th);
if (!skipIfFailed) {
throw th;
} else {
th.printStackTrace();
}
}
}
public JAssets downloadIndex(JMinecraftVersionList.Version version, File output) throws IOException {
if (!output.exists()) {
output.getParentFile().mkdirs();
downloadFileMirrored(DownloadMirror.DOWNLOAD_CLASS_METADATA, version.assetIndex != null
? version.assetIndex.url
: "https://s3.amazonaws.com/Minecraft.Download/indexes/" + version.assets + ".json", output, null,
(curr, max) -> ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT,
(int) Math.max((float)curr/max*100,0), R.string.mcl_launch_downloading_progress, output.getName(), curr/BYTE_TO_MB, max/BYTE_TO_MB)
);
}
return Tools.GLOBAL_GSON.fromJson(Tools.read(output.getAbsolutePath()), JAssets.class);
}
public void downloadVersionJson(String versionName, File verJsonDir, JMinecraftVersionList.Version verInfo) throws IOException, DownloaderException {
if(!LauncherPreferences.PREF_CHECK_LIBRARY_SHA) Log.w("Chk","Checker is off");
boolean isManifestGood = verifyManifest(verJsonDir, verInfo);
byte retryCount = 0;
while(!isManifestGood && retryCount < 5) {
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.mcl_launch_downloading, versionName + ".json");
downloadFileMirrored(DownloadMirror.DOWNLOAD_CLASS_METADATA, verInfo.url, verJsonDir, getByteBuffer(),
(curr, max) -> ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT,
(int) Math.max((float)curr/max*100,0), R.string.mcl_launch_downloading_progress, versionName + ".json", curr/BYTE_TO_MB, max/BYTE_TO_MB)
);
isManifestGood = verifyManifest(verJsonDir, verInfo);
retryCount++;
// Always do the first verification. But skip all the errors from further ones.
if(!LauncherPreferences.PREF_VERIFY_MANIFEST) return;
}
if(!isManifestGood) {
if(DownloadMirror.isMirrored()) throw new DownloaderException(new MirrorTamperedException());
else throw new DownloaderException(new IOException("Manifest check failed after 5 tries"));
}
}
private boolean verifyManifest(File verJsonDir, JMinecraftVersionList.Version verInfo) {
return verJsonDir.exists()
&& (!LauncherPreferences.PREF_CHECK_LIBRARY_SHA
|| verInfo.sha1 == null
|| Tools.compareSHA1(verJsonDir, verInfo.sha1));
}
public static String normalizeVersionId(String versionString) {
JMinecraftVersionList versionList = (JMinecraftVersionList) ExtraCore.getValue(ExtraConstants.RELEASE_TABLE);
if(versionList == null || versionList.versions == null) return versionString;
@ -408,31 +22,8 @@ public class AsyncMinecraftDownloader {
return null;
}
/**@return A byte buffer bound to a thread, useful to recycle it across downloads */
private byte[] getByteBuffer(){
byte[] buffer = mThreadBuffers.get(Thread.currentThread());
if (buffer == null){
buffer = new byte[8192];
mThreadBuffers.put(Thread.currentThread(), buffer);
}
return buffer;
}
public interface DoneListener{
void onDownloadDone();
void onDownloadFailed(Throwable throwable);
}
private static class DownloaderException extends Exception {
public DownloaderException() {}
public DownloaderException(Throwable e) {
super(e);
}
}
}

View File

@ -0,0 +1,373 @@
package net.kdt.pojavlaunch.tasks;
import static net.kdt.pojavlaunch.PojavApplication.sExecutorService;
import android.app.Activity;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import com.kdt.mcgui.ProgressLayout;
import net.kdt.pojavlaunch.JAssetInfo;
import net.kdt.pojavlaunch.JAssets;
import net.kdt.pojavlaunch.JMinecraftVersionList;
import net.kdt.pojavlaunch.JRE17Util;
import net.kdt.pojavlaunch.R;
import net.kdt.pojavlaunch.Tools;
import net.kdt.pojavlaunch.mirrors.DownloadMirror;
import net.kdt.pojavlaunch.prefs.LauncherPreferences;
import net.kdt.pojavlaunch.utils.DownloadUtils;
import net.kdt.pojavlaunch.utils.FileUtils;
import net.kdt.pojavlaunch.value.DependentLibrary;
import net.kdt.pojavlaunch.value.MinecraftClientInfo;
import net.kdt.pojavlaunch.value.MinecraftLibraryArtifact;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
// TODO: implement mirror tamper checking.
public class NewMinecraftDownloader {
public static final String MINECRAFT_RES = "https://resources.download.minecraft.net/";
private AtomicReference<Exception> mThrownDownloaderException;
private ArrayList<DownloaderTask> mScheduledDownloadTasks;
private AtomicLong mDownloadFileCounter;
private AtomicLong mDownloadSizeCounter;
private long mDownloadFileCount;
private static final ThreadLocal<byte[]> sThreadLocalDownloadBuffer = new ThreadLocal<>();
/**
* Start the game version download process on the global executor service.
* @param activity Activity, used for automatic installation of JRE 17 if needed
* @param version The JMinecraftVersionList.Version from the version list, if available
* @param realVersion The version ID (necessary)
* @param listener The download status listener
*/
public void start(@Nullable Activity activity, @Nullable JMinecraftVersionList.Version version,
@NonNull String realVersion, // this was there for a reason
@NonNull AsyncMinecraftDownloader.DoneListener listener) {
sExecutorService.execute(() -> {
try {
downloadGame(activity, version, realVersion);
listener.onDownloadDone();
}catch (Exception e) {
listener.onDownloadFailed(e);
}
ProgressLayout.clearProgress(ProgressLayout.DOWNLOAD_MINECRAFT);
});
}
/**
* Download the game version.
* @param activity Activity, used for automatic installation of JRE 17 if needed
* @param verInfo The JMinecraftVersionList.Version from the version list, if available
* @param versionName The version ID (necessary)
* @throws Exception when an exception occurs in the function body or in any of the downloading threads.
*/
private void downloadGame(Activity activity, JMinecraftVersionList.Version verInfo, String versionName) throws Exception {
// Put up a dummy progress line, for the activity to start the service and do all the other necessary
// work to keep the launcher alive. We will replace this line when we will start downloading stuff.
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.newdl_starting);
mScheduledDownloadTasks = new ArrayList<>();
mDownloadFileCounter = new AtomicLong(0);
mDownloadSizeCounter = new AtomicLong(0);
mThrownDownloaderException = new AtomicReference<>(null);
if(!downloadAndProcessMetadata(activity, verInfo, versionName)) {
throw new RuntimeException(activity.getString(R.string.exception_failed_to_unpack_jre17));
}
ArrayBlockingQueue<Runnable> taskQueue =
new ArrayBlockingQueue<>(mScheduledDownloadTasks.size(), false);
ThreadPoolExecutor downloaderPool =
new ThreadPoolExecutor(4, 4, 500, TimeUnit.MILLISECONDS, taskQueue);
// I have tried pre-filling the queue directly instead of doing this, but it didn't work.
// What a shame.
for(DownloaderTask scheduledTask : mScheduledDownloadTasks) downloaderPool.execute(scheduledTask);
downloaderPool.shutdown();
try {
while (mThrownDownloaderException.get() == null &&
!downloaderPool.awaitTermination(33, TimeUnit.MILLISECONDS)) {
long dlFileCounter = mDownloadFileCounter.get();
int progress = (int)((dlFileCounter * 100L) / mDownloadFileCount);
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, progress,
R.string.newdl_downloading_game_files, dlFileCounter,
mDownloadFileCount, (double)mDownloadSizeCounter.get() / (1024d * 1024d));
}
Exception thrownException = mThrownDownloaderException.get();
if(thrownException != null) throw thrownException;
}catch (InterruptedException e) {
// Interrupted while waiting, which means that the download was cancelled.
// Kill all downloading threads immediately, and ignore any exceptions thrown by them
downloaderPool.shutdownNow();
}
}
private File createGameJsonPath(String versionId) {
return new File(Tools.DIR_HOME_VERSION, versionId + File.separator + versionId + ".json");
}
private File createGameJarPath(String versionId) {
return new File(Tools.DIR_HOME_VERSION, versionId + File.separator + versionId + ".jar");
}
private File downloadGameJson(JMinecraftVersionList.Version verInfo) throws IOException {
File targetFile = createGameJsonPath(verInfo.id);
if(verInfo.sha1 == null && targetFile.canRead() && targetFile.isFile())
return targetFile;
FileUtils.ensureParentDirectory(targetFile);
DownloadUtils.ensureSha1(targetFile, LauncherPreferences.PREF_VERIFY_MANIFEST ? verInfo.sha1 : null, ()-> {
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0,
R.string.newdl_downloading_metadata, targetFile.getName());
DownloadMirror.downloadFileMirrored(DownloadMirror.DOWNLOAD_CLASS_METADATA, verInfo.url, targetFile);
return null;
});
return targetFile;
}
private JAssets downloadAssetsIndex(JMinecraftVersionList.Version verInfo) throws IOException{
JMinecraftVersionList.AssetIndex assetIndex = verInfo.assetIndex;
if(assetIndex == null || verInfo.assets == null) return null;
File targetFile = new File(Tools.ASSETS_PATH, "indexes"+ File.separator + verInfo.assets + ".json");
FileUtils.ensureParentDirectory(targetFile);
DownloadUtils.ensureSha1(targetFile, assetIndex.sha1, ()-> {
ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0,
R.string.newdl_downloading_metadata, targetFile.getName());
DownloadMirror.downloadFileMirrored(DownloadMirror.DOWNLOAD_CLASS_METADATA, assetIndex.url, targetFile);
return null;
});
return Tools.GLOBAL_GSON.fromJson(Tools.read(targetFile), JAssets.class);
}
private MinecraftClientInfo getClientInfo(JMinecraftVersionList.Version verInfo) {
Map<String, MinecraftClientInfo> downloads = verInfo.downloads;
if(downloads == null) return null;
return downloads.get("client");
}
/**
* Download (if necessary) and process a version's metadata, scheduling all downloads that this
* version needs.
* @param activity Activity, used for automatic installation of JRE 17 if needed
* @param verInfo The JMinecraftVersionList.Version from the version list, if available
* @param versionName The version ID (necessary)
* @return false if JRE17 installation failed, true otherwise
* @throws IOException if the download of any of the metadata files fails
*/
private boolean downloadAndProcessMetadata(Activity activity, JMinecraftVersionList.Version verInfo, String versionName) throws IOException {
File versionJsonFile;
if(verInfo != null) versionJsonFile = downloadGameJson(verInfo);
else versionJsonFile = createGameJsonPath(versionName);
if(versionJsonFile.canRead()) {
verInfo = Tools.GLOBAL_GSON.fromJson(Tools.read(versionJsonFile), JMinecraftVersionList.Version.class);
} else {
throw new IOException("Unable to read Version JSON for version "+versionName);
}
if(activity != null && !JRE17Util.installNewJreIfNeeded(activity, verInfo)){
return false;
}
JAssets assets = downloadAssetsIndex(verInfo);
if(assets != null) scheduleAssetDownloads(assets);
MinecraftClientInfo minecraftClientInfo = getClientInfo(verInfo);
if(minecraftClientInfo != null) {
growDownloadList(1);
scheduleDownload(createGameJarPath(verInfo.id),
DownloadMirror.DOWNLOAD_CLASS_LIBRARIES,
minecraftClientInfo.url,
minecraftClientInfo.sha1,
minecraftClientInfo.size,
false
);
}
if(verInfo.libraries != null) scheduleLibraryDownloads(verInfo.libraries);
if(verInfo.logging != null) scheduleLoggingAssetDownloadIfNeeded(verInfo.logging);
if(Tools.isValidString(verInfo.inheritsFrom)) {
JMinecraftVersionList.Version inheritedVersion = AsyncMinecraftDownloader.getListedVersion(verInfo.inheritsFrom);
// Infinite inheritance !?! :noway:
return downloadAndProcessMetadata(activity, inheritedVersion, verInfo.inheritsFrom);
}
return true;
}
private void growDownloadList(int addedElementCount) {
mScheduledDownloadTasks.ensureCapacity(mScheduledDownloadTasks.size() + addedElementCount);
}
private void scheduleDownload(File targetFile, int downloadClass, String url, String sha1,
long size, boolean skipIfFailed) {
mDownloadFileCount++;
mScheduledDownloadTasks.add(
new DownloaderTask(targetFile, downloadClass, url, sha1, size, skipIfFailed)
);
}
private void scheduleLibraryDownloads(DependentLibrary[] dependentLibraries) {
Tools.preProcessLibraries(dependentLibraries);
growDownloadList(dependentLibraries.length);
for(DependentLibrary dependentLibrary : dependentLibraries) {
String libArtifactPath = Tools.artifactToPath(dependentLibrary);
String sha1 = null, url = null;
long size = 0;
boolean skipIfFailed = false;
if(dependentLibrary.downloads != null && dependentLibrary.downloads.artifact != null) {
MinecraftLibraryArtifact artifact = dependentLibrary.downloads.artifact;
sha1 = artifact.sha1;
url = artifact.url;
size = artifact.size;
}
if(url == null) {
url = (dependentLibrary.url == null
? "https://libraries.minecraft.net/"
: dependentLibrary.url.replace("http://","https://")) + libArtifactPath;
skipIfFailed = true;
}
if(!LauncherPreferences.PREF_CHECK_LIBRARY_SHA) sha1 = null;
scheduleDownload(new File(Tools.DIR_HOME_LIBRARY, libArtifactPath),
DownloadMirror.DOWNLOAD_CLASS_LIBRARIES,
url, sha1, size, skipIfFailed
);
}
}
private void scheduleAssetDownloads(JAssets assets) {
Map<String, JAssetInfo> assetObjects = assets.objects;
if(assetObjects == null) return;
Set<String> assetNames = assetObjects.keySet();
growDownloadList(assetNames.size());
for(String asset : assetNames) {
JAssetInfo assetInfo = assetObjects.get(asset);
if(assetInfo == null) continue;
File targetFile;
String hashedPath = assetInfo.hash.substring(0, 2) + File.separator + assetInfo.hash;
if(assets.virtual || assets.mapToResources) {
targetFile = new File(Tools.OBSOLETE_RESOURCES_PATH, asset);
} else {
targetFile = new File(Tools.ASSETS_PATH, "objects" + File.separator + hashedPath);
}
String sha1 = LauncherPreferences.PREF_CHECK_LIBRARY_SHA ? assetInfo.hash : null;
scheduleDownload(targetFile,
DownloadMirror.DOWNLOAD_CLASS_ASSETS,
MINECRAFT_RES + hashedPath,
sha1,
assetInfo.size,
false);
}
}
private void scheduleLoggingAssetDownloadIfNeeded(JMinecraftVersionList.LoggingConfig loggingConfig) {
if(loggingConfig.client == null || loggingConfig.client.file == null) return;
JMinecraftVersionList.FileProperties loggingFileProperties = loggingConfig.client.file;
File internalLoggingConfig = new File(Tools.DIR_DATA + File.separator + "security",
loggingFileProperties.id.replace("client", "log4j-rce-patch"));
if(internalLoggingConfig.exists()) return;
File destination = new File(Tools.DIR_GAME_NEW, loggingFileProperties.id);
scheduleDownload(destination,
DownloadMirror.DOWNLOAD_CLASS_LIBRARIES,
loggingFileProperties.url,
loggingFileProperties.sha1,
loggingFileProperties.size,
false);
}
private static byte[] getLocalBuffer() {
byte[] tlb = sThreadLocalDownloadBuffer.get();
if(tlb != null) return tlb;
tlb = new byte[65535];
sThreadLocalDownloadBuffer.set(tlb);
return tlb;
}
class DownloaderTask implements Runnable, Tools.DownloaderFeedback {
private final File mTargetPath;
private final String mTargetUrl;
private String mTargetSha1;
private final int mDownloadClass;
private final boolean mSkipIfFailed;
private int mLastCurr;
private final long mDownloadSize;
DownloaderTask(File targetPath, int downloadClass, String targetUrl, String targetSha1,
long downloadSize, boolean skipIfFailed) {
this.mTargetPath = targetPath;
this.mTargetUrl = targetUrl;
this.mTargetSha1 = targetSha1;
this.mDownloadClass = downloadClass;
this.mDownloadSize = downloadSize;
this.mSkipIfFailed = skipIfFailed;
}
@Override
public void run() {
try {
runCatching();
}catch (Exception e) {
mThrownDownloaderException.set(e);
}
}
private void runCatching() throws Exception {
FileUtils.ensureParentDirectory(mTargetPath);
if(Tools.isValidString(mTargetSha1)) {
verifyFileSha1();
}else {
mTargetSha1 = null; // Nullify SHA1 as DownloadUtils.ensureSha1 only checks for null,
// not for string validity
if(mTargetPath.exists()) finishWithoutDownloading();
else downloadFile();
}
}
private void verifyFileSha1() throws Exception {
if(mTargetPath.isFile() && mTargetPath.canRead() && Tools.compareSHA1(mTargetPath, mTargetSha1)) {
finishWithoutDownloading();
} else {
// Rely on the download function to throw an IOE in case if the file is not
// writable/not a file/etc...
downloadFile();
}
}
private void downloadFile() throws Exception {
try {
DownloadUtils.ensureSha1(mTargetPath, mTargetSha1, () -> {
DownloadMirror.downloadFileMirrored(mDownloadClass, mTargetUrl, mTargetPath,
getLocalBuffer(), this);
return null;
});
}catch (Exception e) {
if(!mSkipIfFailed) throw e;
}
mDownloadFileCounter.incrementAndGet();
}
private void finishWithoutDownloading() {
mDownloadFileCounter.incrementAndGet();
mDownloadSizeCounter.addAndGet(mDownloadSize);
}
@Override
public void updateProgress(int curr, int max) {
mDownloadSizeCounter.addAndGet(curr - mLastCurr);
mLastCurr = curr;
}
}
}

View File

@ -368,4 +368,7 @@
<string name="preference_vsync_in_zink_title">Allow V-Sync with Zink</string>
<string name="preference_vsync_in_zink_description">Allows the launcher to use internal system APIs to enable V-Sync for Zink. Turn this off if your launcher suddenly crashes with Zink after a system update.</string>
<string name="exception_failed_to_unpack_jre17">Failed to install JRE 17</string>
<string name="newdl_starting">Reading game metadata…</string>
<string name="newdl_downloading_metadata">Downloading game metadata (%s)</string>
<string name="newdl_downloading_game_files">Downloading game files… (%d/%d, %.2f MB)</string>
</resources>