From 5d80d9baecd3892dd1bced2a46a2fb3c8b64d8f5 Mon Sep 17 00:00:00 2001 From: Maksim Belov <45949002+artdeell@users.noreply.github.com> Date: Mon, 30 Dec 2024 20:31:59 +0300 Subject: [PATCH] Feat[downloader]: downloader improvements (#6428) - Add download size queries through a HEAD request - Use the file size for progress instead of file count when all file size are available - Add download speed meter --- .../pojavlaunch/mirrors/DownloadMirror.java | 23 ++++++- .../tasks/MinecraftDownloader.java | 68 ++++++++++++++----- .../pojavlaunch/tasks/SpeedCalculator.java | 44 ++++++++++++ .../kdt/pojavlaunch/utils/DownloadUtils.java | 17 +++++ .../src/main/res/values/strings.xml | 3 +- 5 files changed, 136 insertions(+), 19 deletions(-) create mode 100644 app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/SpeedCalculator.java diff --git a/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/mirrors/DownloadMirror.java b/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/mirrors/DownloadMirror.java index d53b0617e..f24159a32 100644 --- a/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/mirrors/DownloadMirror.java +++ b/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/mirrors/DownloadMirror.java @@ -43,7 +43,7 @@ public class DownloadMirror { return; }catch (FileNotFoundException e) { Log.w("DownloadMirror", "Cannot find the file on the mirror", e); - Log.i("DownloadMirror", "Failling back to default source"); + Log.i("DownloadMirror", "Falling back to default source"); } DownloadUtils.downloadFileMonitored(urlInput, outputFile, buffer, monitor); } @@ -63,11 +63,30 @@ public class DownloadMirror { return; }catch (FileNotFoundException e) { Log.w("DownloadMirror", "Cannot find the file on the mirror", e); - Log.i("DownloadMirror", "Failling back to default source"); + Log.i("DownloadMirror", "Falling back to default source"); } DownloadUtils.downloadFile(urlInput, outputFile); } + /** + * Get the content length of a file on the current mirror. If the file is missing on the mirror, + * or the mirror does not give out the length, request the length from the original source + * @param downloadClass Class of the download. Can either be DOWNLOAD_CLASS_LIBRARIES, + * DOWNLOAD_CLASS_METADATA or DOWNLOAD_CLASS_ASSETS + * @param urlInput The original (Mojang) URL for the download + * @return the length of the file denoted by the URL in bytes, or -1 if not available + */ + public static long getContentLengthMirrored(int downloadClass, String urlInput) throws IOException { + long length = DownloadUtils.getContentLength(getMirrorMapping(downloadClass, urlInput)); + if(length < 1) { + Log.w("DownloadMirror", "Unable to get content length from mirror"); + Log.i("DownloadMirror", "Falling back to default source"); + return DownloadUtils.getContentLength(urlInput); + }else { + return length; + } + } + /** * Check if the current download source is a mirror and not an official source. * @return true if the source is a mirror, false otherwise diff --git a/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/MinecraftDownloader.java b/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/MinecraftDownloader.java index 5abaa57fd..95074cc80 100644 --- a/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/MinecraftDownloader.java +++ b/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/MinecraftDownloader.java @@ -37,14 +37,18 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; public class MinecraftDownloader { + private static final double ONE_MEGABYTE = (1024d * 1024d); public static final String MINECRAFT_RES = "https://resources.download.minecraft.net/"; private AtomicReference mDownloaderThreadException; private ArrayList mScheduledDownloadTasks; - private AtomicLong mDownloadFileCounter; - private AtomicLong mDownloadSizeCounter; - private long mDownloadFileCount; + private AtomicLong mProcessedFileCounter; + private AtomicLong mProcessedSizeCounter; // Total bytes of processed files (passed SHA1 or downloaded) + private AtomicLong mInternetUsageCounter; // How many bytes downloaded over Internet + private long mTotalFileCount; + private long mTotalSize; private File mSourceJarFile; // The source client JAR picked during the inheritance process private File mTargetJarFile; // The destination client JAR to which the source will be copied to. + private boolean mUseFileCounter; // Whether a file counter or a size counter should be used for progress private static final ThreadLocal sThreadLocalDownloadBuffer = new ThreadLocal<>(); @@ -80,12 +84,15 @@ public class MinecraftDownloader { // Put up a dummy progress line, for the activity to start the service and do all the other necessary // work to keep the launcher alive. We will replace this line when we will start downloading stuff. ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.newdl_starting); + SpeedCalculator speedCalculator = new SpeedCalculator(); mTargetJarFile = createGameJarPath(versionName); mScheduledDownloadTasks = new ArrayList<>(); - mDownloadFileCounter = new AtomicLong(0); - mDownloadSizeCounter = new AtomicLong(0); + mProcessedFileCounter = new AtomicLong(0); + mProcessedSizeCounter = new AtomicLong(0); + mInternetUsageCounter = new AtomicLong(0); mDownloaderThreadException = new AtomicReference<>(null); + mUseFileCounter = false; if(!downloadAndProcessMetadata(activity, verInfo, versionName)) { throw new RuntimeException(activity.getString(R.string.exception_failed_to_unpack_jre17)); @@ -104,11 +111,9 @@ public class MinecraftDownloader { try { while (mDownloaderThreadException.get() == null && !downloaderPool.awaitTermination(33, TimeUnit.MILLISECONDS)) { - long dlFileCounter = mDownloadFileCounter.get(); - int progress = (int)((dlFileCounter * 100L) / mDownloadFileCount); - ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, progress, - R.string.newdl_downloading_game_files, dlFileCounter, - mDownloadFileCount, (double)mDownloadSizeCounter.get() / (1024d * 1024d)); + double speed = speedCalculator.feed(mInternetUsageCounter.get()) / ONE_MEGABYTE; + if(mUseFileCounter) reportProgressFileCounter(speed); + else reportProgressSizeCounter(speed); } Exception thrownException = mDownloaderThreadException.get(); if(thrownException != null) { @@ -123,6 +128,23 @@ public class MinecraftDownloader { } } + private void reportProgressFileCounter(double speed) { + long dlFileCounter = mProcessedFileCounter.get(); + int progress = (int)((dlFileCounter * 100L) / mTotalFileCount); + ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, progress, + R.string.newdl_downloading_game_files, dlFileCounter, + mTotalFileCount, speed); + } + + private void reportProgressSizeCounter(double speed) { + long dlFileSize = mProcessedSizeCounter.get(); + double dlSizeMegabytes = (double) dlFileSize / ONE_MEGABYTE; + double dlTotalMegabytes = (double) mTotalSize / ONE_MEGABYTE; + int progress = (int)((dlFileSize * 100L) / mTotalSize); + ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, progress, + R.string.newdl_downloading_game_files_size, dlSizeMegabytes, dlTotalMegabytes, speed); + } + private File createGameJsonPath(String versionId) { return new File(Tools.DIR_HOME_VERSION, versionId + File.separator + versionId + ".json"); } @@ -233,7 +255,19 @@ public class MinecraftDownloader { private void scheduleDownload(File targetFile, int downloadClass, String url, String sha1, long size, boolean skipIfFailed) throws IOException { FileUtils.ensureParentDirectory(targetFile); - mDownloadFileCount++; + mTotalFileCount++; + if(size < 0) { + size = DownloadMirror.getContentLengthMirrored(downloadClass, url); + } + if(size < 0) { + // If we were unable to get the content length ourselves, we automatically fall back + // to tracking the progress using the file counter. + size = 0; + mUseFileCounter = true; + Log.i("MinecraftDownloader", "Failed to determine size of "+targetFile.getName()+", switching to file counter"); + }else { + mTotalSize += size; + } mScheduledDownloadTasks.add( new DownloaderTask(targetFile, downloadClass, url, sha1, size, skipIfFailed) ); @@ -401,18 +435,20 @@ public class MinecraftDownloader { }catch (Exception e) { if(!mSkipIfFailed) throw e; } - mDownloadFileCounter.incrementAndGet(); + mProcessedFileCounter.incrementAndGet(); } private void finishWithoutDownloading() { - mDownloadFileCounter.incrementAndGet(); - mDownloadSizeCounter.addAndGet(mDownloadSize); + mProcessedFileCounter.incrementAndGet(); + mProcessedSizeCounter.addAndGet(mDownloadSize); } @Override public void updateProgress(int curr, int max) { - mDownloadSizeCounter.addAndGet(curr - mLastCurr); - mLastCurr = curr; + int delta = curr - mLastCurr; + mProcessedSizeCounter.addAndGet(delta); + mInternetUsageCounter.addAndGet(delta); + mLastCurr = curr; } } } diff --git a/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/SpeedCalculator.java b/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/SpeedCalculator.java new file mode 100644 index 000000000..136b0c178 --- /dev/null +++ b/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/SpeedCalculator.java @@ -0,0 +1,44 @@ +package net.kdt.pojavlaunch.tasks; + +/** + * A simple class to calculate the average Internet speed using a simple moving average. + */ +public class SpeedCalculator { + private long mLastMillis; + private long mLastBytes; + private int mIndex; + private final double[] mPreviousInputs; + private double mSum; + + public SpeedCalculator() { + this(64); + } + + public SpeedCalculator(int averageDepth) { + mPreviousInputs = new double[averageDepth]; + } + + private double addToAverage(double speed) { + mSum -= mPreviousInputs[mIndex]; + mSum += speed; + mPreviousInputs[mIndex] = speed; + if(++mIndex == mPreviousInputs.length) mIndex = 0; + double dLength = mPreviousInputs.length; + return (mSum + (dLength / 2d)) / dLength; + } + + /** + * Update the current amount of bytes downloaded. + * @param bytes the new amount of bytes downloaded + * @return the current download speed in bytes per second + */ + public double feed(long bytes) { + long millis = System.currentTimeMillis(); + long deltaBytes = bytes - mLastBytes; + long deltaMillis = millis - mLastMillis; + mLastBytes = bytes; + mLastMillis = millis; + double speed = (double)deltaBytes / ((double)deltaMillis / 1000d); + return addToAverage(speed); + } +} diff --git a/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/utils/DownloadUtils.java b/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/utils/DownloadUtils.java index b4df71409..dbdbadaf0 100644 --- a/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/utils/DownloadUtils.java +++ b/app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/utils/DownloadUtils.java @@ -155,6 +155,23 @@ public class DownloadUtils { return result; } + /** + * Get the content length for a given URL. + * @param url the URL to get the length for + * @return the length in bytes or -1 if not available + * @throws IOException if an I/O error occurs. + */ + public static long getContentLength(String url) throws IOException { + HttpURLConnection urlConnection = (HttpURLConnection) new URL(url).openConnection(); + urlConnection.setRequestMethod("HEAD"); + urlConnection.setDoInput(false); + urlConnection.setDoOutput(false); + urlConnection.connect(); + int responseCode = urlConnection.getResponseCode(); + if(responseCode >= 200 && responseCode <= 299) return urlConnection.getContentLength(); + return -1; + } + public interface ParseCallback { T process(String input) throws ParseException; } diff --git a/app_pojavlauncher/src/main/res/values/strings.xml b/app_pojavlauncher/src/main/res/values/strings.xml index 91104432d..8814ef333 100644 --- a/app_pojavlauncher/src/main/res/values/strings.xml +++ b/app_pojavlauncher/src/main/res/values/strings.xml @@ -374,7 +374,8 @@ Failed to install JRE 17 Reading game metadata… Downloading game metadata (%s) - Downloading game files… (%d/%d, %.2f MB) + Downloading game… (%d/%d, %.2f MB/s) + Downloading game… (%.2f/%.2f MB, %.2f MB/s) Select image region V. lock H. lock