From b1d8edc72a61de07367d6c91e06065ffd7f2cbbc Mon Sep 17 00:00:00 2001 From: Glavo Date: Fri, 8 Aug 2025 16:24:48 +0800 Subject: [PATCH] =?UTF-8?q?FetchTask=20=E5=90=AF=E7=94=A8=E9=80=8F?= =?UTF-8?q?=E6=98=8E=20GZip=20=E5=8E=8B=E7=BC=A9=20(#4219)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/jackhuang/hmcl/task/FetchTask.java | 97 ++++++++++++------- .../hmcl/util/io/ContentEncoding.java | 6 +- 2 files changed, 63 insertions(+), 40 deletions(-) diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java index 9bfbc12cd..61e9806a9 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java @@ -21,26 +21,24 @@ import org.jackhuang.hmcl.event.Event; import org.jackhuang.hmcl.event.EventBus; import org.jackhuang.hmcl.util.CacheRepository; import org.jackhuang.hmcl.util.DigestUtils; +import org.jackhuang.hmcl.util.StringUtils; import org.jackhuang.hmcl.util.ToStringBuilder; +import org.jackhuang.hmcl.util.io.ContentEncoding; import org.jackhuang.hmcl.util.io.IOUtils; import org.jackhuang.hmcl.util.io.NetworkUtils; import org.jackhuang.hmcl.util.io.ResponseCodeException; import org.jetbrains.annotations.NotNull; -import java.io.Closeable; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.net.HttpURLConnection; import java.net.URI; -import java.net.URL; import java.net.URLConnection; import java.nio.file.Path; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import static org.jackhuang.hmcl.util.Lang.threadPool; import static org.jackhuang.hmcl.util.logging.Logger.LOG; @@ -107,7 +105,7 @@ public abstract class FetchTask extends Task { break download; } - List redirects = null; + List redirects = null; String bmclapiHash = null; try { beforeDownload(uri); @@ -117,6 +115,7 @@ public abstract class FetchTask extends Task { if (conn instanceof HttpURLConnection) { var httpConnection = (HttpURLConnection) conn; + httpConnection.setRequestProperty("Accept-Encoding", "gzip"); if (checkETag) repository.injectConnection(httpConnection); Map> requestProperties = httpConnection.getRequestProperties(); @@ -143,25 +142,25 @@ public abstract class FetchTask extends Task { throw new IOException("Too much redirects"); } - URL prevUrl = httpConnection.getURL(); String location = httpConnection.getHeaderField("Location"); httpConnection.disconnect(); - if (location == null || location.isBlank()) { + if (StringUtils.isBlank(location)) throw new IOException("Redirected to an empty location"); - } - URL target = new URL(prevUrl, NetworkUtils.encodeLocation(location)); - redirects.add(target.toString()); + URI target = NetworkUtils.toURI(httpConnection.getURL()) + .resolve(NetworkUtils.toURI(location)); + redirects.add(target); - HttpURLConnection redirected = (HttpURLConnection) target.openConnection(); + if (!NetworkUtils.isHttpUri(target)) + throw new IOException("Redirected to not http URI: " + target); + + HttpURLConnection redirected = NetworkUtils.createHttpConnection(target); redirected.setUseCaches(checkETag); - redirected.setConnectTimeout(NetworkUtils.TIME_OUT); - redirected.setReadTimeout(NetworkUtils.TIME_OUT); - redirected.setInstanceFollowRedirects(false); requestProperties - .forEach((key, value) -> value.forEach(element -> - redirected.addRequestProperty(key, element))); + .forEach((key, values) -> + values.forEach(element -> + redirected.addRequestProperty(key, element))); httpConnection = redirected; } else { break; @@ -192,36 +191,36 @@ public abstract class FetchTask extends Task { } } - long contentLength = conn.getContentLength(); - try (Context context = getContext(conn, checkETag, bmclapiHash); - InputStream stream = conn.getInputStream()) { - int lastDownloaded = 0, downloaded = 0; + long contentLength = conn.getContentLengthLong(); + var encoding = ContentEncoding.fromConnection(conn); + try (var context = getContext(conn, checkETag, bmclapiHash); + var counter = new CounterInputStream(conn.getInputStream()); + var input = encoding.wrap(counter)) { + long lastDownloaded = 0L; byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE]; while (true) { if (isCancelled()) break; - int len = stream.read(buffer); + int len = input.read(buffer); if (len == -1) break; context.write(buffer, 0, len); - downloaded += len; - if (contentLength >= 0) { // Update progress information per second - updateProgress(downloaded, contentLength); + updateProgress(counter.downloaded, contentLength); } - updateDownloadSpeed(downloaded - lastDownloaded); - lastDownloaded = downloaded; + updateDownloadSpeed(counter.downloaded - lastDownloaded); + lastDownloaded = counter.downloaded; } if (isCancelled()) break download; - updateDownloadSpeed(downloaded - lastDownloaded); + updateDownloadSpeed(counter.downloaded - lastDownloaded); - if (contentLength >= 0 && downloaded != contentLength) - throw new IOException("Unexpected file size: " + downloaded + ", expected: " + contentLength); + if (contentLength >= 0 && counter.downloaded != contentLength) + throw new IOException("Unexpected file size: " + counter.downloaded + ", expected: " + contentLength); context.withResult(true); } @@ -246,7 +245,7 @@ public abstract class FetchTask extends Task { } private static final Timer timer = new Timer("DownloadSpeedRecorder", true); - private static final AtomicInteger downloadSpeed = new AtomicInteger(0); + private static final AtomicLong downloadSpeed = new AtomicLong(0L); public static final EventBus speedEvent = new EventBus(); static { @@ -258,14 +257,38 @@ public abstract class FetchTask extends Task { }, 0, 1000); } - private static void updateDownloadSpeed(int speed) { + private static void updateDownloadSpeed(long speed) { downloadSpeed.addAndGet(speed); } - public static class SpeedEvent extends Event { - private final int speed; + private static final class CounterInputStream extends FilterInputStream { + long downloaded; - public SpeedEvent(Object source, int speed) { + CounterInputStream(InputStream in) { + super(in); + } + + @Override + public int read() throws IOException { + int b = in.read(); + if (b >= 0) + downloaded++; + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int n = in.read(b, off, len); + if (n >= 0) + downloaded += n; + return n; + } + } + + public static class SpeedEvent extends Event { + private final long speed; + + public SpeedEvent(Object source, long speed) { super(source); this.speed = speed; @@ -276,7 +299,7 @@ public abstract class FetchTask extends Task { * * @return download speed */ - public int getSpeed() { + public long getSpeed() { return speed; } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/ContentEncoding.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/ContentEncoding.java index f257b0065..686e7507f 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/ContentEncoding.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/ContentEncoding.java @@ -28,7 +28,7 @@ import java.util.zip.GZIPInputStream; * @author Glavo */ public enum ContentEncoding { - NONE { + IDENTITY { @Override public InputStream wrap(InputStream inputStream) { return inputStream; @@ -43,8 +43,8 @@ public enum ContentEncoding { public static @NotNull ContentEncoding fromConnection(URLConnection connection) throws IOException { String encoding = connection.getContentEncoding(); - if (encoding == null || encoding.isEmpty()) { - return NONE; + if (encoding == null || encoding.isEmpty() || "identity".equals(encoding)) { + return IDENTITY; } else if ("gzip".equalsIgnoreCase(encoding)) { return GZIP; } else {