From 3873459aaa6d228c3edcca0f3bcc828eeda570b4 Mon Sep 17 00:00:00 2001 From: Glavo Date: Sat, 2 Aug 2025 16:04:26 +0800 Subject: [PATCH] =?UTF-8?q?=E9=80=9A=E8=BF=87=20x-bmclapi-hash=20=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E8=B5=84=E6=BA=90=20(#4169)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/jackhuang/hmcl/auth/offline/Skin.java | 2 +- .../jackhuang/hmcl/task/CacheFileTask.java | 2 +- .../org/jackhuang/hmcl/task/FetchTask.java | 104 ++++++++++-------- .../jackhuang/hmcl/task/FileDownloadTask.java | 45 +++++--- .../java/org/jackhuang/hmcl/task/GetTask.java | 2 +- .../jackhuang/hmcl/util/CacheRepository.java | 19 ++-- .../org/jackhuang/hmcl/util/DigestUtils.java | 14 ++- .../jackhuang/hmcl/util/io/NetworkUtils.java | 17 ++- 8 files changed, 122 insertions(+), 83 deletions(-) diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/auth/offline/Skin.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/auth/offline/Skin.java index 1678b186c..ec477a3c2 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/auth/offline/Skin.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/auth/offline/Skin.java @@ -244,7 +244,7 @@ public class Skin { } @Override - protected Context getContext(URLConnection connection, boolean checkETag) throws IOException { + protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException { return new Context() { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/CacheFileTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/CacheFileTask.java index 6f67c7c7a..e594f29d4 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/CacheFileTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/CacheFileTask.java @@ -63,7 +63,7 @@ public final class CacheFileTask extends FetchTask { } @Override - protected Context getContext(URLConnection connection, boolean checkETag) throws IOException { + protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException { assert checkETag; Path temp = Files.createTempFile("hmcl-download-", null); diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java index 15402087d..1444370cf 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java @@ -20,6 +20,7 @@ package org.jackhuang.hmcl.task; import org.jackhuang.hmcl.event.Event; import org.jackhuang.hmcl.event.EventBus; import org.jackhuang.hmcl.util.CacheRepository; +import org.jackhuang.hmcl.util.DigestUtils; import org.jackhuang.hmcl.util.ToStringBuilder; import org.jackhuang.hmcl.util.io.IOUtils; import org.jackhuang.hmcl.util.io.NetworkUtils; @@ -32,6 +33,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URI; +import java.net.URL; import java.net.URLConnection; import java.nio.file.Path; import java.util.*; @@ -73,7 +75,7 @@ public abstract class FetchTask extends Task { protected abstract EnumCheckETag shouldCheckETag(); - protected abstract Context getContext(URLConnection connection, boolean checkETag) throws IOException; + protected abstract Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException; @Override public void execute() throws Exception { @@ -100,18 +102,66 @@ public abstract class FetchTask extends Task { } List redirects = null; + String bmclapiHash = null; try { beforeDownload(uri); - updateProgress(0); URLConnection conn = NetworkUtils.createConnection(uri); - if (checkETag) repository.injectConnection(conn); if (conn instanceof HttpURLConnection) { - redirects = new ArrayList<>(); + var httpConnection = (HttpURLConnection) conn; - conn = NetworkUtils.resolveConnection((HttpURLConnection) conn, redirects); + if (checkETag) repository.injectConnection(httpConnection); + Map> requestProperties = httpConnection.getRequestProperties(); + + bmclapiHash = httpConnection.getHeaderField("x-bmclapi-hash"); + if (DigestUtils.isSha1Digest(bmclapiHash)) { + Optional cache = repository.checkExistentFile(null, "SHA-1", bmclapiHash); + if (cache.isPresent()) { + useCachedResult(cache.get()); + LOG.info("Using cached file for " + NetworkUtils.dropQuery(uri)); + return; + } + } else { + bmclapiHash = null; + } + + while (true) { + int code = httpConnection.getResponseCode(); + if (code >= 300 && code <= 308 && code != 306 && code != 304) { + if (redirects == null) { + redirects = new ArrayList<>(); + } else if (redirects.size() >= 20) { + httpConnection.disconnect(); + throw new IOException("Too much redirects"); + } + + URL prevUrl = httpConnection.getURL(); + String location = httpConnection.getHeaderField("Location"); + + httpConnection.disconnect(); + if (location == null || location.isBlank()) { + throw new IOException("Redirected to an empty location"); + } + + URL target = new URL(prevUrl, NetworkUtils.encodeLocation(location)); + redirects.add(target.toString()); + + HttpURLConnection redirected = (HttpURLConnection) target.openConnection(); + redirected.setUseCaches(checkETag); + redirected.setConnectTimeout(NetworkUtils.TIME_OUT); + redirected.setReadTimeout(NetworkUtils.TIME_OUT); + redirected.setInstanceFollowRedirects(false); + requestProperties + .forEach((key, value) -> value.forEach(element -> + redirected.addRequestProperty(key, element))); + httpConnection = redirected; + } else { + break; + } + } + conn = httpConnection; int responseCode = ((HttpURLConnection) conn).getResponseCode(); if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED) { @@ -137,7 +187,8 @@ public abstract class FetchTask extends Task { } long contentLength = conn.getContentLength(); - try (Context context = getContext(conn, checkETag); InputStream stream = conn.getInputStream()) { + try (Context context = getContext(conn, checkETag, bmclapiHash); + InputStream stream = conn.getInputStream()) { int lastDownloaded = 0, downloaded = 0; byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE]; while (true) { @@ -173,13 +224,13 @@ public abstract class FetchTask extends Task { } catch (FileNotFoundException ex) { failedURI = uri; exception = ex; - LOG.warning("Failed to download " + uri + ", not found" + ((redirects == null || redirects.isEmpty()) ? "" : ", redirects: " + redirects), ex); + LOG.warning("Failed to download " + uri + ", not found" + (redirects == null ? "" : ", redirects: " + redirects), ex); break; // we will not try this URL again } catch (IOException ex) { failedURI = uri; exception = ex; - LOG.warning("Failed to download " + uri + ", repeat times: " + (++repeat) + ((redirects == null || redirects.isEmpty()) ? "" : ", redirects: " + redirects), ex); + LOG.warning("Failed to download " + uri + ", repeat times: " + (++repeat) + (redirects == null ? "" : ", redirects: " + redirects), ex); } } } @@ -249,43 +300,6 @@ public abstract class FetchTask extends Task { CACHED } - protected static final class DownloadState { - private final int startPosition; - private final int endPosition; - private final int currentPosition; - private final boolean finished; - - public DownloadState(int startPosition, int endPosition, int currentPosition) { - if (currentPosition < startPosition || currentPosition > endPosition) { - throw new IllegalArgumentException("Illegal download state: start " + startPosition + ", end " + endPosition + ", cur " + currentPosition); - } - this.startPosition = startPosition; - this.endPosition = endPosition; - this.currentPosition = currentPosition; - finished = currentPosition == endPosition; - } - - public int getStartPosition() { - return startPosition; - } - - public int getEndPosition() { - return endPosition; - } - - public int getCurrentPosition() { - return currentPosition; - } - - public boolean isFinished() { - return finished; - } - } - - protected static final class DownloadMission { - - } - public static int DEFAULT_CONCURRENCY = Math.min(Runtime.getRuntime().availableProcessors() * 4, 64); private static int downloadExecutorConcurrency = DEFAULT_CONCURRENCY; private static volatile ThreadPoolExecutor DOWNLOAD_EXECUTOR; diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java index ec6895aa0..12ab51bae 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java @@ -17,6 +17,7 @@ */ package org.jackhuang.hmcl.task; +import org.jackhuang.hmcl.util.DigestUtils; import org.jackhuang.hmcl.util.Hex; import org.jackhuang.hmcl.util.io.ChecksumMismatchException; import org.jackhuang.hmcl.util.io.CompressingUtils; @@ -38,7 +39,6 @@ import java.util.Objects; import java.util.Optional; import static java.util.Objects.requireNonNull; -import static org.jackhuang.hmcl.util.DigestUtils.getDigest; import static org.jackhuang.hmcl.util.logging.Logger.LOG; /** @@ -70,15 +70,9 @@ public class FileDownloadTask extends FetchTask { return checksum; } - public MessageDigest createDigest() { - return getDigest(algorithm); - } - - public void performCheck(MessageDigest digest) throws ChecksumMismatchException { - String actualChecksum = Hex.encodeHex(digest.digest()); - if (!checksum.equalsIgnoreCase(actualChecksum)) { - throw new ChecksumMismatchException(algorithm, checksum, actualChecksum); - } + @Override + public String toString() { + return String.format("IntegrityCheck[algorithm='%s', checksum='%s']", algorithm, checksum); } } @@ -200,11 +194,25 @@ public class FileDownloadTask extends FetchTask { } @Override - protected Context getContext(URLConnection connection, boolean checkETag) throws IOException { + protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException { Path temp = Files.createTempFile(null, null); - MessageDigest digest = integrityCheck == null ? null : integrityCheck.createDigest(); - OutputStream fileOutput = Files.newOutputStream(temp); + String algorithm; + String checksum; + if (integrityCheck != null) { + algorithm = integrityCheck.getAlgorithm(); + checksum = integrityCheck.getChecksum(); + } else if (bmclapiHash != null) { + algorithm = "SHA-1"; + checksum = bmclapiHash; + } else { + algorithm = null; + checksum = null; + } + + MessageDigest digest = algorithm != null ? DigestUtils.getDigest(algorithm) : null; + + OutputStream fileOutput = Files.newOutputStream(temp); return new Context() { @Override public void write(byte[] buffer, int offset, int len) throws IOException { @@ -245,13 +253,16 @@ public class FileDownloadTask extends FetchTask { } // Integrity check - if (integrityCheck != null) { - integrityCheck.performCheck(digest); + if (checksum != null) { + String actualChecksum = Hex.encodeHex(digest.digest()); + if (!checksum.equalsIgnoreCase(actualChecksum)) { + throw new ChecksumMismatchException(algorithm, checksum, actualChecksum); + } } - if (caching && integrityCheck != null) { + if (caching && algorithm != null) { try { - repository.cacheFile(file, integrityCheck.getAlgorithm(), integrityCheck.getChecksum()); + repository.cacheFile(file, algorithm, checksum); } catch (IOException e) { LOG.warning("Failed to cache file", e); } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java index 119b8f42a..dba5d75f7 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java @@ -72,7 +72,7 @@ public final class GetTask extends FetchTask { } @Override - protected Context getContext(URLConnection connection, boolean checkETag) { + protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) { return new Context() { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java index 0e7fae9c7..f2f36b977 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java @@ -24,8 +24,10 @@ import org.jackhuang.hmcl.util.function.ExceptionalSupplier; import org.jackhuang.hmcl.util.gson.JsonUtils; import org.jackhuang.hmcl.util.io.FileUtils; import org.jackhuang.hmcl.util.io.NetworkUtils; +import org.jetbrains.annotations.Nullable; import java.io.*; +import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.URLConnection; @@ -92,6 +94,7 @@ public class CacheRepository { } protected Path getFile(String algorithm, String hash) { + hash = hash.toLowerCase(Locale.ROOT); return getCacheDirectory().resolve(algorithm).resolve(hash.substring(0, 2)).resolve(hash); } @@ -121,7 +124,7 @@ public class CacheRepository { return cache; } - public Optional checkExistentFile(Path original, String algorithm, String hash) { + public Optional checkExistentFile(@Nullable Path original, String algorithm, String hash) { if (fileExists(algorithm, hash)) return Optional.of(getFile(algorithm, hash)); @@ -177,7 +180,7 @@ public class CacheRepository { } } - public void injectConnection(URLConnection conn) { + public void injectConnection(HttpURLConnection conn) { conn.setUseCaches(true); URI uri; @@ -258,11 +261,13 @@ public class CacheRepository { if (oldItem == null) { return newItem; } else if (force || oldItem.compareTo(newItem) < 0) { - Path cached = getFile(SHA1, oldItem.hash); - try { - Files.deleteIfExists(cached); - } catch (IOException e) { - LOG.warning("Cannot delete old file"); + if (!oldItem.hash.equalsIgnoreCase(newItem.hash)) { + Path cached = getFile(SHA1, oldItem.hash); + try { + Files.deleteIfExists(cached); + } catch (IOException e) { + LOG.warning("Cannot delete old file"); + } } return newItem; } else { diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/DigestUtils.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/DigestUtils.java index 25da29056..654413610 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/DigestUtils.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/DigestUtils.java @@ -25,7 +25,6 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; /** - * * @author huangyuhui */ public final class DigestUtils { @@ -35,6 +34,19 @@ public final class DigestUtils { private static final int STREAM_BUFFER_LENGTH = 1024; + public static boolean isSha1Digest(String digest) { + if (digest == null || digest.length() != 40) return false; + + for (int i = 0; i < digest.length(); i++) { + char ch = digest.charAt(i); + if ((ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') && (ch < 'A' || ch > 'F')) { + return false; + } + } + + return true; + } + public static MessageDigest getDigest(String algorithm) { try { return MessageDigest.getInstance(algorithm); diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java index 034508ad4..5fe499a4b 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java @@ -39,7 +39,7 @@ import static org.jackhuang.hmcl.util.logging.Logger.LOG; public final class NetworkUtils { public static final String PARAMETER_SEPARATOR = "&"; public static final String NAME_VALUE_SEPARATOR = "="; - private static final int TIME_OUT = 8000; + public static final int TIME_OUT = 8000; private NetworkUtils() { } @@ -114,7 +114,11 @@ public final class NetworkUtils { URLConnection connection = uri.toURL().openConnection(); connection.setConnectTimeout(TIME_OUT); connection.setReadTimeout(TIME_OUT); - connection.setRequestProperty("Accept-Language", Locale.getDefault().toLanguageTag()); + if (connection instanceof HttpURLConnection) { + var httpConnection = (HttpURLConnection) connection; + httpConnection.setRequestProperty("Accept-Language", Locale.getDefault().toLanguageTag()); + httpConnection.setInstanceFollowRedirects(false); + } return connection; } @@ -154,10 +158,6 @@ public final class NetworkUtils { return sb.toString(); } - public static HttpURLConnection resolveConnection(HttpURLConnection conn) throws IOException { - return resolveConnection(conn, null); - } - /** * This method is a work-around that aims to solve problem when "Location" in * stupid server's response is not encoded. @@ -167,7 +167,7 @@ public final class NetworkUtils { * @throws IOException if an I/O error occurs. * @see Issue with libcurl */ - public static HttpURLConnection resolveConnection(HttpURLConnection conn, List redirects) throws IOException { + public static HttpURLConnection resolveConnection(HttpURLConnection conn) throws IOException { final boolean useCache = conn.getUseCaches(); int redirect = 0; while (true) { @@ -182,9 +182,6 @@ public final class NetworkUtils { String newURL = conn.getHeaderField("Location"); conn.disconnect(); - if (redirects != null) { - redirects.add(newURL); - } if (redirect > 20) { throw new IOException("Too much redirects"); }