通过 x-bmclapi-hash 缓存资源 (#4169)

HMCLCore/src/main/java/org/jackhuang/hmcl/auth/offline/Skin.java

@@ -244,7 +244,7 @@ public class Skin {
         }
 
         @Override
-        protected Context getContext(URLConnection connection, boolean checkETag) throws IOException {
+        protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException {
             return new Context() {
                 final ByteArrayOutputStream baos = new ByteArrayOutputStream();
 

HMCLCore/src/main/java/org/jackhuang/hmcl/task/CacheFileTask.java

@@ -63,7 +63,7 @@ public final class CacheFileTask extends FetchTask<Path> {
     }
 
     @Override
-    protected Context getContext(URLConnection connection, boolean checkETag) throws IOException {
+    protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException {
         assert checkETag;
 
         Path temp = Files.createTempFile("hmcl-download-", null);

HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java

@@ -20,6 +20,7 @@ package org.jackhuang.hmcl.task;
 import org.jackhuang.hmcl.event.Event;
 import org.jackhuang.hmcl.event.EventBus;
 import org.jackhuang.hmcl.util.CacheRepository;
+import org.jackhuang.hmcl.util.DigestUtils;
 import org.jackhuang.hmcl.util.ToStringBuilder;
 import org.jackhuang.hmcl.util.io.IOUtils;
 import org.jackhuang.hmcl.util.io.NetworkUtils;
@@ -32,6 +33,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URI;
+import java.net.URL;
 import java.net.URLConnection;
 import java.nio.file.Path;
 import java.util.*;
@@ -73,7 +75,7 @@ public abstract class FetchTask<T> extends Task<T> {
 
     protected abstract EnumCheckETag shouldCheckETag();
 
-    protected abstract Context getContext(URLConnection connection, boolean checkETag) throws IOException;
+    protected abstract Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException;
 
     @Override
     public void execute() throws Exception {
@@ -100,18 +102,66 @@ public abstract class FetchTask<T> extends Task<T> {
                 }
 
                 List<String> redirects = null;
+                String bmclapiHash = null;
                 try {
                     beforeDownload(uri);
-
                     updateProgress(0);
 
                     URLConnection conn = NetworkUtils.createConnection(uri);
-                    if (checkETag) repository.injectConnection(conn);
 
                     if (conn instanceof HttpURLConnection) {
-                        redirects = new ArrayList<>();
+                        var httpConnection = (HttpURLConnection) conn;
 
-                        conn = NetworkUtils.resolveConnection((HttpURLConnection) conn, redirects);
+                        if (checkETag) repository.injectConnection(httpConnection);
+                        Map<String, List<String>> requestProperties = httpConnection.getRequestProperties();
+
+                        bmclapiHash = httpConnection.getHeaderField("x-bmclapi-hash");
+                        if (DigestUtils.isSha1Digest(bmclapiHash)) {
+                            Optional<Path> cache = repository.checkExistentFile(null, "SHA-1", bmclapiHash);
+                            if (cache.isPresent()) {
+                                useCachedResult(cache.get());
+                                LOG.info("Using cached file for " + NetworkUtils.dropQuery(uri));
+                                return;
+                            }
+                        } else {
+                            bmclapiHash = null;
+                        }
+
+                        while (true) {
+                            int code = httpConnection.getResponseCode();
+                            if (code >= 300 && code <= 308 && code != 306 && code != 304) {
+                                if (redirects == null) {
+                                    redirects = new ArrayList<>();
+                                } else if (redirects.size() >= 20) {
+                                    httpConnection.disconnect();
+                                    throw new IOException("Too much redirects");
+                                }
+
+                                URL prevUrl = httpConnection.getURL();
+                                String location = httpConnection.getHeaderField("Location");
+
+                                httpConnection.disconnect();
+                                if (location == null || location.isBlank()) {
+                                    throw new IOException("Redirected to an empty location");
+                                }
+
+                                URL target = new URL(prevUrl, NetworkUtils.encodeLocation(location));
+                                redirects.add(target.toString());
+
+                                HttpURLConnection redirected = (HttpURLConnection) target.openConnection();
+                                redirected.setUseCaches(checkETag);
+                                redirected.setConnectTimeout(NetworkUtils.TIME_OUT);
+                                redirected.setReadTimeout(NetworkUtils.TIME_OUT);
+                                redirected.setInstanceFollowRedirects(false);
+                                requestProperties
+                                        .forEach((key, value) -> value.forEach(element ->
+                                                redirected.addRequestProperty(key, element)));
+                                httpConnection = redirected;
+                            } else {
+                                break;
+                            }
+                        }
+                        conn = httpConnection;
                         int responseCode = ((HttpURLConnection) conn).getResponseCode();
 
                         if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED) {
@@ -137,7 +187,8 @@ public abstract class FetchTask<T> extends Task<T> {
                     }
 
                     long contentLength = conn.getContentLength();
-                    try (Context context = getContext(conn, checkETag); InputStream stream = conn.getInputStream()) {
+                    try (Context context = getContext(conn, checkETag, bmclapiHash);
+                         InputStream stream = conn.getInputStream()) {
                         int lastDownloaded = 0, downloaded = 0;
                         byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE];
                         while (true) {
@@ -173,13 +224,13 @@ public abstract class FetchTask<T> extends Task<T> {
                 } catch (FileNotFoundException ex) {
                     failedURI = uri;
                     exception = ex;
-                    LOG.warning("Failed to download " + uri + ", not found" + ((redirects == null || redirects.isEmpty()) ? "" : ", redirects: " + redirects), ex);
+                    LOG.warning("Failed to download " + uri + ", not found" + (redirects == null ? "" : ", redirects: " + redirects), ex);
 
                     break; // we will not try this URL again
                 } catch (IOException ex) {
                     failedURI = uri;
                     exception = ex;
-                    LOG.warning("Failed to download " + uri + ", repeat times: " + (++repeat) + ((redirects == null || redirects.isEmpty()) ? "" : ", redirects: " + redirects), ex);
+                    LOG.warning("Failed to download " + uri + ", repeat times: " + (++repeat) + (redirects == null ? "" : ", redirects: " + redirects), ex);
                 }
             }
         }
@@ -249,43 +300,6 @@ public abstract class FetchTask<T> extends Task<T> {
         CACHED
     }
 
-    protected static final class DownloadState {
-        private final int startPosition;
-        private final int endPosition;
-        private final int currentPosition;
-        private final boolean finished;
-
-        public DownloadState(int startPosition, int endPosition, int currentPosition) {
-            if (currentPosition < startPosition || currentPosition > endPosition) {
-                throw new IllegalArgumentException("Illegal download state: start " + startPosition + ", end " + endPosition + ", cur " + currentPosition);
-            }
-            this.startPosition = startPosition;
-            this.endPosition = endPosition;
-            this.currentPosition = currentPosition;
-            finished = currentPosition == endPosition;
-        }
-
-        public int getStartPosition() {
-            return startPosition;
-        }
-
-        public int getEndPosition() {
-            return endPosition;
-        }
-
-        public int getCurrentPosition() {
-            return currentPosition;
-        }
-
-        public boolean isFinished() {
-            return finished;
-        }
-    }
-
-    protected static final class DownloadMission {
-
-    }
-
     public static int DEFAULT_CONCURRENCY = Math.min(Runtime.getRuntime().availableProcessors() * 4, 64);
     private static int downloadExecutorConcurrency = DEFAULT_CONCURRENCY;
     private static volatile ThreadPoolExecutor DOWNLOAD_EXECUTOR;

HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java

@@ -17,6 +17,7 @@
  */
 package org.jackhuang.hmcl.task;
 
+import org.jackhuang.hmcl.util.DigestUtils;
 import org.jackhuang.hmcl.util.Hex;
 import org.jackhuang.hmcl.util.io.ChecksumMismatchException;
 import org.jackhuang.hmcl.util.io.CompressingUtils;
@@ -38,7 +39,6 @@ import java.util.Objects;
 import java.util.Optional;
 
 import static java.util.Objects.requireNonNull;
-import static org.jackhuang.hmcl.util.DigestUtils.getDigest;
 import static org.jackhuang.hmcl.util.logging.Logger.LOG;
 
 /**
@@ -70,15 +70,9 @@ public class FileDownloadTask extends FetchTask<Void> {
             return checksum;
         }
 
-        public MessageDigest createDigest() {
+        @Override
-            return getDigest(algorithm);
+        public String toString() {
-        }
+            return String.format("IntegrityCheck[algorithm='%s', checksum='%s']", algorithm, checksum);
-
-        public void performCheck(MessageDigest digest) throws ChecksumMismatchException {
-            String actualChecksum = Hex.encodeHex(digest.digest());
-            if (!checksum.equalsIgnoreCase(actualChecksum)) {
-                throw new ChecksumMismatchException(algorithm, checksum, actualChecksum);
-            }
         }
     }
 
@@ -200,11 +194,25 @@ public class FileDownloadTask extends FetchTask<Void> {
     }
 
     @Override
-    protected Context getContext(URLConnection connection, boolean checkETag) throws IOException {
+    protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException {
         Path temp = Files.createTempFile(null, null);
-        MessageDigest digest = integrityCheck == null ? null : integrityCheck.createDigest();
-        OutputStream fileOutput = Files.newOutputStream(temp);
 
+        String algorithm;
+        String checksum;
+        if (integrityCheck != null) {
+            algorithm = integrityCheck.getAlgorithm();
+            checksum = integrityCheck.getChecksum();
+        } else if (bmclapiHash != null) {
+            algorithm = "SHA-1";
+            checksum = bmclapiHash;
+        } else {
+            algorithm = null;
+            checksum = null;
+        }
+
+        MessageDigest digest = algorithm != null ? DigestUtils.getDigest(algorithm) : null;
+
+        OutputStream fileOutput = Files.newOutputStream(temp);
         return new Context() {
             @Override
             public void write(byte[] buffer, int offset, int len) throws IOException {
@@ -245,13 +253,16 @@ public class FileDownloadTask extends FetchTask<Void> {
                 }
 
                 // Integrity check
-                if (integrityCheck != null) {
+                if (checksum != null) {
-                    integrityCheck.performCheck(digest);
+                    String actualChecksum = Hex.encodeHex(digest.digest());
+                    if (!checksum.equalsIgnoreCase(actualChecksum)) {
+                        throw new ChecksumMismatchException(algorithm, checksum, actualChecksum);
+                    }
                 }
 
-                if (caching && integrityCheck != null) {
+                if (caching && algorithm != null) {
                     try {
-                        repository.cacheFile(file, integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
+                        repository.cacheFile(file, algorithm, checksum);
                     } catch (IOException e) {
                         LOG.warning("Failed to cache file", e);
                     }

HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java

@@ -72,7 +72,7 @@ public final class GetTask extends FetchTask<String> {
     }
 
     @Override
-    protected Context getContext(URLConnection connection, boolean checkETag) {
+    protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) {
         return new Context() {
             final ByteArrayOutputStream baos = new ByteArrayOutputStream();
 

HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java

@@ -24,8 +24,10 @@ import org.jackhuang.hmcl.util.function.ExceptionalSupplier;
 import org.jackhuang.hmcl.util.gson.JsonUtils;
 import org.jackhuang.hmcl.util.io.FileUtils;
 import org.jackhuang.hmcl.util.io.NetworkUtils;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.*;
+import java.net.HttpURLConnection;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URLConnection;
@@ -92,6 +94,7 @@ public class CacheRepository {
     }
 
     protected Path getFile(String algorithm, String hash) {
+        hash = hash.toLowerCase(Locale.ROOT);
         return getCacheDirectory().resolve(algorithm).resolve(hash.substring(0, 2)).resolve(hash);
     }
 
@@ -121,7 +124,7 @@ public class CacheRepository {
         return cache;
     }
 
-    public Optional<Path> checkExistentFile(Path original, String algorithm, String hash) {
+    public Optional<Path> checkExistentFile(@Nullable Path original, String algorithm, String hash) {
         if (fileExists(algorithm, hash))
             return Optional.of(getFile(algorithm, hash));
 
@@ -177,7 +180,7 @@ public class CacheRepository {
         }
     }
 
-    public void injectConnection(URLConnection conn) {
+    public void injectConnection(HttpURLConnection conn) {
         conn.setUseCaches(true);
 
         URI uri;
@@ -258,11 +261,13 @@ public class CacheRepository {
             if (oldItem == null) {
                 return newItem;
             } else if (force || oldItem.compareTo(newItem) < 0) {
-                Path cached = getFile(SHA1, oldItem.hash);
+                if (!oldItem.hash.equalsIgnoreCase(newItem.hash)) {
-                try {
+                    Path cached = getFile(SHA1, oldItem.hash);
-                    Files.deleteIfExists(cached);
+                    try {
-                } catch (IOException e) {
+                        Files.deleteIfExists(cached);
-                    LOG.warning("Cannot delete old file");
+                    } catch (IOException e) {
+                        LOG.warning("Cannot delete old file");
+                    }
                 }
                 return newItem;
             } else {

HMCLCore/src/main/java/org/jackhuang/hmcl/util/DigestUtils.java

@@ -25,7 +25,6 @@ import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 
 /**
- *
  * @author huangyuhui
  */
 public final class DigestUtils {
@@ -35,6 +34,19 @@ public final class DigestUtils {
 
     private static final int STREAM_BUFFER_LENGTH = 1024;
 
+    public static boolean isSha1Digest(String digest) {
+        if (digest == null || digest.length() != 40) return false;
+
+        for (int i = 0; i < digest.length(); i++) {
+            char ch = digest.charAt(i);
+            if ((ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') && (ch < 'A' || ch > 'F')) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
     public static MessageDigest getDigest(String algorithm) {
         try {
             return MessageDigest.getInstance(algorithm);

HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java

@@ -39,7 +39,7 @@ import static org.jackhuang.hmcl.util.logging.Logger.LOG;
 public final class NetworkUtils {
     public static final String PARAMETER_SEPARATOR = "&";
     public static final String NAME_VALUE_SEPARATOR = "=";
-    private static final int TIME_OUT = 8000;
+    public static final int TIME_OUT = 8000;
 
     private NetworkUtils() {
     }
@@ -114,7 +114,11 @@ public final class NetworkUtils {
         URLConnection connection = uri.toURL().openConnection();
         connection.setConnectTimeout(TIME_OUT);
         connection.setReadTimeout(TIME_OUT);
-        connection.setRequestProperty("Accept-Language", Locale.getDefault().toLanguageTag());
+        if (connection instanceof HttpURLConnection) {
+            var httpConnection = (HttpURLConnection) connection;
+            httpConnection.setRequestProperty("Accept-Language", Locale.getDefault().toLanguageTag());
+            httpConnection.setInstanceFollowRedirects(false);
+        }
         return connection;
     }
 
@@ -154,10 +158,6 @@ public final class NetworkUtils {
         return sb.toString();
     }
 
-    public static HttpURLConnection resolveConnection(HttpURLConnection conn) throws IOException {
-        return resolveConnection(conn, null);
-    }
-
     /**
      * This method is a work-around that aims to solve problem when "Location" in
      * stupid server's response is not encoded.
@@ -167,7 +167,7 @@ public final class NetworkUtils {
      * @throws IOException if an I/O error occurs.
      * @see <a href="https://github.com/curl/curl/issues/473">Issue with libcurl</a>
      */
-    public static HttpURLConnection resolveConnection(HttpURLConnection conn, List<String> redirects) throws IOException {
+    public static HttpURLConnection resolveConnection(HttpURLConnection conn) throws IOException {
         final boolean useCache = conn.getUseCaches();
         int redirect = 0;
         while (true) {
@@ -182,9 +182,6 @@ public final class NetworkUtils {
                 String newURL = conn.getHeaderField("Location");
                 conn.disconnect();
 
-                if (redirects != null) {
-                    redirects.add(newURL);
-                }
                 if (redirect > 20) {
                     throw new IOException("Too much redirects");
                 }