AstralTransRocketries/HMCL
1
0
Fork 0
mirror of https://github.com/HMCL-dev/HMCL.git synced 2025-08-03 03:16:35 -04:00
Code Issues Packages Projects Releases Wiki Activity

通过 x-bmclapi-hash 缓存资源 (#4169)

Browse Source
This commit is contained in:
Glavo 2025-08-02 16:04:26 +08:00 committed by GitHub
parent 4ee3857427
commit 3873459aaa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 122 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
HMCLCore/src/main/java/org/jackhuang/hmcl/auth/offline/Skin.java
View File

@ -244,7 +244,7 @@ public class Skin {
}
@Override
protected Context getContext(URLConnection connection, boolean checkETag) throws IOException {
protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException {
return new Context() {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();

2
HMCLCore/src/main/java/org/jackhuang/hmcl/task/CacheFileTask.java
View File

@ -63,7 +63,7 @@ public final class CacheFileTask extends FetchTask<Path> {
}
@Override
protected Context getContext(URLConnection connection, boolean checkETag) throws IOException {
protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException {
assert checkETag;
Path temp = Files.createTempFile("hmcl-download-", null);

104
HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java
View File

@ -20,6 +20,7 @@ package org.jackhuang.hmcl.task;
import org.jackhuang.hmcl.event.Event;
import org.jackhuang.hmcl.event.EventBus;
import org.jackhuang.hmcl.util.CacheRepository;
import org.jackhuang.hmcl.util.DigestUtils;
import org.jackhuang.hmcl.util.ToStringBuilder;
import org.jackhuang.hmcl.util.io.IOUtils;
import org.jackhuang.hmcl.util.io.NetworkUtils;
@ -32,6 +33,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Path;
import java.util.*;
@ -73,7 +75,7 @@ public abstract class FetchTask<T> extends Task<T> {
protected abstract EnumCheckETag shouldCheckETag();
protected abstract Context getContext(URLConnection connection, boolean checkETag) throws IOException;
protected abstract Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException;
@Override
public void execute() throws Exception {
@ -100,18 +102,66 @@ public abstract class FetchTask<T> extends Task<T> {
}
List<String> redirects = null;
String bmclapiHash = null;
try {
beforeDownload(uri);
updateProgress(0);
URLConnection conn = NetworkUtils.createConnection(uri);
if (checkETag) repository.injectConnection(conn);
if (conn instanceof HttpURLConnection) {
redirects = new ArrayList<>();
var httpConnection = (HttpURLConnection) conn;
conn = NetworkUtils.resolveConnection((HttpURLConnection) conn, redirects);
if (checkETag) repository.injectConnection(httpConnection);
Map<String, List<String>> requestProperties = httpConnection.getRequestProperties();
bmclapiHash = httpConnection.getHeaderField("x-bmclapi-hash");
if (DigestUtils.isSha1Digest(bmclapiHash)) {
Optional<Path> cache = repository.checkExistentFile(null, "SHA-1", bmclapiHash);
if (cache.isPresent()) {
useCachedResult(cache.get());
LOG.info("Using cached file for " + NetworkUtils.dropQuery(uri));
return;
}
} else {
bmclapiHash = null;
}
while (true) {
int code = httpConnection.getResponseCode();
if (code >= 300 && code <= 308 && code != 306 && code != 304) {
if (redirects == null) {
redirects = new ArrayList<>();
} else if (redirects.size() >= 20) {
httpConnection.disconnect();
throw new IOException("Too much redirects");
}
URL prevUrl = httpConnection.getURL();
String location = httpConnection.getHeaderField("Location");
httpConnection.disconnect();
if (location == null || location.isBlank()) {
throw new IOException("Redirected to an empty location");
}
URL target = new URL(prevUrl, NetworkUtils.encodeLocation(location));
redirects.add(target.toString());
HttpURLConnection redirected = (HttpURLConnection) target.openConnection();
redirected.setUseCaches(checkETag);
redirected.setConnectTimeout(NetworkUtils.TIME_OUT);
redirected.setReadTimeout(NetworkUtils.TIME_OUT);
redirected.setInstanceFollowRedirects(false);
requestProperties
.forEach((key, value) -> value.forEach(element ->
redirected.addRequestProperty(key, element)));
httpConnection = redirected;
} else {
break;
}
}
conn = httpConnection;
int responseCode = ((HttpURLConnection) conn).getResponseCode();
if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED) {
@ -137,7 +187,8 @@ public abstract class FetchTask<T> extends Task<T> {
}
long contentLength = conn.getContentLength();
try (Context context = getContext(conn, checkETag); InputStream stream = conn.getInputStream()) {
try (Context context = getContext(conn, checkETag, bmclapiHash);
InputStream stream = conn.getInputStream()) {
int lastDownloaded = 0, downloaded = 0;
byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE];
while (true) {
@ -173,13 +224,13 @@ public abstract class FetchTask<T> extends Task<T> {
} catch (FileNotFoundException ex) {
failedURI = uri;
exception = ex;
LOG.warning("Failed to download " + uri + ", not found" + ((redirects == null || redirects.isEmpty()) ? "" : ", redirects: " + redirects), ex);
LOG.warning("Failed to download " + uri + ", not found" + (redirects == null ? "" : ", redirects: " + redirects), ex);
break; // we will not try this URL again
} catch (IOException ex) {
failedURI = uri;
exception = ex;
LOG.warning("Failed to download " + uri + ", repeat times: " + (++repeat) + ((redirects == null || redirects.isEmpty()) ? "" : ", redirects: " + redirects), ex);
LOG.warning("Failed to download " + uri + ", repeat times: " + (++repeat) + (redirects == null ? "" : ", redirects: " + redirects), ex);
}
}
}
@ -249,43 +300,6 @@ public abstract class FetchTask<T> extends Task<T> {
CACHED
}
protected static final class DownloadState {
private final int startPosition;
private final int endPosition;
private final int currentPosition;
private final boolean finished;
public DownloadState(int startPosition, int endPosition, int currentPosition) {
if (currentPosition < startPosition || currentPosition > endPosition) {
throw new IllegalArgumentException("Illegal download state: start " + startPosition + ", end " + endPosition + ", cur " + currentPosition);
}
this.startPosition = startPosition;
this.endPosition = endPosition;
this.currentPosition = currentPosition;
finished = currentPosition == endPosition;
}
public int getStartPosition() {
return startPosition;
}
public int getEndPosition() {
return endPosition;
}
public int getCurrentPosition() {
return currentPosition;
}
public boolean isFinished() {
return finished;
}
}
protected static final class DownloadMission {
}
public static int DEFAULT_CONCURRENCY = Math.min(Runtime.getRuntime().availableProcessors() * 4, 64);
private static int downloadExecutorConcurrency = DEFAULT_CONCURRENCY;
private static volatile ThreadPoolExecutor DOWNLOAD_EXECUTOR;

45
HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java
View File

@ -17,6 +17,7 @@
*/
package org.jackhuang.hmcl.task;
import org.jackhuang.hmcl.util.DigestUtils;
import org.jackhuang.hmcl.util.Hex;
import org.jackhuang.hmcl.util.io.ChecksumMismatchException;
import org.jackhuang.hmcl.util.io.CompressingUtils;
@ -38,7 +39,6 @@ import java.util.Objects;
import java.util.Optional;
import static java.util.Objects.requireNonNull;
import static org.jackhuang.hmcl.util.DigestUtils.getDigest;
import static org.jackhuang.hmcl.util.logging.Logger.LOG;
/**
@ -70,15 +70,9 @@ public class FileDownloadTask extends FetchTask<Void> {
return checksum;
}
public MessageDigest createDigest() {
return getDigest(algorithm);
}
public void performCheck(MessageDigest digest) throws ChecksumMismatchException {
String actualChecksum = Hex.encodeHex(digest.digest());
if (!checksum.equalsIgnoreCase(actualChecksum)) {
throw new ChecksumMismatchException(algorithm, checksum, actualChecksum);
}
@Override
public String toString() {
return String.format("IntegrityCheck[algorithm='%s', checksum='%s']", algorithm, checksum);
}
}
@ -200,11 +194,25 @@ public class FileDownloadTask extends FetchTask<Void> {
}
@Override
protected Context getContext(URLConnection connection, boolean checkETag) throws IOException {
protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) throws IOException {
Path temp = Files.createTempFile(null, null);
MessageDigest digest = integrityCheck == null ? null : integrityCheck.createDigest();
OutputStream fileOutput = Files.newOutputStream(temp);
String algorithm;
String checksum;
if (integrityCheck != null) {
algorithm = integrityCheck.getAlgorithm();
checksum = integrityCheck.getChecksum();
} else if (bmclapiHash != null) {
algorithm = "SHA-1";
checksum = bmclapiHash;
} else {
algorithm = null;
checksum = null;
}
MessageDigest digest = algorithm != null ? DigestUtils.getDigest(algorithm) : null;
OutputStream fileOutput = Files.newOutputStream(temp);
return new Context() {
@Override
public void write(byte[] buffer, int offset, int len) throws IOException {
@ -245,13 +253,16 @@ public class FileDownloadTask extends FetchTask<Void> {
}
// Integrity check
if (integrityCheck != null) {
integrityCheck.performCheck(digest);
if (checksum != null) {
String actualChecksum = Hex.encodeHex(digest.digest());
if (!checksum.equalsIgnoreCase(actualChecksum)) {
throw new ChecksumMismatchException(algorithm, checksum, actualChecksum);
}
}
if (caching && integrityCheck != null) {
if (caching && algorithm != null) {
try {
repository.cacheFile(file, integrityCheck.getAlgorithm(), integrityCheck.getChecksum());
repository.cacheFile(file, algorithm, checksum);
} catch (IOException e) {
LOG.warning("Failed to cache file", e);
}

2
HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java
View File

@ -72,7 +72,7 @@ public final class GetTask extends FetchTask<String> {
}
@Override
protected Context getContext(URLConnection connection, boolean checkETag) {
protected Context getContext(URLConnection connection, boolean checkETag, String bmclapiHash) {
return new Context() {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();

19
HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java
View File

@ -24,8 +24,10 @@ import org.jackhuang.hmcl.util.function.ExceptionalSupplier;
import org.jackhuang.hmcl.util.gson.JsonUtils;
import org.jackhuang.hmcl.util.io.FileUtils;
import org.jackhuang.hmcl.util.io.NetworkUtils;
import org.jetbrains.annotations.Nullable;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLConnection;
@ -92,6 +94,7 @@ public class CacheRepository {
}
protected Path getFile(String algorithm, String hash) {
hash = hash.toLowerCase(Locale.ROOT);
return getCacheDirectory().resolve(algorithm).resolve(hash.substring(0, 2)).resolve(hash);
}
@ -121,7 +124,7 @@ public class CacheRepository {
return cache;
}
public Optional<Path> checkExistentFile(Path original, String algorithm, String hash) {
public Optional<Path> checkExistentFile(@Nullable Path original, String algorithm, String hash) {
if (fileExists(algorithm, hash))
return Optional.of(getFile(algorithm, hash));
@ -177,7 +180,7 @@ public class CacheRepository {
}
}
public void injectConnection(URLConnection conn) {
public void injectConnection(HttpURLConnection conn) {
conn.setUseCaches(true);
URI uri;
@ -258,11 +261,13 @@ public class CacheRepository {
if (oldItem == null) {
return newItem;
} else if (force || oldItem.compareTo(newItem) < 0) {
Path cached = getFile(SHA1, oldItem.hash);
try {
Files.deleteIfExists(cached);
} catch (IOException e) {
LOG.warning("Cannot delete old file");
if (!oldItem.hash.equalsIgnoreCase(newItem.hash)) {
Path cached = getFile(SHA1, oldItem.hash);
try {
Files.deleteIfExists(cached);
} catch (IOException e) {
LOG.warning("Cannot delete old file");
}
}
return newItem;
} else {

14
HMCLCore/src/main/java/org/jackhuang/hmcl/util/DigestUtils.java
View File

@ -25,7 +25,6 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
/**
*
* @author huangyuhui
*/
public final class DigestUtils {
@ -35,6 +34,19 @@ public final class DigestUtils {
private static final int STREAM_BUFFER_LENGTH = 1024;
public static boolean isSha1Digest(String digest) {
if (digest == null || digest.length() != 40) return false;
for (int i = 0; i < digest.length(); i++) {
char ch = digest.charAt(i);
if ((ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') && (ch < 'A' || ch > 'F')) {
return false;
}
}
return true;
}
public static MessageDigest getDigest(String algorithm) {
try {
return MessageDigest.getInstance(algorithm);

17
HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java
View File

@ -39,7 +39,7 @@ import static org.jackhuang.hmcl.util.logging.Logger.LOG;
public final class NetworkUtils {
public static final String PARAMETER_SEPARATOR = "&";
public static final String NAME_VALUE_SEPARATOR = "=";
private static final int TIME_OUT = 8000;
public static final int TIME_OUT = 8000;
private NetworkUtils() {
}
@ -114,7 +114,11 @@ public final class NetworkUtils {
URLConnection connection = uri.toURL().openConnection();
connection.setConnectTimeout(TIME_OUT);
connection.setReadTimeout(TIME_OUT);
connection.setRequestProperty("Accept-Language", Locale.getDefault().toLanguageTag());
if (connection instanceof HttpURLConnection) {
var httpConnection = (HttpURLConnection) connection;
httpConnection.setRequestProperty("Accept-Language", Locale.getDefault().toLanguageTag());
httpConnection.setInstanceFollowRedirects(false);
}
return connection;
}
@ -154,10 +158,6 @@ public final class NetworkUtils {
return sb.toString();
}
public static HttpURLConnection resolveConnection(HttpURLConnection conn) throws IOException {
return resolveConnection(conn, null);
}
/**
* This method is a work-around that aims to solve problem when "Location" in
* stupid server's response is not encoded.
@ -167,7 +167,7 @@ public final class NetworkUtils {
* @throws IOException if an I/O error occurs.
* @see <a href="https://github.com/curl/curl/issues/473">Issue with libcurl</a>
*/
public static HttpURLConnection resolveConnection(HttpURLConnection conn, List<String> redirects) throws IOException {
public static HttpURLConnection resolveConnection(HttpURLConnection conn) throws IOException {
final boolean useCache = conn.getUseCaches();
int redirect = 0;
while (true) {
@ -182,9 +182,6 @@ public final class NetworkUtils {
String newURL = conn.getHeaderField("Location");
conn.disconnect();
if (redirects != null) {
redirects.add(newURL);
}
if (redirect > 20) {
throw new IOException("Too much redirects");
}