diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java index 76d1452d7..060475f46 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java @@ -179,8 +179,10 @@ public class FileDownloadTask extends Task { public void execute() throws Exception { URL currentURL = url; + boolean checkETag; // Check cache if (integrityCheck != null && caching) { + checkETag = false; Optional cache = repository.checkExistentFile(candidate, integrityCheck.getAlgorithm(), integrityCheck.getChecksum()); if (cache.isPresent()) { try { @@ -191,6 +193,8 @@ public class FileDownloadTask extends Task { Logging.LOG.log(Level.WARNING, "Failed to copy cache files", e); } } + } else { + checkETag = true; } Logging.LOG.log(Level.FINER, "Downloading " + currentURL + " to " + file); @@ -213,10 +217,16 @@ public class FileDownloadTask extends Task { updateProgress(0); HttpURLConnection con = NetworkUtils.createConnection(url); + if (checkETag) repository.injectConnection(con); con.connect(); - if (con.getResponseCode() / 100 != 2) + if (con.getResponseCode() == 304) { + // Handle cache + Path cache = repository.getCachedRemoteFile(con); + FileUtils.copyFile(cache.toFile(), file); + } else if (con.getResponseCode() / 100 != 2) { throw new IOException("Server error, response code: " + con.getResponseCode()); + } int contentLength = con.getContentLength(); if (contentLength < 1) @@ -289,6 +299,21 @@ public class FileDownloadTask extends Task { integrityCheck.performCheck(digest); } + if (caching) { + try { + if (integrityCheck == null) + repository.cacheFile(file.toPath(), CacheRepository.SHA1, Hex.encodeHex(DigestUtils.digest(CacheRepository.SHA1, file.toPath()))); + else + repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum()); + } catch (IOException e) { + Logging.LOG.log(Level.WARNING, "Failed to cache file", e); + } + } + + if (checkETag) { + repository.cacheRemoteFile(file.toPath(), con); + } + return; } catch (IOException | IllegalStateException e) { if (temp != null) @@ -301,17 +326,6 @@ public class FileDownloadTask extends Task { if (exception != null) throw new IOException("Unable to download file " + currentURL + ". " + exception.getMessage(), exception); - - if (caching) { - try { - if (integrityCheck == null) - repository.cacheFile(file.toPath(), CacheRepository.SHA1, Hex.encodeHex(DigestUtils.digest(CacheRepository.SHA1, file.toPath()))); - else - repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum()); - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Failed to cache file", e); - } - } } } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java index 151dce699..5ee639722 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java @@ -17,7 +17,9 @@ */ package org.jackhuang.hmcl.task; +import org.jackhuang.hmcl.util.CacheRepository; import org.jackhuang.hmcl.util.Logging; +import org.jackhuang.hmcl.util.io.FileUtils; import org.jackhuang.hmcl.util.io.IOUtils; import org.jackhuang.hmcl.util.io.NetworkUtils; @@ -27,6 +29,8 @@ import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.logging.Level; import static java.nio.charset.StandardCharsets.UTF_8; @@ -41,6 +45,7 @@ public final class GetTask extends TaskResult { private final Charset charset; private final int retry; private final String id; + private CacheRepository repository = CacheRepository.getInstance(); public GetTask(URL url) { this(url, ID); @@ -73,15 +78,33 @@ public final class GetTask extends TaskResult { return id; } + public GetTask setCacheRepository(CacheRepository repository) { + this.repository = repository; + return this; + } + @Override public void execute() throws Exception { Exception exception = null; + boolean checkETag = true; for (int time = 0; time < retry; ++time) { if (time > 0) Logging.LOG.log(Level.WARNING, "Failed to download, repeat times: " + time); try { updateProgress(0); HttpURLConnection conn = NetworkUtils.createConnection(url); + if (checkETag) repository.injectConnection(conn); + conn.connect(); + + if (conn.getResponseCode() == 304) { + // Handle cache + Path cache = repository.getCachedRemoteFile(conn); + setResult(FileUtils.readText(cache)); + return; + } else if (conn.getResponseCode() / 100 != 2) { + throw new IOException("Server error, response code: " + conn.getResponseCode()); + } + InputStream input = conn.getInputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); byte[] buf = new byte[IOUtils.DEFAULT_BUFFER_SIZE]; @@ -100,7 +123,12 @@ public final class GetTask extends TaskResult { if (size > 0 && size != read) throw new IllegalStateException("Not completed! Readed: " + read + ", total size: " + size); - setResult(baos.toString(charset.name())); + String result = baos.toString(charset.name()); + setResult(result); + + if (checkETag) { + repository.cacheText(result, conn); + } return; } catch (IOException ex) { exception = ex; diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java index e6dfb9de6..0a9f1c6b3 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/CacheRepository.java @@ -17,21 +17,63 @@ */ package org.jackhuang.hmcl.util; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.net.URLConnection; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Optional; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.logging.Level; +import java.util.stream.Stream; +import com.google.gson.JsonParseException; +import com.google.gson.annotations.SerializedName; import org.jackhuang.hmcl.util.function.ExceptionalSupplier; +import org.jackhuang.hmcl.util.gson.JsonUtils; import org.jackhuang.hmcl.util.io.FileUtils; +import org.jackhuang.hmcl.util.io.IOUtils; + +import static java.nio.charset.StandardCharsets.UTF_8; public class CacheRepository { private Path commonDirectory; private Path cacheDirectory; + private Path indexFile; + private Map index; + private final ReadWriteLock lock = new ReentrantReadWriteLock(); public void changeDirectory(Path commonDir) { commonDirectory = commonDir; cacheDirectory = commonDir.resolve("cache"); + indexFile = cacheDirectory.resolve("etag.json"); + + lock.writeLock().lock(); + try { + if (Files.isRegularFile(indexFile)) { + ETagIndex raw = JsonUtils.GSON.fromJson(FileUtils.readText(indexFile.toFile()), ETagIndex.class); + if (raw == null) + index = new HashMap<>(); + else + index = joinETagIndexes(raw.eTag); + } else + index = new HashMap<>(); + } catch (IOException | JsonParseException e) { + Logging.LOG.log(Level.WARNING, "Unable to read index file", e); + index = new HashMap<>(); + } finally { + lock.writeLock().unlock(); + } } public Path getCommonDirectory() { @@ -100,6 +142,181 @@ public class CacheRepository { return cache; } + public Path getCachedRemoteFile(URLConnection conn) throws IOException { + String url = conn.getURL().toString(); + lock.readLock().lock(); + ETagItem eTagItem; + try { + eTagItem = index.get(url); + } finally { + lock.readLock().unlock(); + } + if (eTagItem == null) throw new IOException("Cannot find the URL"); + if (StringUtils.isBlank(eTagItem.hash) || !fileExists(SHA1, eTagItem.hash)) throw new FileNotFoundException(); + Path file = getFile(SHA1, eTagItem.hash); + if (Files.getLastModifiedTime(file).toMillis() != eTagItem.localLastModified) { + String hash = Hex.encodeHex(DigestUtils.digest(SHA1, file)); + if (!Objects.equals(hash, eTagItem.hash)) + throw new IOException("This file is modified"); + } + return file; + } + + public void injectConnection(URLConnection conn) { + String url = conn.getURL().toString(); + lock.readLock().lock(); + ETagItem eTagItem; + try { + eTagItem = index.get(url); + } finally { + lock.readLock().unlock(); + } + if (eTagItem == null) return; + if (eTagItem.eTag != null) + conn.setRequestProperty("If-None-Match", eTagItem.eTag); + // if (eTagItem.getRemoteLastModified() != null) + // conn.setRequestProperty("If-Modified-Since", eTagItem.getRemoteLastModified()); + } + + public synchronized void cacheRemoteFile(Path downloaded, URLConnection conn) throws IOException { + String eTag = conn.getHeaderField("ETag"); + if (eTag == null) return; + String url = conn.getURL().toString(); + String lastModified = conn.getHeaderField("Last-Modified"); + String hash = Hex.encodeHex(DigestUtils.digest(SHA1, downloaded)); + Path cached = cacheFile(downloaded, SHA1, hash); + ETagItem eTagItem = new ETagItem(url, eTag, hash, Files.getLastModifiedTime(cached).toMillis(), lastModified); + Lock writeLock = lock.writeLock(); + writeLock.lock(); + try { + index.put(url, eTagItem); + saveETagIndex(); + } finally { + writeLock.unlock(); + } + } + + public synchronized void cacheText(String text, URLConnection conn) throws IOException { + String eTag = conn.getHeaderField("ETag"); + if (eTag == null) return; + String url = conn.getURL().toString(); + String lastModified = conn.getHeaderField("Last-Modified"); + String hash = Hex.encodeHex(DigestUtils.digest(SHA1, text)); + Path cached = getFile(SHA1, hash); + FileUtils.writeText(cached.toFile(), text); + ETagItem eTagItem = new ETagItem(url, eTag, hash, Files.getLastModifiedTime(cached).toMillis(), lastModified); + Lock writeLock = lock.writeLock(); + writeLock.lock(); + try { + index.put(url, eTagItem); + saveETagIndex(); + } finally { + writeLock.unlock(); + } + } + + @SafeVarargs + private final Map joinETagIndexes(Collection... indexes) { + Map eTags = new ConcurrentHashMap<>(); + + Stream stream = Arrays.stream(indexes).filter(Objects::nonNull).map(Collection::stream) + .reduce(Stream.empty(), Stream::concat); + + stream.forEach(eTag -> { + eTags.compute(eTag.url, (key, oldValue) -> { + if (oldValue == null || oldValue.compareTo(eTag) < 0) + return eTag; + else + return oldValue; + }); + }); + + return eTags; + } + + public void saveETagIndex() throws IOException { + try (RandomAccessFile file = new RandomAccessFile(indexFile.toFile(), "rw"); FileChannel channel = file.getChannel()) { + FileLock lock = channel.lock(); + try { + ETagIndex indexOnDisk = JsonUtils.GSON.fromJson(new String(IOUtils.readFullyWithoutClosing(Channels.newInputStream(channel)), UTF_8), ETagIndex.class); + Map newIndex = joinETagIndexes(indexOnDisk == null ? null : indexOnDisk.eTag, index.values()); + channel.truncate(0); + OutputStream os = Channels.newOutputStream(channel); + ETagIndex writeTo = new ETagIndex(newIndex.values()); + IOUtils.write(JsonUtils.GSON.toJson(writeTo).getBytes(UTF_8), os); + this.index = newIndex; + } finally { + lock.release(); + } + } + } + + private class ETagIndex { + private final Collection eTag; + + public ETagIndex() { + this.eTag = new HashSet<>(); + } + + public ETagIndex(Collection eTags) { + this.eTag = new HashSet<>(eTags); + } + } + + private class ETagItem { + private final String url; + private final String eTag; + private final String hash; + @SerializedName("local") + private final long localLastModified; + @SerializedName("remote") + private final String remoteLastModified; + + /** + * For Gson. + */ + public ETagItem() { + this(null, null, null, 0, null); + } + + public ETagItem(String url, String eTag, String hash, long localLastModified, String remoteLastModified) { + this.url = url; + this.eTag = eTag; + this.hash = hash; + this.localLastModified = localLastModified; + this.remoteLastModified = remoteLastModified; + } + + public int compareTo(ETagItem other) { + if (!url.equals(other.url)) + throw new IllegalArgumentException(); + + ZonedDateTime thisTime = Lang.ignoringException(() -> ZonedDateTime.parse(remoteLastModified, DateTimeFormatter.RFC_1123_DATE_TIME), null); + ZonedDateTime otherTime = Lang.ignoringException(() -> ZonedDateTime.parse(other.remoteLastModified, DateTimeFormatter.RFC_1123_DATE_TIME), null); + if (thisTime == null && otherTime == null) return 0; + else if (thisTime == null) return -1; + else if (otherTime == null) return 1; + else return thisTime.compareTo(otherTime); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ETagItem eTagItem = (ETagItem) o; + return localLastModified == eTagItem.localLastModified && + Objects.equals(url, eTagItem.url) && + Objects.equals(eTag, eTagItem.eTag) && + Objects.equals(hash, eTagItem.hash) && + Objects.equals(remoteLastModified, eTagItem.remoteLastModified); + } + + @Override + public int hashCode() { + return Objects.hash(url, eTag, hash, localLastModified, remoteLastModified); + } + } + private static CacheRepository instance = new CacheRepository(); public static CacheRepository getInstance() { diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/IOUtils.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/IOUtils.java index 17293a0a2..936da3e1e 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/IOUtils.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/IOUtils.java @@ -17,10 +17,7 @@ */ package org.jackhuang.hmcl.util.io; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.nio.charset.Charset; /** @@ -60,6 +57,14 @@ public final class IOUtils { return readFully(stream).toString(charset.name()); } + public static void write(String text, OutputStream outputStream) throws IOException { + write(text.getBytes(), outputStream); + } + + public static void write(byte[] bytes, OutputStream outputStream) throws IOException { + copyTo(new ByteArrayInputStream(bytes), outputStream); + } + public static void copyTo(InputStream src, OutputStream dest) throws IOException { copyTo(src, dest, new byte[DEFAULT_BUFFER_SIZE]); }