diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/fabric/FabricInstallTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/fabric/FabricInstallTask.java index 699ff0155..e577896a5 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/fabric/FabricInstallTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/fabric/FabricInstallTask.java @@ -49,8 +49,8 @@ public final class FabricInstallTask extends Task { this.version = version; this.remote = remoteVersion; - launchMetaTask = new GetTask(dependencyManager.getDownloadProvider().injectURLsWithCandidates(remoteVersion.getUrls())) - .setCacheRepository(dependencyManager.getCacheRepository()); + launchMetaTask = new GetTask(dependencyManager.getDownloadProvider().injectURLsWithCandidates(remoteVersion.getUrls())); + launchMetaTask.setCacheRepository(dependencyManager.getCacheRepository()); } @Override diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/forge/ForgeInstallTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/forge/ForgeInstallTask.java index 3e8fd366c..a558b65cc 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/forge/ForgeInstallTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/forge/ForgeInstallTask.java @@ -72,9 +72,9 @@ public final class ForgeInstallTask extends Task { dependent = new FileDownloadTask( dependencyManager.getDownloadProvider().injectURLsWithCandidates(remote.getUrls()), - installer.toFile(), null) - .setCacheRepository(dependencyManager.getCacheRepository()) - .setCaching(true); + installer.toFile(), null); + dependent.setCacheRepository(dependencyManager.getCacheRepository()); + dependent.setCaching(true); dependent.addIntegrityCheckHandler(FileDownloadTask.ZIP_INTEGRITY_CHECK_HANDLER); } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameAssetDownloadTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameAssetDownloadTask.java index f6b32d62f..775f0fa93 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameAssetDownloadTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameAssetDownloadTask.java @@ -113,11 +113,11 @@ public final class GameAssetDownloadTask extends Task { FileDownloadTask task = new FileDownloadTask(urls, file, new FileDownloadTask.IntegrityCheck("SHA-1", assetObject.getHash())); task.setName(assetObject.getHash()); - dependencies.add(task - .setCacheRepository(dependencyManager.getCacheRepository()) - .setCaching(true) - .setCandidate(dependencyManager.getCacheRepository().getCommonDirectory() - .resolve("assets").resolve("objects").resolve(assetObject.getLocation())).withCounter()); + task.setCandidate(dependencyManager.getCacheRepository().getCommonDirectory() + .resolve("assets").resolve("objects").resolve(assetObject.getLocation())); + task.setCacheRepository(dependencyManager.getCacheRepository()); + task.setCaching(true); + dependencies.add(task.withCounter()); } else { dependencyManager.getCacheRepository().tryCacheFile(file.toPath(), CacheRepository.SHA1, assetObject.getHash()); } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameAssetIndexDownloadTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameAssetIndexDownloadTask.java index 3f56f2b5c..a3731b87c 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameAssetIndexDownloadTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameAssetIndexDownloadTask.java @@ -63,10 +63,12 @@ public final class GameAssetIndexDownloadTask extends Task { // We should not check the hash code of asset index file since this file is not consistent // And Mojang will modify this file anytime. So assetIndex.hash might be outdated. - dependencies.add(new FileDownloadTask( + FileDownloadTask task = new FileDownloadTask( dependencyManager.getDownloadProvider().injectURLWithCandidates(assetIndexInfo.getUrl()), assetIndexFile - ).setCacheRepository(dependencyManager.getCacheRepository())); + ); + task.setCacheRepository(dependencyManager.getCacheRepository()); + dependencies.add(task); } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameDownloadTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameDownloadTask.java index 0b906dc54..5844bde9a 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameDownloadTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/GameDownloadTask.java @@ -59,9 +59,9 @@ public final class GameDownloadTask extends Task { FileDownloadTask task = new FileDownloadTask( dependencyManager.getDownloadProvider().injectURLWithCandidates(version.getDownloadInfo().getUrl()), jar, - IntegrityCheck.of(CacheRepository.SHA1, version.getDownloadInfo().getSha1())) - .setCaching(true) - .setCacheRepository(dependencyManager.getCacheRepository()); + IntegrityCheck.of(CacheRepository.SHA1, version.getDownloadInfo().getSha1())); + task.setCaching(true); + task.setCacheRepository(dependencyManager.getCacheRepository()); if (gameVersion != null) task.setCandidate(dependencyManager.getCacheRepository().getCommonDirectory().resolve("jars").resolve(gameVersion + ".jar")); diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/LibraryDownloadTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/LibraryDownloadTask.java index f50644948..8447e9b9c 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/LibraryDownloadTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/game/LibraryDownloadTask.java @@ -129,16 +129,16 @@ public class LibraryDownloadTask extends Task { URL packXz = NetworkUtils.toURL(dependencyManager.getDownloadProvider().injectURL(url) + ".pack.xz"); if (NetworkUtils.urlExists(packXz)) { List urls = dependencyManager.getDownloadProvider().injectURLWithCandidates(url + ".pack.xz"); - task = new FileDownloadTask(urls, xzFile, null) - .setCacheRepository(cacheRepository) - .setCaching(true); + task = new FileDownloadTask(urls, xzFile, null); + task.setCacheRepository(cacheRepository); + task.setCaching(true); xz = true; } else { List urls = dependencyManager.getDownloadProvider().injectURLWithCandidates(url); task = new FileDownloadTask(urls, jar, - library.getDownload().getSha1() != null ? new IntegrityCheck("SHA-1", library.getDownload().getSha1()) : null) - .setCacheRepository(cacheRepository) - .setCaching(true); + library.getDownload().getSha1() != null ? new IntegrityCheck("SHA-1", library.getDownload().getSha1()) : null); + task.setCacheRepository(cacheRepository); + task.setCaching(true); task.addIntegrityCheckHandler(FileDownloadTask.ZIP_INTEGRITY_CHECK_HANDLER); xz = false; } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/optifine/OptiFineInstallTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/optifine/OptiFineInstallTask.java index 43751f954..8764ed495 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/download/optifine/OptiFineInstallTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/download/optifine/OptiFineInstallTask.java @@ -92,11 +92,12 @@ public final class OptiFineInstallTask extends Task { dest = Files.createTempFile("optifine-installer", ".jar"); if (installer == null) { - dependents.add(new FileDownloadTask( + FileDownloadTask task = new FileDownloadTask( dependencyManager.getDownloadProvider().injectURLsWithCandidates(remote.getUrls()), - dest.toFile(), null) - .setCacheRepository(dependencyManager.getCacheRepository()) - .setCaching(true)); + dest.toFile(), null); + task.setCacheRepository(dependencyManager.getCacheRepository()); + task.setCaching(true); + dependents.add(task); } else { FileUtils.copyFile(installer, dest); } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/mod/curse/CurseCompletionTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/mod/curse/CurseCompletionTask.java index 582556dff..ac62aa2d8 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/mod/curse/CurseCompletionTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/mod/curse/CurseCompletionTask.java @@ -149,9 +149,10 @@ public final class CurseCompletionTask extends Task { for (CurseManifestFile file : newManifest.getFiles()) if (StringUtils.isNotBlank(file.getFileName())) { if (!modManager.hasSimpleMod(file.getFileName())) { - dependencies.add(new FileDownloadTask(file.getUrl(), modManager.getSimpleModPath(file.getFileName()).toFile()) - .setCacheRepository(dependency.getCacheRepository()) - .setCaching(true).withCounter()); + FileDownloadTask task = new FileDownloadTask(file.getUrl(), modManager.getSimpleModPath(file.getFileName()).toFile()); + task.setCacheRepository(dependency.getCacheRepository()); + task.setCaching(true); + dependencies.add(task.withCounter()); } } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java new file mode 100644 index 000000000..3f49b6c06 --- /dev/null +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java @@ -0,0 +1,228 @@ +/* + * Hello Minecraft! Launcher + * Copyright (C) 2020 huangyuhui and contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.jackhuang.hmcl.task; + +import org.jackhuang.hmcl.event.Event; +import org.jackhuang.hmcl.event.EventBus; +import org.jackhuang.hmcl.util.CacheRepository; +import org.jackhuang.hmcl.util.Logging; +import org.jackhuang.hmcl.util.ToStringBuilder; +import org.jackhuang.hmcl.util.io.IOUtils; +import org.jackhuang.hmcl.util.io.NetworkUtils; +import org.jackhuang.hmcl.util.io.ResponseCodeException; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; + +public abstract class FetchTask extends Task { + protected final List urls; + protected final int retry; + protected boolean caching; + protected CacheRepository repository = CacheRepository.getInstance(); + + public FetchTask(List urls, int retry) { + if (urls == null || urls.isEmpty()) + throw new IllegalArgumentException("At least one URL is required"); + + this.urls = new ArrayList<>(urls); + this.retry = retry; + + setExecutor(Schedulers.io()); + } + + public void setCaching(boolean caching) { + this.caching = caching; + } + + public void setCacheRepository(CacheRepository repository) { + this.repository = repository; + } + + protected void beforeDownload(URL url) throws IOException {} + + protected abstract void useCachedResult(Path cachedFile) throws IOException; + + protected abstract EnumCheckETag shouldCheckETag(); + + protected abstract Context getContext(URLConnection conn, boolean checkETag) throws IOException; + + @Override + public void execute() throws Exception { + Exception exception = null; + URL failedURL = null; + boolean checkETag; + switch (shouldCheckETag()) { + case CHECK_E_TAG: checkETag = true; break; + case NOT_CHECK_E_TAG: checkETag = false; break; + default: return; + } + + int repeat = 0; + download: for (URL url : urls) { + for (int retryTime = 0; retryTime < retry; retryTime++) { + if (isCancelled()) { + break download; + } + + try { + beforeDownload(url); + + updateProgress(0); + + URLConnection conn = NetworkUtils.createConnection(url); + if (checkETag) repository.injectConnection(conn); + + if (conn instanceof HttpURLConnection) { + conn = NetworkUtils.resolveConnection((HttpURLConnection) conn); + int responseCode = ((HttpURLConnection) conn).getResponseCode(); + + if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED) { + // Handle cache + try { + Path cache = repository.getCachedRemoteFile(conn); + useCachedResult(cache); + return; + } catch (IOException e) { + Logging.LOG.log(Level.WARNING, "Unable to use cached file, redownload " + url, e); + repository.removeRemoteEntry(conn); + // Now we must reconnect the server since 304 may result in empty content, + // if we want to redownload the file, we must reconnect the server without etag settings. + retryTime--; + continue; + } + } else if (responseCode / 100 == 4) { + break; // we will not try this URL again + } else if (responseCode / 100 != 2) { + throw new ResponseCodeException(url, responseCode); + } + } + + long contentLength = conn.getContentLength(); + try (Context context = getContext(conn, checkETag); InputStream stream = conn.getInputStream()) { + int lastDownloaded = 0, downloaded = 0; + byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE]; + while (true) { + if (isCancelled()) break; + + int len = stream.read(buffer); + if (len == -1) break; + + context.write(buffer, 0, len); + + downloaded += len; + + if (contentLength >= 0) { + // Update progress information per second + updateProgress(downloaded, contentLength); + } + + updateDownloadSpeed(downloaded - lastDownloaded); + lastDownloaded = downloaded; + } + + updateDownloadSpeed(downloaded - lastDownloaded); + + if (contentLength >= 0 && downloaded != contentLength) + throw new IOException("Unexpected file size: " + downloaded + ", expected: " + contentLength); + + if (isCancelled()) break download; + + context.withResult(true); + } + + return; + } catch (IOException ex) { + failedURL = url; + exception = ex; + Logging.LOG.log(Level.WARNING, "Failed to download " + url + ", repeat times: " + (++repeat), ex); + } + } + } + + if (exception != null) + throw new DownloadException(failedURL, exception); + } + + private static final Timer timer = new Timer("DownloadSpeedRecorder", true); + private static final AtomicInteger downloadSpeed = new AtomicInteger(0); + public static final EventBus speedEvent = new EventBus(); + + static { + timer.schedule(new TimerTask() { + @Override + public void run() { + speedEvent.channel(SpeedEvent.class).fireEvent(new SpeedEvent(speedEvent, downloadSpeed.getAndSet(0))); + } + }, 0, 1000); + } + + private static void updateDownloadSpeed(int speed) { + downloadSpeed.addAndGet(speed); + } + + public static class SpeedEvent extends Event { + private final int speed; + + public SpeedEvent(Object source, int speed) { + super(source); + + this.speed = speed; + } + + /** + * Download speed in byte/sec. + * @return download speed + */ + public int getSpeed() { + return speed; + } + + @Override + public String toString() { + return new ToStringBuilder(this).append("speed", speed).toString(); + } + } + + protected static abstract class Context implements Closeable { + private boolean success; + + public abstract void write(byte[] buffer, int offset, int len) throws IOException; + + public final void withResult(boolean success) { + this.success = success; + } + + protected boolean isSuccess() { + return success; + } + } + + protected enum EnumCheckETag { + CHECK_E_TAG, + NOT_CHECK_E_TAG, + CACHED + } +} diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java index f72d95bb4..33bd26cc9 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java @@ -17,26 +17,23 @@ */ package org.jackhuang.hmcl.task; -import org.jackhuang.hmcl.event.Event; -import org.jackhuang.hmcl.event.EventBus; -import org.jackhuang.hmcl.util.CacheRepository; import org.jackhuang.hmcl.util.Logging; -import org.jackhuang.hmcl.util.ToStringBuilder; -import org.jackhuang.hmcl.util.io.*; +import org.jackhuang.hmcl.util.io.ChecksumMismatchException; +import org.jackhuang.hmcl.util.io.CompressingUtils; +import org.jackhuang.hmcl.util.io.FileUtils; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; import java.math.BigInteger; -import java.net.HttpURLConnection; import java.net.URL; +import java.net.URLConnection; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import static java.util.Objects.requireNonNull; @@ -47,7 +44,7 @@ import static org.jackhuang.hmcl.util.DigestUtils.getDigest; * * @author huangyuhui */ -public class FileDownloadTask extends Task { +public class FileDownloadTask extends FetchTask { public static class IntegrityCheck { private String algorithm; @@ -83,13 +80,9 @@ public class FileDownloadTask extends Task { } } - private final List urls; private final File file; private final IntegrityCheck integrityCheck; - private final int retry; private Path candidate; - private boolean caching; - private CacheRepository repository = CacheRepository.getInstance(); private RandomAccessFile rFile; private InputStream stream; private final ArrayList integrityCheckHandlers = new ArrayList<>(); @@ -148,35 +141,11 @@ public class FileDownloadTask extends Task { * @param retry the times for retrying if downloading fails. */ public FileDownloadTask(List urls, File file, IntegrityCheck integrityCheck, int retry) { - if (urls == null || urls.isEmpty()) - throw new IllegalArgumentException("At least one URL is required"); - - this.urls = new ArrayList<>(urls); + super(urls, retry); this.file = file; this.integrityCheck = integrityCheck; - this.retry = retry; setName(file.getName()); - setExecutor(Schedulers.io()); - } - - private void closeFiles() { - if (rFile != null) - try { - rFile.close(); - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Failed to close file: " + rFile, e); - } - - rFile = null; - - if (stream != null) - try { - stream.close(); - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Failed to close stream", e); - } - stream = null; } public File getFile() { @@ -188,217 +157,105 @@ public class FileDownloadTask extends Task { return this; } - public FileDownloadTask setCaching(boolean caching) { - this.caching = caching; - return this; - } - - public FileDownloadTask setCacheRepository(CacheRepository repository) { - this.repository = repository; - return this; - } - public void addIntegrityCheckHandler(IntegrityCheckHandler handler) { integrityCheckHandlers.add(Objects.requireNonNull(handler)); } @Override - public void execute() throws Exception { - boolean checkETag; + protected EnumCheckETag shouldCheckETag() { // Check cache if (integrityCheck != null && caching) { - checkETag = false; Optional cache = repository.checkExistentFile(candidate, integrityCheck.getAlgorithm(), integrityCheck.getChecksum()); if (cache.isPresent()) { try { FileUtils.copyFile(cache.get().toFile(), file); Logging.LOG.log(Level.FINER, "Successfully verified file " + file + " from " + urls.get(0)); - return; + return EnumCheckETag.CACHED; } catch (IOException e) { Logging.LOG.log(Level.WARNING, "Failed to copy cache files", e); } } + return EnumCheckETag.NOT_CHECK_E_TAG; } else { - checkETag = true; + return EnumCheckETag.CHECK_E_TAG; } + } - Exception exception = null; - URL failedURL = null; + @Override + protected void beforeDownload(URL url) { + Logging.LOG.log(Level.FINER, "Downloading " + url + " to " + file); + } - int repeat = 0; - download: for (URL url : urls) { - for (int retryTime = 0; retryTime < retry; retryTime++) { - if (isCancelled()) { - break download; + @Override + protected void useCachedResult(Path cache) throws IOException { + FileUtils.copyFile(cache.toFile(), file); + } + + @Override + protected Context getContext(URLConnection conn, boolean checkETag) throws IOException { + Path temp = Files.createTempFile(null, null); + RandomAccessFile rFile = new RandomAccessFile(temp.toFile(), "rw"); + MessageDigest digest = integrityCheck == null ? null : integrityCheck.createDigest(); + + return new Context() { + @Override + public void write(byte[] buffer, int offset, int len) throws IOException { + if (digest != null) { + digest.update(buffer, offset, len); } - Logging.LOG.log(Level.FINER, "Downloading " + url + " to " + file); - Path temp = null; + rFile.write(buffer, offset, len); + } + + @Override + public void close() throws IOException { + try { + rFile.close(); + } catch (IOException e) { + Logging.LOG.log(Level.WARNING, "Failed to close file: " + rFile, e); + } + + if (!isSuccess()) { + try { + Files.delete(temp); + } catch (IOException e) { + Logging.LOG.log(Level.WARNING, "Failed to delete file: " + rFile, e); + } + return; + } + + for (IntegrityCheckHandler handler : integrityCheckHandlers) { + handler.checkIntegrity(temp, file.toPath()); + } + + Files.deleteIfExists(file.toPath()); + if (!FileUtils.makeDirectory(file.getAbsoluteFile().getParentFile())) + throw new IOException("Unable to make parent directory " + file); try { - updateProgress(0); + FileUtils.moveFile(temp.toFile(), file); + } catch (Exception e) { + throw new IOException("Unable to move temp file from " + temp + " to " + file, e); + } - HttpURLConnection con = NetworkUtils.createConnection(url); - if (checkETag) repository.injectConnection(con); - con = NetworkUtils.resolveConnection(con); + // Integrity check + if (integrityCheck != null) { + integrityCheck.performCheck(digest); + } - if (con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { - // Handle cache - try { - Path cache = repository.getCachedRemoteFile(con); - FileUtils.copyFile(cache.toFile(), file); - return; - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Unable to use cached file, redownload it", e); - repository.removeRemoteEntry(con); - // Now we must reconnect the server since 304 may result in empty content, - // if we want to redownload the file, we must reconnect the server without etag settings. - retryTime--; - continue; - } - } else if (con.getResponseCode() / 100 == 4) { - break; // we will not try this URL again - } else if (con.getResponseCode() / 100 != 2) { - throw new ResponseCodeException(url, con.getResponseCode()); - } - - int contentLength = con.getContentLength(); - if (contentLength < 0) - throw new IOException("The content length is invalid."); - - if (!FileUtils.makeDirectory(file.getAbsoluteFile().getParentFile())) - throw new IOException("Could not make directory " + file.getAbsoluteFile().getParent()); - - temp = Files.createTempFile(null, null); - rFile = new RandomAccessFile(temp.toFile(), "rw"); - - MessageDigest digest = integrityCheck == null ? null : integrityCheck.createDigest(); - - stream = con.getInputStream(); - int lastDownloaded = 0, downloaded = 0; - byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE]; - while (true) { - if (isCancelled()) { - break; - } - - int read = stream.read(buffer); - if (read == -1) - break; - - if (digest != null) { - digest.update(buffer, 0, read); - } - - // Write buffer to file. - rFile.write(buffer, 0, read); - downloaded += read; - - // Update progress information per second - updateProgress(downloaded, contentLength); - - updateDownloadSpeed(downloaded - lastDownloaded); - lastDownloaded = downloaded; - } - - updateDownloadSpeed(downloaded - lastDownloaded); - - closeFiles(); - - if (downloaded != contentLength) - throw new IOException("Unexpected file size: " + downloaded + ", expected: " + contentLength); - - // Restore temp file to original name. - if (isCancelled()) { - temp.toFile().delete(); - break download; - } - - for (IntegrityCheckHandler handler : integrityCheckHandlers) { - handler.checkIntegrity(temp, file.toPath()); - } - - Files.deleteIfExists(file.toPath()); - if (!FileUtils.makeDirectory(file.getAbsoluteFile().getParentFile())) - throw new IOException("Unable to make parent directory " + file); + if (caching && integrityCheck != null) { try { - FileUtils.moveFile(temp.toFile(), file); - } catch (Exception e) { - throw new IOException("Unable to move temp file from " + temp + " to " + file, e); + repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum()); + } catch (IOException e) { + Logging.LOG.log(Level.WARNING, "Failed to cache file", e); } + } - // Integrity check - if (integrityCheck != null) { - integrityCheck.performCheck(digest); - } - - if (caching && integrityCheck != null) { - try { - repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum()); - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Failed to cache file", e); - } - } - - if (checkETag) { - repository.cacheRemoteFile(file.toPath(), con); - } - - return; - } catch (IOException e) { - if (temp != null) - temp.toFile().delete(); - failedURL = url; - exception = e; - Logging.LOG.log(Level.WARNING, "Failed to download " + url + ", repeat times: " + (++repeat), e); - } finally { - closeFiles(); + if (checkETag) { + repository.cacheRemoteFile(file.toPath(), conn); } } - } - - if (exception != null) - throw new DownloadException(failedURL, exception); - } - - private static final Timer timer = new Timer("DownloadSpeedRecorder", true); - private static final AtomicInteger downloadSpeed = new AtomicInteger(0); - public static final EventBus speedEvent = new EventBus(); - - static { - timer.schedule(new TimerTask() { - @Override - public void run() { - speedEvent.channel(SpeedEvent.class).fireEvent(new SpeedEvent(speedEvent, downloadSpeed.getAndSet(0))); - } - }, 0, 1000); - } - - private static void updateDownloadSpeed(int speed) { - downloadSpeed.addAndGet(speed); - } - - public static class SpeedEvent extends Event { - private final int speed; - - public SpeedEvent(Object source, int speed) { - super(source); - - this.speed = speed; - } - - /** - * Download speed in byte/sec. - * @return download speed - */ - public int getSpeed() { - return speed; - } - - @Override - public String toString() { - return new ToStringBuilder(this).append("speed", speed).toString(); - } + }; } public interface IntegrityCheckHandler { diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java index 5ebe123f0..c47efda2c 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java @@ -17,23 +17,15 @@ */ package org.jackhuang.hmcl.task; -import org.jackhuang.hmcl.util.CacheRepository; -import org.jackhuang.hmcl.util.Logging; import org.jackhuang.hmcl.util.io.FileUtils; -import org.jackhuang.hmcl.util.io.IOUtils; -import org.jackhuang.hmcl.util.io.NetworkUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; import java.net.URL; +import java.net.URLConnection; import java.nio.charset.Charset; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; +import java.util.*; import static java.nio.charset.StandardCharsets.UTF_8; @@ -41,12 +33,9 @@ import static java.nio.charset.StandardCharsets.UTF_8; * * @author huangyuhui */ -public final class GetTask extends Task { +public final class GetTask extends FetchTask { - private final List urls; private final Charset charset; - private final int retry; - private CacheRepository repository = CacheRepository.getInstance(); public GetTask(URL url) { this(url, UTF_8); @@ -65,68 +54,35 @@ public final class GetTask extends Task { } public GetTask(List urls, Charset charset, int retry) { - this.urls = new ArrayList<>(urls); + super(urls, retry); this.charset = charset; - this.retry = retry; setName(urls.get(0).toString()); - setExecutor(Schedulers.io()); - } - - public GetTask setCacheRepository(CacheRepository repository) { - this.repository = repository; - return this; } @Override - public void execute() throws Exception { - Exception exception = null; - URL failedURL = null; - boolean checkETag = true; - for (int time = 0; time < retry * urls.size(); ++time) { - URL url = urls.get(time / retry); - if (isCancelled()) { - break; + protected EnumCheckETag shouldCheckETag() { + return EnumCheckETag.CHECK_E_TAG; + } + + @Override + protected void useCachedResult(Path cachedFile) throws IOException { + setResult(FileUtils.readText(cachedFile)); + } + + @Override + protected Context getContext(URLConnection conn, boolean checkETag) { + return new Context() { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + @Override + public void write(byte[] buffer, int offset, int len) { + baos.write(buffer, offset, len); } - try { - updateProgress(0); - HttpURLConnection conn = NetworkUtils.createConnection(url); - if (checkETag) repository.injectConnection(conn); - conn = NetworkUtils.resolveConnection(conn); - - if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { - // Handle cache - try { - Path cache = repository.getCachedRemoteFile(conn); - setResult(FileUtils.readText(cache)); - return; - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Unable to use cached file, redownload it", e); - repository.removeRemoteEntry(conn); - continue; - } - } else if (conn.getResponseCode() / 100 != 2) { - throw new IOException("Server error, response code: " + conn.getResponseCode()); - } - - InputStream input = conn.getInputStream(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - byte[] buf = new byte[IOUtils.DEFAULT_BUFFER_SIZE]; - int size = conn.getContentLength(), read = 0, len; - while ((len = input.read(buf)) != -1) { - baos.write(buf, 0, len); - read += len; - - if (size >= 0) - updateProgress(read, size); - - if (Thread.currentThread().isInterrupted()) - return; - } - - if (size > 0 && size != read) - throw new IOException("Not completed! Readed: " + read + ", total size: " + size); + @Override + public void close() throws IOException { + if (!isSuccess()) return; String result = baos.toString(charset.name()); setResult(result); @@ -134,15 +90,8 @@ public final class GetTask extends Task { if (checkETag) { repository.cacheText(result, conn); } - return; - } catch (IOException ex) { - failedURL = url; - exception = ex; - Logging.LOG.log(Level.WARNING, "Failed to download " + url + ", repeat times: " + (time + 1), ex); } - } - if (exception != null) - throw new DownloadException(failedURL, exception); + }; } } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/Task.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/Task.java index ddb02d7c7..54d32264d 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/Task.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/Task.java @@ -310,7 +310,7 @@ public abstract class Task { return progress.getReadOnlyProperty(); } - protected void updateProgress(int progress, int total) { + protected void updateProgress(long progress, long total) { updateProgress(1.0 * progress / total); } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java index 13e22294d..0f58686d7 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java @@ -54,14 +54,18 @@ public final class NetworkUtils { return sb.toString(); } - public static HttpURLConnection createConnection(URL url) throws IOException { - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + public static URLConnection createConnection(URL url) throws IOException { + URLConnection connection = url.openConnection(); connection.setUseCaches(false); connection.setConnectTimeout(15000); connection.setReadTimeout(15000); return connection; } + public static HttpURLConnection createHttpConnection(URL url) throws IOException { + return (HttpURLConnection) createConnection(url); + } + /** * @see Curl * @param location the url to be URL encoded @@ -129,7 +133,7 @@ public final class NetworkUtils { } public static String doGet(URL url) throws IOException { - HttpURLConnection con = createConnection(url); + HttpURLConnection con = createHttpConnection(url); con = resolveConnection(con); return IOUtils.readFullyAsString(con.getInputStream()); } @@ -151,7 +155,7 @@ public final class NetworkUtils { public static String doPost(URL url, String post, String contentType) throws IOException { byte[] bytes = post.getBytes(UTF_8); - HttpURLConnection con = createConnection(url); + HttpURLConnection con = createHttpConnection(url); con.setRequestMethod("POST"); con.setDoOutput(true); con.setRequestProperty("Content-Type", contentType + "; charset=utf-8"); @@ -176,7 +180,7 @@ public final class NetworkUtils { } public static String detectFileName(URL url) throws IOException { - HttpURLConnection conn = resolveConnection(createConnection(url)); + HttpURLConnection conn = resolveConnection(createHttpConnection(url)); int code = conn.getResponseCode(); if (code / 100 == 4) throw new FileNotFoundException(); @@ -214,7 +218,7 @@ public final class NetworkUtils { } public static boolean urlExists(URL url) throws IOException { - HttpURLConnection con = createConnection(url); + HttpURLConnection con = createHttpConnection(url); con = resolveConnection(con); int responseCode = con.getResponseCode(); con.disconnect();