diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadManager.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadManager.java index 8a5c851ea..2c9aba2c0 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadManager.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadManager.java @@ -10,50 +10,68 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.logging.Level; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.jackhuang.hmcl.util.Logging.LOG; class DownloadManager { - static DownloadState ne(int contentLength, int initialParts) { - - } - static DownloadTaskState download(List urls, Path file, int initialParts) throws IOException { Path downloadingFile = file.resolveSibling(FileUtils.getName(file) + ".download"); Path stateFile = file.resolveSibling(FileUtils.getName(file) + ".status"); - DownloadState state; + DownloadState state = null; if (Files.exists(downloadingFile) && Files.exists(stateFile)) { // Resume downloading from state try { String status = FileUtils.readText(stateFile); state = JsonUtils.fromNonNullJson(status, DownloadState.class); } catch (JsonParseException e) { - state = + LOG.log(Level.WARNING, "Failed to parse download state file", e); } } + + if (state == null || !urls.equals(state.urls)) { + return DownloadTaskState.newWithLengthUnknown(urls, initialParts); + } else { + return new DownloadTaskState(state); + } } protected static class DownloadTaskState { private final List urls; private final List segments; + private final List threads; + private String fastestUrl; + private int retry = 0; + private boolean cancelled = false; DownloadTaskState(DownloadState state) { urls = new ArrayList<>(state.urls); segments = new ArrayList<>(state.segments); + threads = IntStream.range(0, state.segments.size()).mapToObj(x -> (Thread) null).collect(Collectors.toList()); } DownloadTaskState(List urls, int contentLength, int initialParts) { - urls = new ArrayList<>(urls); + if (urls == null || urls.size() == 0) { + throw new IllegalArgumentException("DownloadTaskState requires at least one url candidate"); + } + + this.urls = new ArrayList<>(urls); segments = new ArrayList<>(initialParts); + threads = new ArrayList<>(initialParts); int partLength = contentLength / initialParts; for (int i = 0; i < initialParts; i++) { int begin = partLength * i; int end = Math.min((partLength + 1) * i, contentLength); segments.add(new DownloadSegment(begin, end, 0)); + threads.add(null); } } public static DownloadTaskState newWithLengthUnknown(List urls, int initialParts) { - return + return new DownloadTaskState(urls, 0, initialParts); } public List getUrls() { @@ -63,6 +81,52 @@ class DownloadManager { public List getSegments() { return segments; } + + public String getFirstUrl() { + return urls.get(0); + } + + /** + * Next url for download runnable to retry. + * + * If some download runnable fails to connect to url, it will call this method + * to acquire next url for retry. Making all download runnable try different + * candidates concurrently to speed up finding fastest download source. + * + * @return next url to retry + */ + public synchronized String getNextUrlToRetry() { + String url = urls.get(retry); + retry = (retry + 1) % urls.size(); + return url; + } + + /** + * One candidate that is accessible and best network connection qualified. + * + * When some download runnable have started downloading, DownloadManager will + * monitor download speed and make failed download runnable connect to the + * fastest url directly without retry. + * + * In some times, the fastest url may be the first url suceeded in connection. + * + * @return fastest url, null if no url have successfully connected yet. + */ + public synchronized String getFastestUrl() { + return fastestUrl; + } + + public synchronized void setFastestUrl(String fastestUrl) { + this.fastestUrl = fastestUrl; + } + + public synchronized void cancel() { + cancelled = true; + } + + public synchronized boolean isCancelled() { + return cancelled; + } } protected static class DownloadState { diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadTask.java index ffe71c6b6..1e6b5ffe5 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadTask.java @@ -1,5 +1,156 @@ package org.jackhuang.hmcl.task; -class DownloadTask { +import org.jackhuang.hmcl.util.CacheRepository; +import org.jackhuang.hmcl.util.Logging; +import org.jackhuang.hmcl.util.io.IOUtils; +import org.jackhuang.hmcl.util.io.NetworkUtils; +import org.jackhuang.hmcl.util.io.ResponseCodeException; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.nio.file.Path; +import java.util.logging.Level; + +abstract class DownloadTask implements Runnable { + + private final DownloadManager.DownloadTaskState state; + private final RandomAccessFile file; + private URLConnection conn; + private DownloadManager.DownloadSegment segment; + protected boolean caching; + protected CacheRepository repository = CacheRepository.getInstance(); + + public DownloadTask(DownloadManager.DownloadTaskState state, RandomAccessFile file, DownloadManager.DownloadSegment segment) { + this.state = state; + this.file = file; + this.segment = segment; + } + + public void setCaching(boolean caching) { + this.caching = caching; + } + + public void setCacheRepository(CacheRepository repository) { + this.repository = repository; + } + + protected void beforeDownload(URL url) throws IOException { + } + + protected abstract void useCachedResult(Path cachedFile) throws IOException; + + protected abstract FetchTask.EnumCheckETag shouldCheckETag(); + + protected abstract FetchTask.Context getContext(URLConnection conn, boolean checkETag) throws IOException; + + @Override + public void run() { + Exception exception = null; + URL failedURL = null; + boolean checkETag; + switch (shouldCheckETag()) { + case CHECK_E_TAG: + checkETag = true; + break; + case NOT_CHECK_E_TAG: + checkETag = false; + break; + default: + return; + } + + int repeat = 0; + while (true) { + if (state.isCancelled()) { + break; + } + + String url = repeat == 0 ? state.getFirstUrl() : state.getNextUrlToRetry(); + repeat++; + + if (url == null) { + break; + } + + try { + beforeDownload(url); + + updateProgress(0); + + URLConnection conn = NetworkUtils.createConnection(NetworkUtils.toURL(url); + if (checkETag) repository.injectConnection(conn); + + if (conn instanceof HttpURLConnection) { + conn = NetworkUtils.resolveConnection((HttpURLConnection) conn); + int responseCode = ((HttpURLConnection) conn).getResponseCode(); + + if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED) { + // Handle cache + try { + Path cache = repository.getCachedRemoteFile(conn); + useCachedResult(cache); + return; + } catch (IOException e) { + Logging.LOG.log(Level.WARNING, "Unable to use cached file, redownload " + url, e); + repository.removeRemoteEntry(conn); + // Now we must reconnect the server since 304 may result in empty content, + // if we want to redownload the file, we must reconnect the server without etag settings. + repeat--; + continue; + } + } else if (responseCode / 100 == 4) { + break; // we will not try this URL again + } else if (responseCode / 100 != 2) { + throw new ResponseCodeException(url, responseCode); + } + } + + long contentLength = conn.getContentLength(); + try (FetchTask.Context context = getContext(conn, checkETag); InputStream stream = conn.getInputStream()) { + int lastDownloaded = 0, downloaded = 0; + byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE]; + while (true) { + if (state.isCancelled()) break; + + int len = stream.read(buffer); + if (len == -1) break; + + context.write(buffer, 0, len); + + downloaded += len; + + if (contentLength >= 0) { + // Update progress information per second + updateProgress(downloaded, contentLength); + } + + updateDownloadSpeed(downloaded - lastDownloaded); + lastDownloaded = downloaded; + } + + if (state.isCancelled()) break; + + updateDownloadSpeed(downloaded - lastDownloaded); + + if (contentLength >= 0 && downloaded != contentLength) + throw new IOException("Unexpected file size: " + downloaded + ", expected: " + contentLength); + + context.withResult(true); + } + + return; + } catch (IOException ex) { + failedURL = url; + exception = ex; + Logging.LOG.log(Level.WARNING, "Failed to download " + url + ", repeat times: " + repeat, ex); + } + } + + if (exception != null) + throw new DownloadException(failedURL, exception); + } } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java index c47efda2c..1be029720 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java @@ -76,7 +76,7 @@ public final class GetTask extends FetchTask { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); @Override - public void write(byte[] buffer, int offset, int len) { + public synchronized void write(byte[] buffer, int offset, int len) { baos.write(buffer, offset, len); }