From 9f486f5549aadfc7ceca585287ac320d74c6590d Mon Sep 17 00:00:00 2001 From: huanghongxun Date: Sat, 19 Sep 2020 13:19:40 +0800 Subject: [PATCH] feat: WIP download task --- .../hmcl/task/CompletableFutureTask.java | 1 + .../jackhuang/hmcl/task/DownloadManager.java | 478 +++++++++++++++--- .../hmcl/task/DownloadSegmentTask.java | 186 +++++++ .../org/jackhuang/hmcl/task/DownloadTask.java | 156 ------ .../org/jackhuang/hmcl/task/FetchTask.java | 276 ---------- .../jackhuang/hmcl/task/FileDownloadTask.java | 202 +++----- .../java/org/jackhuang/hmcl/task/GetTask.java | 66 +-- .../jackhuang/hmcl/util/io/NetworkUtils.java | 37 +- 8 files changed, 733 insertions(+), 669 deletions(-) create mode 100644 HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadSegmentTask.java delete mode 100644 HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadTask.java delete mode 100644 HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/CompletableFutureTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/CompletableFutureTask.java index b3c19031f..97c64465c 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/CompletableFutureTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/CompletableFutureTask.java @@ -8,5 +8,6 @@ public abstract class CompletableFutureTask extends Task { @Override public void execute() throws Exception { + throw new AssertionError("Cannot reach here"); } } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadManager.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadManager.java index 2c9aba2c0..1e49498a8 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadManager.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadManager.java @@ -1,66 +1,250 @@ package org.jackhuang.hmcl.task; import com.google.gson.JsonParseException; +import org.jackhuang.hmcl.event.Event; +import org.jackhuang.hmcl.event.EventBus; +import org.jackhuang.hmcl.util.CacheRepository; +import org.jackhuang.hmcl.util.ToStringBuilder; import org.jackhuang.hmcl.util.gson.JsonUtils; import org.jackhuang.hmcl.util.io.FileUtils; +import org.jetbrains.annotations.Nullable; +import java.io.File; import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.jackhuang.hmcl.util.Logging.LOG; -class DownloadManager { +public class DownloadManager { - static DownloadTaskState download(List urls, Path file, int initialParts) throws IOException { - Path downloadingFile = file.resolveSibling(FileUtils.getName(file) + ".download"); - Path stateFile = file.resolveSibling(FileUtils.getName(file) + ".status"); - DownloadState state = null; - if (Files.exists(downloadingFile) && Files.exists(stateFile)) { - // Resume downloading from state - try { - String status = FileUtils.readText(stateFile); - state = JsonUtils.fromNonNullJson(status, DownloadState.class); - } catch (JsonParseException e) { - LOG.log(Level.WARNING, "Failed to parse download state file", e); - } + public static class DownloadTaskStateBuilder { + private List urls; + private Path file; + private int retry = 3; + private int initialParts = 1; + + /** + * Set the url of remote file to be downloaded. + * @param url url of the remote file to be downloaded + * @return this + */ + public DownloadTaskStateBuilder setUrl(URL url) { + this.urls = Collections.singletonList(url); + return this; } - if (state == null || !urls.equals(state.urls)) { - return DownloadTaskState.newWithLengthUnknown(urls, initialParts); - } else { - return new DownloadTaskState(state); + /** + * Set urls of the remote file to be downloaded, will be attempted in order. + * @param urls urls of remote files to be downloaded + * @return this + */ + public DownloadTaskStateBuilder setUrls(List urls) { + this.urls = urls; + return this; + } + + /** + * Set location to save remote file. + * @param file location to save the remote file. + * @return this + */ + public DownloadTaskStateBuilder setFile(Path file) { + this.file = file; + return this; + } + + /** + * Set location to save remote file. + * @param file location to save the remote file. + * @return this + */ + public DownloadTaskStateBuilder setFile(File file) { + this.file = file.toPath(); + return this; + } + + /** + * Set retry times of one url. + * @param retry retry times of one url. + * @return this + */ + public DownloadTaskStateBuilder setRetry(int retry) { + this.retry = retry; + return this; + } + + /** + * Splits the remote file into multiple parts, and download in different + * threads. + * + * @param initialParts number of threads to download the file. + * @return this + */ + public DownloadTaskStateBuilder setInitialParts(int initialParts) { + this.initialParts = initialParts; + return this; + } + + public DownloadTaskState build() throws IOException { + if (file == null) { + return DownloadTaskState.newWithLengthUnknown(urls, Files.createTempFile(null, null), retry, initialParts); + } + + Path downloadingFile = file.resolveSibling(FileUtils.getName(file) + ".download"); + Path stateFile = file.resolveSibling(FileUtils.getName(file) + ".status"); + DownloadState state = null; + if (Files.exists(downloadingFile) && Files.exists(stateFile)) { + // Resume downloading from state + try { + String status = FileUtils.readText(stateFile); + state = JsonUtils.fromNonNullJson(status, DownloadState.class); + } catch (JsonParseException e) { + LOG.log(Level.WARNING, "Failed to parse download state file", e); + } + } + + if (state == null || !urls.equals(state.urls)) { + return DownloadTaskState.newWithLengthUnknown(urls, file, retry, initialParts); + } else { + return new DownloadTaskState(state, file, retry); + } } } - protected static class DownloadTaskState { - private final List urls; - private final List segments; - private final List threads; - private String fastestUrl; - private int retry = 0; - private boolean cancelled = false; + protected static class SafeRegion implements AutoCloseable { + final ReentrantLock lock = new ReentrantLock(); - DownloadTaskState(DownloadState state) { - urls = new ArrayList<>(state.urls); - segments = new ArrayList<>(state.segments); - threads = IntStream.range(0, state.segments.size()).mapToObj(x -> (Thread) null).collect(Collectors.toList()); + void begin() { + lock.lock(); } - DownloadTaskState(List urls, int contentLength, int initialParts) { + void end() { + lock.unlock(); + } + + @Override + public void close() { + end(); + } + } + + protected static abstract class DownloadTask extends CompletableFutureTask { + protected final DownloadTaskState state; + protected boolean caching = false; + protected CacheRepository repository = CacheRepository.getInstance(); + private final CompletableFuture future = new CompletableFuture<>(); + private EnumCheckETag checkETag; + + public DownloadTask(DownloadTaskState state) { + this.state = state; + } + + public final void setCaching(boolean caching) { + this.caching = caching; + } + + public final void setCacheRepository(CacheRepository repository) { + this.repository = repository; + } + + public final DownloadTaskState getDownloadState() { + return state; + } + + protected abstract void write(byte[] buffer, int offset, int len) throws IOException; + + protected EnumCheckETag shouldCheckETag() { + return EnumCheckETag.NOT_CHECK_E_TAG; + } + + protected final EnumCheckETag getCheckETag() { return checkETag; } + + protected void onBeforeConnection(URL url) {} + + protected abstract void onStart() throws IOException; + + /** + * Make cached file as result of this task. + * + * @param cachedFile verified cached file + * @throws IOException if an I/O error occurred. + */ + protected void finishWithCachedResult(Path cachedFile) throws IOException { + state.finished = true; + + future.complete(getResult()); + } + + public void finish() throws IOException { + state.finished = true; + + future.complete(getResult()); + } + + @Override + public final CompletableFuture getCompletableFuture() { + return CompletableFuture.runAsync(() -> { + checkETag = shouldCheckETag(); + + for (Runnable runnable : state.threads) + download().submit(runnable); + }).thenCompose(unused -> future); + } + + protected enum EnumCheckETag { + CHECK_E_TAG, + NOT_CHECK_E_TAG, + CACHED + } + } + + protected static final class DownloadTaskState { + private final List urls; + private final Path file; + private final List segments; + private final List threads; + private URL fastestUrl; + private final int retry; + private int retryUrl = 0; + private boolean cancelled = false; + private boolean finished = false; + private int contentLength; + private final int initialParts; + + private final SafeRegion connectionCheckRegion = new SafeRegion(); + private final SafeRegion writeRegion = new SafeRegion(); + + DownloadTaskState(DownloadState state, Path file, int retry) { + this.urls = new ArrayList<>(state.urls); + this.file = file; + this.retry = retry; + this.segments = new ArrayList<>(state.segments); + this.threads = IntStream.range(0, state.segments.size()).mapToObj(x -> (Thread) null).collect(Collectors.toList()); + this.contentLength = state.getContentLength(); + this.initialParts = state.getSegments().size(); + } + + DownloadTaskState(List urls, Path file, int retry, int contentLength, int initialParts) { if (urls == null || urls.size() == 0) { throw new IllegalArgumentException("DownloadTaskState requires at least one url candidate"); } this.urls = new ArrayList<>(urls); - segments = new ArrayList<>(initialParts); - threads = new ArrayList<>(initialParts); + this.file = file; + this.retry = retry; + this.initialParts = initialParts; + this.segments = new ArrayList<>(initialParts); + this.threads = new ArrayList<>(initialParts); int partLength = contentLength / initialParts; for (int i = 0; i < initialParts; i++) { int begin = partLength * i; @@ -70,22 +254,64 @@ class DownloadManager { } } - public static DownloadTaskState newWithLengthUnknown(List urls, int initialParts) { - return new DownloadTaskState(urls, 0, initialParts); + public static DownloadTaskState newWithLengthUnknown(List urls, Path file, int retry, int initialParts) { + return new DownloadTaskState(urls, file, retry, 0, initialParts); } - public List getUrls() { + public synchronized List getUrls() { return urls; } + public Path getFile() { + return file; + } + + public Path getDownloadingFile() { + return file.resolveSibling(FileUtils.getName(file) + ".download"); + } + + public Path getStateFile() { + return file.resolveSibling(FileUtils.getName(file) + ".status"); + } + public List getSegments() { return segments; } - public String getFirstUrl() { + protected synchronized void setContentLength(int contentLength) { + if (this.contentLength != 0) { + throw new IllegalStateException("ContentLength already set"); + } + + this.contentLength = contentLength; + if (contentLength < 0) { + return; + } + + int partLength = contentLength / initialParts; + for (int i = 0; i < segments.size(); i++) { + int begin = partLength * i; + int end = Math.min((partLength + 1) * i, contentLength); + segments.get(i).setDownloadRange(begin, end); + } + } + + public synchronized int getContentLength() { + return contentLength; + } + + public synchronized URL getFirstUrl() { return urls.get(0); } + public synchronized boolean isFinished() { + return finished; + } + + protected synchronized void setFinished(boolean finished) { + this.finished = finished; + } + /** * Next url for download runnable to retry. * @@ -93,11 +319,16 @@ class DownloadManager { * to acquire next url for retry. Making all download runnable try different * candidates concurrently to speed up finding fastest download source. * + * If all URLs are tried and tested definitely negative for downloading, + * returns null. + * * @return next url to retry */ - public synchronized String getNextUrlToRetry() { - String url = urls.get(retry); - retry = (retry + 1) % urls.size(); + @Nullable + public synchronized URL getNextUrlToRetry() { + if (retryUrl < 0 || retryUrl >= urls.size()) return null; + URL url = urls.get(retryUrl); + retryUrl = (retryUrl + 1) % urls.size(); return url; } @@ -112,11 +343,11 @@ class DownloadManager { * * @return fastest url, null if no url have successfully connected yet. */ - public synchronized String getFastestUrl() { + public synchronized URL getFastestUrl() { return fastestUrl; } - public synchronized void setFastestUrl(String fastestUrl) { + public synchronized void setFastestUrl(URL fastestUrl) { this.fastestUrl = fastestUrl; } @@ -127,37 +358,64 @@ class DownloadManager { public synchronized boolean isCancelled() { return cancelled; } + + public SafeRegion checkingConnection() { + connectionCheckRegion.begin(); + return connectionCheckRegion; + } + + public SafeRegion writing() { + writeRegion.begin(); + return writeRegion; + } + + public synchronized void forbidURL(URL url) { + int index; + while ((index = urls.indexOf(url)) != -1) { + if (retryUrl >= index) { + retryUrl--; + } + urls.remove(index); + } + } } - protected static class DownloadState { - private final List urls; + protected static final class DownloadState { + private final List urls; private final List segments; + private final int contentLength; /** * Constructor for Gson */ public DownloadState() { - this(Collections.emptyList(), Collections.emptyList()); + this(Collections.emptyList(), Collections.emptyList(), 0); } - public DownloadState(List urls, List segments) { + public DownloadState(List urls, List segments, int contentLength) { this.urls = urls; this.segments = segments; + this.contentLength = contentLength; } - public List getUrls() { + public List getUrls() { return urls; } public List getSegments() { return segments; } + + public int getContentLength() { + return contentLength; + } } - protected static class DownloadSegment { - private final int startPosition; - private final int endPosition; - private int currentPosition; + protected static final class DownloadSegment { + private int startPosition; + private int endPosition; + private int downloaded; + private URLConnection connection; /** * Constructor for Gson @@ -166,13 +424,13 @@ class DownloadManager { this(0, 0, 0); } - public DownloadSegment(int startPosition, int endPosition, int currentPosition) { - if (currentPosition < startPosition || currentPosition > endPosition) { - throw new IllegalArgumentException("Illegal download state: start " + startPosition + ", end " + endPosition + ", cur " + currentPosition); + public DownloadSegment(int startPosition, int endPosition, int downloaded) { + if (downloaded > endPosition - startPosition) { + throw new IllegalArgumentException("Illegal download state: start " + startPosition + ", end " + endPosition + ", total downloaded " + downloaded); } this.startPosition = startPosition; this.endPosition = endPosition; - this.currentPosition = currentPosition; + this.downloaded = downloaded; } public int getStartPosition() { @@ -183,19 +441,117 @@ class DownloadManager { return endPosition; } - public int getCurrentPosition() { - return currentPosition; + public void setDownloadRange(int start, int end) { + this.startPosition = start; + this.endPosition = end; + this.downloaded = 0; } - public void setCurrentPosition(int currentPosition) { - this.currentPosition = currentPosition; + public int getDownloaded() { + return downloaded; + } + + public void setDownloaded(int downloaded) { + this.downloaded = downloaded; + } + + public int getLength() { + return endPosition - startPosition; + } + + public URLConnection getConnection() { + return connection; + } + + protected void setConnection(URLConnection connection) { + this.connection = connection; } public boolean isFinished() { - return currentPosition == endPosition; + return downloaded == getLength(); } public boolean isWaiting() { return startPosition == endPosition && startPosition == 0; } } + private static final Timer timer = new Timer("DownloadSpeedRecorder", true); + private static final AtomicInteger downloadSpeed = new AtomicInteger(0); + public static final EventBus speedEvent = new EventBus(); + + static { + timer.schedule(new TimerTask() { + @Override + public void run() { + speedEvent.channel(SpeedEvent.class).fireEvent(new SpeedEvent(speedEvent, downloadSpeed.getAndSet(0))); + } + }, 0, 1000); + } + + public static void updateDownloadSpeed(int speed) { + downloadSpeed.addAndGet(speed); + } + + public static class SpeedEvent extends Event { + private final int speed; + + public SpeedEvent(Object source, int speed) { + super(source); + + this.speed = speed; + } + + /** + * Download speed in byte/sec. + * @return download speed + */ + public int getSpeed() { + return speed; + } + + @Override + public String toString() { + return new ToStringBuilder(this).append("speed", speed).toString(); + } + } + + private static int downloadExecutorConcurrency = Math.min(Runtime.getRuntime().availableProcessors() * 4, 64); + private static volatile ExecutorService DOWNLOAD_EXECUTOR; + + /** + * Get singleton instance of the thread pool for file downloading. + * + * @return Thread pool for FetchTask + */ + protected static ExecutorService download() { + if (DOWNLOAD_EXECUTOR == null) { + synchronized (Schedulers.class) { + if (DOWNLOAD_EXECUTOR == null) { + DOWNLOAD_EXECUTOR = new ThreadPoolExecutor(0, downloadExecutorConcurrency, 10, TimeUnit.SECONDS, new SynchronousQueue<>(), + runnable -> { + Thread thread = Executors.defaultThreadFactory().newThread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + } + + return DOWNLOAD_EXECUTOR; + } + + public static void setDownloadExecutorConcurrency(int concurrency) { + synchronized (Schedulers.class) { + downloadExecutorConcurrency = concurrency; + if (DOWNLOAD_EXECUTOR != null) { + DOWNLOAD_EXECUTOR.shutdownNow(); + DOWNLOAD_EXECUTOR = null; + } + } + } + + public static int getDownloadExecutorConcurrency() { + synchronized (Schedulers.class) { + return downloadExecutorConcurrency; + } + } } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadSegmentTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadSegmentTask.java new file mode 100644 index 000000000..39234091a --- /dev/null +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadSegmentTask.java @@ -0,0 +1,186 @@ +package org.jackhuang.hmcl.task; + +import org.jackhuang.hmcl.util.Logging; +import org.jackhuang.hmcl.util.io.IOUtils; +import org.jackhuang.hmcl.util.io.NetworkUtils; +import org.jackhuang.hmcl.util.io.ResponseCodeException; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.nio.file.Path; +import java.util.logging.Level; + +class DownloadSegmentTask { + + private final DownloadManager.DownloadTask task; + private final DownloadManager.DownloadTaskState state; + private final RandomAccessFile file; + private int tryTime = 0; + private URLConnection conn; + private URL lastURL; + private DownloadManager.DownloadSegment segment; + + public DownloadSegmentTask(DownloadManager.DownloadTask task, RandomAccessFile file, DownloadManager.DownloadSegment segment) { + this.task = task; + this.state = task.getDownloadState(); + this.file = file; + this.segment = segment; + } + + private URLConnection createConnection(boolean retryLastConnection) throws IOException { + if (retryLastConnection && lastURL != null) { + return NetworkUtils.createConnection(lastURL, 4000); + } + + // 1. try connection given by DownloadTask + if (this.conn != null) { + URLConnection conn = this.conn; + lastURL = conn.getURL(); + this.conn = null; + return conn; + } + + // 2. try suggested URL at the first time + if (tryTime == 0) { + lastURL = state.getFirstUrl(); + return NetworkUtils.createConnection(lastURL, 4000); + } + + // 3. try fastest URL if measured + URL fastestURL = state.getFastestUrl(); + if (fastestURL != null) { + lastURL = fastestURL; + return NetworkUtils.createConnection(lastURL, 4000); + } + + // 4. try other URL, DownloadTaskState will make all DownloadSegmentTask + // try different URL to speed up connection. + URL nextURLToTry = state.getNextUrlToRetry(); + if (nextURLToTry == null) { + return null; + } + tryTime++; + lastURL = nextURLToTry; + return NetworkUtils.createConnection(lastURL, 4000); + } + + public void run() throws DownloadException { + Exception exception = null; + URL failedURL = null; + boolean checkETag; + switch (task.getCheckETag()) { + case CHECK_E_TAG: + checkETag = true; + break; + case NOT_CHECK_E_TAG: + checkETag = false; + break; + default: + return; + } + + boolean retryLastConnection = false; + while (true) { + if (state.isCancelled() || state.isFinished()) { + break; + } + + try { + URLConnection conn = createConnection(retryLastConnection); + if (conn == null) { + break; + } + + if (checkETag) task.repository.injectConnection(conn); + + task.onBeforeConnection(lastURL); + segment.setDownloaded(0); + + try (DownloadManager.SafeRegion region = state.checkingConnection()) { + // If other DownloadSegmentTask finishedWithCachedResult + // then this task should stop. + if (state.isFinished()) { + return; + } + + if (state.getContentLength() == 0) { + state.setContentLength(conn.getContentLength()); + } + + // TODO: reset connection with range + + if (conn instanceof HttpURLConnection) { + conn = NetworkUtils.resolveConnection((HttpURLConnection) conn); + int responseCode = ((HttpURLConnection) conn).getResponseCode(); + + if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED) { + // Handle cache + try { + Path cache = task.repository.getCachedRemoteFile(conn); + task.finishWithCachedResult(cache); + return; + } catch (IOException e) { + Logging.LOG.log(Level.WARNING, "Unable to use cached file, re-download " + lastURL, e); + task.repository.removeRemoteEntry(conn); + // Now we must reconnect the server since 304 may result in empty content, + // if we want to re-download the file, we must reconnect the server without etag settings. + retryLastConnection = true; + continue; + } + } else if (responseCode / 100 == 4) { + state.forbidURL(lastURL); + break; // we will not try this URL again + } else if (responseCode / 100 != 2) { + throw new ResponseCodeException(lastURL, responseCode); + } + } + } + + try (InputStream stream = conn.getInputStream()) { + int lastDownloaded = 0, downloaded = 0; + byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE]; + while (true) { + if (state.isCancelled()) break; + + int len = stream.read(buffer); + if (len == -1) break; + + try (DownloadManager.SafeRegion region = state.writing()) { + task.write(buffer, 0, len); + } + + downloaded += len; + + if (conn.getContentLength() >= 0) { + // Update progress information per second + segment.setDownloaded(downloaded); + } + + DownloadManager.updateDownloadSpeed(downloaded - lastDownloaded); + lastDownloaded = downloaded; + } + + if (state.isCancelled()) break; + + if (conn.getContentLength() >= 0 && !segment.isFinished()) + throw new IOException("Unexpected segment size: " + downloaded + ", expected: " + segment.getLength()); + } + + segment.setConnection(conn); + + return; + } catch (IOException ex) { + failedURL = lastURL; + exception = ex; + Logging.LOG.log(Level.WARNING, "Failed to download " + failedURL + ", repeat times: " + tryTime, ex); + } + } + + if (exception != null) + throw new DownloadException(failedURL, exception); + } +} diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadTask.java deleted file mode 100644 index 1e6b5ffe5..000000000 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/DownloadTask.java +++ /dev/null @@ -1,156 +0,0 @@ -package org.jackhuang.hmcl.task; - -import org.jackhuang.hmcl.util.CacheRepository; -import org.jackhuang.hmcl.util.Logging; -import org.jackhuang.hmcl.util.io.IOUtils; -import org.jackhuang.hmcl.util.io.NetworkUtils; -import org.jackhuang.hmcl.util.io.ResponseCodeException; - -import java.io.IOException; -import java.io.InputStream; -import java.io.RandomAccessFile; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLConnection; -import java.nio.file.Path; -import java.util.logging.Level; - -abstract class DownloadTask implements Runnable { - - private final DownloadManager.DownloadTaskState state; - private final RandomAccessFile file; - private URLConnection conn; - private DownloadManager.DownloadSegment segment; - protected boolean caching; - protected CacheRepository repository = CacheRepository.getInstance(); - - public DownloadTask(DownloadManager.DownloadTaskState state, RandomAccessFile file, DownloadManager.DownloadSegment segment) { - this.state = state; - this.file = file; - this.segment = segment; - } - - public void setCaching(boolean caching) { - this.caching = caching; - } - - public void setCacheRepository(CacheRepository repository) { - this.repository = repository; - } - - protected void beforeDownload(URL url) throws IOException { - } - - protected abstract void useCachedResult(Path cachedFile) throws IOException; - - protected abstract FetchTask.EnumCheckETag shouldCheckETag(); - - protected abstract FetchTask.Context getContext(URLConnection conn, boolean checkETag) throws IOException; - - @Override - public void run() { - Exception exception = null; - URL failedURL = null; - boolean checkETag; - switch (shouldCheckETag()) { - case CHECK_E_TAG: - checkETag = true; - break; - case NOT_CHECK_E_TAG: - checkETag = false; - break; - default: - return; - } - - int repeat = 0; - while (true) { - if (state.isCancelled()) { - break; - } - - String url = repeat == 0 ? state.getFirstUrl() : state.getNextUrlToRetry(); - repeat++; - - if (url == null) { - break; - } - - try { - beforeDownload(url); - - updateProgress(0); - - URLConnection conn = NetworkUtils.createConnection(NetworkUtils.toURL(url); - if (checkETag) repository.injectConnection(conn); - - if (conn instanceof HttpURLConnection) { - conn = NetworkUtils.resolveConnection((HttpURLConnection) conn); - int responseCode = ((HttpURLConnection) conn).getResponseCode(); - - if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED) { - // Handle cache - try { - Path cache = repository.getCachedRemoteFile(conn); - useCachedResult(cache); - return; - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Unable to use cached file, redownload " + url, e); - repository.removeRemoteEntry(conn); - // Now we must reconnect the server since 304 may result in empty content, - // if we want to redownload the file, we must reconnect the server without etag settings. - repeat--; - continue; - } - } else if (responseCode / 100 == 4) { - break; // we will not try this URL again - } else if (responseCode / 100 != 2) { - throw new ResponseCodeException(url, responseCode); - } - } - - long contentLength = conn.getContentLength(); - try (FetchTask.Context context = getContext(conn, checkETag); InputStream stream = conn.getInputStream()) { - int lastDownloaded = 0, downloaded = 0; - byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE]; - while (true) { - if (state.isCancelled()) break; - - int len = stream.read(buffer); - if (len == -1) break; - - context.write(buffer, 0, len); - - downloaded += len; - - if (contentLength >= 0) { - // Update progress information per second - updateProgress(downloaded, contentLength); - } - - updateDownloadSpeed(downloaded - lastDownloaded); - lastDownloaded = downloaded; - } - - if (state.isCancelled()) break; - - updateDownloadSpeed(downloaded - lastDownloaded); - - if (contentLength >= 0 && downloaded != contentLength) - throw new IOException("Unexpected file size: " + downloaded + ", expected: " + contentLength); - - context.withResult(true); - } - - return; - } catch (IOException ex) { - failedURL = url; - exception = ex; - Logging.LOG.log(Level.WARNING, "Failed to download " + url + ", repeat times: " + repeat, ex); - } - } - - if (exception != null) - throw new DownloadException(failedURL, exception); - } -} diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java deleted file mode 100644 index 11f78d3c8..000000000 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Hello Minecraft! Launcher - * Copyright (C) 2020 huangyuhui and contributors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.jackhuang.hmcl.task; - -import org.jackhuang.hmcl.event.Event; -import org.jackhuang.hmcl.event.EventBus; -import org.jackhuang.hmcl.util.CacheRepository; -import org.jackhuang.hmcl.util.Logging; -import org.jackhuang.hmcl.util.ToStringBuilder; -import org.jackhuang.hmcl.util.io.IOUtils; -import org.jackhuang.hmcl.util.io.NetworkUtils; -import org.jackhuang.hmcl.util.io.ResponseCodeException; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLConnection; -import java.nio.file.Path; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; - -public abstract class FetchTask extends Task { - protected final List urls; - protected final int retry; - protected boolean caching; - protected CacheRepository repository = CacheRepository.getInstance(); - - public FetchTask(List urls, int retry) { - if (urls == null || urls.isEmpty()) - throw new IllegalArgumentException("At least one URL is required"); - - this.urls = new ArrayList<>(urls); - this.retry = retry; - - setExecutor(Schedulers.io()); - } - - public void setCaching(boolean caching) { - this.caching = caching; - } - - public void setCacheRepository(CacheRepository repository) { - this.repository = repository; - } - - protected void beforeDownload(URL url) throws IOException {} - - protected abstract void useCachedResult(Path cachedFile) throws IOException; - - protected abstract EnumCheckETag shouldCheckETag(); - - protected abstract Context getContext(URLConnection conn, boolean checkETag) throws IOException; - - @Override - public void execute() throws Exception { - Exception exception = null; - URL failedURL = null; - boolean checkETag; - switch (shouldCheckETag()) { - case CHECK_E_TAG: checkETag = true; break; - case NOT_CHECK_E_TAG: checkETag = false; break; - default: return; - } - - int repeat = 0; - download: for (URL url : urls) { - for (int retryTime = 0; retryTime < retry; retryTime++) { - if (isCancelled()) { - break download; - } - - try { - beforeDownload(url); - - updateProgress(0); - - URLConnection conn = NetworkUtils.createConnection(url); - if (checkETag) repository.injectConnection(conn); - - if (conn instanceof HttpURLConnection) { - conn = NetworkUtils.resolveConnection((HttpURLConnection) conn); - int responseCode = ((HttpURLConnection) conn).getResponseCode(); - - if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED) { - // Handle cache - try { - Path cache = repository.getCachedRemoteFile(conn); - useCachedResult(cache); - return; - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Unable to use cached file, redownload " + url, e); - repository.removeRemoteEntry(conn); - // Now we must reconnect the server since 304 may result in empty content, - // if we want to redownload the file, we must reconnect the server without etag settings. - retryTime--; - continue; - } - } else if (responseCode / 100 == 4) { - break; // we will not try this URL again - } else if (responseCode / 100 != 2) { - throw new ResponseCodeException(url, responseCode); - } - } - - long contentLength = conn.getContentLength(); - try (Context context = getContext(conn, checkETag); InputStream stream = conn.getInputStream()) { - int lastDownloaded = 0, downloaded = 0; - byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE]; - while (true) { - if (isCancelled()) break; - - int len = stream.read(buffer); - if (len == -1) break; - - context.write(buffer, 0, len); - - downloaded += len; - - if (contentLength >= 0) { - // Update progress information per second - updateProgress(downloaded, contentLength); - } - - updateDownloadSpeed(downloaded - lastDownloaded); - lastDownloaded = downloaded; - } - - if (isCancelled()) break download; - - updateDownloadSpeed(downloaded - lastDownloaded); - - if (contentLength >= 0 && downloaded != contentLength) - throw new IOException("Unexpected file size: " + downloaded + ", expected: " + contentLength); - - context.withResult(true); - } - - return; - } catch (IOException ex) { - failedURL = url; - exception = ex; - Logging.LOG.log(Level.WARNING, "Failed to download " + url + ", repeat times: " + (++repeat), ex); - } - } - } - - if (exception != null) - throw new DownloadException(failedURL, exception); - } - - private static final Timer timer = new Timer("DownloadSpeedRecorder", true); - private static final AtomicInteger downloadSpeed = new AtomicInteger(0); - public static final EventBus speedEvent = new EventBus(); - - static { - timer.schedule(new TimerTask() { - @Override - public void run() { - speedEvent.channel(SpeedEvent.class).fireEvent(new SpeedEvent(speedEvent, downloadSpeed.getAndSet(0))); - } - }, 0, 1000); - } - - private static void updateDownloadSpeed(int speed) { - downloadSpeed.addAndGet(speed); - } - - public static class SpeedEvent extends Event { - private final int speed; - - public SpeedEvent(Object source, int speed) { - super(source); - - this.speed = speed; - } - - /** - * Download speed in byte/sec. - * @return download speed - */ - public int getSpeed() { - return speed; - } - - @Override - public String toString() { - return new ToStringBuilder(this).append("speed", speed).toString(); - } - } - - protected static abstract class Context implements Closeable { - private boolean success; - - public abstract void write(byte[] buffer, int offset, int len) throws IOException; - - public final void withResult(boolean success) { - this.success = success; - } - - protected boolean isSuccess() { - return success; - } - } - - protected enum EnumCheckETag { - CHECK_E_TAG, - NOT_CHECK_E_TAG, - CACHED - } - - protected class DownloadMission { - - - - } - - private static int downloadExecutorConcurrency = Math.min(Runtime.getRuntime().availableProcessors() * 4, 64); - private static volatile ExecutorService DOWNLOAD_EXECUTOR; - - /** - * Get singleton instance of the thread pool for file downloading. - * - * @return Thread pool for FetchTask - */ - protected static ExecutorService download() { - if (DOWNLOAD_EXECUTOR == null) { - synchronized (Schedulers.class) { - if (DOWNLOAD_EXECUTOR == null) { - DOWNLOAD_EXECUTOR = new ThreadPoolExecutor(0, downloadExecutorConcurrency, 10, TimeUnit.SECONDS, new SynchronousQueue<>(), - runnable -> { - Thread thread = Executors.defaultThreadFactory().newThread(runnable); - thread.setDaemon(true); - return thread; - }); - } - } - } - - return DOWNLOAD_EXECUTOR; - } - - public static void setDownloadExecutorConcurrency(int concurrency) { - synchronized (Schedulers.class) { - downloadExecutorConcurrency = concurrency; - if (DOWNLOAD_EXECUTOR != null) { - DOWNLOAD_EXECUTOR.shutdownNow(); - DOWNLOAD_EXECUTOR = null; - } - } - } - - public static int getDownloadExecutorConcurrency() { - synchronized (Schedulers.class) { - return downloadExecutorConcurrency; - } - } -} diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java index 33bd26cc9..ed30edc33 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/FileDownloadTask.java @@ -22,18 +22,17 @@ import org.jackhuang.hmcl.util.io.ChecksumMismatchException; import org.jackhuang.hmcl.util.io.CompressingUtils; import org.jackhuang.hmcl.util.io.FileUtils; -import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.io.RandomAccessFile; import java.math.BigInteger; import java.net.URL; -import java.net.URLConnection; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; -import java.util.*; +import java.util.ArrayList; +import java.util.Objects; +import java.util.Optional; import java.util.logging.Level; import static java.util.Objects.requireNonNull; @@ -44,11 +43,11 @@ import static org.jackhuang.hmcl.util.DigestUtils.getDigest; * * @author huangyuhui */ -public class FileDownloadTask extends FetchTask { +public class FileDownloadTask extends DownloadManager.DownloadTask { public static class IntegrityCheck { - private String algorithm; - private String checksum; + private final String algorithm; + private final String checksum; public IntegrityCheck(String algorithm, String checksum) { this.algorithm = requireNonNull(algorithm); @@ -80,78 +79,30 @@ public class FileDownloadTask extends FetchTask { } } - private final File file; private final IntegrityCheck integrityCheck; private Path candidate; private RandomAccessFile rFile; - private InputStream stream; private final ArrayList integrityCheckHandlers = new ArrayList<>(); - /** - * @param url the URL of remote file. - * @param file the location that download to. - */ - public FileDownloadTask(URL url, File file) { - this(url, file, null); + public FileDownloadTask(DownloadManager.DownloadTaskState state) { + this(state, null); } - /** - * @param url the URL of remote file. - * @param file the location that download to. - * @param integrityCheck the integrity check to perform, null if no integrity check is to be performed - */ - public FileDownloadTask(URL url, File file, IntegrityCheck integrityCheck) { - this(Collections.singletonList(url), file, integrityCheck); - } - - /** - * @param url the URL of remote file. - * @param file the location that download to. - * @param integrityCheck the integrity check to perform, null if no integrity check is to be performed - * @param retry the times for retrying if downloading fails. - */ - public FileDownloadTask(URL url, File file, IntegrityCheck integrityCheck, int retry) { - this(Collections.singletonList(url), file, integrityCheck, retry); - } - - /** - * Constructor. - * @param urls urls of remote file, will be attempted in order. - * @param file the location that download to. - */ - public FileDownloadTask(List urls, File file) { - this(urls, file, null); - } - - /** - * Constructor. - * @param urls urls of remote file, will be attempted in order. - * @param file the location that download to. - * @param integrityCheck the integrity check to perform, null if no integrity check is to be performed - */ - public FileDownloadTask(List urls, File file, IntegrityCheck integrityCheck) { - this(urls, file, integrityCheck, 3); - } - - /** - * Constructor. - * @param urls urls of remote file, will be attempted in order. - * @param file the location that download to. - * @param integrityCheck the integrity check to perform, null if no integrity check is to be performed - * @param retry the times for retrying if downloading fails. - */ - public FileDownloadTask(List urls, File file, IntegrityCheck integrityCheck, int retry) { - super(urls, retry); - this.file = file; + public FileDownloadTask(DownloadManager.DownloadTaskState state, IntegrityCheck integrityCheck) { + super(state); this.integrityCheck = integrityCheck; - setName(file.getName()); - } - - public File getFile() { - return file; + setName(FileUtils.getName(state.getFile())); } + /** + * Set candidate that the content should equal to the file to be downloaded. + * + * If candidate set and verified, the file will be taken, and download will not happen. + * + * @param candidate path to candidate + * @return this + */ public FileDownloadTask setCandidate(Path candidate) { this.candidate = candidate; return this; @@ -168,8 +119,8 @@ public class FileDownloadTask extends FetchTask { Optional cache = repository.checkExistentFile(candidate, integrityCheck.getAlgorithm(), integrityCheck.getChecksum()); if (cache.isPresent()) { try { - FileUtils.copyFile(cache.get().toFile(), file); - Logging.LOG.log(Level.FINER, "Successfully verified file " + file + " from " + urls.get(0)); + FileUtils.copyFile(cache.get(), state.getFile()); + Logging.LOG.log(Level.FINER, "Successfully verified file " + state.getFile() + " from " + state.getFirstUrl()); return EnumCheckETag.CACHED; } catch (IOException e) { Logging.LOG.log(Level.WARNING, "Failed to copy cache files", e); @@ -182,80 +133,75 @@ public class FileDownloadTask extends FetchTask { } @Override - protected void beforeDownload(URL url) { - Logging.LOG.log(Level.FINER, "Downloading " + url + " to " + file); + protected void onBeforeConnection(URL url) { + Logging.LOG.log(Level.FINER, "Downloading " + url + " to " + state.getFile()); } @Override - protected void useCachedResult(Path cache) throws IOException { - FileUtils.copyFile(cache.toFile(), file); + protected void finishWithCachedResult(Path cache) throws IOException { + FileUtils.copyFile(cache, state.getFile()); } @Override - protected Context getContext(URLConnection conn, boolean checkETag) throws IOException { - Path temp = Files.createTempFile(null, null); - RandomAccessFile rFile = new RandomAccessFile(temp.toFile(), "rw"); + protected void write(byte[] buffer, int offset, int len) throws IOException { + rFile.write(buffer, offset, len); + } + + @Override + protected void onStart() throws IOException { + rFile = new RandomAccessFile(state.getDownloadingFile().toFile(), "rw"); + } + + @Override + public void finish() throws IOException { MessageDigest digest = integrityCheck == null ? null : integrityCheck.createDigest(); + // TODO: digest - return new Context() { - @Override - public void write(byte[] buffer, int offset, int len) throws IOException { - if (digest != null) { - digest.update(buffer, offset, len); - } + try { + rFile.close(); + } catch (IOException e) { + Logging.LOG.log(Level.WARNING, "Failed to close file: " + rFile, e); + } - rFile.write(buffer, offset, len); + if (!state.isFinished()) { + try { + Files.delete(state.getDownloadingFile()); + } catch (IOException e) { + Logging.LOG.log(Level.WARNING, "Failed to delete file: " + rFile, e); } + return; + } - @Override - public void close() throws IOException { - try { - rFile.close(); - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Failed to close file: " + rFile, e); - } + for (IntegrityCheckHandler handler : integrityCheckHandlers) { + handler.checkIntegrity(state.getDownloadingFile(), state.getFile()); + } - if (!isSuccess()) { - try { - Files.delete(temp); - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Failed to delete file: " + rFile, e); - } - return; - } + Files.deleteIfExists(state.getFile()); + if (!FileUtils.makeDirectory(state.getFile().toAbsolutePath().getParent().toFile())) + throw new IOException("Unable to make parent directory " + state.getFile()); - for (IntegrityCheckHandler handler : integrityCheckHandlers) { - handler.checkIntegrity(temp, file.toPath()); - } + try { + FileUtils.moveFile(state.getDownloadingFile().toFile(), state.getFile().toFile()); + } catch (Exception e) { + throw new IOException("Unable to move temp file from " + state.getDownloadingFile() + " to " + state.getFile(), e); + } - Files.deleteIfExists(file.toPath()); - if (!FileUtils.makeDirectory(file.getAbsoluteFile().getParentFile())) - throw new IOException("Unable to make parent directory " + file); + // Integrity check + if (integrityCheck != null) { + integrityCheck.performCheck(digest); + } - try { - FileUtils.moveFile(temp.toFile(), file); - } catch (Exception e) { - throw new IOException("Unable to move temp file from " + temp + " to " + file, e); - } - - // Integrity check - if (integrityCheck != null) { - integrityCheck.performCheck(digest); - } - - if (caching && integrityCheck != null) { - try { - repository.cacheFile(file.toPath(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum()); - } catch (IOException e) { - Logging.LOG.log(Level.WARNING, "Failed to cache file", e); - } - } - - if (checkETag) { - repository.cacheRemoteFile(file.toPath(), conn); - } + if (caching && integrityCheck != null) { + try { + repository.cacheFile(state.getFile(), integrityCheck.getAlgorithm(), integrityCheck.getChecksum()); + } catch (IOException e) { + Logging.LOG.log(Level.WARNING, "Failed to cache file", e); } - }; + } + + if (getCheckETag() == EnumCheckETag.CHECK_E_TAG) { + repository.cacheRemoteFile(state.getFile(), state.getSegments().get(0).getConnection()); + } } public interface IntegrityCheckHandler { diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java index 1be029720..b914771f5 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java @@ -21,11 +21,8 @@ import org.jackhuang.hmcl.util.io.FileUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.net.URL; -import java.net.URLConnection; import java.nio.charset.Charset; import java.nio.file.Path; -import java.util.*; import static java.nio.charset.StandardCharsets.UTF_8; @@ -33,31 +30,20 @@ import static java.nio.charset.StandardCharsets.UTF_8; * * @author huangyuhui */ -public final class GetTask extends FetchTask { +public final class GetTask extends DownloadManager.DownloadTask { private final Charset charset; + private ByteArrayOutputStream baos; - public GetTask(URL url) { - this(url, UTF_8); + public GetTask(DownloadManager.DownloadTaskState state) { + this(state, UTF_8); } - public GetTask(URL url, Charset charset) { - this(url, charset, 3); - } - - public GetTask(URL url, Charset charset, int retry) { - this(Collections.singletonList(url), charset, retry); - } - - public GetTask(List url) { - this(url, UTF_8, 3); - } - - public GetTask(List urls, Charset charset, int retry) { - super(urls, retry); + public GetTask(DownloadManager.DownloadTaskState state, Charset charset) { + super(state); this.charset = charset; - setName(urls.get(0).toString()); + setName(state.getFirstUrl().toString()); } @Override @@ -66,32 +52,34 @@ public final class GetTask extends FetchTask { } @Override - protected void useCachedResult(Path cachedFile) throws IOException { + protected void finishWithCachedResult(Path cachedFile) throws IOException { setResult(FileUtils.readText(cachedFile)); + + super.finishWithCachedResult(cachedFile); } @Override - protected Context getContext(URLConnection conn, boolean checkETag) { - return new Context() { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + protected void write(byte[] buffer, int offset, int len) { + baos.write(buffer, offset, len); + } - @Override - public synchronized void write(byte[] buffer, int offset, int len) { - baos.write(buffer, offset, len); - } + @Override + protected void onStart() { + baos = new ByteArrayOutputStream(); + } - @Override - public void close() throws IOException { - if (!isSuccess()) return; + @Override + public void finish() throws IOException { + if (!state.isFinished()) return; - String result = baos.toString(charset.name()); - setResult(result); + String result = baos.toString(charset.name()); + setResult(result); - if (checkETag) { - repository.cacheText(result, conn); - } - } - }; + if (getCheckETag() == EnumCheckETag.CHECK_E_TAG) { + repository.cacheText(result, state.getSegments().get(0).getConnection()); + } + + super.finish(); } } diff --git a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java index 5337bffd4..5639e4191 100644 --- a/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java +++ b/HMCLCore/src/main/java/org/jackhuang/hmcl/util/io/NetworkUtils.java @@ -17,6 +17,8 @@ */ package org.jackhuang.hmcl.util.io; +import org.jetbrains.annotations.NotNull; + import java.io.*; import java.net.*; import java.util.List; @@ -55,23 +57,27 @@ public final class NetworkUtils { return sb.toString(); } - public static URLConnection createConnection(URL url) throws IOException { + public static URLConnection createConnection(@NotNull URL url) throws IOException { + return createConnection(url, 15000); + } + + public static URLConnection createConnection(@NotNull URL url, int timeout) throws IOException { URLConnection connection = url.openConnection(); connection.setUseCaches(false); - connection.setConnectTimeout(15000); - connection.setReadTimeout(15000); + connection.setConnectTimeout(timeout); + connection.setReadTimeout(timeout); connection.setRequestProperty("Accept-Language", Locale.getDefault().toString()); return connection; } - public static HttpURLConnection createHttpConnection(URL url) throws IOException { + public static HttpURLConnection createHttpConnection(@NotNull URL url) throws IOException { return (HttpURLConnection) createConnection(url); } /** - * @see Curl * @param location the url to be URL encoded * @return encoded URL + * @see Curl */ public static String encodeLocation(String location) { StringBuilder sb = new StringBuilder(); @@ -98,18 +104,31 @@ public final class NetworkUtils { /** * This method is a work-around that aims to solve problem when "Location" in stupid server's response is not encoded. - * @see Issue with libcurl + * * @param conn the stupid http connection. * @return manually redirected http connection. * @throws IOException if an I/O error occurs. + * @see Issue with libcurl */ public static HttpURLConnection resolveConnection(HttpURLConnection conn) throws IOException { + return resolveConnection(conn, 15000); + } + + /** + * This method is a work-around that aims to solve problem when "Location" in stupid server's response is not encoded. + * + * @param conn the stupid http connection. + * @param timeout timeout in milliseconds in once connection. + * @return manually redirected http connection. + * @throws IOException if an I/O error occurs. + * @see Issue with libcurl + */ + public static HttpURLConnection resolveConnection(HttpURLConnection conn, int timeout) throws IOException { int redirect = 0; while (true) { - conn.setUseCaches(false); - conn.setConnectTimeout(15000); - conn.setReadTimeout(15000); + conn.setConnectTimeout(timeout); + conn.setReadTimeout(timeout); conn.setInstanceFollowRedirects(false); Map> properties = conn.getRequestProperties(); String method = conn.getRequestMethod();