diff --git a/src/main/java/de/bixilon/minosoft/Minosoft.java b/src/main/java/de/bixilon/minosoft/Minosoft.java index 0cb1e76a9..111dfef2e 100644 --- a/src/main/java/de/bixilon/minosoft/Minosoft.java +++ b/src/main/java/de/bixilon/minosoft/Minosoft.java @@ -16,6 +16,7 @@ package de.bixilon.minosoft; import com.google.common.collect.HashBiMap; import de.bixilon.minosoft.config.Configuration; import de.bixilon.minosoft.config.ConfigurationPaths; +import de.bixilon.minosoft.data.assets.AssetsManager; import de.bixilon.minosoft.data.mappings.versions.Versions; import de.bixilon.minosoft.gui.LocaleManager; import de.bixilon.minosoft.gui.main.AccountListCell; @@ -26,6 +27,7 @@ import de.bixilon.minosoft.logging.Log; import de.bixilon.minosoft.logging.LogLevels; import de.bixilon.minosoft.modding.event.EventManager; import de.bixilon.minosoft.modding.loading.ModLoader; +import de.bixilon.minosoft.util.CountUpAndDownLatch; import de.bixilon.minosoft.util.Util; import de.bixilon.minosoft.util.mojang.api.MojangAccount; @@ -38,7 +40,8 @@ import java.util.concurrent.CountDownLatch; public final class Minosoft { public static final HashSet eventManagers = new HashSet<>(); - private static final CountDownLatch startStatus = new CountDownLatch(3); // number of critical components (wait for them before other "big" actions) + public static final CountUpAndDownLatch assetsLatch = new CountUpAndDownLatch(1); // count of files still to download, will be used to show progress + private static final CountDownLatch startStatusLatch = new CountDownLatch(4); // number of critical components (wait for them before other "big" actions) public static HashBiMap accountList; public static MojangAccount selectedAccount; public static ArrayList serverList; @@ -91,14 +94,18 @@ public final class Minosoft { }); startCallables.add(() -> { ModLoader.loadMods(); - countDownStart(); // (another) critical component was loaded + countDownStart(); return true; }); - startCallables.add(() -> { Launcher.start(); return true; }); + startCallables.add(() -> { + AssetsManager.downloadAllAssets(assetsLatch); + countDownStart(); + return true; + }); // If you add another "critical" component (wait for them at startup): You MUST adjust increment the number of the counter in `startStatus` (See in the first lines of this file) try { Util.executeInThreadPool("Start", startCallables); @@ -108,8 +115,8 @@ public final class Minosoft { } private static void countDownStart() { - startStatus.countDown(); - Launcher.setProgressBar((int) startStatus.getCount()); + startStatusLatch.countDown(); + Launcher.setProgressBar((int) startStatusLatch.getCount()); } public static void checkClientToken() { @@ -161,13 +168,13 @@ public final class Minosoft { */ public static void waitForStartup() { try { - startStatus.await(); + startStatusLatch.await(); } catch (InterruptedException e) { e.printStackTrace(); } } public static int getStartUpJobsLeft() { - return (int) startStatus.getCount(); + return (int) startStatusLatch.getCount(); } } diff --git a/src/main/java/de/bixilon/minosoft/data/assets/AssetsManager.java b/src/main/java/de/bixilon/minosoft/data/assets/AssetsManager.java new file mode 100644 index 000000000..d2d5d771b --- /dev/null +++ b/src/main/java/de/bixilon/minosoft/data/assets/AssetsManager.java @@ -0,0 +1,211 @@ +/* + * Codename Minosoft + * Copyright (C) 2020 Moritz Zwerger + * + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with this program. If not, see . + * + * This software is not affiliated with Mojang AB, the original developer of Minecraft. + */ + +package de.bixilon.minosoft.data.assets; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.stream.JsonReader; +import de.bixilon.minosoft.Config; +import de.bixilon.minosoft.logging.Log; +import de.bixilon.minosoft.util.CountUpAndDownLatch; +import de.bixilon.minosoft.util.HTTP; +import de.bixilon.minosoft.util.Util; + +import java.io.*; +import java.util.HashMap; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class AssetsManager { + public static final String ASSETS_INDEX_VERSION = "1.16"; // version.json -> assetIndex -> id + public static final String ASSETS_INDEX_HASH = "8cf727ca683b8b133293a605571772306f0ee6b3"; // version.json -> assetIndex -> sha1 + public static final String ASSETS_CLIENT_JAR_VERSION = "1.16.4-pre1"; // version.json -> id + public static final String ASSETS_CLIENT_JAR_HASH = "32f54ca3a6857bf6b72359e2cff6087a0da00a6f"; // sha1 hash of file generated by minosoft (client jar file mappings: name -> hash) + public static final String[] RELEVANT_ASSETS = {"minecraft/lang/", "minecraft/sounds/", "minecraft/textures/", "minecraft/font/"}; + + private static final HashMap assets = new HashMap<>(); + + public static void downloadAssetsIndex() throws IOException { + Util.downloadFileAsGz(String.format("https://launchermeta.mojang.com/v1/packages/%s/%s.json", ASSETS_INDEX_HASH, ASSETS_INDEX_VERSION), getAssetDiskPath(ASSETS_INDEX_HASH)); + } + + private static HashMap parseAssetsIndex(String hash) throws IOException { + InputStreamReader reader = readAssetByHash(hash); + JsonObject json = JsonParser.parseReader(new JsonReader(reader)).getAsJsonObject(); + if (json.has("objects")) { + json = json.getAsJsonObject("objects"); + } + HashMap ret = new HashMap<>(); + for (String key : json.keySet()) { + JsonElement value = json.get(key); + if (value.isJsonPrimitive()) { + ret.put(key, value.getAsString()); + continue; + } + ret.put(key, value.getAsJsonObject().get("hash").getAsString()); + } + return ret; + } + + private static HashMap parseAssetsIndex() throws IOException { + HashMap mappings = parseAssetsIndex(ASSETS_INDEX_HASH); + mappings.putAll(parseAssetsIndex(ASSETS_CLIENT_JAR_HASH)); + return mappings; + } + + public static void downloadAllAssets(CountUpAndDownLatch latch) throws IOException { + if (assets.size() > 0) { + return; + } + downloadAssetsIndex(); + assets.putAll(parseAssetsIndex(ASSETS_INDEX_HASH)); + latch.setCount(assets.size() + 1); // set size of mappings + 1 (for client jar assets) + // download assets + assets.keySet().parallelStream().forEach((filename) -> { + try { + AssetsManager.downloadAsset(assets.get(filename)); + latch.countDown(); + } catch (Exception e) { + e.printStackTrace(); + } + }); + generateJarAssets(); + assets.putAll(parseAssetsIndex(ASSETS_CLIENT_JAR_HASH)); + latch.countDown(); + } + + public static boolean doesAssetExist(String name) { + return assets.containsKey(name); + } + + public static HashMap getAssets() { + return assets; + } + + public static InputStreamReader readAsset(String name) throws IOException { + return readAssetByHash(assets.get(name)); + } + + public static InputStream readAssetAsStream(String name) throws IOException { + return readAssetAsStreamByHash(assets.get(name)); + } + + private static JsonElement readJsonAsset(String name) throws IOException { + return readJsonAssetByHash(assets.get(name)); + } + + private static void downloadAsset(String hash) throws Exception { + downloadAsset(String.format("https://resources.download.minecraft.net/%s/%s", hash.substring(0, 2), hash), hash); + } + + private static InputStreamReader readAssetByHash(String hash) throws IOException { + return new InputStreamReader(readAssetAsStreamByHash(hash)); + } + + private static InputStream readAssetAsStreamByHash(String hash) throws IOException { + return new GZIPInputStream(new FileInputStream(getAssetDiskPath(hash))); + } + + private static JsonElement readJsonAssetByHash(String hash) throws IOException { + return JsonParser.parseReader(readAssetByHash(hash)); + } + + public static void generateJarAssets() throws IOException { + long startTime = System.currentTimeMillis(); + Log.verbose("Generating client.jar assets..."); + JsonObject manifest = HTTP.getJson("https://launchermeta.mojang.com/mc/game/version_manifest.json").getAsJsonObject(); + String assetsVersionJsonUrl = null; + for (JsonElement versionElement : manifest.getAsJsonArray("versions")) { + JsonObject version = versionElement.getAsJsonObject(); + if (version.get("id").getAsString().equals(ASSETS_CLIENT_JAR_VERSION)) { + assetsVersionJsonUrl = version.get("url").getAsString(); + break; + } + } + if (assetsVersionJsonUrl == null) { + throw new RuntimeException(String.format("Invalid version manifest or invalid ASSETS_CLIENT_JAR_VERSION (%s)", ASSETS_CLIENT_JAR_VERSION)); + } + String versionJsonHash = assetsVersionJsonUrl.replace("https://launchermeta.mojang.com/v1/packages/", "").replace(String.format("/%s.json", ASSETS_CLIENT_JAR_VERSION), ""); + downloadAsset(assetsVersionJsonUrl, versionJsonHash); + // download jar + JsonObject clientJarJson = readJsonAssetByHash(versionJsonHash).getAsJsonObject().getAsJsonObject("downloads").getAsJsonObject("client"); + downloadAsset(clientJarJson.get("url").getAsString(), clientJarJson.get("sha1").getAsString()); + + HashMap clientJarAssetsHashMap = new HashMap<>(); + ZipInputStream versionJar = new ZipInputStream(readAssetAsStreamByHash(clientJarJson.get("sha1").getAsString())); + ZipEntry currentFile; + while ((currentFile = versionJar.getNextEntry()) != null) { + if (!currentFile.getName().startsWith("assets") || currentFile.isDirectory()) { + continue; + } + boolean relevant = false; + for (String prefix : RELEVANT_ASSETS) { + if (currentFile.getName().startsWith("assets/" + prefix)) { + relevant = true; + break; + } + } + if (!relevant) { + continue; + } + // ToDo: use input steam twice ? + ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream(); + int len; + byte[] buffer = new byte[2048]; + while ((len = versionJar.read(buffer)) > 0) { + outputBuffer.write(buffer, 0, len); + } + String hash = saveAsset(outputBuffer.toByteArray()); + + clientJarAssetsHashMap.put(currentFile.getName().substring("assets/".length()), hash); + } + JsonObject clientJarAssetsMapping = new JsonObject(); + clientJarAssetsHashMap.forEach(clientJarAssetsMapping::addProperty); + String json = new GsonBuilder().create().toJson(clientJarAssetsMapping); + String assetHash = saveAsset(json.getBytes()); + Log.verbose(String.format("Generated jar assets in %dms (elements=%d, hash=%s)", (System.currentTimeMillis() - startTime), clientJarAssetsHashMap.size(), assetHash)); + } + + private static String saveAsset(byte[] data) throws IOException { + String hash = Util.sha1(data); + String destination = getAssetDiskPath(hash); + File outFile = new File(destination); + if (outFile.exists() && outFile.length() > 0) { + return hash; + } + Util.createParentFolderIfNotExist(destination); + OutputStream out = new GZIPOutputStream(new FileOutputStream(destination)); + out.write(data); + out.close(); + return hash; + } + + private static void downloadAsset(String url, String hash) throws IOException { + String destination = getAssetDiskPath(hash); + File file = new File(destination); + if (file.exists() && file.length() > 0) { + return; // ToDo: check sha1 + } + Log.verbose(String.format("Downloading %s -> %s", url, hash)); + Util.downloadFileAsGz(url, destination); + } + + private static String getAssetDiskPath(String hash) { + return Config.homeDir + String.format("assets/objects/%s/%s.gz", hash.substring(0, 2), hash); + } +} \ No newline at end of file diff --git a/src/main/java/de/bixilon/minosoft/util/CountUpAndDownLatch.java b/src/main/java/de/bixilon/minosoft/util/CountUpAndDownLatch.java new file mode 100644 index 000000000..8ef0ce5b2 --- /dev/null +++ b/src/main/java/de/bixilon/minosoft/util/CountUpAndDownLatch.java @@ -0,0 +1,72 @@ +/* + * Codename Minosoft + * Copyright (C) 2020 Moritz Zwerger + * + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with this program. If not, see . + * + * This software is not affiliated with Mojang AB, the original developer of Minecraft. + */ + +package de.bixilon.minosoft.util; + +import java.util.concurrent.CountDownLatch; + +// Thanks https://stackoverflow.com/questions/14255019/latch-that-can-be-incremented +public class CountUpAndDownLatch { + private final Object lock = new Object(); + private CountDownLatch latch; + + public CountUpAndDownLatch(int count) { + this.latch = new CountDownLatch(count); + } + + public void countDownOrWaitIfZero() throws InterruptedException { + synchronized (lock) { + while (latch.getCount() == 0) { + lock.wait(); + } + latch.countDown(); + lock.notifyAll(); + } + } + + public void waitUntilZero() throws InterruptedException { + synchronized (lock) { + while (latch.getCount() != 0) { + lock.wait(); + } + } + } + + public void countUp() { //should probably check for Integer.MAX_VALUE + synchronized (lock) { + latch = new CountDownLatch((int) latch.getCount() + 1); + lock.notifyAll(); + } + } + + public void countDown() { //should probably check for Integer.MAX_VALUE + synchronized (lock) { + latch.countDown(); + lock.notifyAll(); + } + } + + public int getCount() { + synchronized (lock) { + return (int) latch.getCount(); + } + } + + public void setCount(int value) { + synchronized (lock) { + latch = new CountDownLatch(value); + lock.notifyAll(); + } + } + +} \ No newline at end of file diff --git a/src/main/java/de/bixilon/minosoft/util/HTTP.java b/src/main/java/de/bixilon/minosoft/util/HTTP.java index c0fced2b7..fddc077df 100644 --- a/src/main/java/de/bixilon/minosoft/util/HTTP.java +++ b/src/main/java/de/bixilon/minosoft/util/HTTP.java @@ -13,7 +13,9 @@ package de.bixilon.minosoft.util; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import de.bixilon.minosoft.logging.Log; import de.bixilon.minosoft.logging.LogLevels; @@ -48,4 +50,12 @@ public final class HTTP { } return null; } + + public static JsonElement getJson(String url) { + HttpResponse response = get(url); + if (response == null || response.statusCode() != 200) { + return null; + } + return JsonParser.parseString(response.body()); + } } diff --git a/src/main/java/de/bixilon/minosoft/util/Util.java b/src/main/java/de/bixilon/minosoft/util/Util.java index 58bf056a2..25e008a12 100644 --- a/src/main/java/de/bixilon/minosoft/util/Util.java +++ b/src/main/java/de/bixilon/minosoft/util/Util.java @@ -104,18 +104,30 @@ public final class Util { return ret; } - public static String sha1(String string) { + public static String sha1(byte[] data) { try { MessageDigest crypt = MessageDigest.getInstance("SHA-1"); crypt.reset(); - crypt.update(string.getBytes(StandardCharsets.UTF_8)); - return new String(crypt.digest()); + crypt.update(data); + return byteArrayToHexString(crypt.digest()); } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } return null; } + public static String byteArrayToHexString(byte[] b) { + StringBuilder result = new StringBuilder(); + for (byte value : b) { + result.append(Integer.toString((value & 0xff) + 0x100, 16).substring(1)); + } + return result.toString(); + } + + public static String sha1(String string) { + return sha1(string.getBytes(StandardCharsets.UTF_8)); + } + public static HashMap readTarGzFile(String fileName) throws IOException { File inputFile = new File(fileName); TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(inputFile))); @@ -191,15 +203,25 @@ public final class Util { } public static void downloadFile(String url, String destination) throws IOException { + createParentFolderIfNotExist(destination); + downloadFile(url, new FileOutputStream(destination)); + } + + public static void downloadFileAsGz(String url, String destination) throws IOException { + createParentFolderIfNotExist(destination); + downloadFile(url, new GZIPOutputStream(new FileOutputStream(destination))); + } + + + public static void downloadFile(String url, OutputStream output) throws IOException { BufferedInputStream inputStream = new BufferedInputStream(new URL(url).openStream()); - FileOutputStream fileOutputStream = new FileOutputStream(destination); byte[] buffer = new byte[1024]; int length; while ((length = inputStream.read(buffer, 0, 1024)) != -1) { - fileOutputStream.write(buffer, 0, length); + output.write(buffer, 0, length); } inputStream.close(); - fileOutputStream.close(); + output.close(); } public static void executeInThreadPool(String name, Collection> callables) throws InterruptedException { @@ -210,4 +232,10 @@ public final class Util { public static ThreadFactory getThreadFactory(String threadName) { return new ThreadFactoryBuilder().setNameFormat(threadName + "#%d").build(); } + + public static void createParentFolderIfNotExist(String destination) { + File file = new File(destination); + file.getParentFile().mkdirs(); + } + } \ No newline at end of file