AstralTransRocketries/minosoft
1
0
Fork 0
mirror of https://gitlab.bixilon.de/bixilon/minosoft.git synced 2025-09-12 00:47:26 -04:00
Code Issues Packages Projects Releases Wiki Activity

AssetsManager (download all assets from mojang, generate jar resources, etc)

Browse Source
This commit is contained in:
Bixilon 2020-10-19 21:55:45 +02:00
parent e1a0fdd289
commit 3231b0266e
No known key found for this signature in database
GPG Key ID: 5CAD791931B09AC4
5 changed files with 341 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/main/java/de/bixilon/minosoft/Minosoft.java
View File

@ -16,6 +16,7 @@ package de.bixilon.minosoft;
import com.google.common.collect.HashBiMap;
import de.bixilon.minosoft.config.Configuration;
import de.bixilon.minosoft.config.ConfigurationPaths;
import de.bixilon.minosoft.data.assets.AssetsManager;
import de.bixilon.minosoft.data.mappings.versions.Versions;
import de.bixilon.minosoft.gui.LocaleManager;
import de.bixilon.minosoft.gui.main.AccountListCell;
@ -26,6 +27,7 @@ import de.bixilon.minosoft.logging.Log;
import de.bixilon.minosoft.logging.LogLevels;
import de.bixilon.minosoft.modding.event.EventManager;
import de.bixilon.minosoft.modding.loading.ModLoader;
import de.bixilon.minosoft.util.CountUpAndDownLatch;
import de.bixilon.minosoft.util.Util;
import de.bixilon.minosoft.util.mojang.api.MojangAccount;
@ -38,7 +40,8 @@ import java.util.concurrent.CountDownLatch;
public final class Minosoft {
public static final HashSet<EventManager> eventManagers = new HashSet<>();
private static final CountDownLatch startStatus = new CountDownLatch(3); // number of critical components (wait for them before other "big" actions)
public static final CountUpAndDownLatch assetsLatch = new CountUpAndDownLatch(1); // count of files still to download, will be used to show progress
private static final CountDownLatch startStatusLatch = new CountDownLatch(4); // number of critical components (wait for them before other "big" actions)
public static HashBiMap<String, MojangAccount> accountList;
public static MojangAccount selectedAccount;
public static ArrayList<Server> serverList;
@ -91,14 +94,18 @@ public final class Minosoft {
});
startCallables.add(() -> {
ModLoader.loadMods();
countDownStart(); // (another) critical component was loaded
countDownStart();
return true;
});
startCallables.add(() -> {
Launcher.start();
return true;
});
startCallables.add(() -> {
AssetsManager.downloadAllAssets(assetsLatch);
countDownStart();
return true;
});
// If you add another "critical" component (wait for them at startup): You MUST adjust increment the number of the counter in `startStatus` (See in the first lines of this file)
try {
Util.executeInThreadPool("Start", startCallables);
@ -108,8 +115,8 @@ public final class Minosoft {
}
private static void countDownStart() {
startStatus.countDown();
Launcher.setProgressBar((int) startStatus.getCount());
startStatusLatch.countDown();
Launcher.setProgressBar((int) startStatusLatch.getCount());
}
public static void checkClientToken() {
@ -161,13 +168,13 @@ public final class Minosoft {
*/
public static void waitForStartup() {
try {
startStatus.await();
startStatusLatch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static int getStartUpJobsLeft() {
return (int) startStatus.getCount();
return (int) startStatusLatch.getCount();
}
}

211
src/main/java/de/bixilon/minosoft/data/assets/AssetsManager.java Normal file
View File

@ -0,0 +1,211 @@
/*
* Codename Minosoft
* Copyright (C) 2020 Moritz Zwerger
*
* This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* This software is not affiliated with Mojang AB, the original developer of Minecraft.
*/
package de.bixilon.minosoft.data.assets;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.stream.JsonReader;
import de.bixilon.minosoft.Config;
import de.bixilon.minosoft.logging.Log;
import de.bixilon.minosoft.util.CountUpAndDownLatch;
import de.bixilon.minosoft.util.HTTP;
import de.bixilon.minosoft.util.Util;
import java.io.*;
import java.util.HashMap;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
public class AssetsManager {
public static final String ASSETS_INDEX_VERSION = "1.16"; // version.json -> assetIndex -> id
public static final String ASSETS_INDEX_HASH = "8cf727ca683b8b133293a605571772306f0ee6b3"; // version.json -> assetIndex -> sha1
public static final String ASSETS_CLIENT_JAR_VERSION = "1.16.4-pre1"; // version.json -> id
public static final String ASSETS_CLIENT_JAR_HASH = "32f54ca3a6857bf6b72359e2cff6087a0da00a6f"; // sha1 hash of file generated by minosoft (client jar file mappings: name -> hash)
public static final String[] RELEVANT_ASSETS = {"minecraft/lang/", "minecraft/sounds/", "minecraft/textures/", "minecraft/font/"};
private static final HashMap<String, String> assets = new HashMap<>();
public static void downloadAssetsIndex() throws IOException {
Util.downloadFileAsGz(String.format("https://launchermeta.mojang.com/v1/packages/%s/%s.json", ASSETS_INDEX_HASH, ASSETS_INDEX_VERSION), getAssetDiskPath(ASSETS_INDEX_HASH));
}
private static HashMap<String, String> parseAssetsIndex(String hash) throws IOException {
InputStreamReader reader = readAssetByHash(hash);
JsonObject json = JsonParser.parseReader(new JsonReader(reader)).getAsJsonObject();
if (json.has("objects")) {
json = json.getAsJsonObject("objects");
}
HashMap<String, String> ret = new HashMap<>();
for (String key : json.keySet()) {
JsonElement value = json.get(key);
if (value.isJsonPrimitive()) {
ret.put(key, value.getAsString());
continue;
}
ret.put(key, value.getAsJsonObject().get("hash").getAsString());
}
return ret;
}
private static HashMap<String, String> parseAssetsIndex() throws IOException {
HashMap<String, String> mappings = parseAssetsIndex(ASSETS_INDEX_HASH);
mappings.putAll(parseAssetsIndex(ASSETS_CLIENT_JAR_HASH));
return mappings;
}
public static void downloadAllAssets(CountUpAndDownLatch latch) throws IOException {
if (assets.size() > 0) {
return;
}
downloadAssetsIndex();
assets.putAll(parseAssetsIndex(ASSETS_INDEX_HASH));
latch.setCount(assets.size() + 1); // set size of mappings + 1 (for client jar assets)
// download assets
assets.keySet().parallelStream().forEach((filename) -> {
try {
AssetsManager.downloadAsset(assets.get(filename));
latch.countDown();
} catch (Exception e) {
e.printStackTrace();
}
});
generateJarAssets();
assets.putAll(parseAssetsIndex(ASSETS_CLIENT_JAR_HASH));
latch.countDown();
}
public static boolean doesAssetExist(String name) {
return assets.containsKey(name);
}
public static HashMap<String, String> getAssets() {
return assets;
}
public static InputStreamReader readAsset(String name) throws IOException {
return readAssetByHash(assets.get(name));
}
public static InputStream readAssetAsStream(String name) throws IOException {
return readAssetAsStreamByHash(assets.get(name));
}
private static JsonElement readJsonAsset(String name) throws IOException {
return readJsonAssetByHash(assets.get(name));
}
private static void downloadAsset(String hash) throws Exception {
downloadAsset(String.format("https://resources.download.minecraft.net/%s/%s", hash.substring(0, 2), hash), hash);
}
private static InputStreamReader readAssetByHash(String hash) throws IOException {
return new InputStreamReader(readAssetAsStreamByHash(hash));
}
private static InputStream readAssetAsStreamByHash(String hash) throws IOException {
return new GZIPInputStream(new FileInputStream(getAssetDiskPath(hash)));
}
private static JsonElement readJsonAssetByHash(String hash) throws IOException {
return JsonParser.parseReader(readAssetByHash(hash));
}
public static void generateJarAssets() throws IOException {
long startTime = System.currentTimeMillis();
Log.verbose("Generating client.jar assets...");
JsonObject manifest = HTTP.getJson("https://launchermeta.mojang.com/mc/game/version_manifest.json").getAsJsonObject();
String assetsVersionJsonUrl = null;
for (JsonElement versionElement : manifest.getAsJsonArray("versions")) {
JsonObject version = versionElement.getAsJsonObject();
if (version.get("id").getAsString().equals(ASSETS_CLIENT_JAR_VERSION)) {
assetsVersionJsonUrl = version.get("url").getAsString();
break;
}
}
if (assetsVersionJsonUrl == null) {
throw new RuntimeException(String.format("Invalid version manifest or invalid ASSETS_CLIENT_JAR_VERSION (%s)", ASSETS_CLIENT_JAR_VERSION));
}
String versionJsonHash = assetsVersionJsonUrl.replace("https://launchermeta.mojang.com/v1/packages/", "").replace(String.format("/%s.json", ASSETS_CLIENT_JAR_VERSION), "");
downloadAsset(assetsVersionJsonUrl, versionJsonHash);
// download jar
JsonObject clientJarJson = readJsonAssetByHash(versionJsonHash).getAsJsonObject().getAsJsonObject("downloads").getAsJsonObject("client");
downloadAsset(clientJarJson.get("url").getAsString(), clientJarJson.get("sha1").getAsString());
HashMap<String, String> clientJarAssetsHashMap = new HashMap<>();
ZipInputStream versionJar = new ZipInputStream(readAssetAsStreamByHash(clientJarJson.get("sha1").getAsString()));
ZipEntry currentFile;
while ((currentFile = versionJar.getNextEntry()) != null) {
if (!currentFile.getName().startsWith("assets") || currentFile.isDirectory()) {
continue;
}
boolean relevant = false;
for (String prefix : RELEVANT_ASSETS) {
if (currentFile.getName().startsWith("assets/" + prefix)) {
relevant = true;
break;
}
}
if (!relevant) {
continue;
}
// ToDo: use input steam twice ?
ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream();
int len;
byte[] buffer = new byte[2048];
while ((len = versionJar.read(buffer)) > 0) {
outputBuffer.write(buffer, 0, len);
}
String hash = saveAsset(outputBuffer.toByteArray());
clientJarAssetsHashMap.put(currentFile.getName().substring("assets/".length()), hash);
}
JsonObject clientJarAssetsMapping = new JsonObject();
clientJarAssetsHashMap.forEach(clientJarAssetsMapping::addProperty);
String json = new GsonBuilder().create().toJson(clientJarAssetsMapping);
String assetHash = saveAsset(json.getBytes());
Log.verbose(String.format("Generated jar assets in %dms (elements=%d, hash=%s)", (System.currentTimeMillis() - startTime), clientJarAssetsHashMap.size(), assetHash));
}
private static String saveAsset(byte[] data) throws IOException {
String hash = Util.sha1(data);
String destination = getAssetDiskPath(hash);
File outFile = new File(destination);
if (outFile.exists() && outFile.length() > 0) {
return hash;
}
Util.createParentFolderIfNotExist(destination);
OutputStream out = new GZIPOutputStream(new FileOutputStream(destination));
out.write(data);
out.close();
return hash;
}
private static void downloadAsset(String url, String hash) throws IOException {
String destination = getAssetDiskPath(hash);
File file = new File(destination);
if (file.exists() && file.length() > 0) {
return; // ToDo: check sha1
}
Log.verbose(String.format("Downloading %s -> %s", url, hash));
Util.downloadFileAsGz(url, destination);
}
private static String getAssetDiskPath(String hash) {
return Config.homeDir + String.format("assets/objects/%s/%s.gz", hash.substring(0, 2), hash);
}
}

72
src/main/java/de/bixilon/minosoft/util/CountUpAndDownLatch.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Codename Minosoft
* Copyright (C) 2020 Moritz Zwerger
*
* This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* This software is not affiliated with Mojang AB, the original developer of Minecraft.
*/
package de.bixilon.minosoft.util;
import java.util.concurrent.CountDownLatch;
// Thanks https://stackoverflow.com/questions/14255019/latch-that-can-be-incremented
public class CountUpAndDownLatch {
private final Object lock = new Object();
private CountDownLatch latch;
public CountUpAndDownLatch(int count) {
this.latch = new CountDownLatch(count);
}
public void countDownOrWaitIfZero() throws InterruptedException {
synchronized (lock) {
while (latch.getCount() == 0) {
lock.wait();
}
latch.countDown();
lock.notifyAll();
}
}
public void waitUntilZero() throws InterruptedException {
synchronized (lock) {
while (latch.getCount() != 0) {
lock.wait();
}
}
}
public void countUp() { //should probably check for Integer.MAX_VALUE
synchronized (lock) {
latch = new CountDownLatch((int) latch.getCount() + 1);
lock.notifyAll();
}
}
public void countDown() { //should probably check for Integer.MAX_VALUE
synchronized (lock) {
latch.countDown();
lock.notifyAll();
}
}
public int getCount() {
synchronized (lock) {
return (int) latch.getCount();
}
}
public void setCount(int value) {
synchronized (lock) {
latch = new CountDownLatch(value);
lock.notifyAll();
}
}
}

10
src/main/java/de/bixilon/minosoft/util/HTTP.java
View File

@ -13,7 +13,9 @@
package de.bixilon.minosoft.util;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import de.bixilon.minosoft.logging.Log;
import de.bixilon.minosoft.logging.LogLevels;
@ -48,4 +50,12 @@ public final class HTTP {
}
return null;
}
public static JsonElement getJson(String url) {
HttpResponse<String> response = get(url);
if (response == null || response.statusCode() != 200) {
return null;
}
return JsonParser.parseString(response.body());
}
}

40
src/main/java/de/bixilon/minosoft/util/Util.java
View File

@ -104,18 +104,30 @@ public final class Util {
return ret;
}
public static String sha1(String string) {
public static String sha1(byte[] data) {
try {
MessageDigest crypt = MessageDigest.getInstance("SHA-1");
crypt.reset();
crypt.update(string.getBytes(StandardCharsets.UTF_8));
return new String(crypt.digest());
crypt.update(data);
return byteArrayToHexString(crypt.digest());
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
return null;
}
public static String byteArrayToHexString(byte[] b) {
StringBuilder result = new StringBuilder();
for (byte value : b) {
result.append(Integer.toString((value & 0xff) + 0x100, 16).substring(1));
}
return result.toString();
}
public static String sha1(String string) {
return sha1(string.getBytes(StandardCharsets.UTF_8));
}
public static HashMap<String, String> readTarGzFile(String fileName) throws IOException {
File inputFile = new File(fileName);
TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(inputFile)));
@ -191,15 +203,25 @@ public final class Util {
}
public static void downloadFile(String url, String destination) throws IOException {
createParentFolderIfNotExist(destination);
downloadFile(url, new FileOutputStream(destination));
}
public static void downloadFileAsGz(String url, String destination) throws IOException {
createParentFolderIfNotExist(destination);
downloadFile(url, new GZIPOutputStream(new FileOutputStream(destination)));
}
public static void downloadFile(String url, OutputStream output) throws IOException {
BufferedInputStream inputStream = new BufferedInputStream(new URL(url).openStream());
FileOutputStream fileOutputStream = new FileOutputStream(destination);
byte[] buffer = new byte[1024];
int length;
while ((length = inputStream.read(buffer, 0, 1024)) != -1) {
fileOutputStream.write(buffer, 0, length);
output.write(buffer, 0, length);
}
inputStream.close();
fileOutputStream.close();
output.close();
}
public static <T> void executeInThreadPool(String name, Collection<Callable<T>> callables) throws InterruptedException {
@ -210,4 +232,10 @@ public final class Util {
public static ThreadFactory getThreadFactory(String threadName) {
return new ThreadFactoryBuilder().setNameFormat(threadName + "#%d").build();
}
public static void createParentFolderIfNotExist(String destination) {
File file = new File(destination);
file.getParentFile().mkdirs();
}
}