AstralTransRocketries/minosoft
1
0
Fork 0
mirror of https://gitlab.bixilon.de/bixilon/minosoft.git synced 2025-09-08 06:50:22 -04:00
Code Issues Packages Projects Releases Wiki Activity

use new assets system for version mappings, allow download of already compressed data, ...

Browse Source
This commit is contained in:
Bixilon 2020-11-10 20:19:12 +01:00
parent a02b17784f
commit fb5c16aaa0
No known key found for this signature in database
GPG Key ID: 5CAD791931B09AC4
10 changed files with 103 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/Config.md
View File

@ -45,7 +45,7 @@ There is a config file located in:
},
"download": {
"urls": {
"mappings": "https://gitlab.com/Bixilon/minosoft/-/raw/master/data/mcdata/%s.tar.gz?inline\u003dfalse"
"resources": "https://gitlab.com/Bixilon/minosoft/-/raw/development/data/resources/%s/%s.tar.gz?inline=false"
}
},
"debug": {
@ -98,8 +98,8 @@ There is a config file located in:
```
## Download
- `url`:
- `mappings`: URL for the mappings. This is data generated by [mappingsDownloader.py](/util/mappingsDownloader.py).
The URL must contain .tar.gz files named after minecraft versions (e.g. `1.14.4.tar.gz`).
- `mappings`: URL for resources. For example mappings. The data is in a folder starting with the first digs of the hash, followed by the hash
The URL must contain .tar.gz files named after minecraft versions (e.g. `0a/0aeb75059ef955d4cf2b9823d15775d0eacb13d5.tar.gz`).
## Debug
- `verify-assets`: If true, minosoft will check the sha1 of every asset. Must be false, if you want to modify assets. (Should be true, can be false, if you want to improve the start time)

4
src/main/java/de/bixilon/minosoft/config/Configuration.java
View File

@ -131,7 +131,7 @@ public class Configuration {
case ACCOUNT_SELECTED -> config.getAsJsonObject("accounts").get("selected").getAsString();
case GENERAL_LOG_LEVEL -> config.getAsJsonObject("general").get("log-level").getAsString();
case GENERAL_LANGUAGE -> config.getAsJsonObject("general").get("language").getAsString();
case MAPPINGS_URL -> config.getAsJsonObject("download").getAsJsonObject("urls").get("mappings").getAsString();
case RESOURCES_URL -> config.getAsJsonObject("download").getAsJsonObject("urls").get("resources").getAsString();
case CLIENT_TOKEN -> config.getAsJsonObject("accounts").get("client-token").getAsString();
};
}
@ -141,7 +141,7 @@ public class Configuration {
case ACCOUNT_SELECTED -> config.getAsJsonObject("accounts").addProperty("selected", value);
case GENERAL_LOG_LEVEL -> config.getAsJsonObject("general").addProperty("log-level", value);
case GENERAL_LANGUAGE -> config.getAsJsonObject("general").addProperty("language", value);
case MAPPINGS_URL -> config.getAsJsonObject("download").getAsJsonObject("urls").addProperty("mappings", value);
case RESOURCES_URL -> config.getAsJsonObject("download").getAsJsonObject("urls").addProperty("resources", value);
case CLIENT_TOKEN -> config.getAsJsonObject("accounts").addProperty("client-token", value);
}
}

2
src/main/java/de/bixilon/minosoft/config/ConfigurationPaths.java
View File

@ -17,7 +17,7 @@ public abstract class ConfigurationPaths {
public enum StringPaths {
GENERAL_LOG_LEVEL,
CLIENT_TOKEN,
MAPPINGS_URL,
RESOURCES_URL,
ACCOUNT_SELECTED,
GENERAL_LANGUAGE,
}

62
src/main/java/de/bixilon/minosoft/data/assets/AssetsManager.java
View File

@ -18,7 +18,6 @@ import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.stream.JsonReader;
import de.bixilon.minosoft.Minosoft;
import de.bixilon.minosoft.config.ConfigurationPaths;
import de.bixilon.minosoft.config.StaticConfiguration;
@ -51,8 +50,10 @@ public class AssetsManager {
}
private static HashMap<String, String> parseAssetsIndex(String hash) throws IOException {
InputStreamReader reader = readAssetByHash(hash);
JsonObject json = JsonParser.parseReader(new JsonReader(reader)).getAsJsonObject();
return parseAssetsIndex(readJsonAssetByHash(hash).getAsJsonObject());
}
private static HashMap<String, String> parseAssetsIndex(JsonObject json) {
if (json.has("objects")) {
json = json.getAsJsonObject("objects");
}
@ -68,11 +69,6 @@ public class AssetsManager {
return ret;
}
private static HashMap<String, String> parseAssetsIndex() throws IOException {
HashMap<String, String> mappings = parseAssetsIndex(ASSETS_INDEX_HASH);
mappings.putAll(parseAssetsIndex(ASSETS_CLIENT_JAR_HASH));
return mappings;
}
public static void downloadAllAssets(CountUpAndDownLatch latch) throws IOException {
if (assets.size() > 0) {
@ -84,23 +80,31 @@ public class AssetsManager {
Log.printException(e, LogLevels.DEBUG);
Log.warn("Could not download assets index. Please check your internet connection");
}
assets.putAll(parseAssetsIndex(ASSETS_INDEX_HASH));
latch.addCount(assets.size() + 1); // set size of mappings + 1 (for client jar assets)
assets.putAll(verifyAssets(AssetsSource.MOJANG, latch, parseAssetsIndex(ASSETS_INDEX_HASH)));
assets.putAll(verifyAssets(AssetsSource.MINOSOFT_GIT, latch, parseAssetsIndex(Util.readJsonAsset("mapping/resources.json"))));
latch.addCount(1); // client jar
// download assets
generateJarAssets();
assets.putAll(parseAssetsIndex(ASSETS_CLIENT_JAR_HASH));
latch.countDown();
}
private static HashMap<String, String> verifyAssets(AssetsSource source, CountUpAndDownLatch latch, HashMap<String, String> assets) {
latch.addCount(assets.size());
assets.keySet().parallelStream().forEach((filename) -> {
try {
String hash = assets.get(filename);
if (!verifyAssetHash(hash)) {
AssetsManager.downloadAsset(hash);
boolean compressed = source == AssetsSource.MOJANG;
if (!verifyAssetHash(hash, compressed)) {
AssetsManager.downloadAsset(source, hash);
}
latch.countDown();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
});
generateJarAssets();
assets.putAll(parseAssetsIndex(ASSETS_CLIENT_JAR_HASH));
latch.countDown();
return assets;
}
public static boolean doesAssetExist(String name) {
@ -123,8 +127,11 @@ public class AssetsManager {
return readJsonAssetByHash(assets.get(name));
}
private static void downloadAsset(String hash) throws Exception {
downloadAsset(String.format("https://resources.download.minecraft.net/%s/%s", hash.substring(0, 2), hash), hash);
private static void downloadAsset(AssetsSource source, String hash) throws Exception {
switch (source) {
case MOJANG -> downloadAsset(String.format("https://resources.download.minecraft.net/%s/%s", hash.substring(0, 2), hash), hash);
case MINOSOFT_GIT -> downloadAsset(String.format(Minosoft.getConfig().getString(ConfigurationPaths.StringPaths.RESOURCES_URL), hash.substring(0, 2), hash), hash, false);
}
}
private static InputStreamReader readAssetByHash(String hash) throws IOException {
@ -147,7 +154,7 @@ public class AssetsManager {
return file.length();
}
private static boolean verifyAssetHash(String hash) {
private static boolean verifyAssetHash(String hash, boolean compressed) {
// file does not exist
if (getAssetSize(hash) == -1) {
return false;
@ -156,12 +163,19 @@ public class AssetsManager {
return true;
}
try {
return hash.equals(Util.sha1Gzip(new File(getAssetDiskPath(hash))));
if (compressed) {
return hash.equals(Util.sha1Gzip(new File(getAssetDiskPath(hash))));
}
return hash.equals(Util.sha1(new File(getAssetDiskPath(hash))));
} catch (IOException ignored) {
}
return false;
}
private static boolean verifyAssetHash(String hash) {
return verifyAssetHash(hash, true);
}
public static void generateJarAssets() throws IOException {
long startTime = System.currentTimeMillis();
Log.verbose("Generating client.jar assets...");
@ -265,11 +279,19 @@ public class AssetsManager {
}
private static void downloadAsset(String url, String hash) throws IOException {
downloadAsset(url, hash, true);
}
private static void downloadAsset(String url, String hash, boolean compressed) throws IOException {
if (verifyAssetHash(hash)) {
return;
}
Log.verbose(String.format("Downloading %s -> %s", url, hash));
Util.downloadFileAsGz(url, getAssetDiskPath(hash));
if (compressed) {
Util.downloadFileAsGz(url, getAssetDiskPath(hash));
return;
}
Util.downloadFile(url, getAssetDiskPath(hash));
}
private static String getAssetDiskPath(String hash) {

19
src/main/java/de/bixilon/minosoft/data/assets/AssetsSource.java Normal file
View File

@ -0,0 +1,19 @@
/*
* Minosoft
* Copyright (C) 2020 Moritz Zwerger
*
* This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with this program.If not, see <https://www.gnu.org/licenses/>.
*
* This software is not affiliated with Mojang AB, the original developer of Minecraft.
*/
package de.bixilon.minosoft.data.assets;
public enum AssetsSource {
MOJANG,
MINOSOFT_GIT
}

36
src/main/java/de/bixilon/minosoft/data/mappings/versions/Versions.java
View File

@ -16,23 +16,18 @@ package de.bixilon.minosoft.data.mappings.versions;
import com.google.common.collect.HashBiMap;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import de.bixilon.minosoft.Minosoft;
import de.bixilon.minosoft.config.ConfigurationPaths;
import de.bixilon.minosoft.config.StaticConfiguration;
import de.bixilon.minosoft.data.Mappings;
import de.bixilon.minosoft.data.assets.AssetsManager;
import de.bixilon.minosoft.logging.Log;
import de.bixilon.minosoft.protocol.protocol.ConnectionStates;
import de.bixilon.minosoft.protocol.protocol.Packets;
import de.bixilon.minosoft.protocol.protocol.ProtocolDefinition;
import de.bixilon.minosoft.util.Util;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.zip.ZipException;
public class Versions {
static final HashBiMap<Integer, Version> versionIdMap = HashBiMap.create();
@ -127,34 +122,7 @@ public class Versions {
Log.verbose(String.format("Loading mappings for version %s...", version));
long startTime = System.currentTimeMillis();
// check if mapping folder exist
File mappingFolder = new File(StaticConfiguration.HOME_DIR + "assets/mapping");
if (!mappingFolder.exists()) {
if (mappingFolder.mkdirs()) {
Log.verbose("Created mappings folder.");
} else {
Log.fatal(String.format("Failed creating mappings folder (%s). Exiting...", mappingFolder.getAbsolutePath()));
System.exit(1);
}
}
String fileName = StaticConfiguration.HOME_DIR + String.format("assets/mapping/%s.tar.gz", version.getVersionName());
HashMap<String, JsonObject> files;
try {
files = Util.readJsonTarGzFile(fileName);
} catch (FileNotFoundException e) {
long downloadStartTime = System.currentTimeMillis();
Log.info(String.format("Mappings for %s are not available on disk. Downloading them...", version.getVersionName()));
Util.downloadFile(String.format(Minosoft.getConfig().getString(ConfigurationPaths.StringPaths.MAPPINGS_URL), version.getVersionName()), fileName);
try {
files = Util.readJsonTarGzFile(fileName);
} catch (ZipException e2) {
// bullshit downloaded, delete file
new File(fileName).delete();
throw e2;
}
Log.info(String.format("Mappings for %s downloaded successfully in %dms!", version.getVersionName(), (System.currentTimeMillis() - downloadStartTime)));
}
HashMap<String, JsonObject> files = Util.readJsonTarStream(AssetsManager.readAssetAsStream(String.format("mappings/%s", version.getVersionName())));
for (Map.Entry<String, Mappings> mappingSet : mappingsHashMap.entrySet()) {
JsonObject data = files.get(mappingSet.getKey() + ".json").getAsJsonObject("minecraft");

5
src/main/java/de/bixilon/minosoft/util/Util.java
View File

@ -179,9 +179,8 @@ public final class Util {
return stringBuilder.toString();
}
public static HashMap<String, JsonObject> readJsonTarGzFile(String fileName) throws IOException {
File inputFile = new File(fileName);
TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(inputFile)));
public static HashMap<String, JsonObject> readJsonTarStream(InputStream inputStream) throws IOException {
TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(inputStream);
HashMap<String, JsonObject> ret = new HashMap<>();
TarArchiveEntry entry;
while ((entry = tarArchiveInputStream.getNextTarEntry()) != null) {

190
src/main/resources/assets/mapping/resources.json
View File

File diff suppressed because one or more lines are too long

2
src/main/resources/config/config.json
View File

@ -21,7 +21,7 @@
},
"download": {
"urls": {
"mappings": "https://gitlab.com/Bixilon/minosoft/-/raw/master/data/mcdata/%s.tar.gz?inline=false"
"resources": "https://gitlab.com/Bixilon/minosoft/-/raw/development/data/resources/%s/%s.tar.gz?inline=false"
}
},
"debug": {

35
util/mappingsDownloader.py
View File

@ -16,7 +16,8 @@
# You should have received a copy of the GNU General Public License along with this program.If not, see <https://www.gnu.org/licenses/>.
#
# This software is not affiliated with Mojang AB, the original developer of Minecraft.
#
import hashlib
import os
import requests
import shutil
@ -27,21 +28,38 @@ import ujson
print("Minecraft mappings downloader (and generator)")
PRE_FLATTENING_UPDATE_VERSION = "17w46a"
DATA_FOLDER = "./mcdata/"
DATA_FOLDER = "../data/resources/"
TEMP_FOLDER = DATA_FOLDER + "tmp/"
FILES_PER_VERSION = ["blocks.json", "registries.json"]
DOWNLOAD_BASE_URL = "https://apimon.de/mcdata/"
manifest = requests.get('https://launchermeta.mojang.com/mc/game/version_manifest.json').json()
failed = []
defaultMappings = ujson.load(open("mappingsDefaults.json"))
resourceMappingsIndex = ujson.load(open("../src/main/resources/assets/mapping/resources.json"))
def sha1File(fileName):
with open(fileName, 'rb') as f:
sha1 = hashlib.sha1()
while True:
data = f.read(4096)
if not data:
break
sha1.update(data)
return sha1.hexdigest()
if not os.path.isdir(DATA_FOLDER):
os.mkdir(DATA_FOLDER)
if not os.path.isdir(TEMP_FOLDER):
os.mkdir(TEMP_FOLDER)
for version in manifest["versions"]:
if version["id"] == PRE_FLATTENING_UPDATE_VERSION:
break
versionBaseFolder = DATA_FOLDER + version["id"] + "/"
if os.path.isfile(DATA_FOLDER + version["id"] + ".tar.gz"):
versionBaseFolder = TEMP_FOLDER + version["id"] + "/"
resourcesJsonPath = ("mappings/%s" % version["id"])
if resourcesJsonPath in resourceMappingsIndex and os.path.isfile(DATA_FOLDER + resourceMappingsIndex[resourcesJsonPath][:2] + "/" + resourceMappingsIndex[resourcesJsonPath] + ".tar.gz"):
print("Skipping %s" % (version["id"]))
continue
if not os.path.isdir(versionBaseFolder):
@ -100,10 +118,17 @@ for version in manifest["versions"]:
print("Skipping %s for %s" % (fileName, version["id"]))
if not version["id"] in failed:
# compress the data to version.tar.gz
tar = tarfile.open(DATA_FOLDER + version["id"] + ".tar.gz", "w:gz")
tar = tarfile.open(versionBaseFolder + version["id"] + ".tar.gz", "w:gz")
for fileName in FILES_PER_VERSION:
tar.add(versionBaseFolder + fileName, arcname=fileName)
tar.close()
sha1 = sha1File(versionBaseFolder + version["id"] + ".tar.gz")
if not os.path.isdir(DATA_FOLDER + sha1[:2]):
os.mkdir(DATA_FOLDER + sha1[:2])
os.rename(versionBaseFolder + version["id"] + ".tar.gz", DATA_FOLDER + sha1[:2] + "/" + sha1 + ".tar.gz")
resourceMappingsIndex[resourcesJsonPath] = sha1
shutil.rmtree(versionBaseFolder)
with open("../src/main/resources/assets/mapping/resources.json", 'w') as file:
ujson.dump(resourceMappingsIndex, file)
print("Done, %s failed" % failed)