import minecraft_launcher_lib import os import shutil import subprocess import sys import ujson import urllib.request import zipfile from datetime import datetime from pathlib import Path # ToDo: "19w11b", "19w11a", "19w09a", "19w08b", "19w08a", "19w07a", "19w06a", "19w05a", "19w04b", "19w04a", "19w03c", "19w03b", "19w03a", "19w02a", "18w50a", "18w49a", "18w48b" "18w48a", "18w47b", "18w47a", "18w46a", "18w45a", "18w44a", "18w43c", "18w43b" DOWNLOAD_UNTIL_VERSION = "18w43b" SKIP_VERSIONS = ["1.14.2 Pre-Release 1"] DATA_ROOT = os.path.abspath("data/") + "/" HASH_FOLDER = DATA_ROOT + "hash/" JSON_ASSETS_HASH_INDEX_FILE = DATA_ROOT + "json_index" MBF_ASSETS_HASH_INDEX_FILE = DATA_ROOT + "mbf_index" OUT_FOLDER = DATA_ROOT + "version/" DATA_FOLDER = DATA_ROOT + "data/" DEPENDENCIES_FOLDER = DATA_FOLDER + "dependencies/" LIBRARIES_PATH = DATA_ROOT + "libraries/" if not os.path.isdir(DATA_FOLDER): os.mkdir(DATA_FOLDER) JAVA_PATH = "java" TINY_REMAPPER_DOWNLOAD_PATH = "https://maven.fabricmc.net/net/fabricmc/tiny-remapper/0.10.4/tiny-remapper-0.10.4-fat.jar" TINY_REMAPPER_PATH = LIBRARIES_PATH + "tiny-remapper-0.10.4-fat.jar" KOTLIN_VERSION = "1.7.21" JACKSON_VERSION = "2.14.0" LIBRARIES = [ ("org.objenesis", "objenesis", "3.3"), ("org.jetbrains.kotlin", "kotlin-stdlib", KOTLIN_VERSION), ("com.fasterxml.jackson.core", "jackson-core", JACKSON_VERSION), ("com.fasterxml.jackson.core", "jackson-databind", JACKSON_VERSION), ("com.fasterxml.jackson.core", "jackson-annotations", JACKSON_VERSION), ("com.github.luben", "zstd-jni", "1.5.4-1"), ("de.bixilon", "mbf-kotlin", "1.0"), ("de.bixilon", "kutil", "1.20.1"), ] VERSION_MANIFEST_URL = "https://launchermeta.mojang.com/mc/game/version_manifest.json" YARN_MANIFEST_URL = "https://meta.fabricmc.net/v2/versions/yarn" TINY_MAPPINGS_BASE_URL = "https://maven.fabricmc.net/net/fabricmc/yarn/" INTERMEDIARY_BASE_URL = "https://maven.fabricmc.net/net/fabricmc/intermediary/" COMPILE_VERSION = "23w41a" MAJOR_VERSIONS = [COMPILE_VERSION, "1.19.3", "1.18.2", "1.17.1", "1.16.5", "1.15.2", "1.14.4"] failedVersionIds = [] partlyFailedVersionIds = [] DOWNLOAD_AND_MAP_ONLY_MODE = False # Used for mappings, then building (in ci pipelines) ONLY_VERSION_TO_DO = "" DONT_COMPILE = False ONLY_MAJOR_VERSIONS = False if "--only-map" in sys.argv: DOWNLOAD_AND_MAP_ONLY_MODE = True if "--dont-compile" in sys.argv: DONT_COMPILE = True for arg in sys.argv: if arg.startswith("--only-version="): ONLY_VERSION_TO_DO = arg[len("--only-version="):] if ONLY_VERSION_TO_DO == "": if "--major-only" in sys.argv: ONLY_MAJOR_VERSIONS = True print("Starting PixLyzer-yarn generator") print("Settings: ") print(" Only map: %s" % DOWNLOAD_AND_MAP_ONLY_MODE) print(" Only version: %s" % ONLY_VERSION_TO_DO) print(" Don't compile: %s" % DONT_COMPILE) print("Downloading version manifest") VERSION_MANIFEST = ujson.loads(urllib.request.urlopen(VERSION_MANIFEST_URL).read().decode("utf-8")) YARN_MANIFEST = ujson.loads(urllib.request.urlopen(YARN_MANIFEST_URL).read().decode("utf-8")) def searchVersion(versionId): global VERSION_MANIFEST for versionEntry in VERSION_MANIFEST["versions"]: if versionEntry["id"] == versionId: return versionEntry raise Exception("Unknown version: %s" % versionId) startWithVersion = VERSION_MANIFEST["versions"][0]["id"] if ONLY_VERSION_TO_DO == "" and not ONLY_MAJOR_VERSIONS: startWithVersion = input("Enter version to start with: ") if startWithVersion == "": startWithVersion = VERSION_MANIFEST["versions"][0]["id"] print("No version provided, starting with %s" % startWithVersion) searchVersion(startWithVersion) def runAndWait(command): process = subprocess.Popen(command, cwd=r'../', shell=True) exitCode = process.wait() if exitCode != 0: print(process.stdout.read().decode('utf-8')) print(process.stderr.read().decode('utf-8')) return exitCode def getVersionJson(versionEntry): # check cache global DATA_FOLDER versionId = versionEntry["id"] path = DATA_FOLDER + versionId + "_yarn/" + versionId + ".json" if not os.path.isfile(path): # download print("Debug: Downloading %s.json" % versionEntry["id"]) Path(path).parent.mkdir(parents=True, exist_ok=True) urllib.request.urlretrieve(versionEntry["url"], path) json = ujson.load(open(path)) del json["arguments"] del json["logging"] return json def checkFile(filename, url, versionId): if not os.path.isfile(filename): Path(filename).parent.mkdir(parents=True, exist_ok=True) print("Downloading %s for %s" % (filename, versionId)) urllib.request.urlretrieve(url.replace(" ", "%20"), filename + ".tmp") os.rename(filename + ".tmp", filename) def checkTinyMappings(path, url, version): if os.path.isfile(path): return checkFile(path + ".jar", url, version) with zipfile.ZipFile(path + ".jar") as zipFile: with zipFile.open('mappings/mappings.tiny') as mappingsEntry, open(path, 'wb') as outFile: shutil.copyfileobj(mappingsEntry, outFile) os.remove(path + ".jar") def mapJar(inputFile, outputFile, mappings, mapFrom, mapTo): runAndWait( JAVA_PATH + " -jar " + TINY_REMAPPER_PATH + " \"" + inputFile + "\"" + " \"" + outputFile + "\"" + " \"" + mappings + "\"" + " \"" + mapFrom + "\"" + " \"" + mapTo + "\"" + " --renameinvalidlocals --rebuildsourcefilenames" ) print("Done") def get_yarn_build(version_name: str) -> str: for entry in YARN_MANIFEST: if entry["gameVersion"] == version_name: return entry["maven"].split(":")[2] raise Exception("Can not find yarn build for %s" % version_name) def checkDeobfuscatedClientJar(jarBasePath, version, versionJson): exhibitionismJarPath = jarBasePath + "-exhibitionism.jar" if not os.path.isfile(exhibitionismJarPath): yarnJarPath = jarBasePath + "-named.jar" if not os.path.isfile(yarnJarPath): clientJarPath = jarBasePath + "_client.jar" intermediaryMappedJarPath = jarBasePath + "_intermediary.jar" if not os.path.isfile(intermediaryMappedJarPath): checkFile(clientJarPath, versionJson["downloads"]["client"]["url"], version["id"]) intermediaryMappingsPath = jarBasePath + "_intermediary.tiny" checkTinyMappings(intermediaryMappingsPath, INTERMEDIARY_BASE_URL + version["id"] + "/intermediary-" + version["id"] + "-v2.jar", version["id"]) print("Mapping intermediary jar for %s" % (version["id"])) mapJar( inputFile=clientJarPath, outputFile=intermediaryMappedJarPath, mappings=intermediaryMappingsPath, mapFrom="official", mapTo="intermediary", ) os.remove(clientJarPath) os.remove(intermediaryMappingsPath) yarnMappingsPath = jarBasePath + "_yarn.tiny" buildName = get_yarn_build(version["id"]) checkTinyMappings(yarnMappingsPath, TINY_MAPPINGS_BASE_URL + buildName + "/yarn-" + buildName + "-v2.jar", version["id"]) print("Mapping named jar for %s" % (version["id"])) mapJar( inputFile=intermediaryMappedJarPath, outputFile=yarnJarPath, mappings=yarnMappingsPath, mapFrom="intermediary", mapTo="named", ) os.remove(intermediaryMappedJarPath) os.remove(yarnMappingsPath) if not os.path.isfile(jarBasePath + "-exhibitionism.jar"): if not os.path.isfile("exhibitionism.jar"): raise Exception("Can not find exhibitionism.jar!") runAndWait("java -cp \"./wrapper/exhibitionism.jar\" kr.heartpattern.exhibitionism.AppKt --input \"%s\" --output \"%s\" --parallel 8" % (yarnJarPath, exhibitionismJarPath + ".tmp")) os.rename(exhibitionismJarPath + ".tmp", exhibitionismJarPath) os.remove(yarnJarPath) def checkJars(version, versionJson): global DATA_FOLDER jarBasePath = DATA_FOLDER + version["id"] + "_yarn/" + version["id"] checkDeobfuscatedClientJar(jarBasePath, version, versionJson) def replaceInFile(file, search, replace): with open(file, "rt") as fin: with open(file, "wt") as fout: for line in fin: fout.write(line.replace(search, replace)) def download_dependencies(version): print("Installing dependencies (to %s)...." % DEPENDENCIES_FOLDER) minecraft_launcher_lib.install.install_libraries(version, DEPENDENCIES_FOLDER, {}) print("Classpath: ") print(minecraft_launcher_lib.command.get_libraries(version, DEPENDENCIES_FOLDER).replace(DEPENDENCIES_FOLDER + "versions/%s/%s.jar" % (version["id"], version["id"]), DATA_FOLDER + version["id"] + "_yarn/" + version["id"] + "-exhibitionism.jar")) print("All dependencies set!") def compilePixLyzer(): global DONT_COMPILE if DONT_COMPILE: return print("Compiling PixLyzer...") compileVersion = {} # TODO WTF searched = searchVersion(COMPILE_VERSION) for versionEntry in searched: compileVersion[versionEntry] = searched[versionEntry] versionJson = getVersionJson(compileVersion) checkJars(compileVersion, versionJson) download_dependencies(versionJson) if runAndWait('mvn clean verify') != 0: raise Exception("Can not compile PixLyzer") print("PixLyzer is now compiled") print("Checking libraries...") ADDITIONAL_CLASSPATH = "" os.chdir(os.path.dirname(__file__)) TARGET_FOLDER = os.getcwd() + "/libraries/" if not os.path.isdir(TARGET_FOLDER): os.mkdir(TARGET_FOLDER) checkFile(TINY_REMAPPER_PATH, url=TINY_REMAPPER_DOWNLOAD_PATH, versionId="libraries") for library in LIBRARIES: filename = "%s-%s.jar" % (library[1], library[2]) checkFile(TARGET_FOLDER + filename, url="https://repo1.maven.org/maven2/%s/%s/%s/%s" % (library[0].replace(".", "/"), library[1], library[2], filename), versionId="libraries") ADDITIONAL_CLASSPATH += TARGET_FOLDER + filename + ":" if ADDITIONAL_CLASSPATH.endswith(":"): ADDITIONAL_CLASSPATH = ADDITIONAL_CLASSPATH[:-1] compilePixLyzer() startWorking = False for version in VERSION_MANIFEST["versions"]: if ONLY_VERSION_TO_DO != "": if version["id"] == ONLY_VERSION_TO_DO: startWorking = True else: continue if version["id"] == startWithVersion: startWorking = True if not startWorking: continue if ONLY_MAJOR_VERSIONS: if version["id"] not in MAJOR_VERSIONS: continue if version["id"] in SKIP_VERSIONS: print("Skipping %s" % version["id"]) continue if version["id"] == DOWNLOAD_UNTIL_VERSION: print("Breaking at %s" % version["id"]) break versionJson = getVersionJson(version) checkJars(version, versionJson) download_dependencies(versionJson) if DOWNLOAD_AND_MAP_ONLY_MODE: break versionStartTime = datetime.now() print("Generating data for %s" % version["id"]) classpath = minecraft_launcher_lib.command.get_libraries(versionJson, DEPENDENCIES_FOLDER).replace(DEPENDENCIES_FOLDER + "versions/%s/%s.jar" % (version["id"], version["id"]), DATA_FOLDER + version["id"] + "_yarn/" + version["id"] + "-exhibitionism.jar") classpath += ":" + ADDITIONAL_CLASSPATH classpath += ":target/classes" # execute if runAndWait("%s --add-opens java.base/java.lang=ALL-UNNAMED -Xverify:none -cp \"%s\" de.bixilon.pixlyzer.PixLyzer \"%s\" \"%s\" \"%s\" \"%s\" \"%s\"" % (JAVA_PATH, classpath, OUT_FOLDER + version["id"], HASH_FOLDER, JSON_ASSETS_HASH_INDEX_FILE, MBF_ASSETS_HASH_INDEX_FILE, version["id"])) != 0: print("PixLyzer did not run successfully for %s" % version["id"]) break print("Done generating %s in %s" % (version["id"], (datetime.now() - versionStartTime))) if ONLY_VERSION_TO_DO != "": break print("Done with downloading and generating all data")