mirror of
https://gitlab.bixilon.de/bixilon/pixlyzer.git
synced 2025-08-03 10:16:56 -04:00
341 lines
12 KiB
Python
Executable File
341 lines
12 KiB
Python
Executable File
import minecraft_launcher_lib
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import ujson
|
|
import urllib.request
|
|
import zipfile
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
# ToDo: "19w11b", "19w11a", "19w09a", "19w08b", "19w08a", "19w07a", "19w06a", "19w05a", "19w04b", "19w04a", "19w03c", "19w03b", "19w03a", "19w02a", "18w50a", "18w49a", "18w48b" "18w48a", "18w47b", "18w47a", "18w46a", "18w45a", "18w44a", "18w43c", "18w43b"
|
|
|
|
DOWNLOAD_UNTIL_VERSION = "18w43b"
|
|
SKIP_VERSIONS = ["1.14.2 Pre-Release 1"]
|
|
|
|
DATA_ROOT = os.path.abspath("data/") + "/"
|
|
HASH_FOLDER = DATA_ROOT + "hash/"
|
|
JSON_ASSETS_HASH_INDEX_FILE = DATA_ROOT + "json_index"
|
|
MBF_ASSETS_HASH_INDEX_FILE = DATA_ROOT + "mbf_index"
|
|
OUT_FOLDER = DATA_ROOT + "version/"
|
|
DATA_FOLDER = DATA_ROOT + "data/"
|
|
DEPENDENCIES_FOLDER = DATA_FOLDER + "dependencies/"
|
|
LIBRARIES_PATH = DATA_ROOT + "libraries/"
|
|
|
|
if not os.path.isdir(DATA_FOLDER):
|
|
os.mkdir(DATA_FOLDER)
|
|
|
|
JAVA_PATH = "java"
|
|
TINY_REMAPPER_DOWNLOAD_PATH = "https://maven.fabricmc.net/net/fabricmc/tiny-remapper/0.10.4/tiny-remapper-0.10.4-fat.jar"
|
|
TINY_REMAPPER_PATH = LIBRARIES_PATH + "tiny-remapper-0.10.4-fat.jar"
|
|
|
|
KOTLIN_VERSION = "1.7.21"
|
|
JACKSON_VERSION = "2.14.0"
|
|
|
|
LIBRARIES = [
|
|
("org.objenesis", "objenesis", "3.3"),
|
|
("org.jetbrains.kotlin", "kotlin-stdlib", KOTLIN_VERSION),
|
|
("com.fasterxml.jackson.core", "jackson-core", JACKSON_VERSION),
|
|
("com.fasterxml.jackson.core", "jackson-databind", JACKSON_VERSION),
|
|
("com.fasterxml.jackson.core", "jackson-annotations", JACKSON_VERSION),
|
|
("com.github.luben", "zstd-jni", "1.5.4-1"),
|
|
("de.bixilon", "mbf-kotlin", "1.0"),
|
|
("de.bixilon", "kutil", "1.20.1"),
|
|
]
|
|
|
|
VERSION_MANIFEST_URL = "https://launchermeta.mojang.com/mc/game/version_manifest.json"
|
|
YARN_MANIFEST_URL = "https://meta.fabricmc.net/v2/versions/yarn"
|
|
|
|
TINY_MAPPINGS_BASE_URL = "https://maven.fabricmc.net/net/fabricmc/yarn/"
|
|
INTERMEDIARY_BASE_URL = "https://maven.fabricmc.net/net/fabricmc/intermediary/"
|
|
|
|
COMPILE_VERSION = "23w41a"
|
|
|
|
MAJOR_VERSIONS = [COMPILE_VERSION, "1.19.3", "1.18.2", "1.17.1", "1.16.5", "1.15.2", "1.14.4"]
|
|
|
|
failedVersionIds = []
|
|
partlyFailedVersionIds = []
|
|
|
|
DOWNLOAD_AND_MAP_ONLY_MODE = False # Used for mappings, then building (in ci pipelines)
|
|
ONLY_VERSION_TO_DO = ""
|
|
DONT_COMPILE = False
|
|
ONLY_MAJOR_VERSIONS = False
|
|
|
|
if "--only-map" in sys.argv:
|
|
DOWNLOAD_AND_MAP_ONLY_MODE = True
|
|
if "--dont-compile" in sys.argv:
|
|
DONT_COMPILE = True
|
|
|
|
for arg in sys.argv:
|
|
if arg.startswith("--only-version="):
|
|
ONLY_VERSION_TO_DO = arg[len("--only-version="):]
|
|
|
|
if ONLY_VERSION_TO_DO == "":
|
|
if "--major-only" in sys.argv:
|
|
ONLY_MAJOR_VERSIONS = True
|
|
|
|
print("Starting PixLyzer-yarn generator")
|
|
print("Settings: ")
|
|
print(" Only map: %s" % DOWNLOAD_AND_MAP_ONLY_MODE)
|
|
print(" Only version: %s" % ONLY_VERSION_TO_DO)
|
|
print(" Don't compile: %s" % DONT_COMPILE)
|
|
print("Downloading version manifest")
|
|
VERSION_MANIFEST = ujson.loads(urllib.request.urlopen(VERSION_MANIFEST_URL).read().decode("utf-8"))
|
|
|
|
YARN_MANIFEST = ujson.loads(urllib.request.urlopen(YARN_MANIFEST_URL).read().decode("utf-8"))
|
|
|
|
|
|
def searchVersion(versionId):
|
|
global VERSION_MANIFEST
|
|
for versionEntry in VERSION_MANIFEST["versions"]:
|
|
if versionEntry["id"] == versionId:
|
|
return versionEntry
|
|
raise Exception("Unknown version: %s" % versionId)
|
|
|
|
|
|
startWithVersion = VERSION_MANIFEST["versions"][0]["id"]
|
|
if ONLY_VERSION_TO_DO == "" and not ONLY_MAJOR_VERSIONS:
|
|
startWithVersion = input("Enter version to start with: ")
|
|
if startWithVersion == "":
|
|
startWithVersion = VERSION_MANIFEST["versions"][0]["id"]
|
|
print("No version provided, starting with %s" % startWithVersion)
|
|
|
|
searchVersion(startWithVersion)
|
|
|
|
|
|
def runAndWait(command):
|
|
process = subprocess.Popen(command, cwd=r'../', shell=True)
|
|
exitCode = process.wait()
|
|
if exitCode != 0:
|
|
print(process.stdout.read().decode('utf-8'))
|
|
print(process.stderr.read().decode('utf-8'))
|
|
return exitCode
|
|
|
|
|
|
def getVersionJson(versionEntry):
|
|
# check cache
|
|
global DATA_FOLDER
|
|
versionId = versionEntry["id"]
|
|
path = DATA_FOLDER + versionId + "_yarn/" + versionId + ".json"
|
|
if not os.path.isfile(path):
|
|
# download
|
|
print("Debug: Downloading %s.json" % versionEntry["id"])
|
|
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
urllib.request.urlretrieve(versionEntry["url"], path)
|
|
|
|
json = ujson.load(open(path))
|
|
|
|
del json["arguments"]
|
|
del json["logging"]
|
|
|
|
return json
|
|
|
|
|
|
def checkFile(filename, url, versionId):
|
|
if not os.path.isfile(filename):
|
|
Path(filename).parent.mkdir(parents=True, exist_ok=True)
|
|
print("Downloading %s for %s" % (filename, versionId))
|
|
urllib.request.urlretrieve(url.replace(" ", "%20"), filename + ".tmp")
|
|
os.rename(filename + ".tmp", filename)
|
|
|
|
|
|
def checkTinyMappings(path, url, version):
|
|
if os.path.isfile(path):
|
|
return
|
|
checkFile(path + ".jar", url, version)
|
|
|
|
with zipfile.ZipFile(path + ".jar") as zipFile:
|
|
with zipFile.open('mappings/mappings.tiny') as mappingsEntry, open(path, 'wb') as outFile:
|
|
shutil.copyfileobj(mappingsEntry, outFile)
|
|
os.remove(path + ".jar")
|
|
|
|
|
|
def mapJar(inputFile, outputFile, mappings, mapFrom, mapTo):
|
|
runAndWait(
|
|
JAVA_PATH + " -jar " + TINY_REMAPPER_PATH +
|
|
" \"" + inputFile + "\"" +
|
|
" \"" + outputFile + "\"" +
|
|
" \"" + mappings + "\"" +
|
|
" \"" + mapFrom + "\"" +
|
|
" \"" + mapTo + "\"" +
|
|
" --renameinvalidlocals --rebuildsourcefilenames"
|
|
)
|
|
print("Done")
|
|
|
|
|
|
def get_yarn_build(version_name: str) -> str:
|
|
for entry in YARN_MANIFEST:
|
|
if entry["gameVersion"] == version_name:
|
|
return entry["maven"].split(":")[2]
|
|
|
|
raise Exception("Can not find yarn build for %s" % version_name)
|
|
|
|
|
|
def checkDeobfuscatedClientJar(jarBasePath, version, versionJson):
|
|
exhibitionismJarPath = jarBasePath + "-exhibitionism.jar"
|
|
if not os.path.isfile(exhibitionismJarPath):
|
|
yarnJarPath = jarBasePath + "-named.jar"
|
|
if not os.path.isfile(yarnJarPath):
|
|
clientJarPath = jarBasePath + "_client.jar"
|
|
|
|
intermediaryMappedJarPath = jarBasePath + "_intermediary.jar"
|
|
if not os.path.isfile(intermediaryMappedJarPath):
|
|
checkFile(clientJarPath, versionJson["downloads"]["client"]["url"], version["id"])
|
|
intermediaryMappingsPath = jarBasePath + "_intermediary.tiny"
|
|
checkTinyMappings(intermediaryMappingsPath, INTERMEDIARY_BASE_URL + version["id"] + "/intermediary-" + version["id"] + "-v2.jar", version["id"])
|
|
|
|
print("Mapping intermediary jar for %s" % (version["id"]))
|
|
mapJar(
|
|
inputFile=clientJarPath,
|
|
outputFile=intermediaryMappedJarPath,
|
|
mappings=intermediaryMappingsPath,
|
|
mapFrom="official",
|
|
mapTo="intermediary",
|
|
)
|
|
os.remove(clientJarPath)
|
|
os.remove(intermediaryMappingsPath)
|
|
|
|
yarnMappingsPath = jarBasePath + "_yarn.tiny"
|
|
buildName = get_yarn_build(version["id"])
|
|
checkTinyMappings(yarnMappingsPath, TINY_MAPPINGS_BASE_URL + buildName + "/yarn-" + buildName + "-v2.jar", version["id"])
|
|
|
|
print("Mapping named jar for %s" % (version["id"]))
|
|
mapJar(
|
|
inputFile=intermediaryMappedJarPath,
|
|
outputFile=yarnJarPath,
|
|
mappings=yarnMappingsPath,
|
|
mapFrom="intermediary",
|
|
mapTo="named",
|
|
)
|
|
|
|
os.remove(intermediaryMappedJarPath)
|
|
os.remove(yarnMappingsPath)
|
|
|
|
if not os.path.isfile(jarBasePath + "-exhibitionism.jar"):
|
|
if not os.path.isfile("exhibitionism.jar"):
|
|
raise Exception("Can not find exhibitionism.jar!")
|
|
|
|
runAndWait("java -cp \"./wrapper/exhibitionism.jar\" kr.heartpattern.exhibitionism.AppKt --input \"%s\" --output \"%s\" --parallel 8" % (yarnJarPath, exhibitionismJarPath + ".tmp"))
|
|
os.rename(exhibitionismJarPath + ".tmp", exhibitionismJarPath)
|
|
os.remove(yarnJarPath)
|
|
|
|
|
|
def checkJars(version, versionJson):
|
|
global DATA_FOLDER
|
|
jarBasePath = DATA_FOLDER + version["id"] + "_yarn/" + version["id"]
|
|
checkDeobfuscatedClientJar(jarBasePath, version, versionJson)
|
|
|
|
|
|
def replaceInFile(file, search, replace):
|
|
with open(file, "rt") as fin:
|
|
with open(file, "wt") as fout:
|
|
for line in fin:
|
|
fout.write(line.replace(search, replace))
|
|
|
|
|
|
def download_dependencies(version):
|
|
print("Installing dependencies (to %s)...." % DEPENDENCIES_FOLDER)
|
|
minecraft_launcher_lib.install.install_libraries(version, DEPENDENCIES_FOLDER, {})
|
|
|
|
print("Classpath: ")
|
|
print(minecraft_launcher_lib.command.get_libraries(version, DEPENDENCIES_FOLDER).replace(DEPENDENCIES_FOLDER + "versions/%s/%s.jar" % (version["id"], version["id"]), DATA_FOLDER + version["id"] + "_yarn/" + version["id"] + "-exhibitionism.jar"))
|
|
print("All dependencies set!")
|
|
|
|
|
|
def compilePixLyzer():
|
|
global DONT_COMPILE
|
|
if DONT_COMPILE:
|
|
return
|
|
print("Compiling PixLyzer...")
|
|
compileVersion = {}
|
|
# TODO WTF
|
|
searched = searchVersion(COMPILE_VERSION)
|
|
for versionEntry in searched:
|
|
compileVersion[versionEntry] = searched[versionEntry]
|
|
|
|
versionJson = getVersionJson(compileVersion)
|
|
|
|
checkJars(compileVersion, versionJson)
|
|
download_dependencies(versionJson)
|
|
|
|
if runAndWait('mvn clean verify') != 0:
|
|
raise Exception("Can not compile PixLyzer")
|
|
print("PixLyzer is now compiled")
|
|
|
|
|
|
print("Checking libraries...")
|
|
|
|
ADDITIONAL_CLASSPATH = ""
|
|
os.chdir(os.path.dirname(__file__))
|
|
TARGET_FOLDER = os.getcwd() + "/libraries/"
|
|
if not os.path.isdir(TARGET_FOLDER):
|
|
os.mkdir(TARGET_FOLDER)
|
|
|
|
checkFile(TINY_REMAPPER_PATH, url=TINY_REMAPPER_DOWNLOAD_PATH, versionId="libraries")
|
|
|
|
for library in LIBRARIES:
|
|
filename = "%s-%s.jar" % (library[1], library[2])
|
|
checkFile(TARGET_FOLDER + filename, url="https://repo1.maven.org/maven2/%s/%s/%s/%s" %
|
|
(library[0].replace(".", "/"), library[1], library[2], filename),
|
|
versionId="libraries")
|
|
ADDITIONAL_CLASSPATH += TARGET_FOLDER + filename + ":"
|
|
|
|
if ADDITIONAL_CLASSPATH.endswith(":"):
|
|
ADDITIONAL_CLASSPATH = ADDITIONAL_CLASSPATH[:-1]
|
|
|
|
compilePixLyzer()
|
|
|
|
startWorking = False
|
|
|
|
for version in VERSION_MANIFEST["versions"]:
|
|
if ONLY_VERSION_TO_DO != "":
|
|
if version["id"] == ONLY_VERSION_TO_DO:
|
|
startWorking = True
|
|
else:
|
|
continue
|
|
|
|
if version["id"] == startWithVersion:
|
|
startWorking = True
|
|
if not startWorking:
|
|
continue
|
|
|
|
if ONLY_MAJOR_VERSIONS:
|
|
if version["id"] not in MAJOR_VERSIONS:
|
|
continue
|
|
|
|
if version["id"] in SKIP_VERSIONS:
|
|
print("Skipping %s" % version["id"])
|
|
continue
|
|
|
|
if version["id"] == DOWNLOAD_UNTIL_VERSION:
|
|
print("Breaking at %s" % version["id"])
|
|
break
|
|
|
|
versionJson = getVersionJson(version)
|
|
|
|
checkJars(version, versionJson)
|
|
download_dependencies(versionJson)
|
|
|
|
if DOWNLOAD_AND_MAP_ONLY_MODE:
|
|
break
|
|
|
|
versionStartTime = datetime.now()
|
|
print("Generating data for %s" % version["id"])
|
|
|
|
classpath = minecraft_launcher_lib.command.get_libraries(versionJson, DEPENDENCIES_FOLDER).replace(DEPENDENCIES_FOLDER + "versions/%s/%s.jar" % (version["id"], version["id"]), DATA_FOLDER + version["id"] + "_yarn/" + version["id"] + "-exhibitionism.jar")
|
|
classpath += ":" + ADDITIONAL_CLASSPATH
|
|
classpath += ":target/classes"
|
|
|
|
# execute
|
|
if runAndWait("%s --add-opens java.base/java.lang=ALL-UNNAMED -Xverify:none -cp \"%s\" de.bixilon.pixlyzer.PixLyzer \"%s\" \"%s\" \"%s\" \"%s\" \"%s\"" % (JAVA_PATH, classpath, OUT_FOLDER + version["id"], HASH_FOLDER, JSON_ASSETS_HASH_INDEX_FILE, MBF_ASSETS_HASH_INDEX_FILE, version["id"])) != 0:
|
|
print("PixLyzer did not run successfully for %s" % version["id"])
|
|
break
|
|
|
|
print("Done generating %s in %s" % (version["id"], (datetime.now() - versionStartTime)))
|
|
|
|
if ONLY_VERSION_TO_DO != "":
|
|
break
|
|
|
|
print("Done with downloading and generating all data")
|