refactor: improve fabric scripts

generateFabric.py

@@ -1,5 +1,6 @@
 from meta.fabricutil import *
-from meta.common import ensure_component_dir, polymc_path, upstream_path
+from meta.common import ensure_component_dir, polymc_path, upstream_path, transform_maven_key
+from meta.common.fabric import JARS_DIR, INSTALLER_INFO_DIR, META_DIR, INTERMEDIARY_COMPONENT, LOADER_COMPONENT
 
 PMC_DIR = polymc_path()
 UPSTREAM_DIR = upstream_path()
@@ -7,91 +8,101 @@ UPSTREAM_DIR = upstream_path()
 ensure_component_dir("net.fabricmc.fabric-loader")
 ensure_component_dir("net.fabricmc.intermediary")
 
-# turn loader versions into packages
-loaderRecommended = []
-loaderVersions = []
-intermediaryRecommended = []
-intermediaryVersions = []
 
-
+def load_jar_info(artifact_key):
-def loadJarInfo(mavenKey):
+    with open(os.path.join(UPSTREAM_DIR, JARS_DIR, f"{artifact_key}.json"), 'r',
-    with open(UPSTREAM_DIR + "/fabric/jars/" + mavenKey.replace(":", ".") + ".json", 'r',
               encoding='utf-8') as jarInfoFile:
         return FabricJarInfo(json.load(jarInfoFile))
 
 
-def processLoaderVersion(loaderVersion, it, loaderData):
+def load_installer_info(version):
-    verStable = it["stable"]
+    with open(os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version}.json"), 'r',
-    if (len(loaderRecommended) < 1) and verStable:
+              encoding='utf-8') as loaderVersionFile:
-        loaderRecommended.append(loaderVersion)
+        data = json.load(loaderVersionFile)
-    versionJarInfo = loadJarInfo(it["maven"])
+        return FabricInstallerDataV1(data)
-    version = PolyMCVersionFile(name="Fabric Loader", uid="net.fabricmc.fabric-loader", version=loaderVersion)
+
-    version.releaseTime = versionJarInfo.releaseTime
+
-    version.requires = [DependencyEntry(uid='net.fabricmc.intermediary')]
+def process_loader_version(entry) -> PolyMCVersionFile:
-    version.order = 10
+    jar_info = load_jar_info(transform_maven_key(entry["maven"]))
-    version.type = "release"
+    installer_info = load_installer_info(entry["version"])
-    if isinstance(loaderData.mainClass, dict):
+
-        version.mainClass = loaderData.mainClass["client"]
+    v = PolyMCVersionFile(name="Fabric Loader", uid="net.fabricmc.fabric-loader", version=entry["version"])
+    v.releaseTime = jar_info.releaseTime
+    v.requires = [DependencyEntry(uid='net.fabricmc.intermediary')]
+    v.order = 10
+    v.type = "release"
+    if isinstance(installer_info.mainClass, dict):
+        v.mainClass = installer_info.mainClass["client"]
     else:
-        version.mainClass = loaderData.mainClass
+        v.mainClass = installer_info.mainClass
-    version.libraries = []
+    v.libraries = []
-    version.libraries.extend(loaderData.libraries.common)
+    v.libraries.extend(installer_info.libraries.common)
-    version.libraries.extend(loaderData.libraries.client)
+    v.libraries.extend(installer_info.libraries.client)
-    loaderLib = PolyMCLibrary(name=GradleSpecifier(it["maven"]), url="https://maven.fabricmc.net")
+    loader_lib = PolyMCLibrary(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net")
-    version.libraries.append(loaderLib)
+    v.libraries.append(loader_lib)
-    loaderVersions.append(version)
+    return v
 
 
-def processIntermediaryVersion(it):
+def process_intermediary_version(entry) -> PolyMCVersionFile:
-    intermediaryRecommended.append(it["version"])
+    jar_info = load_jar_info(transform_maven_key(entry["maven"]))
-    versionJarInfo = loadJarInfo(it["maven"])
+
-    version = PolyMCVersionFile(name="Intermediary Mappings", uid="net.fabricmc.intermediary", version=it["version"])
+    v = PolyMCVersionFile(name="Intermediary Mappings", uid="net.fabricmc.intermediary", version=entry["version"])
-    version.releaseTime = versionJarInfo.releaseTime
+    v.releaseTime = jar_info.releaseTime
-    version.requires = [DependencyEntry(uid='net.minecraft', equals=it["version"])]
+    v.requires = [DependencyEntry(uid='net.minecraft', equals=entry["version"])]
-    version.order = 11
+    v.order = 11
-    version.type = "release"
+    v.type = "release"
-    version.libraries = []
+    v.libraries = []
-    version.volatile = True
+    v.volatile = True
-    mappingLib = PolyMCLibrary(name=GradleSpecifier(it["maven"]), url="https://maven.fabricmc.net")
+    intermediary_lib = PolyMCLibrary(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net")
-    version.libraries.append(mappingLib)
+    v.libraries.append(intermediary_lib)
-    intermediaryVersions.append(version)
+    return v
 
 
-with open(UPSTREAM_DIR + "/fabric/meta-v2/loader.json", 'r', encoding='utf-8') as loaderVersionIndexFile:
+def main():
-    loaderVersionIndex = json.load(loaderVersionIndexFile)
+    recommended_loader_versions = []
-    for it in loaderVersionIndex:
+    recommended_intermediary_versions = []
-        version = it["version"]
-        with open(UPSTREAM_DIR + "/fabric/loader-installer-json/" + version + ".json", 'r',
-                  encoding='utf-8') as loaderVersionFile:
-            ldata = json.load(loaderVersionFile)
-            ldata = FabricInstallerDataV1(ldata)
-            processLoaderVersion(version, it, ldata)
 
-with open(UPSTREAM_DIR + "/fabric/meta-v2/intermediary.json", 'r', encoding='utf-8') as intermediaryVersionIndexFile:
+    with open(os.path.join(UPSTREAM_DIR, META_DIR, "loader.json"), 'r', encoding='utf-8') as f:
-    intermediaryVersionIndex = json.load(intermediaryVersionIndexFile)
+        loader_version_index = json.load(f)
-    for it in intermediaryVersionIndex:
+        for entry in loader_version_index:
-        processIntermediaryVersion(it)
+            version = entry["version"]
+            print(f"Processing loader {version}")
 
-for version in loaderVersions:
+            v = process_loader_version(entry)
-    outFilepath = PMC_DIR + "/net.fabricmc.fabric-loader/%s.json" % version.version
-    with open(outFilepath, 'w') as outfile:
-        json.dump(version.to_json(), outfile, sort_keys=True, indent=4)
 
-sharedData = PolyMCSharedPackageData(uid='net.fabricmc.fabric-loader', name='Fabric Loader')
+            if not recommended_loader_versions:  # first (newest) loader is recommended
-sharedData.recommended = loaderRecommended
+                recommended_loader_versions.append(version)
-sharedData.description = "Fabric Loader is a tool to load Fabric-compatible mods in game environments."
-sharedData.projectUrl = "https://fabricmc.net"
-sharedData.authors = ["Fabric Developers"]
-sharedData.write()
 
-for version in intermediaryVersions:
+            with open(os.path.join(PMC_DIR, LOADER_COMPONENT, f"{v.version}.json"), 'w') as outfile:
-    outFilepath = PMC_DIR + "/net.fabricmc.intermediary/%s.json" % version.version
+                json.dump(v.to_json(), outfile, sort_keys=True, indent=4)
-    with open(outFilepath, 'w') as outfile:
-        json.dump(version.to_json(), outfile, sort_keys=True, indent=4)
 
-sharedData = PolyMCSharedPackageData(uid='net.fabricmc.intermediary', name='Intermediary Mappings')
+    with open(os.path.join(UPSTREAM_DIR, META_DIR, "intermediary.json"), 'r', encoding='utf-8') as f:
-sharedData.recommended = intermediaryRecommended
+        intermediary_version_index = json.load(f)
-sharedData.description = "Intermediary mappings allow using Fabric Loader with mods for Minecraft in a more compatible manner."
+        for entry in intermediary_version_index:
-sharedData.projectUrl = "https://fabricmc.net"
+            version = entry["version"]
-sharedData.authors = ["Fabric Developers"]
+            print(f"Processing intermediary {version}")
-sharedData.write()
+
+            v = process_intermediary_version(entry)
+
+            recommended_intermediary_versions.append(version)  # all intermediaries are recommended
+
+            with open(os.path.join(PMC_DIR, INTERMEDIARY_COMPONENT, f"{v.version}.json"), 'w') as outfile:
+                json.dump(v.to_json(), outfile, sort_keys=True, indent=4)
+
+    package = PolyMCSharedPackageData(uid=LOADER_COMPONENT, name='Fabric Loader')
+    package.recommended = recommended_loader_versions
+    package.description = "Fabric Loader is a tool to load Fabric-compatible mods in game environments."
+    package.projectUrl = "https://fabricmc.net"
+    package.authors = ["Fabric Developers"]
+    package.write()
+
+    package = PolyMCSharedPackageData(uid=INTERMEDIARY_COMPONENT, name='Intermediary Mappings')
+    package.recommended = recommended_intermediary_versions
+    package.description = "Intermediary mappings allow using Fabric Loader with mods for Minecraft in a more compatible manner."
+    package.projectUrl = "https://fabricmc.net"
+    package.authors = ["Fabric Developers"]
+    package.write()
+
+
+if __name__ == '__main__':
+    main()

meta/common/__init__.py

@@ -1,5 +1,7 @@
 import os
 
+DATETIME_FORMAT_HTTP = "%a, %d %b %Y %H:%M:%S %Z"
+
 
 def polymc_path():
     if "PMC_DIR" in os.environ:
@@ -13,7 +15,17 @@ def upstream_path():
     return "upstream"
 
 
+def ensure_upstream_dir(path):
+    path = os.path.join(upstream_path(), path)
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+
 def ensure_component_dir(component_id):
     path = os.path.join(polymc_path(), component_id)
     if not os.path.exists(path):
         os.makedirs(path)
+
+
+def transform_maven_key(maven_key: str):
+    return maven_key.replace(":", ".")

meta/common/fabric.py

@@ -0,0 +1,10 @@
+from os.path import join
+
+BASE_DIR = "fabric"
+
+JARS_DIR = join(BASE_DIR, "jars")
+INSTALLER_INFO_DIR = join(BASE_DIR, "loader-installer-json")
+META_DIR = join(BASE_DIR, "loader-installer-json")
+
+LOADER_COMPONENT = "net.fabricmc.fabric-loader"
+INTERMEDIARY_COMPONENT = "net.fabricmc.intermediary"

meta/metautil.py

@@ -4,8 +4,9 @@ import os
 
 import iso8601
 from .jsonobject import *
+from .common import polymc_path
 
-PMC_DIR = os.environ["PMC_DIR"]
+PMC_DIR = polymc_path()
 
 
 class ISOTimestampProperty(AbstractDateProperty):

updateFabric.py

@@ -5,33 +5,32 @@ import requests
 from cachecontrol import CacheControl
 from cachecontrol.caches import FileCache
 from meta.fabricutil import *
+from meta.common import DATETIME_FORMAT_HTTP, upstream_path, ensure_upstream_dir, transform_maven_key
+from meta.common.fabric import JARS_DIR, INSTALLER_INFO_DIR, META_DIR
 
-DATETIME_FORMAT_HTTP = "%a, %d %b %Y %H:%M:%S %Z"
+UPSTREAM_DIR = upstream_path()
 
-UPSTREAM_DIR = os.environ["UPSTREAM_DIR"]
+ensure_upstream_dir(JARS_DIR)
+ensure_upstream_dir(INSTALLER_INFO_DIR)
+ensure_upstream_dir(META_DIR)
 
 forever_cache = FileCache('caches/http_cache', forever=True)
 sess = CacheControl(requests.Session(), forever_cache)
 
 
-def mkdirs(path):
-    if not os.path.exists(path):
-        os.makedirs(path)
-
-
 def filehash(filename, hashtype, blocksize=65536):
-    hash = hashtype()
+    h = hashtype()
     with open(filename, "rb") as f:
         for block in iter(lambda: f.read(blocksize), b""):
-            hash.update(block)
+            h.update(block)
-    return hash.hexdigest()
+    return h.hexdigest()
 
 
-def get_maven_url(mavenKey, server, ext):
+def get_maven_url(maven_key, server, ext):
-    mavenParts = mavenKey.split(":", 3)
+    parts = maven_key.split(":", 3)
-    mavenVerUrl = server + mavenParts[0].replace(".", "/") + "/" + mavenParts[1] + "/" + mavenParts[2] + "/"
+    maven_ver_url = server + parts[0].replace(".", "/") + "/" + parts[1] + "/" + parts[2] + "/"
-    mavenUrl = mavenVerUrl + mavenParts[1] + "-" + mavenParts[2] + ext
+    maven_url = maven_ver_url + parts[1] + "-" + parts[2] + ext
-    return mavenUrl
+    return maven_url
 
 
 def get_json_file(path, url):
@@ -56,12 +55,11 @@ def head_file(url):
 
 
 def get_binary_file(path, url):
-    with open(path, 'w', encoding='utf-8') as f:
+    with open(path, 'wb') as f:
         r = sess.get(url)
         r.raise_for_status()
-        with open(path, 'wb') as f:
+        for chunk in r.iter_content(chunk_size=128):
-            for chunk in r.iter_content(chunk_size=128):
+            f.write(chunk)
-                f.write(chunk)
 
 
 def compute_jar_file(path, url):
@@ -101,21 +99,24 @@ def compute_jar_file(path, url):
         json.dump(data.to_json(), outfile, sort_keys=True, indent=4)
 
 
-mkdirs(UPSTREAM_DIR + "/fabric/meta-v2")
+def main():
-mkdirs(UPSTREAM_DIR + "/fabric/loader-installer-json")
+    # get the version list for each component we are interested in
-mkdirs(UPSTREAM_DIR + "/fabric/jars")
+    for component in ["intermediary", "loader"]:
+        index = get_json_file(os.path.join(UPSTREAM_DIR, META_DIR, f"{component}.json"),
+                              "https://meta.fabricmc.net/v2/versions/" + component)
+        for it in index:
+            print(f"Processing {component} {it['version']} ")
+            jar_maven_url = get_maven_url(it["maven"], "https://maven.fabricmc.net/", ".jar")
+            compute_jar_file(os.path.join(UPSTREAM_DIR, JARS_DIR, transform_maven_key(it["maven"])), jar_maven_url)
 
-# get the version list for each component we are interested in
+    # for each loader, download installer JSON file from maven
-for component in ["intermediary", "loader"]:
+    with open(os.path.join(UPSTREAM_DIR, META_DIR, "loader.json"), 'r', encoding='utf-8') as loaderVersionIndexFile:
-    index = get_json_file(UPSTREAM_DIR + "/fabric/meta-v2/" + component + ".json",
+        loader_version_index = json.load(loaderVersionIndexFile)
-                          "https://meta.fabricmc.net/v2/versions/" + component)
+        for it in loader_version_index:
-    for it in index:
+            print(f"Downloading installer info for loader {it['version']} ")
-        jarMavenUrl = get_maven_url(it["maven"], "https://maven.fabricmc.net/", ".jar")
+            maven_url = get_maven_url(it["maven"], "https://maven.fabricmc.net/", ".json")
-        compute_jar_file(UPSTREAM_DIR + "/fabric/jars/" + it["maven"].replace(":", "."), jarMavenUrl)
+            get_json_file(os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{it['version']}.json"), maven_url)
 
-# for each loader, download installer JSON file from maven
+
-with open(UPSTREAM_DIR + "/fabric/meta-v2/loader.json", 'r', encoding='utf-8') as loaderVersionIndexFile:
+if __name__ == '__main__':
-    loaderVersionIndex = json.load(loaderVersionIndexFile)
+    main()
-    for it in loaderVersionIndex:
-        mavenUrl = get_maven_url(it["maven"], "https://maven.fabricmc.net/", ".json")
-        get_json_file(UPSTREAM_DIR + "/fabric/loader-installer-json/" + it["version"] + ".json", mavenUrl)