refactor: switch Fabric over to pydantic

Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.10.2-bullseye
 ARG UID=1337
 ARG GID=1337
 
-RUN pip install cachecontrol iso8601 requests lockfile jsonobject six \
+RUN pip install cachecontrol iso8601 requests lockfile jsonobject six pydantic \
     && apt-get update && apt-get install -y rsync cron
 
 # add our cronjob

generateFabric.py

@@ -1,6 +1,11 @@
-from meta.fabricutil import *
+import json
+import os
+
 from meta.common import ensure_component_dir, polymc_path, upstream_path, transform_maven_key
 from meta.common.fabric import JARS_DIR, INSTALLER_INFO_DIR, META_DIR, INTERMEDIARY_COMPONENT, LOADER_COMPONENT
+from meta.model import MetaVersionFile, Dependency, Library, MetaPackageData
+from meta.model.fabric import FabricJarInfo, FabricInstallerDataV1, FabricMainClasses
+from meta.model.types import GradleSpecifier
 
 PMC_DIR = polymc_path()
 UPSTREAM_DIR = upstream_path()
@@ -9,51 +14,46 @@ ensure_component_dir("net.fabricmc.fabric-loader")
 ensure_component_dir("net.fabricmc.intermediary")
 
 
-def load_jar_info(artifact_key):
+def load_jar_info(artifact_key) -> FabricJarInfo:
-    with open(os.path.join(UPSTREAM_DIR, JARS_DIR, f"{artifact_key}.json"), 'r',
+    return FabricJarInfo.parse_file(os.path.join(UPSTREAM_DIR, JARS_DIR, f"{artifact_key}.json"))
-              encoding='utf-8') as jarInfoFile:
-        return FabricJarInfo(json.load(jarInfoFile))
 
 
-def load_installer_info(version):
+def load_installer_info(version) -> FabricInstallerDataV1:
-    with open(os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version}.json"), 'r',
+    return FabricInstallerDataV1.parse_file(os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version}.json"))
-              encoding='utf-8') as loaderVersionFile:
-        data = json.load(loaderVersionFile)
-        return FabricInstallerDataV1(data)
 
 
-def process_loader_version(entry) -> PolyMCVersionFile:
+def process_loader_version(entry) -> MetaVersionFile:
     jar_info = load_jar_info(transform_maven_key(entry["maven"]))
     installer_info = load_installer_info(entry["version"])
 
-    v = PolyMCVersionFile(name="Fabric Loader", uid="net.fabricmc.fabric-loader", version=entry["version"])
+    v = MetaVersionFile(name="Fabric Loader", uid="net.fabricmc.fabric-loader", version=entry["version"])
-    v.releaseTime = jar_info.releaseTime
+    v.release_time = jar_info.release_time
-    v.requires = [DependencyEntry(uid='net.fabricmc.intermediary')]
+    v.requires = [Dependency(uid='net.fabricmc.intermediary')]
     v.order = 10
     v.type = "release"
-    if isinstance(installer_info.mainClass, dict):
+    if isinstance(installer_info.main_class, FabricMainClasses):
-        v.mainClass = installer_info.mainClass["client"]
+        v.main_class = installer_info.main_class.client
     else:
-        v.mainClass = installer_info.mainClass
+        v.main_class = installer_info.main_class
     v.libraries = []
     v.libraries.extend(installer_info.libraries.common)
     v.libraries.extend(installer_info.libraries.client)
-    loader_lib = PolyMCLibrary(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net")
+    loader_lib = Library(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net")
     v.libraries.append(loader_lib)
     return v
 
 
-def process_intermediary_version(entry) -> PolyMCVersionFile:
+def process_intermediary_version(entry) -> MetaVersionFile:
     jar_info = load_jar_info(transform_maven_key(entry["maven"]))
 
-    v = PolyMCVersionFile(name="Intermediary Mappings", uid="net.fabricmc.intermediary", version=entry["version"])
+    v = MetaVersionFile(name="Intermediary Mappings", uid="net.fabricmc.intermediary", version=entry["version"])
-    v.releaseTime = jar_info.releaseTime
+    v.release_time = jar_info.release_time
-    v.requires = [DependencyEntry(uid='net.minecraft', equals=entry["version"])]
+    v.requires = [Dependency(uid='net.minecraft', equals=entry["version"])]
     v.order = 11
     v.type = "release"
     v.libraries = []
     v.volatile = True
-    intermediary_lib = PolyMCLibrary(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net")
+    intermediary_lib = Library(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net")
     v.libraries.append(intermediary_lib)
     return v
 
@@ -73,8 +73,7 @@ def main():
             if not recommended_loader_versions:  # first (newest) loader is recommended
                 recommended_loader_versions.append(version)
 
-            with open(os.path.join(PMC_DIR, LOADER_COMPONENT, f"{v.version}.json"), 'w') as outfile:
+            v.write(os.path.join(PMC_DIR, LOADER_COMPONENT, f"{v.version}.json"))
-                json.dump(v.to_json(), outfile, sort_keys=True, indent=4)
 
     with open(os.path.join(UPSTREAM_DIR, META_DIR, "intermediary.json"), 'r', encoding='utf-8') as f:
         intermediary_version_index = json.load(f)
@@ -86,22 +85,21 @@ def main():
 
             recommended_intermediary_versions.append(version)  # all intermediaries are recommended
 
-            with open(os.path.join(PMC_DIR, INTERMEDIARY_COMPONENT, f"{v.version}.json"), 'w') as outfile:
+            v.write(os.path.join(PMC_DIR, INTERMEDIARY_COMPONENT, f"{v.version}.json"))
-                json.dump(v.to_json(), outfile, sort_keys=True, indent=4)
 
-    package = PolyMCSharedPackageData(uid=LOADER_COMPONENT, name='Fabric Loader')
+    package = MetaPackageData(uid=LOADER_COMPONENT, name='Fabric Loader')
     package.recommended = recommended_loader_versions
     package.description = "Fabric Loader is a tool to load Fabric-compatible mods in game environments."
-    package.projectUrl = "https://fabricmc.net"
+    package.project_url = "https://fabricmc.net"
     package.authors = ["Fabric Developers"]
-    package.write()
+    package.write(os.path.join(PMC_DIR, LOADER_COMPONENT, "package.json"))
 
-    package = PolyMCSharedPackageData(uid=INTERMEDIARY_COMPONENT, name='Intermediary Mappings')
+    package = MetaPackageData(uid=INTERMEDIARY_COMPONENT, name='Intermediary Mappings')
     package.recommended = recommended_intermediary_versions
     package.description = "Intermediary mappings allow using Fabric Loader with mods for Minecraft in a more compatible manner."
-    package.projectUrl = "https://fabricmc.net"
+    package.project_url = "https://fabricmc.net"
     package.authors = ["Fabric Developers"]
-    package.write()
+    package.write(os.path.join(PMC_DIR, INTERMEDIARY_COMPONENT, "package.json"))
 
 
 if __name__ == '__main__':

meta/common/__init__.py

@@ -1,8 +1,13 @@
 import os
+import datetime
 
 DATETIME_FORMAT_HTTP = "%a, %d %b %Y %H:%M:%S %Z"
 
 
+def serialize_datetime(dt: datetime.datetime):
+    return dt.replace(tzinfo=datetime.timezone.utc).isoformat()
+
+
 def polymc_path():
     if "PMC_DIR" in os.environ:
         return os.environ["PMC_DIR"]

meta/fabricutil.py

@@ -1,32 +0,0 @@
-from .metautil import *
-
-
-class FabricInstallerArguments(JsonObject):
-    client = ListProperty(StringProperty)
-    common = ListProperty(StringProperty)
-    server = ListProperty(StringProperty)
-
-
-class FabricInstallerLaunchwrapper(JsonObject):
-    tweakers = ObjectProperty(FabricInstallerArguments, required=True)
-
-
-class FabricInstallerLibraries(JsonObject):
-    client = ListProperty(PolyMCLibrary)
-    common = ListProperty(PolyMCLibrary)
-    server = ListProperty(PolyMCLibrary)
-
-
-class FabricInstallerDataV1(JsonObject):
-    version = IntegerProperty(required=True)
-    libraries = ObjectProperty(FabricInstallerLibraries, required=True)
-    mainClass = DefaultProperty()
-    arguments = ObjectProperty(FabricInstallerArguments, required=False)
-    launchwrapper = ObjectProperty(FabricInstallerLaunchwrapper, required=False)
-
-
-class FabricJarInfo(JsonObject):
-    releaseTime = ISOTimestampProperty()
-    size = IntegerProperty()
-    sha256 = StringProperty()
-    sha1 = StringProperty()

meta/model/__init__.py

@@ -0,0 +1,154 @@
+import os.path
+from datetime import datetime
+from typing import Optional, List, Dict, Any
+
+import pydantic
+from pydantic import Field, AnyHttpUrl, validator
+
+from .types import GradleSpecifier
+from ..common import serialize_datetime
+
+META_FORMAT_VERSION = 1
+
+
+class MetaBase(pydantic.BaseModel):
+    def dict(self, **kwargs) -> Dict[str, Any]:
+        for k in ["by_alias"]:
+            if k in kwargs:
+                del kwargs[k]
+
+        return super(MetaBase, self).dict(by_alias=True, **kwargs)
+
+    def json(self, **kwargs: Any) -> str:
+        for k in ["exclude_none", "sort_keys", "indent"]:
+            if k in kwargs:
+                del kwargs[k]
+
+        return super(MetaBase, self).json(exclude_none=True, sort_keys=True, indent=4, **kwargs)
+
+    def write(self, file_path):
+        with open(file_path, "w") as f:
+            f.write(self.json())
+
+    class Config:
+        allow_population_by_field_name = True
+
+        json_encoders = {
+            datetime: serialize_datetime
+        }
+
+
+class Versioned(MetaBase):
+    @validator("format_version")
+    def format_version_must_be_supported(cls, v):
+        return v > META_FORMAT_VERSION
+
+    format_version: int = Field(META_FORMAT_VERSION, alias="formatVersion")
+
+
+class MojangArtifactBase(MetaBase):
+    sha1: Optional[str]
+    size: Optional[int]
+    url: AnyHttpUrl
+
+
+class MojangAssets(MojangArtifactBase):
+    id: str
+    totalSize: int
+
+
+class MojangArtifact(MojangArtifactBase):
+    path: Optional[str]
+
+
+class MojangLibraryExtractRules(MetaBase):
+    """
+            "rules": [
+                {
+                    "action": "allow"
+                },
+                {
+                    "action": "disallow",
+                    "os": {
+                        "name": "osx"
+                    }
+                }
+            ]
+    """
+    exclude: List[str]  # TODO maybe drop this completely?
+
+
+class MojangLibraryDownloads(MetaBase):
+    artifact: Optional[MojangArtifact]
+    classifiers: Dict[Any, MojangArtifact]
+
+
+class OSRule(MetaBase):
+    @validator("name")
+    def name_must_be_os(cls, v):
+        return v in ["osx", "linux", "windows"]
+
+    name: str
+    version: Optional[str]
+
+
+class MojangRule(MetaBase):
+    @validator("action")
+    def action_must_be_allow_disallow(cls, v):
+        return v in ["allow", "disallow"]
+
+    action: str
+    os: Optional[OSRule]
+
+
+class MojangLibrary(MetaBase):
+    extract: Optional[MojangLibraryExtractRules]
+    name: GradleSpecifier
+    downloads: Optional[MojangLibraryDownloads]
+    natives: Optional[Dict[str, str]]
+    rules: Optional[List[MojangRule]]
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
+class Dependency(MetaBase):
+    uid: str
+    equals: Optional[str]
+    suggests: Optional[str]
+
+
+class Library(MojangLibrary):
+    url: Optional[str]
+    mmcHint: Optional[AnyHttpUrl] = Field(None, alias="MMC-hint")
+
+
+class MetaVersionFile(Versioned):
+    name: str
+    version: str
+    uid: str
+    type: Optional[str]
+    order: Optional[int]
+    volatile: Optional[bool]
+    requires: Optional[List[Dependency]]
+    conflicts: Optional[List[Dependency]]
+    libraries: Optional[List[Library]]
+    asset_index: Optional[MojangAssets] = Field(alias="assetIndex")
+    maven_files: Optional[List[Library]] = Field(alias="mavenFiles")
+    main_jar: Optional[Library] = Field(alias="mainJar")
+    jar_mods: Optional[List[Library]] = Field(alias="jarMods")
+    main_class: Optional[str] = Field(alias="mainClass")
+    applet_class: Optional[str] = Field(alias="appletClass")
+    minecraft_arguments: Optional[str] = Field(alias="minecraftArguments")
+    release_time: Optional[datetime] = Field(alias="releaseTime")
+    additional_traits: Optional[List[str]] = Field(alias="+traits")
+    additional_tweakers: Optional[List[str]] = Field(alias="+tweakers")
+
+
+class MetaPackageData(Versioned):
+    name: str
+    uid: str
+    recommended: Optional[List[str]]
+    authors: Optional[List[str]]
+    description: Optional[str]
+    project_url: Optional[AnyHttpUrl] = Field(alias="projectUrl")

meta/model/fabric.py

@@ -0,0 +1,43 @@
+from datetime import datetime
+from typing import Optional, List, Union, Dict
+
+from pydantic import Field
+
+from . import Library, MetaBase
+
+
+class FabricInstallerArguments(MetaBase):
+    client: Optional[List[str]]
+    common: Optional[List[str]]
+    server: Optional[List[str]]
+
+
+class FabricInstallerLaunchwrapper(MetaBase):
+    tweakers: FabricInstallerArguments
+
+
+class FabricInstallerLibraries(MetaBase):
+    client: Optional[List[Library]]
+    common: Optional[List[Library]]
+    server: Optional[List[Library]]
+
+
+class FabricMainClasses(MetaBase):
+    client: Optional[str]
+    common: Optional[str]
+    server: Optional[str]
+
+
+class FabricInstallerDataV1(MetaBase):
+    version: int
+    libraries: FabricInstallerLibraries
+    main_class: Optional[Union[str, FabricMainClasses]] = Field(alias="mainClass")
+    arguments: Optional[FabricInstallerArguments]
+    launchwrapper: Optional[FabricInstallerLaunchwrapper]
+
+
+class FabricJarInfo(MetaBase):
+    release_time: Optional[datetime] = Field(alias="releaseTime")
+    size: Optional[int]
+    sha256: Optional[str]
+    sha1: Optional[str]

meta/model/types.py

@@ -0,0 +1,47 @@
+class GradleSpecifier(str):
+    """
+        A gradle specifier - a maven coordinate. Like one of these:
+        "org.lwjgl.lwjgl:lwjgl:2.9.0"
+        "net.java.jinput:jinput:2.0.5"
+        "net.minecraft:launchwrapper:1.5"
+    """
+
+    def __init__(self, name: str):
+        ext_split = name.split('@')
+
+        components = ext_split[0].split(':')
+        self.group = components[0]
+        self.artifact = components[1]
+        self.version = components[2]
+
+        self.extension = 'jar'
+        if len(ext_split) == 2:
+            self.extension = ext_split[1]
+
+        self.classifier = None
+        if len(components) == 4:
+            self.classifier = components[3]
+
+    def __new__(cls, name: str):
+        return super(GradleSpecifier, cls).__new__(cls, name)
+
+    def filename(self):
+        if self.classifier:
+            return "%s-%s-%s.%s" % (self.artifact, self.version, self.classifier, self.extension)
+        else:
+            return "%s-%s.%s" % (self.artifact, self.version, self.extension)
+
+    def base(self):
+        return "%s/%s/%s/" % (self.group.replace('.', '/'), self.artifact, self.version)
+
+    def path(self):
+        return self.base() + self.filename()
+
+    def __repr__(self):
+        return f"GradleSpecifier('{self}')"
+
+    def is_lwjgl(self):
+        return self.group in ("org.lwjgl", "org.lwjgl.lwjgl", "net.java.jinput", "net.java.jutils")
+
+    def is_log4j(self):
+        return self.group == "org.apache.logging.log4j"

updateFabric.py

@@ -1,10 +1,13 @@
 import hashlib
 import zipfile
+import json
+import os
+from datetime import datetime
 
 import requests
 from cachecontrol import CacheControl
 from cachecontrol.caches import FileCache
-from meta.fabricutil import *
+from meta.model.fabric import FabricJarInfo
 from meta.common import DATETIME_FORMAT_HTTP, upstream_path, ensure_upstream_dir, transform_maven_key
 from meta.common.fabric import JARS_DIR, INSTALLER_INFO_DIR, META_DIR
 
@@ -68,7 +71,7 @@ def compute_jar_file(path, url):
     try:
         # Let's not download a Jar file if we don't need to.
         headers = head_file(url)
-        tstamp = datetime.datetime.strptime(headers["Last-Modified"], DATETIME_FORMAT_HTTP)
+        tstamp = datetime.strptime(headers["Last-Modified"], DATETIME_FORMAT_HTTP)
         sha1 = get_plaintext(url + ".sha1")
         sha256 = get_plaintext(url + ".sha256")
         size = int(headers["Content-Length"])
@@ -78,11 +81,11 @@ def compute_jar_file(path, url):
 
         jar_path = path + ".jar"
         get_binary_file(jar_path, url)
-        tstamp = datetime.datetime.fromtimestamp(0)
+        tstamp = datetime.fromtimestamp(0)
         with zipfile.ZipFile(jar_path, 'r') as jar:
             allinfo = jar.infolist()
             for info in allinfo:
-                tstamp_new = datetime.datetime(*info.date_time)
+                tstamp_new = datetime(*info.date_time)
                 if tstamp_new > tstamp:
                     tstamp = tstamp_new
 
@@ -90,13 +93,8 @@ def compute_jar_file(path, url):
         sha256 = filehash(jar_path, hashlib.sha256)
         size = os.path.getsize(jar_path)
 
-    data = FabricJarInfo()
+    data = FabricJarInfo(releaseTime=tstamp, sha1=sha1, sha256=sha256, size=size)
-    data.releaseTime = tstamp
+    data.write(path + ".json")
-    data.sha1 = sha1
-    data.sha256 = sha256
-    data.size = size
-    with open(path + ".json", 'w') as outfile:
-        json.dump(data.to_json(), outfile, sort_keys=True, indent=4)
 
 
 def main():