diff --git a/Dockerfile b/Dockerfile index 6fa5e4b..e7a5820 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.10.2-bullseye ARG UID=1337 ARG GID=1337 -RUN pip install cachecontrol iso8601 requests lockfile jsonobject six \ +RUN pip install cachecontrol iso8601 requests lockfile jsonobject six pydantic \ && apt-get update && apt-get install -y rsync cron # add our cronjob diff --git a/generateFabric.py b/generateFabric.py index 333145c..0979529 100755 --- a/generateFabric.py +++ b/generateFabric.py @@ -1,6 +1,11 @@ -from meta.fabricutil import * +import json +import os + from meta.common import ensure_component_dir, polymc_path, upstream_path, transform_maven_key from meta.common.fabric import JARS_DIR, INSTALLER_INFO_DIR, META_DIR, INTERMEDIARY_COMPONENT, LOADER_COMPONENT +from meta.model import MetaVersionFile, Dependency, Library, MetaPackageData +from meta.model.fabric import FabricJarInfo, FabricInstallerDataV1, FabricMainClasses +from meta.model.types import GradleSpecifier PMC_DIR = polymc_path() UPSTREAM_DIR = upstream_path() @@ -9,51 +14,46 @@ ensure_component_dir("net.fabricmc.fabric-loader") ensure_component_dir("net.fabricmc.intermediary") -def load_jar_info(artifact_key): - with open(os.path.join(UPSTREAM_DIR, JARS_DIR, f"{artifact_key}.json"), 'r', - encoding='utf-8') as jarInfoFile: - return FabricJarInfo(json.load(jarInfoFile)) +def load_jar_info(artifact_key) -> FabricJarInfo: + return FabricJarInfo.parse_file(os.path.join(UPSTREAM_DIR, JARS_DIR, f"{artifact_key}.json")) -def load_installer_info(version): - with open(os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version}.json"), 'r', - encoding='utf-8') as loaderVersionFile: - data = json.load(loaderVersionFile) - return FabricInstallerDataV1(data) +def load_installer_info(version) -> FabricInstallerDataV1: + return FabricInstallerDataV1.parse_file(os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version}.json")) -def process_loader_version(entry) -> PolyMCVersionFile: +def process_loader_version(entry) -> MetaVersionFile: jar_info = load_jar_info(transform_maven_key(entry["maven"])) installer_info = load_installer_info(entry["version"]) - v = PolyMCVersionFile(name="Fabric Loader", uid="net.fabricmc.fabric-loader", version=entry["version"]) - v.releaseTime = jar_info.releaseTime - v.requires = [DependencyEntry(uid='net.fabricmc.intermediary')] + v = MetaVersionFile(name="Fabric Loader", uid="net.fabricmc.fabric-loader", version=entry["version"]) + v.release_time = jar_info.release_time + v.requires = [Dependency(uid='net.fabricmc.intermediary')] v.order = 10 v.type = "release" - if isinstance(installer_info.mainClass, dict): - v.mainClass = installer_info.mainClass["client"] + if isinstance(installer_info.main_class, FabricMainClasses): + v.main_class = installer_info.main_class.client else: - v.mainClass = installer_info.mainClass + v.main_class = installer_info.main_class v.libraries = [] v.libraries.extend(installer_info.libraries.common) v.libraries.extend(installer_info.libraries.client) - loader_lib = PolyMCLibrary(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net") + loader_lib = Library(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net") v.libraries.append(loader_lib) return v -def process_intermediary_version(entry) -> PolyMCVersionFile: +def process_intermediary_version(entry) -> MetaVersionFile: jar_info = load_jar_info(transform_maven_key(entry["maven"])) - v = PolyMCVersionFile(name="Intermediary Mappings", uid="net.fabricmc.intermediary", version=entry["version"]) - v.releaseTime = jar_info.releaseTime - v.requires = [DependencyEntry(uid='net.minecraft', equals=entry["version"])] + v = MetaVersionFile(name="Intermediary Mappings", uid="net.fabricmc.intermediary", version=entry["version"]) + v.release_time = jar_info.release_time + v.requires = [Dependency(uid='net.minecraft', equals=entry["version"])] v.order = 11 v.type = "release" v.libraries = [] v.volatile = True - intermediary_lib = PolyMCLibrary(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net") + intermediary_lib = Library(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net") v.libraries.append(intermediary_lib) return v @@ -73,8 +73,7 @@ def main(): if not recommended_loader_versions: # first (newest) loader is recommended recommended_loader_versions.append(version) - with open(os.path.join(PMC_DIR, LOADER_COMPONENT, f"{v.version}.json"), 'w') as outfile: - json.dump(v.to_json(), outfile, sort_keys=True, indent=4) + v.write(os.path.join(PMC_DIR, LOADER_COMPONENT, f"{v.version}.json")) with open(os.path.join(UPSTREAM_DIR, META_DIR, "intermediary.json"), 'r', encoding='utf-8') as f: intermediary_version_index = json.load(f) @@ -86,22 +85,21 @@ def main(): recommended_intermediary_versions.append(version) # all intermediaries are recommended - with open(os.path.join(PMC_DIR, INTERMEDIARY_COMPONENT, f"{v.version}.json"), 'w') as outfile: - json.dump(v.to_json(), outfile, sort_keys=True, indent=4) + v.write(os.path.join(PMC_DIR, INTERMEDIARY_COMPONENT, f"{v.version}.json")) - package = PolyMCSharedPackageData(uid=LOADER_COMPONENT, name='Fabric Loader') + package = MetaPackageData(uid=LOADER_COMPONENT, name='Fabric Loader') package.recommended = recommended_loader_versions package.description = "Fabric Loader is a tool to load Fabric-compatible mods in game environments." - package.projectUrl = "https://fabricmc.net" + package.project_url = "https://fabricmc.net" package.authors = ["Fabric Developers"] - package.write() + package.write(os.path.join(PMC_DIR, LOADER_COMPONENT, "package.json")) - package = PolyMCSharedPackageData(uid=INTERMEDIARY_COMPONENT, name='Intermediary Mappings') + package = MetaPackageData(uid=INTERMEDIARY_COMPONENT, name='Intermediary Mappings') package.recommended = recommended_intermediary_versions package.description = "Intermediary mappings allow using Fabric Loader with mods for Minecraft in a more compatible manner." - package.projectUrl = "https://fabricmc.net" + package.project_url = "https://fabricmc.net" package.authors = ["Fabric Developers"] - package.write() + package.write(os.path.join(PMC_DIR, INTERMEDIARY_COMPONENT, "package.json")) if __name__ == '__main__': diff --git a/meta/common/__init__.py b/meta/common/__init__.py index 77aa996..f0f620f 100644 --- a/meta/common/__init__.py +++ b/meta/common/__init__.py @@ -1,8 +1,13 @@ import os +import datetime DATETIME_FORMAT_HTTP = "%a, %d %b %Y %H:%M:%S %Z" +def serialize_datetime(dt: datetime.datetime): + return dt.replace(tzinfo=datetime.timezone.utc).isoformat() + + def polymc_path(): if "PMC_DIR" in os.environ: return os.environ["PMC_DIR"] diff --git a/meta/fabricutil.py b/meta/fabricutil.py deleted file mode 100644 index 2f38f8f..0000000 --- a/meta/fabricutil.py +++ /dev/null @@ -1,32 +0,0 @@ -from .metautil import * - - -class FabricInstallerArguments(JsonObject): - client = ListProperty(StringProperty) - common = ListProperty(StringProperty) - server = ListProperty(StringProperty) - - -class FabricInstallerLaunchwrapper(JsonObject): - tweakers = ObjectProperty(FabricInstallerArguments, required=True) - - -class FabricInstallerLibraries(JsonObject): - client = ListProperty(PolyMCLibrary) - common = ListProperty(PolyMCLibrary) - server = ListProperty(PolyMCLibrary) - - -class FabricInstallerDataV1(JsonObject): - version = IntegerProperty(required=True) - libraries = ObjectProperty(FabricInstallerLibraries, required=True) - mainClass = DefaultProperty() - arguments = ObjectProperty(FabricInstallerArguments, required=False) - launchwrapper = ObjectProperty(FabricInstallerLaunchwrapper, required=False) - - -class FabricJarInfo(JsonObject): - releaseTime = ISOTimestampProperty() - size = IntegerProperty() - sha256 = StringProperty() - sha1 = StringProperty() diff --git a/meta/model/__init__.py b/meta/model/__init__.py new file mode 100644 index 0000000..91e40bf --- /dev/null +++ b/meta/model/__init__.py @@ -0,0 +1,154 @@ +import os.path +from datetime import datetime +from typing import Optional, List, Dict, Any + +import pydantic +from pydantic import Field, AnyHttpUrl, validator + +from .types import GradleSpecifier +from ..common import serialize_datetime + +META_FORMAT_VERSION = 1 + + +class MetaBase(pydantic.BaseModel): + def dict(self, **kwargs) -> Dict[str, Any]: + for k in ["by_alias"]: + if k in kwargs: + del kwargs[k] + + return super(MetaBase, self).dict(by_alias=True, **kwargs) + + def json(self, **kwargs: Any) -> str: + for k in ["exclude_none", "sort_keys", "indent"]: + if k in kwargs: + del kwargs[k] + + return super(MetaBase, self).json(exclude_none=True, sort_keys=True, indent=4, **kwargs) + + def write(self, file_path): + with open(file_path, "w") as f: + f.write(self.json()) + + class Config: + allow_population_by_field_name = True + + json_encoders = { + datetime: serialize_datetime + } + + +class Versioned(MetaBase): + @validator("format_version") + def format_version_must_be_supported(cls, v): + return v > META_FORMAT_VERSION + + format_version: int = Field(META_FORMAT_VERSION, alias="formatVersion") + + +class MojangArtifactBase(MetaBase): + sha1: Optional[str] + size: Optional[int] + url: AnyHttpUrl + + +class MojangAssets(MojangArtifactBase): + id: str + totalSize: int + + +class MojangArtifact(MojangArtifactBase): + path: Optional[str] + + +class MojangLibraryExtractRules(MetaBase): + """ + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx" + } + } + ] + """ + exclude: List[str] # TODO maybe drop this completely? + + +class MojangLibraryDownloads(MetaBase): + artifact: Optional[MojangArtifact] + classifiers: Dict[Any, MojangArtifact] + + +class OSRule(MetaBase): + @validator("name") + def name_must_be_os(cls, v): + return v in ["osx", "linux", "windows"] + + name: str + version: Optional[str] + + +class MojangRule(MetaBase): + @validator("action") + def action_must_be_allow_disallow(cls, v): + return v in ["allow", "disallow"] + + action: str + os: Optional[OSRule] + + +class MojangLibrary(MetaBase): + extract: Optional[MojangLibraryExtractRules] + name: GradleSpecifier + downloads: Optional[MojangLibraryDownloads] + natives: Optional[Dict[str, str]] + rules: Optional[List[MojangRule]] + + class Config: + arbitrary_types_allowed = True + + +class Dependency(MetaBase): + uid: str + equals: Optional[str] + suggests: Optional[str] + + +class Library(MojangLibrary): + url: Optional[str] + mmcHint: Optional[AnyHttpUrl] = Field(None, alias="MMC-hint") + + +class MetaVersionFile(Versioned): + name: str + version: str + uid: str + type: Optional[str] + order: Optional[int] + volatile: Optional[bool] + requires: Optional[List[Dependency]] + conflicts: Optional[List[Dependency]] + libraries: Optional[List[Library]] + asset_index: Optional[MojangAssets] = Field(alias="assetIndex") + maven_files: Optional[List[Library]] = Field(alias="mavenFiles") + main_jar: Optional[Library] = Field(alias="mainJar") + jar_mods: Optional[List[Library]] = Field(alias="jarMods") + main_class: Optional[str] = Field(alias="mainClass") + applet_class: Optional[str] = Field(alias="appletClass") + minecraft_arguments: Optional[str] = Field(alias="minecraftArguments") + release_time: Optional[datetime] = Field(alias="releaseTime") + additional_traits: Optional[List[str]] = Field(alias="+traits") + additional_tweakers: Optional[List[str]] = Field(alias="+tweakers") + + +class MetaPackageData(Versioned): + name: str + uid: str + recommended: Optional[List[str]] + authors: Optional[List[str]] + description: Optional[str] + project_url: Optional[AnyHttpUrl] = Field(alias="projectUrl") diff --git a/meta/model/fabric.py b/meta/model/fabric.py new file mode 100644 index 0000000..cd326e5 --- /dev/null +++ b/meta/model/fabric.py @@ -0,0 +1,43 @@ +from datetime import datetime +from typing import Optional, List, Union, Dict + +from pydantic import Field + +from . import Library, MetaBase + + +class FabricInstallerArguments(MetaBase): + client: Optional[List[str]] + common: Optional[List[str]] + server: Optional[List[str]] + + +class FabricInstallerLaunchwrapper(MetaBase): + tweakers: FabricInstallerArguments + + +class FabricInstallerLibraries(MetaBase): + client: Optional[List[Library]] + common: Optional[List[Library]] + server: Optional[List[Library]] + + +class FabricMainClasses(MetaBase): + client: Optional[str] + common: Optional[str] + server: Optional[str] + + +class FabricInstallerDataV1(MetaBase): + version: int + libraries: FabricInstallerLibraries + main_class: Optional[Union[str, FabricMainClasses]] = Field(alias="mainClass") + arguments: Optional[FabricInstallerArguments] + launchwrapper: Optional[FabricInstallerLaunchwrapper] + + +class FabricJarInfo(MetaBase): + release_time: Optional[datetime] = Field(alias="releaseTime") + size: Optional[int] + sha256: Optional[str] + sha1: Optional[str] diff --git a/meta/model/types.py b/meta/model/types.py new file mode 100644 index 0000000..d631565 --- /dev/null +++ b/meta/model/types.py @@ -0,0 +1,47 @@ +class GradleSpecifier(str): + """ + A gradle specifier - a maven coordinate. Like one of these: + "org.lwjgl.lwjgl:lwjgl:2.9.0" + "net.java.jinput:jinput:2.0.5" + "net.minecraft:launchwrapper:1.5" + """ + + def __init__(self, name: str): + ext_split = name.split('@') + + components = ext_split[0].split(':') + self.group = components[0] + self.artifact = components[1] + self.version = components[2] + + self.extension = 'jar' + if len(ext_split) == 2: + self.extension = ext_split[1] + + self.classifier = None + if len(components) == 4: + self.classifier = components[3] + + def __new__(cls, name: str): + return super(GradleSpecifier, cls).__new__(cls, name) + + def filename(self): + if self.classifier: + return "%s-%s-%s.%s" % (self.artifact, self.version, self.classifier, self.extension) + else: + return "%s-%s.%s" % (self.artifact, self.version, self.extension) + + def base(self): + return "%s/%s/%s/" % (self.group.replace('.', '/'), self.artifact, self.version) + + def path(self): + return self.base() + self.filename() + + def __repr__(self): + return f"GradleSpecifier('{self}')" + + def is_lwjgl(self): + return self.group in ("org.lwjgl", "org.lwjgl.lwjgl", "net.java.jinput", "net.java.jutils") + + def is_log4j(self): + return self.group == "org.apache.logging.log4j" \ No newline at end of file diff --git a/updateFabric.py b/updateFabric.py index f48858d..37740f1 100755 --- a/updateFabric.py +++ b/updateFabric.py @@ -1,10 +1,13 @@ import hashlib import zipfile +import json +import os +from datetime import datetime import requests from cachecontrol import CacheControl from cachecontrol.caches import FileCache -from meta.fabricutil import * +from meta.model.fabric import FabricJarInfo from meta.common import DATETIME_FORMAT_HTTP, upstream_path, ensure_upstream_dir, transform_maven_key from meta.common.fabric import JARS_DIR, INSTALLER_INFO_DIR, META_DIR @@ -68,7 +71,7 @@ def compute_jar_file(path, url): try: # Let's not download a Jar file if we don't need to. headers = head_file(url) - tstamp = datetime.datetime.strptime(headers["Last-Modified"], DATETIME_FORMAT_HTTP) + tstamp = datetime.strptime(headers["Last-Modified"], DATETIME_FORMAT_HTTP) sha1 = get_plaintext(url + ".sha1") sha256 = get_plaintext(url + ".sha256") size = int(headers["Content-Length"]) @@ -78,11 +81,11 @@ def compute_jar_file(path, url): jar_path = path + ".jar" get_binary_file(jar_path, url) - tstamp = datetime.datetime.fromtimestamp(0) + tstamp = datetime.fromtimestamp(0) with zipfile.ZipFile(jar_path, 'r') as jar: allinfo = jar.infolist() for info in allinfo: - tstamp_new = datetime.datetime(*info.date_time) + tstamp_new = datetime(*info.date_time) if tstamp_new > tstamp: tstamp = tstamp_new @@ -90,13 +93,8 @@ def compute_jar_file(path, url): sha256 = filehash(jar_path, hashlib.sha256) size = os.path.getsize(jar_path) - data = FabricJarInfo() - data.releaseTime = tstamp - data.sha1 = sha1 - data.sha256 = sha256 - data.size = size - with open(path + ".json", 'w') as outfile: - json.dump(data.to_json(), outfile, sort_keys=True, indent=4) + data = FabricJarInfo(releaseTime=tstamp, sha1=sha1, sha256=sha256, size=size) + data.write(path + ".json") def main():