refactor: switch Fabric over to pydantic

This commit is contained in:
Sefa Eyeoglu 2022-03-29 13:53:59 +02:00
parent 6d9561c28d
commit a5bb3a091d
No known key found for this signature in database
GPG Key ID: C10411294912A422
8 changed files with 290 additions and 77 deletions

View File

@ -2,7 +2,7 @@ FROM python:3.10.2-bullseye
ARG UID=1337
ARG GID=1337
RUN pip install cachecontrol iso8601 requests lockfile jsonobject six \
RUN pip install cachecontrol iso8601 requests lockfile jsonobject six pydantic \
&& apt-get update && apt-get install -y rsync cron
# add our cronjob

View File

@ -1,6 +1,11 @@
from meta.fabricutil import *
import json
import os
from meta.common import ensure_component_dir, polymc_path, upstream_path, transform_maven_key
from meta.common.fabric import JARS_DIR, INSTALLER_INFO_DIR, META_DIR, INTERMEDIARY_COMPONENT, LOADER_COMPONENT
from meta.model import MetaVersionFile, Dependency, Library, MetaPackageData
from meta.model.fabric import FabricJarInfo, FabricInstallerDataV1, FabricMainClasses
from meta.model.types import GradleSpecifier
PMC_DIR = polymc_path()
UPSTREAM_DIR = upstream_path()
@ -9,51 +14,46 @@ ensure_component_dir("net.fabricmc.fabric-loader")
ensure_component_dir("net.fabricmc.intermediary")
def load_jar_info(artifact_key):
with open(os.path.join(UPSTREAM_DIR, JARS_DIR, f"{artifact_key}.json"), 'r',
encoding='utf-8') as jarInfoFile:
return FabricJarInfo(json.load(jarInfoFile))
def load_jar_info(artifact_key) -> FabricJarInfo:
return FabricJarInfo.parse_file(os.path.join(UPSTREAM_DIR, JARS_DIR, f"{artifact_key}.json"))
def load_installer_info(version):
with open(os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version}.json"), 'r',
encoding='utf-8') as loaderVersionFile:
data = json.load(loaderVersionFile)
return FabricInstallerDataV1(data)
def load_installer_info(version) -> FabricInstallerDataV1:
return FabricInstallerDataV1.parse_file(os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version}.json"))
def process_loader_version(entry) -> PolyMCVersionFile:
def process_loader_version(entry) -> MetaVersionFile:
jar_info = load_jar_info(transform_maven_key(entry["maven"]))
installer_info = load_installer_info(entry["version"])
v = PolyMCVersionFile(name="Fabric Loader", uid="net.fabricmc.fabric-loader", version=entry["version"])
v.releaseTime = jar_info.releaseTime
v.requires = [DependencyEntry(uid='net.fabricmc.intermediary')]
v = MetaVersionFile(name="Fabric Loader", uid="net.fabricmc.fabric-loader", version=entry["version"])
v.release_time = jar_info.release_time
v.requires = [Dependency(uid='net.fabricmc.intermediary')]
v.order = 10
v.type = "release"
if isinstance(installer_info.mainClass, dict):
v.mainClass = installer_info.mainClass["client"]
if isinstance(installer_info.main_class, FabricMainClasses):
v.main_class = installer_info.main_class.client
else:
v.mainClass = installer_info.mainClass
v.main_class = installer_info.main_class
v.libraries = []
v.libraries.extend(installer_info.libraries.common)
v.libraries.extend(installer_info.libraries.client)
loader_lib = PolyMCLibrary(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net")
loader_lib = Library(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net")
v.libraries.append(loader_lib)
return v
def process_intermediary_version(entry) -> PolyMCVersionFile:
def process_intermediary_version(entry) -> MetaVersionFile:
jar_info = load_jar_info(transform_maven_key(entry["maven"]))
v = PolyMCVersionFile(name="Intermediary Mappings", uid="net.fabricmc.intermediary", version=entry["version"])
v.releaseTime = jar_info.releaseTime
v.requires = [DependencyEntry(uid='net.minecraft', equals=entry["version"])]
v = MetaVersionFile(name="Intermediary Mappings", uid="net.fabricmc.intermediary", version=entry["version"])
v.release_time = jar_info.release_time
v.requires = [Dependency(uid='net.minecraft', equals=entry["version"])]
v.order = 11
v.type = "release"
v.libraries = []
v.volatile = True
intermediary_lib = PolyMCLibrary(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net")
intermediary_lib = Library(name=GradleSpecifier(entry["maven"]), url="https://maven.fabricmc.net")
v.libraries.append(intermediary_lib)
return v
@ -73,8 +73,7 @@ def main():
if not recommended_loader_versions: # first (newest) loader is recommended
recommended_loader_versions.append(version)
with open(os.path.join(PMC_DIR, LOADER_COMPONENT, f"{v.version}.json"), 'w') as outfile:
json.dump(v.to_json(), outfile, sort_keys=True, indent=4)
v.write(os.path.join(PMC_DIR, LOADER_COMPONENT, f"{v.version}.json"))
with open(os.path.join(UPSTREAM_DIR, META_DIR, "intermediary.json"), 'r', encoding='utf-8') as f:
intermediary_version_index = json.load(f)
@ -86,22 +85,21 @@ def main():
recommended_intermediary_versions.append(version) # all intermediaries are recommended
with open(os.path.join(PMC_DIR, INTERMEDIARY_COMPONENT, f"{v.version}.json"), 'w') as outfile:
json.dump(v.to_json(), outfile, sort_keys=True, indent=4)
v.write(os.path.join(PMC_DIR, INTERMEDIARY_COMPONENT, f"{v.version}.json"))
package = PolyMCSharedPackageData(uid=LOADER_COMPONENT, name='Fabric Loader')
package = MetaPackageData(uid=LOADER_COMPONENT, name='Fabric Loader')
package.recommended = recommended_loader_versions
package.description = "Fabric Loader is a tool to load Fabric-compatible mods in game environments."
package.projectUrl = "https://fabricmc.net"
package.project_url = "https://fabricmc.net"
package.authors = ["Fabric Developers"]
package.write()
package.write(os.path.join(PMC_DIR, LOADER_COMPONENT, "package.json"))
package = PolyMCSharedPackageData(uid=INTERMEDIARY_COMPONENT, name='Intermediary Mappings')
package = MetaPackageData(uid=INTERMEDIARY_COMPONENT, name='Intermediary Mappings')
package.recommended = recommended_intermediary_versions
package.description = "Intermediary mappings allow using Fabric Loader with mods for Minecraft in a more compatible manner."
package.projectUrl = "https://fabricmc.net"
package.project_url = "https://fabricmc.net"
package.authors = ["Fabric Developers"]
package.write()
package.write(os.path.join(PMC_DIR, INTERMEDIARY_COMPONENT, "package.json"))
if __name__ == '__main__':

View File

@ -1,8 +1,13 @@
import os
import datetime
DATETIME_FORMAT_HTTP = "%a, %d %b %Y %H:%M:%S %Z"
def serialize_datetime(dt: datetime.datetime):
return dt.replace(tzinfo=datetime.timezone.utc).isoformat()
def polymc_path():
if "PMC_DIR" in os.environ:
return os.environ["PMC_DIR"]

View File

@ -1,32 +0,0 @@
from .metautil import *
class FabricInstallerArguments(JsonObject):
client = ListProperty(StringProperty)
common = ListProperty(StringProperty)
server = ListProperty(StringProperty)
class FabricInstallerLaunchwrapper(JsonObject):
tweakers = ObjectProperty(FabricInstallerArguments, required=True)
class FabricInstallerLibraries(JsonObject):
client = ListProperty(PolyMCLibrary)
common = ListProperty(PolyMCLibrary)
server = ListProperty(PolyMCLibrary)
class FabricInstallerDataV1(JsonObject):
version = IntegerProperty(required=True)
libraries = ObjectProperty(FabricInstallerLibraries, required=True)
mainClass = DefaultProperty()
arguments = ObjectProperty(FabricInstallerArguments, required=False)
launchwrapper = ObjectProperty(FabricInstallerLaunchwrapper, required=False)
class FabricJarInfo(JsonObject):
releaseTime = ISOTimestampProperty()
size = IntegerProperty()
sha256 = StringProperty()
sha1 = StringProperty()

154
meta/model/__init__.py Normal file
View File

@ -0,0 +1,154 @@
import os.path
from datetime import datetime
from typing import Optional, List, Dict, Any
import pydantic
from pydantic import Field, AnyHttpUrl, validator
from .types import GradleSpecifier
from ..common import serialize_datetime
META_FORMAT_VERSION = 1
class MetaBase(pydantic.BaseModel):
def dict(self, **kwargs) -> Dict[str, Any]:
for k in ["by_alias"]:
if k in kwargs:
del kwargs[k]
return super(MetaBase, self).dict(by_alias=True, **kwargs)
def json(self, **kwargs: Any) -> str:
for k in ["exclude_none", "sort_keys", "indent"]:
if k in kwargs:
del kwargs[k]
return super(MetaBase, self).json(exclude_none=True, sort_keys=True, indent=4, **kwargs)
def write(self, file_path):
with open(file_path, "w") as f:
f.write(self.json())
class Config:
allow_population_by_field_name = True
json_encoders = {
datetime: serialize_datetime
}
class Versioned(MetaBase):
@validator("format_version")
def format_version_must_be_supported(cls, v):
return v > META_FORMAT_VERSION
format_version: int = Field(META_FORMAT_VERSION, alias="formatVersion")
class MojangArtifactBase(MetaBase):
sha1: Optional[str]
size: Optional[int]
url: AnyHttpUrl
class MojangAssets(MojangArtifactBase):
id: str
totalSize: int
class MojangArtifact(MojangArtifactBase):
path: Optional[str]
class MojangLibraryExtractRules(MetaBase):
"""
"rules": [
{
"action": "allow"
},
{
"action": "disallow",
"os": {
"name": "osx"
}
}
]
"""
exclude: List[str] # TODO maybe drop this completely?
class MojangLibraryDownloads(MetaBase):
artifact: Optional[MojangArtifact]
classifiers: Dict[Any, MojangArtifact]
class OSRule(MetaBase):
@validator("name")
def name_must_be_os(cls, v):
return v in ["osx", "linux", "windows"]
name: str
version: Optional[str]
class MojangRule(MetaBase):
@validator("action")
def action_must_be_allow_disallow(cls, v):
return v in ["allow", "disallow"]
action: str
os: Optional[OSRule]
class MojangLibrary(MetaBase):
extract: Optional[MojangLibraryExtractRules]
name: GradleSpecifier
downloads: Optional[MojangLibraryDownloads]
natives: Optional[Dict[str, str]]
rules: Optional[List[MojangRule]]
class Config:
arbitrary_types_allowed = True
class Dependency(MetaBase):
uid: str
equals: Optional[str]
suggests: Optional[str]
class Library(MojangLibrary):
url: Optional[str]
mmcHint: Optional[AnyHttpUrl] = Field(None, alias="MMC-hint")
class MetaVersionFile(Versioned):
name: str
version: str
uid: str
type: Optional[str]
order: Optional[int]
volatile: Optional[bool]
requires: Optional[List[Dependency]]
conflicts: Optional[List[Dependency]]
libraries: Optional[List[Library]]
asset_index: Optional[MojangAssets] = Field(alias="assetIndex")
maven_files: Optional[List[Library]] = Field(alias="mavenFiles")
main_jar: Optional[Library] = Field(alias="mainJar")
jar_mods: Optional[List[Library]] = Field(alias="jarMods")
main_class: Optional[str] = Field(alias="mainClass")
applet_class: Optional[str] = Field(alias="appletClass")
minecraft_arguments: Optional[str] = Field(alias="minecraftArguments")
release_time: Optional[datetime] = Field(alias="releaseTime")
additional_traits: Optional[List[str]] = Field(alias="+traits")
additional_tweakers: Optional[List[str]] = Field(alias="+tweakers")
class MetaPackageData(Versioned):
name: str
uid: str
recommended: Optional[List[str]]
authors: Optional[List[str]]
description: Optional[str]
project_url: Optional[AnyHttpUrl] = Field(alias="projectUrl")

43
meta/model/fabric.py Normal file
View File

@ -0,0 +1,43 @@
from datetime import datetime
from typing import Optional, List, Union, Dict
from pydantic import Field
from . import Library, MetaBase
class FabricInstallerArguments(MetaBase):
client: Optional[List[str]]
common: Optional[List[str]]
server: Optional[List[str]]
class FabricInstallerLaunchwrapper(MetaBase):
tweakers: FabricInstallerArguments
class FabricInstallerLibraries(MetaBase):
client: Optional[List[Library]]
common: Optional[List[Library]]
server: Optional[List[Library]]
class FabricMainClasses(MetaBase):
client: Optional[str]
common: Optional[str]
server: Optional[str]
class FabricInstallerDataV1(MetaBase):
version: int
libraries: FabricInstallerLibraries
main_class: Optional[Union[str, FabricMainClasses]] = Field(alias="mainClass")
arguments: Optional[FabricInstallerArguments]
launchwrapper: Optional[FabricInstallerLaunchwrapper]
class FabricJarInfo(MetaBase):
release_time: Optional[datetime] = Field(alias="releaseTime")
size: Optional[int]
sha256: Optional[str]
sha1: Optional[str]

47
meta/model/types.py Normal file
View File

@ -0,0 +1,47 @@
class GradleSpecifier(str):
"""
A gradle specifier - a maven coordinate. Like one of these:
"org.lwjgl.lwjgl:lwjgl:2.9.0"
"net.java.jinput:jinput:2.0.5"
"net.minecraft:launchwrapper:1.5"
"""
def __init__(self, name: str):
ext_split = name.split('@')
components = ext_split[0].split(':')
self.group = components[0]
self.artifact = components[1]
self.version = components[2]
self.extension = 'jar'
if len(ext_split) == 2:
self.extension = ext_split[1]
self.classifier = None
if len(components) == 4:
self.classifier = components[3]
def __new__(cls, name: str):
return super(GradleSpecifier, cls).__new__(cls, name)
def filename(self):
if self.classifier:
return "%s-%s-%s.%s" % (self.artifact, self.version, self.classifier, self.extension)
else:
return "%s-%s.%s" % (self.artifact, self.version, self.extension)
def base(self):
return "%s/%s/%s/" % (self.group.replace('.', '/'), self.artifact, self.version)
def path(self):
return self.base() + self.filename()
def __repr__(self):
return f"GradleSpecifier('{self}')"
def is_lwjgl(self):
return self.group in ("org.lwjgl", "org.lwjgl.lwjgl", "net.java.jinput", "net.java.jutils")
def is_log4j(self):
return self.group == "org.apache.logging.log4j"

View File

@ -1,10 +1,13 @@
import hashlib
import zipfile
import json
import os
from datetime import datetime
import requests
from cachecontrol import CacheControl
from cachecontrol.caches import FileCache
from meta.fabricutil import *
from meta.model.fabric import FabricJarInfo
from meta.common import DATETIME_FORMAT_HTTP, upstream_path, ensure_upstream_dir, transform_maven_key
from meta.common.fabric import JARS_DIR, INSTALLER_INFO_DIR, META_DIR
@ -68,7 +71,7 @@ def compute_jar_file(path, url):
try:
# Let's not download a Jar file if we don't need to.
headers = head_file(url)
tstamp = datetime.datetime.strptime(headers["Last-Modified"], DATETIME_FORMAT_HTTP)
tstamp = datetime.strptime(headers["Last-Modified"], DATETIME_FORMAT_HTTP)
sha1 = get_plaintext(url + ".sha1")
sha256 = get_plaintext(url + ".sha256")
size = int(headers["Content-Length"])
@ -78,11 +81,11 @@ def compute_jar_file(path, url):
jar_path = path + ".jar"
get_binary_file(jar_path, url)
tstamp = datetime.datetime.fromtimestamp(0)
tstamp = datetime.fromtimestamp(0)
with zipfile.ZipFile(jar_path, 'r') as jar:
allinfo = jar.infolist()
for info in allinfo:
tstamp_new = datetime.datetime(*info.date_time)
tstamp_new = datetime(*info.date_time)
if tstamp_new > tstamp:
tstamp = tstamp_new
@ -90,13 +93,8 @@ def compute_jar_file(path, url):
sha256 = filehash(jar_path, hashlib.sha256)
size = os.path.getsize(jar_path)
data = FabricJarInfo()
data.releaseTime = tstamp
data.sha1 = sha1
data.sha256 = sha256
data.size = size
with open(path + ".json", 'w') as outfile:
json.dump(data.to_json(), outfile, sort_keys=True, indent=4)
data = FabricJarInfo(releaseTime=tstamp, sha1=sha1, sha256=sha256, size=size)
data.write(path + ".json")
def main():