diff --git a/generateForge.py b/generateForge.py index 7f25544..bb44845 100755 --- a/generateForge.py +++ b/generateForge.py @@ -1,7 +1,7 @@ import os import re import sys -from distutils.version import LooseVersion +from packaging import version as pversion from operator import attrgetter from typing import Collection @@ -14,7 +14,7 @@ from meta.common.forge import ( STATIC_LEGACYINFO_FILE, INSTALLER_INFO_DIR, BAD_VERSIONS, - FORGEWRAPPER_MAVEN, + FORGEWRAPPER_LIBRARY, ) from meta.common.mojang import MINECRAFT_COMPONENT from meta.model import ( @@ -81,7 +81,7 @@ def should_ignore_artifact(libs: Collection[GradleSpecifier], match: GradleSpeci if ver.version == match.version: # Everything is matched perfectly - this one will be ignored return True - elif LooseVersion(ver.version) > LooseVersion(match.version): + elif pversion.parse(ver.version) > pversion.parse(match.version): return True else: # Otherwise it did not match - new version is higher and this is an upgrade @@ -287,16 +287,7 @@ def version_from_build_system_installer( v.libraries = [] - wrapper_lib = Library( - name=GradleSpecifier("io.github.zekerzhayard", "ForgeWrapper", "mmc2") - ) - wrapper_lib.downloads = MojangLibraryDownloads() - wrapper_lib.downloads.artifact = MojangArtifact( - url=FORGEWRAPPER_MAVEN % (wrapper_lib.name.path()), - sha1="4ee5f25cc9c7efbf54aff4c695da1054c1a1d7a3", - size=34444, - ) - v.libraries.append(wrapper_lib) + v.libraries.append(FORGEWRAPPER_LIBRARY) for upstream_lib in installer.libraries: forge_lib = Library.parse_obj(upstream_lib.dict()) @@ -437,7 +428,6 @@ def main(): v = version_from_build_system_installer(installer, profile, version) else: if version.uses_installer(): - # If we do not have the Forge json, we ignore this version if not os.path.isfile(profile_filepath): eprint("Skipping %s with missing profile json" % key) diff --git a/generateNeoForge.py b/generateNeoForge.py new file mode 100644 index 0000000..31df89a --- /dev/null +++ b/generateNeoForge.py @@ -0,0 +1,199 @@ +import os +import re +import sys +from operator import attrgetter +from typing import Collection + +from meta.common import ensure_component_dir, launcher_path, upstream_path, static_path +from meta.common.neoforge import ( + NEOFORGE_COMPONENT, + INSTALLER_MANIFEST_DIR, + VERSION_MANIFEST_DIR, + DERIVED_INDEX_FILE, + INSTALLER_INFO_DIR, +) +from meta.common.forge import FORGEWRAPPER_LIBRARY +from meta.common.mojang import MINECRAFT_COMPONENT +from meta.model import ( + MetaVersion, + Dependency, + Library, + GradleSpecifier, + MojangLibraryDownloads, + MojangArtifact, + MetaPackage, +) +from meta.model.neoforge import ( + NeoForgeVersion, + NeoForgeInstallerProfileV2, + InstallerInfo, + DerivedNeoForgeIndex, +) +from meta.model.mojang import MojangVersion + +LAUNCHER_DIR = launcher_path() +UPSTREAM_DIR = upstream_path() +STATIC_DIR = static_path() + +ensure_component_dir(NEOFORGE_COMPONENT) + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def version_from_build_system_installer( + installer: MojangVersion, + profile: NeoForgeInstallerProfileV2, + version: NeoForgeVersion, +) -> MetaVersion: + v = MetaVersion(name="NeoForge", version=version.rawVersion, uid=NEOFORGE_COMPONENT) + v.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=version.mc_version_sane)] + v.main_class = "io.github.zekerzhayard.forgewrapper.installer.Main" + + # FIXME: Add the size and hash here + v.maven_files = [] + + # load the locally cached installer file info and use it to add the installer entry in the json + info = InstallerInfo.parse_file( + os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version.long_version}.json") + ) + installer_lib = Library( + name=GradleSpecifier( + "net.neoforged", "forge", version.long_version, "installer" + ) + ) + installer_lib.downloads = MojangLibraryDownloads() + installer_lib.downloads.artifact = MojangArtifact( + url="https://maven.neoforged.net/%s" % (installer_lib.name.path()), + sha1=info.sha1hash, + size=info.size, + ) + v.maven_files.append(installer_lib) + + for upstream_lib in profile.libraries: + forge_lib = Library.parse_obj(upstream_lib.dict()) + if forge_lib.name.is_log4j(): + continue + + if ( + forge_lib.name.group == "net.neoforged" + and forge_lib.name.artifact == "forge" + and forge_lib.name.classifier == "universal" + ): + forge_lib.downloads.artifact.url = ( + "https://maven.neoforged.net/%s" % forge_lib.name.path() + ) + v.maven_files.append(forge_lib) + + v.libraries = [] + + v.libraries.append(FORGEWRAPPER_LIBRARY) + + for upstream_lib in installer.libraries: + forge_lib = Library.parse_obj(upstream_lib.dict()) + if forge_lib.name.is_log4j(): + continue + + if forge_lib.name.group == "net.neoforged": + if forge_lib.name.artifact == "forge": + forge_lib.name.classifier = "launcher" + forge_lib.downloads.artifact.path = forge_lib.name.path() + forge_lib.downloads.artifact.url = ( + "https://maven.neoforged.net/%s" % forge_lib.name.path() + ) + forge_lib.name = forge_lib.name + v.libraries.append(forge_lib) + + v.release_time = installer.release_time + v.order = 5 + mc_args = ( + "--username ${auth_player_name} --version ${version_name} --gameDir ${game_directory} " + "--assetsDir ${assets_root} --assetIndex ${assets_index_name} --uuid ${auth_uuid} " + "--accessToken ${auth_access_token} --userType ${user_type} --versionType ${version_type}" + ) + for arg in installer.arguments.game: + mc_args += f" {arg}" + v.minecraft_arguments = mc_args + return v + + +def main(): + # load the locally cached version list + remote_versions = DerivedNeoForgeIndex.parse_file( + os.path.join(UPSTREAM_DIR, DERIVED_INDEX_FILE) + ) + recommended_versions = [] + + for key, entry in remote_versions.versions.items(): + if entry.mc_version is None: + eprint("Skipping %s with invalid MC version" % key) + continue + + version = NeoForgeVersion(entry) + + if version.url() is None: + eprint("Skipping %s with no valid files" % key) + continue + eprint("Processing Forge %s" % version.rawVersion) + version_elements = version.rawVersion.split(".") + if len(version_elements) < 1: + eprint("Skipping version %s with not enough version elements" % key) + continue + + major_version_str = version_elements[0] + if not major_version_str.isnumeric(): + eprint( + "Skipping version %s with non-numeric major version %s" + % (key, major_version_str) + ) + continue + + if entry.recommended: + recommended_versions.append(version.rawVersion) + + # If we do not have the corresponding Minecraft version, we ignore it + if not os.path.isfile( + os.path.join( + LAUNCHER_DIR, MINECRAFT_COMPONENT, f"{version.mc_version_sane}.json" + ) + ): + eprint( + "Skipping %s with no corresponding Minecraft version %s" + % (key, version.mc_version_sane) + ) + continue + + # Path for new-style build system based installers + installer_version_filepath = os.path.join( + UPSTREAM_DIR, VERSION_MANIFEST_DIR, f"{version.long_version}.json" + ) + profile_filepath = os.path.join( + UPSTREAM_DIR, INSTALLER_MANIFEST_DIR, f"{version.long_version}.json" + ) + + eprint(installer_version_filepath) + assert os.path.isfile( + installer_version_filepath + ), f"version {installer_version_filepath} does not have installer version manifest" + installer = MojangVersion.parse_file(installer_version_filepath) + profile = NeoForgeInstallerProfileV2.parse_file(profile_filepath) + v = version_from_build_system_installer(installer, profile, version) + + v.write(os.path.join(LAUNCHER_DIR, NEOFORGE_COMPONENT, f"{v.version}.json")) + + recommended_versions.sort() + + print("Recommended versions:", recommended_versions) + + package = MetaPackage( + uid=NEOFORGE_COMPONENT, + name="NeoForge", + project_url="https://neoforged.net", + ) + package.recommended = recommended_versions + package.write(os.path.join(LAUNCHER_DIR, NEOFORGE_COMPONENT, "package.json")) + + +if __name__ == "__main__": + main() diff --git a/index.py b/index.py index 3868703..23dc233 100755 --- a/index.py +++ b/index.py @@ -48,7 +48,6 @@ for package in sorted(os.listdir(LAUNCHER_DIR)): for filename in os.listdir(LAUNCHER_DIR + "/%s" % package): if filename in ignore: continue - # parse and hash the version file filepath = LAUNCHER_DIR + "/%s/%s" % (package, filename) filehash = hash_file(hashlib.sha256, filepath) diff --git a/meta/common/__init__.py b/meta/common/__init__.py index 7a6514b..454a2cf 100644 --- a/meta/common/__init__.py +++ b/meta/common/__init__.py @@ -6,6 +6,8 @@ import requests from cachecontrol import CacheControl from cachecontrol.caches import FileCache +LAUNCHER_MAVEN = "https://files.prismlauncher.org/maven/%s" + def serialize_datetime(dt: datetime.datetime): if dt.tzinfo is None: diff --git a/meta/common/forge.py b/meta/common/forge.py index be62659..4a8afbe 100644 --- a/meta/common/forge.py +++ b/meta/common/forge.py @@ -1,5 +1,7 @@ from os.path import join +from ..model import GradleSpecifier, make_launcher_library + BASE_DIR = "forge" JARS_DIR = join(BASE_DIR, "jars") @@ -13,5 +15,9 @@ STATIC_LEGACYINFO_FILE = join(BASE_DIR, "forge-legacyinfo.json") FORGE_COMPONENT = "net.minecraftforge" -FORGEWRAPPER_MAVEN = "https://files.prismlauncher.org/maven/%s" +FORGEWRAPPER_LIBRARY = make_launcher_library( + GradleSpecifier("io.github.zekerzhayard", "ForgeWrapper", "1.5.6-prism"), + "b059aa8c4d2508055c6ed2a2561923a5e670a5eb", + 34860, +) BAD_VERSIONS = ["1.12.2-14.23.5.2851"] diff --git a/meta/common/neoforge.py b/meta/common/neoforge.py new file mode 100644 index 0000000..83f4890 --- /dev/null +++ b/meta/common/neoforge.py @@ -0,0 +1,14 @@ +from os.path import join + +from ..model import GradleSpecifier, make_launcher_library + +BASE_DIR = "neoforge" + +JARS_DIR = join(BASE_DIR, "jars") +INSTALLER_INFO_DIR = join(BASE_DIR, "installer_info") +INSTALLER_MANIFEST_DIR = join(BASE_DIR, "installer_manifests") +VERSION_MANIFEST_DIR = join(BASE_DIR, "version_manifests") +FILE_MANIFEST_DIR = join(BASE_DIR, "files_manifests") +DERIVED_INDEX_FILE = join(BASE_DIR, "derived_index.json") + +NEOFORGE_COMPONENT = "net.neoforged" diff --git a/meta/model/__init__.py b/meta/model/__init__.py index 0246cdb..68cd034 100644 --- a/meta/model/__init__.py +++ b/meta/model/__init__.py @@ -1,11 +1,13 @@ import copy from datetime import datetime +from pathlib import Path from typing import Optional, List, Dict, Any, Iterator import pydantic from pydantic import Field, validator from ..common import ( + LAUNCHER_MAVEN, serialize_datetime, replace_old_launchermeta_url, get_all_bases, @@ -146,6 +148,7 @@ class MetaBase(pydantic.BaseModel): ) def write(self, file_path): + Path(file_path).parent.mkdir(parents=True, exist_ok=True) with open(file_path, "w") as f: f.write(self.json()) @@ -328,3 +331,10 @@ class MetaPackage(Versioned): authors: Optional[List[str]] description: Optional[str] project_url: Optional[str] = Field(alias="projectUrl") + + +def make_launcher_library( + name: GradleSpecifier, hash: str, size: int, maven=LAUNCHER_MAVEN +): + artifact = MojangArtifact(url=maven % name.path(), sha1=hash, size=size) + return Library(name=name, downloads=MojangLibraryDownloads(artifact=artifact)) diff --git a/meta/model/neoforge.py b/meta/model/neoforge.py new file mode 100644 index 0000000..a13605c --- /dev/null +++ b/meta/model/neoforge.py @@ -0,0 +1,239 @@ +from datetime import datetime +from typing import Optional, List, Dict + +from pydantic import Field + +from . import MetaBase, GradleSpecifier, MojangLibrary +from .mojang import MojangVersion + + +class NeoForgeFile(MetaBase): + classifier: str + extension: str + + def filename(self, long_version): + return "%s-%s-%s.%s" % ("forge", long_version, self.classifier, self.extension) + + def url(self, long_version): + return "https://maven.neoforged.net/net/neoforged/forge/%s/%s" % ( + long_version, + self.filename(long_version), + ) + + +class NeoForgeEntry(MetaBase): + long_version: str = Field(alias="longversion") + mc_version: str = Field(alias="mcversion") + version: str + build: int + branch: Optional[str] + latest: Optional[bool] + recommended: Optional[bool] + files: Optional[Dict[str, NeoForgeFile]] + + +class NeoForgeMCVersionInfo(MetaBase): + latest: Optional[str] + recommended: Optional[str] + versions: List[str] = Field([]) + + +class DerivedNeoForgeIndex(MetaBase): + versions: Dict[str, NeoForgeEntry] = Field({}) + by_mc_version: Dict[str, NeoForgeMCVersionInfo] = Field({}, alias="by_mcversion") + + +class FMLLib( + MetaBase +): # old ugly stuff. Maybe merge this with Library or MojangLibrary later + filename: str + checksum: str + ours: bool + + +class NeoForgeInstallerProfileInstallSection(MetaBase): + """ + "install": { + "profileName": "NeoForge", + "target":"NeoForge8.9.0.753", + "path":"net.minecraftNeoForge:minecraftNeoForge:8.9.0.753", + "version":"NeoForge 8.9.0.753", + "filePath":"minecraftNeoForge-universal-1.6.1-8.9.0.753.jar", + "welcome":"Welcome to the simple NeoForge installer.", + "minecraft":"1.6.1", + "logo":"/big_logo.png", + "mirrorList": "http://files.minecraftNeoForge.net/mirror-brand.list" + }, + "install": { + "profileName": "NeoForge", + "target":"1.11-NeoForge1.11-13.19.0.2141", + "path":"net.minecraftNeoForge:NeoForge:1.11-13.19.0.2141", + "version":"NeoForge 1.11-13.19.0.2141", + "filePath":"NeoForge-1.11-13.19.0.2141-universal.jar", + "welcome":"Welcome to the simple NeoForge installer.", + "minecraft":"1.11", + "mirrorList" : "http://files.minecraftNeoForge.net/mirror-brand.list", + "logo":"/big_logo.png", + "modList":"none" + }, + """ + + profile_name: str = Field(alias="profileName") + target: str + path: GradleSpecifier + version: str + file_path: str = Field(alias="filePath") + welcome: str + minecraft: str + logo: str + mirror_list: str = Field(alias="mirrorList") + mod_list: Optional[str] = Field(alias="modList") + + +class NeoForgeLibrary(MojangLibrary): + url: Optional[str] + server_req: Optional[bool] = Field(alias="serverreq") + client_req: Optional[bool] = Field(alias="clientreq") + checksums: Optional[List[str]] + comment: Optional[str] + + +class NeoForgeVersionFile(MojangVersion): + libraries: Optional[List[NeoForgeLibrary]] # overrides Mojang libraries + inherits_from: Optional[str] = Field("inheritsFrom") + jar: Optional[str] + + +class NeoForgeOptional(MetaBase): + """ + "optionals": [ + { + "name": "Mercurius", + "client": true, + "server": true, + "default": true, + "inject": true, + "desc": "A mod that collects statistics about Minecraft and your system.
Useful for NeoForge to understand how Minecraft/NeoForge are used.", + "url": "http://www.minecraftNeoForge.net/forum/index.php?topic=43278.0", + "artifact": "net.minecraftNeoForge:MercuriusUpdater:1.11.2", + "maven": "http://maven.minecraftNeoForge.net/" + } + ] + """ + + name: Optional[str] + client: Optional[bool] + server: Optional[bool] + default: Optional[bool] + inject: Optional[bool] + desc: Optional[str] + url: Optional[str] + artifact: Optional[GradleSpecifier] + maven: Optional[str] + + +class DataSpec(MetaBase): + client: Optional[str] + server: Optional[str] + + +class ProcessorSpec(MetaBase): + jar: Optional[str] + classpath: Optional[List[str]] + args: Optional[List[str]] + outputs: Optional[Dict[str, str]] + sides: Optional[List[str]] + + +class NeoForgeInstallerProfileV2(MetaBase): + _comment: Optional[List[str]] + spec: Optional[int] + profile: Optional[str] + version: Optional[str] + icon: Optional[str] + json_data: Optional[str] = Field(alias="json") + path: Optional[GradleSpecifier] + logo: Optional[str] + minecraft: Optional[str] + welcome: Optional[str] + data: Optional[Dict[str, DataSpec]] + processors: Optional[List[ProcessorSpec]] + libraries: Optional[List[MojangLibrary]] + mirror_list: Optional[str] = Field(alias="mirrorList") + server_jar_path: Optional[str] = Field(alias="serverJarPath") + + +class InstallerInfo(MetaBase): + sha1hash: Optional[str] + sha256hash: Optional[str] + size: Optional[int] + + +# A post-processed entry constructed from the reconstructed NeoForge version index +class NeoForgeVersion: + def __init__(self, entry: NeoForgeEntry): + self.build = entry.build + self.rawVersion = entry.version + self.mc_version = entry.mc_version + self.mc_version_sane = self.mc_version.replace("_pre", "-pre", 1) + self.branch = entry.branch + self.installer_filename = None + self.installer_url = None + self.universal_filename = None + self.universal_url = None + self.changelog_url = None + self.long_version = "%s-%s" % (self.mc_version, self.rawVersion) + if self.branch is not None: + self.long_version += "-%s" % self.branch + + # this comment's whole purpose is to say this: cringe + for classifier, file in entry.files.items(): + extension = file.extension + filename = file.filename(self.long_version) + url = file.url(self.long_version) + print(url) + print(self.long_version) + if (classifier == "installer") and (extension == "jar"): + self.installer_filename = filename + self.installer_url = url + if (classifier == "universal" or classifier == "client") and ( + extension == "jar" or extension == "zip" + ): + self.universal_filename = filename + self.universal_url = url + if (classifier == "changelog") and (extension == "txt"): + self.changelog_url = url + + def name(self): + return "neoforge %d" % self.build + + def uses_installer(self): + return self.installer_url is not None + + def filename(self): + if self.uses_installer(): + return self.installer_filename + return self.universal_filename + + def url(self): + if self.uses_installer(): + return self.installer_url + return self.universal_url + + def is_supported(self): + if self.url() is None: + return False + + foo = self.rawVersion.split(".") + if len(foo) < 1: + return False + + major_version = foo[0] + if not major_version.isnumeric(): + return False + + # majorVersion = int(majorVersionStr) + # if majorVersion >= 37: + # return False + + return True diff --git a/nix/dev.nix b/nix/dev.nix index 4aaf883..e753ed8 100644 --- a/nix/dev.nix +++ b/nix/dev.nix @@ -34,6 +34,8 @@ requests packaging pydantic + + coverage ])) ]; }; diff --git a/update.sh b/update.sh index 9286c80..48cce7a 100755 --- a/update.sh +++ b/update.sh @@ -43,6 +43,7 @@ upstream_git checkout "${BRANCH}" || exit 1 python updateMojang.py || fail_in python updateForge.py || fail_in +python updateNeoForge.py || fail_in python updateFabric.py || fail_in python updateQuilt.py || fail_in python updateLiteloader.py || fail_in @@ -50,6 +51,7 @@ python updateLiteloader.py || fail_in if [ "${DEPLOY_TO_GIT}" = true ] ; then upstream_git add mojang/version_manifest_v2.json mojang/versions/* || fail_in upstream_git add forge/*.json forge/version_manifests/*.json forge/installer_manifests/*.json forge/files_manifests/*.json forge/installer_info/*.json || fail_in + upstream_git add neoforge/*.json neoforge/version_manifests/*.json neoforge/installer_manifests/*.json neoforge/files_manifests/*.json neoforge/installer_info/*.json || fail_in upstream_git add fabric/loader-installer-json/*.json fabric/meta-v2/*.json fabric/jars/*.json || fail_in upstream_git add quilt/loader-installer-json/*.json quilt/meta-v3/*.json quilt/jars/*.json || fail_in upstream_git add liteloader/*.json || fail_in @@ -64,6 +66,7 @@ launcher_git checkout "${BRANCH}" || exit 1 python generateMojang.py || fail_out python generateForge.py || fail_out +python generateNeoForge.py || fail_out python generateFabric.py || fail_out python generateQuilt.py || fail_out python generateLiteloader.py || fail_out @@ -72,6 +75,7 @@ python index.py || fail_out if [ "${DEPLOY_TO_GIT}" = true ] ; then launcher_git add index.json org.lwjgl/* org.lwjgl3/* net.minecraft/* || fail_out launcher_git add net.minecraftforge/* || fail_out + launcher_git add net.neoforged/* || fail_out launcher_git add net.fabricmc.fabric-loader/* net.fabricmc.intermediary/* || fail_out launcher_git add org.quiltmc.quilt-loader/* || fail_out # TODO: add Quilt hashed, once it is actually used launcher_git add com.mumfrey.liteloader/* || fail_out diff --git a/updateNeoForge.py b/updateNeoForge.py new file mode 100644 index 0000000..431930c --- /dev/null +++ b/updateNeoForge.py @@ -0,0 +1,282 @@ +""" + Get the source files necessary for generating Forge versions +""" +import copy +import hashlib +import json +import os +import re +import sys +import zipfile +from contextlib import suppress +from datetime import datetime +from pathlib import Path +from pprint import pprint +import urllib.parse + +from pydantic import ValidationError + +from meta.common import upstream_path, ensure_upstream_dir, static_path, default_session +from meta.common.neoforge import ( + JARS_DIR, + INSTALLER_INFO_DIR, + INSTALLER_MANIFEST_DIR, + VERSION_MANIFEST_DIR, + FILE_MANIFEST_DIR, +) +from meta.model.neoforge import ( + NeoForgeFile, + NeoForgeEntry, + NeoForgeMCVersionInfo, + DerivedNeoForgeIndex, + NeoForgeVersion, + NeoForgeInstallerProfileV2, + InstallerInfo, +) +from meta.model.mojang import MojangVersion + +UPSTREAM_DIR = upstream_path() +STATIC_DIR = static_path() + +ensure_upstream_dir(JARS_DIR) +ensure_upstream_dir(INSTALLER_INFO_DIR) +ensure_upstream_dir(INSTALLER_MANIFEST_DIR) +ensure_upstream_dir(VERSION_MANIFEST_DIR) +ensure_upstream_dir(FILE_MANIFEST_DIR) + +sess = default_session() + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def filehash(filename, hashtype, blocksize=65536): + hashtype = hashtype() + with open(filename, "rb") as f: + for block in iter(lambda: f.read(blocksize), b""): + hashtype.update(block) + return hashtype.hexdigest() + + +def find_nth(haystack, needle, n): + start = haystack.find(needle) + while start >= 0 and n > 1: + start = haystack.find(needle, start + len(needle)) + n -= 1 + return start + + +def get_single_forge_files_manifest(longversion): + print(f"Getting NeoForge manifest for {longversion}") + path_thing = UPSTREAM_DIR + "/neoforge/files_manifests/%s.json" % longversion + files_manifest_file = Path(path_thing) + from_file = False + if files_manifest_file.is_file(): + with open(path_thing, "r") as f: + files_json = json.load(f) + from_file = True + else: + file_url = ( + "https://maven.neoforged.net/api/maven/details/releases/net%2Fneoforged%2Fforge%2F" + + urllib.parse.quote(longversion) + ) + r = sess.get(file_url) + r.raise_for_status() + files_json = r.json() + + ret_dict = dict() + + for file in files_json.get("files"): + assert type(file) == dict + name = file["name"] + file_name, file_ext = os.path.splitext(name) + if file_ext in [".md5", ".sha1", ".sha256", ".sha512"]: + continue + + classifier = file["name"][find_nth(name, "-", 3) + 1 : len(file_name)] + + # assert len(extensionObj.items()) == 1 + file_obj = NeoForgeFile(classifier=classifier, extension=file_ext[1:]) + ret_dict[classifier] = file_obj + + if not from_file: + Path(path_thing).parent.mkdir(parents=True, exist_ok=True) + with open(path_thing, "w", encoding="utf-8") as f: + json.dump(files_json, f, sort_keys=True, indent=4) + + return ret_dict + + +def main(): + # get the remote version list fragments + r = sess.get( + "https://maven.neoforged.net/api/maven/versions/releases/net%2Fneoforged%2Fforge" + ) + r.raise_for_status() + main_json = r.json()["versions"] + assert type(main_json) == list + + new_index = DerivedNeoForgeIndex() + + version_expression = re.compile( + "^(?P[0-9a-zA-Z_\\.]+)-(?P[0-9\\.]+\\.(?P[0-9]+))(-(?P[a-zA-Z0-9\\.]+))?$" + ) + + print("") + print("Processing versions:") + for long_version in main_json: + assert type(long_version) == str + mc_version = long_version.split("-")[0] + match = version_expression.match(long_version) + assert match, f"{long_version} doesn't match version regex" + assert match.group("mc") == mc_version + try: + files = get_single_forge_files_manifest(long_version) + except: + continue + build = int(match.group("build")) + version = match.group("ver") + branch = match.group("branch") + + # TODO: what *is* recommended? + is_recommended = False + + entry = NeoForgeEntry( + long_version=long_version, + mc_version=mc_version, + version=version, + build=build, + branch=branch, + # NOTE: we add this later after the fact. The forge promotions file lies about these. + latest=False, + recommended=is_recommended, + files=files, + ) + new_index.versions[long_version] = entry + if not new_index.by_mc_version: + new_index.by_mc_version = dict() + if mc_version not in new_index.by_mc_version: + new_index.by_mc_version.setdefault(mc_version, NeoForgeMCVersionInfo()) + new_index.by_mc_version[mc_version].versions.append(long_version) + # NOTE: we add this later after the fact. The forge promotions file lies about these. + # if entry.latest: + # new_index.by_mc_version[mc_version].latest = long_version + if entry.recommended: + new_index.by_mc_version[mc_version].recommended = long_version + + print("") + print("Dumping index files...") + + with open( + UPSTREAM_DIR + "/neoforge/maven-metadata.json", "w", encoding="utf-8" + ) as f: + json.dump(main_json, f, sort_keys=True, indent=4) + + new_index.write(UPSTREAM_DIR + "/neoforge/derived_index.json") + + print("Grabbing installers and dumping installer profiles...") + # get the installer jars - if needed - and get the installer profiles out of them + for key, entry in new_index.versions.items(): + eprint("Updating NeoForge %s" % key) + if entry.mc_version is None: + eprint("Skipping %d with invalid MC version" % entry.build) + continue + + version = NeoForgeVersion(entry) + if version.url() is None: + eprint("Skipping %d with no valid files" % version.build) + continue + if not version.uses_installer(): + eprint(f"version {version.long_version} does not use installer") + continue + + jar_path = os.path.join(UPSTREAM_DIR, JARS_DIR, version.filename()) + + installer_info_path = ( + UPSTREAM_DIR + "/neoforge/installer_info/%s.json" % version.long_version + ) + profile_path = ( + UPSTREAM_DIR + + "/neoforge/installer_manifests/%s.json" % version.long_version + ) + version_file_path = ( + UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version + ) + + installer_refresh_required = not os.path.isfile( + profile_path + ) or not os.path.isfile(installer_info_path) + + if installer_refresh_required: + # grab the installer if it's not there + if not os.path.isfile(jar_path): + eprint("Downloading %s" % version.url()) + try: + rfile = sess.get(version.url(), stream=True) + rfile.raise_for_status() + Path(jar_path).parent.mkdir(parents=True, exist_ok=True) + with open(jar_path, "wb") as f: + for chunk in rfile.iter_content(chunk_size=128): + f.write(chunk) + except Exception as e: + eprint("Failed to download %s" % version.url()) + eprint("Error is %s" % e) + continue + + eprint("Processing %s" % version.url()) + # harvestables from the installer + if not os.path.isfile(profile_path): + print(jar_path) + with zipfile.ZipFile(jar_path) as jar: + with suppress(KeyError): + with jar.open("version.json") as profile_zip_entry: + version_data = profile_zip_entry.read() + + # Process: does it parse? + MojangVersion.parse_raw(version_data) + + Path(version_file_path).parent.mkdir( + parents=True, exist_ok=True + ) + with open(version_file_path, "wb") as versionJsonFile: + versionJsonFile.write(version_data) + versionJsonFile.close() + + with jar.open("install_profile.json") as profile_zip_entry: + install_profile_data = profile_zip_entry.read() + + # Process: does it parse? + is_parsable = False + exception = None + try: + NeoForgeInstallerProfileV2.parse_raw(install_profile_data) + is_parsable = True + except ValidationError as err: + exception = err + + if not is_parsable: + if version.is_supported(): + raise exception + else: + eprint( + "Version %s is not supported and won't be generated later." + % version.long_version + ) + + Path(profile_path).parent.mkdir(parents=True, exist_ok=True) + with open(profile_path, "wb") as profileFile: + profileFile.write(install_profile_data) + profileFile.close() + + # installer info v1 + if not os.path.isfile(installer_info_path): + installer_info = InstallerInfo() + installer_info.sha1hash = filehash(jar_path, hashlib.sha1) + installer_info.sha256hash = filehash(jar_path, hashlib.sha256) + installer_info.size = os.path.getsize(jar_path) + installer_info.write(installer_info_path) + + +if __name__ == "__main__": + main()