Add support for automatically generating NeoForge metadata

generateNeoForge.py

@@ -0,0 +1,424 @@
+import os
+import re
+import sys
+from distutils.version import LooseVersion
+from operator import attrgetter
+from typing import Collection
+
+from meta.common import ensure_component_dir, launcher_path, upstream_path, static_path
+from meta.common.neoforge import (
+    NEOFORGE_COMPONENT,
+    INSTALLER_MANIFEST_DIR,
+    VERSION_MANIFEST_DIR,
+    DERIVED_INDEX_FILE,
+    STATIC_LEGACYINFO_FILE,
+    INSTALLER_INFO_DIR,
+    BAD_VERSIONS,
+    FORGEWRAPPER_MAVEN,
+)
+from meta.common.forge import (FORGE_COMPONENT)
+from meta.common.mojang import MINECRAFT_COMPONENT
+from meta.model import (
+    MetaVersion,
+    Dependency,
+    Library,
+    GradleSpecifier,
+    MojangLibraryDownloads,
+    MojangArtifact,
+    MetaPackage,
+)
+from meta.model.neoforge import (
+    NeoForgeVersion,
+    NeoForgeInstallerProfile,
+    NeoForgeLegacyInfo,
+    fml_libs_for_version,
+    NeoForgeInstallerProfileV2,
+    InstallerInfo,
+    DerivedNeoForgeIndex,
+    NeoForgeLegacyInfoList,
+)
+from meta.model.mojang import MojangVersion
+
+LAUNCHER_DIR = launcher_path()
+UPSTREAM_DIR = upstream_path()
+STATIC_DIR = static_path()
+
+ensure_component_dir(NEOFORGE_COMPONENT)
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+# Construct a set of libraries out of a Minecraft version file, for filtering.
+mc_version_cache = {}
+
+
+def load_mc_version_filter(version: str):
+    if version in mc_version_cache:
+        return mc_version_cache[version]
+    v = MetaVersion.parse_file(
+        os.path.join(LAUNCHER_DIR, MINECRAFT_COMPONENT, f"{version}.json")
+    )
+    libs = set(map(attrgetter("name"), v.libraries))
+    mc_version_cache[version] = libs
+    return libs
+
+
+"""
+Match a library coordinate to a set of library coordinates.
+ * Block those that pass completely.
+ * For others, block those with lower versions than in the set.
+"""
+
+
+def should_ignore_artifact(libs: Collection[GradleSpecifier], match: GradleSpecifier):
+    for ver in libs:
+        if (
+            ver.group == match.group
+            and ver.artifact == match.artifact
+            and ver.classifier == match.classifier
+        ):
+            if ver.version == match.version:
+                # Everything is matched perfectly - this one will be ignored
+                return True
+            elif LooseVersion(ver.version) > LooseVersion(match.version):
+                return True
+            else:
+                # Otherwise it did not match - new version is higher and this is an upgrade
+                return False
+    # No match found in the set - we need to keep this
+    return False
+
+
+def version_from_profile(
+    profile: NeoForgeInstallerProfile, version: NeoForgeVersion
+) -> MetaVersion:
+    v = MetaVersion(name="NeoForge", version=version.rawVersion, uid=NEOFORGE_COMPONENT)
+    mc_version = profile.install.minecraft
+    v.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=mc_version)]
+    v.main_class = profile.version_info.main_class
+    v.release_time = profile.version_info.time
+
+    args = profile.version_info.minecraft_arguments
+    tweakers = []
+    expression = re.compile(r"--tweakClass ([a-zA-Z0-9.]+)")
+    match = expression.search(args)
+    while match is not None:
+        tweakers.append(match.group(1))
+        args = args[: match.start()] + args[match.end() :]
+        match = expression.search(args)
+    if len(tweakers) > 0:
+        args = args.strip()
+        v.additional_tweakers = tweakers
+    # v.minecraftArguments = args
+
+    v.libraries = []
+    mc_filter = load_mc_version_filter(mc_version)
+    for forge_lib in profile.version_info.libraries:
+        if (
+            forge_lib.name.is_lwjgl()
+            or forge_lib.name.is_log4j()
+            or should_ignore_artifact(mc_filter, forge_lib.name)
+        ):
+            continue
+
+        overridden_name = forge_lib.name
+        if overridden_name.group == "net.minecraftforge":
+            if overridden_name.artifact == "minecraftforge":
+                overridden_name.artifact = "forge"
+                overridden_name.version = "%s-%s" % (
+                    mc_version,
+                    overridden_name.version,
+                )
+
+                overridden_name.classifier = "universal"
+            elif overridden_name.artifact == "forge":
+                overridden_name.classifier = "universal"
+
+        overridden_lib = Library(name=overridden_name)
+        if forge_lib.url == "http://maven.minecraftforge.net/":
+            overridden_lib.url = "https://maven.minecraftforge.net/"
+        else:
+            overridden_lib.url = forge_lib.url
+        # if forge_lib.checksums and len(forge_lib.checksums) == 2:
+        #    overridden_lib.mmcHint = "forge-pack-xz"
+        v.libraries.append(overridden_lib)
+
+    v.order = 5
+    return v
+
+
+def version_from_modernized_installer(
+    installer: MojangVersion, version: NeoForgeVersion
+) -> MetaVersion:
+    v = MetaVersion(name="NeoForge", version=version.rawVersion, uid=NEOFORGE_COMPONENT)
+    mc_version = version.mc_version
+    v.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=mc_version)]
+    v.main_class = installer.main_class
+    v.release_time = installer.release_time
+
+    args = installer.minecraft_arguments
+    tweakers = []
+    expression = re.compile("--tweakClass ([a-zA-Z0-9.]+)")
+    match = expression.search(args)
+    while match is not None:
+        tweakers.append(match.group(1))
+        args = args[: match.start()] + args[match.end() :]
+        match = expression.search(args)
+    if len(tweakers) > 0:
+        args = args.strip()
+        v.additional_tweakers = tweakers
+    # v.minecraftArguments = args
+
+    v.libraries = []
+
+    mc_filter = load_mc_version_filter(mc_version)
+    for upstream_lib in installer.libraries:
+        forge_lib = Library.parse_obj(
+            upstream_lib.dict()
+        )  # "cast" MojangLibrary to Library
+        if (
+            forge_lib.name.is_lwjgl()
+            or forge_lib.name.is_log4j()
+            or should_ignore_artifact(mc_filter, forge_lib.name)
+        ):
+            continue
+
+        if forge_lib.name.group == "net.minecraftforge":
+            if forge_lib.name.artifact == "forge":
+                overridden_name = forge_lib.name
+                overridden_name.classifier = "universal"
+                forge_lib.downloads.artifact.path = overridden_name.path()
+                forge_lib.downloads.artifact.url = (
+                    "https://maven.minecraftforge.net/%s" % overridden_name.path()
+                )
+                forge_lib.name = overridden_name
+
+            elif forge_lib.name.artifact == "minecraftforge":
+                overridden_name = forge_lib.name
+                overridden_name.artifact = "forge"
+                overridden_name.classifier = "universal"
+                overridden_name.version = "%s-%s" % (
+                    mc_version,
+                    overridden_name.version,
+                )
+                forge_lib.downloads.artifact.path = overridden_name.path()
+                forge_lib.downloads.artifact.url = (
+                    "https://maven.minecraftforge.net/%s" % overridden_name.path()
+                )
+                forge_lib.name = overridden_name
+
+        v.libraries.append(forge_lib)
+
+    v.order = 5
+    return v
+
+
+def version_from_legacy(info: NeoForgeLegacyInfo, version: NeoForgeVersion) -> MetaVersion:
+    v = MetaVersion(name="NeoForge", version=version.rawVersion, uid=NEOFORGE_COMPONENT)
+    mc_version = version.mc_version_sane
+    v.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=mc_version)]
+    v.release_time = info.release_time
+    v.order = 5
+    if fml_libs_for_version(
+        mc_version
+    ):  # WHY, WHY DID I WASTE MY TIME REWRITING FMLLIBSMAPPING
+        v.additional_traits = ["legacyFML"]
+
+    classifier = "client"
+    if "universal" in version.url():
+        classifier = "universal"
+
+    main_mod = Library(
+        name=GradleSpecifier(
+            "net.minecraftforge", "forge", version.long_version, classifier
+        )
+    )
+    main_mod.downloads = MojangLibraryDownloads()
+    main_mod.downloads.artifact = MojangArtifact(
+        url=version.url(), sha1=info.sha1, size=info.size
+    )
+    main_mod.downloads.artifact.path = None
+    v.jar_mods = [main_mod]
+    return v
+
+
+def version_from_build_system_installer(
+    installer: MojangVersion, profile: NeoForgeInstallerProfileV2, version: NeoForgeVersion
+) -> MetaVersion:
+    v = MetaVersion(name="NeoForge", version=version.rawVersion, uid=NEOFORGE_COMPONENT)
+    v.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=version.mc_version_sane)]
+    v.main_class = "io.github.zekerzhayard.forgewrapper.installer.Main"
+
+    # FIXME: Add the size and hash here
+    v.maven_files = []
+
+    # load the locally cached installer file info and use it to add the installer entry in the json
+    info = InstallerInfo.parse_file(
+        os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version.long_version}.json")
+    )
+    installer_lib = Library(
+        name=GradleSpecifier(
+            "net.neoforged", "forge", version.long_version, "installer"
+        )
+    )
+    installer_lib.downloads = MojangLibraryDownloads()
+    installer_lib.downloads.artifact = MojangArtifact(
+        url="https://maven.neoforged.net/%s" % (installer_lib.name.path()),
+        sha1=info.sha1hash,
+        size=info.size,
+    )
+    v.maven_files.append(installer_lib)
+
+    for upstream_lib in profile.libraries:
+        forge_lib = Library.parse_obj(upstream_lib.dict())
+        if forge_lib.name.is_log4j():
+            continue
+
+        if (
+            forge_lib.name.group == "net.neoforged"
+            and forge_lib.name.artifact == "forge"
+            and forge_lib.name.classifier == "universal"
+        ):
+            forge_lib.downloads.artifact.url = (
+                "https://maven.neoforged.net/%s" % forge_lib.name.path()
+            )
+        v.maven_files.append(forge_lib)
+
+    v.libraries = []
+
+    wrapper_lib = Library(
+        name=GradleSpecifier("io.github.zekerzhayard", "ForgeWrapper", "1.5.6"))
+    wrapper_lib.downloads = MojangLibraryDownloads()
+    wrapper_lib.downloads.artifact = MojangArtifact(
+        url=FORGEWRAPPER_MAVEN,
+        sha1="b38d28e8b7fde13b1bc0db946a2da6760fecf98d",
+        size=34715,
+    )
+    v.libraries.append(wrapper_lib)
+
+    for upstream_lib in installer.libraries:
+        forge_lib = Library.parse_obj(upstream_lib.dict())
+        if forge_lib.name.is_log4j():
+            continue
+
+        if forge_lib.name.group == "net.neoforged":
+            if forge_lib.name.artifact == "forge":
+                forge_lib.name.classifier = "launcher"
+                forge_lib.downloads.artifact.path = forge_lib.name.path()
+                forge_lib.downloads.artifact.url = (
+                    "https://maven.neoforged.net/%s" % forge_lib.name.path()
+                )
+                forge_lib.name = forge_lib.name
+        v.libraries.append(forge_lib)
+
+    v.release_time = installer.release_time
+    v.order = 5
+    mc_args = (
+        "--username ${auth_player_name} --version ${version_name} --gameDir ${game_directory} "
+        "--assetsDir ${assets_root} --assetIndex ${assets_index_name} --uuid ${auth_uuid} "
+        "--accessToken ${auth_access_token} --userType ${user_type} --versionType ${version_type}"
+    )
+    for arg in installer.arguments.game:
+        mc_args += f" {arg}"
+    v.minecraft_arguments = mc_args
+    return v
+
+
+def main():
+    # load the locally cached version list
+    remote_versions = DerivedNeoForgeIndex.parse_file(
+        os.path.join(UPSTREAM_DIR, DERIVED_INDEX_FILE)
+    )
+    recommended_versions = []
+
+
+    for key, entry in remote_versions.versions.items():
+        if entry.mc_version is None:
+            eprint("Skipping %s with invalid MC version" % key)
+            continue
+
+        version = NeoForgeVersion(entry)
+
+        if version.long_version in BAD_VERSIONS:
+            # Version 1.12.2-14.23.5.2851 is ultra cringe, I can't imagine why you would even spend one second on
+            # actually adding support for this version.
+            # It is cringe, because it's installer info is broken af
+            eprint(f"Skipping bad version {version.long_version}")
+            continue
+
+        if version.url() is None:
+            eprint("Skipping %s with no valid files" % key)
+            continue
+        eprint("Processing Forge %s" % version.rawVersion)
+        version_elements = version.rawVersion.split(".")
+        if len(version_elements) < 1:
+            eprint("Skipping version %s with not enough version elements" % key)
+            continue
+
+        major_version_str = version_elements[0]
+        if not major_version_str.isnumeric():
+            eprint(
+                "Skipping version %s with non-numeric major version %s"
+                % (key, major_version_str)
+            )
+            continue
+
+        if entry.recommended:
+            recommended_versions.append(version.rawVersion)
+
+        # If we do not have the corresponding Minecraft version, we ignore it
+        if not os.path.isfile(
+            os.path.join(
+                LAUNCHER_DIR, MINECRAFT_COMPONENT, f"{version.mc_version_sane}.json"
+            )
+        ):
+            eprint(
+                "Skipping %s with no corresponding Minecraft version %s"
+                % (key, version.mc_version_sane)
+            )
+            continue
+
+        # Path for new-style build system based installers
+        installer_version_filepath = os.path.join(
+            UPSTREAM_DIR, VERSION_MANIFEST_DIR, f"{version.long_version}.json"
+        )
+        profile_filepath = os.path.join(
+            UPSTREAM_DIR, INSTALLER_MANIFEST_DIR, f"{version.long_version}.json"
+        )
+
+        eprint(installer_version_filepath)
+        if os.path.isfile(installer_version_filepath):
+            installer = MojangVersion.parse_file(installer_version_filepath)
+            profile = NeoForgeInstallerProfileV2.parse_file(profile_filepath)
+            v = version_from_build_system_installer(installer, profile, version)
+        else:
+            if version.uses_installer():
+
+                # If we do not have the Forge json, we ignore this version
+                if not os.path.isfile(profile_filepath):
+                    eprint("Skipping %s with missing profile json" % key)
+                    continue
+                profile = NeoForgeInstallerProfile.parse_file(profile_filepath)
+                v = version_from_profile(profile, version)
+
+        v.write(os.path.join(LAUNCHER_DIR, NEOFORGE_COMPONENT, f"{v.version}.json"))
+        v.version = "NEO-"+v.version
+        v.write(os.path.join(LAUNCHER_DIR, FORGE_COMPONENT, f"{v.version}.json"))
+
+        recommended_versions.sort()
+
+        print("Recommended versions:", recommended_versions)
+
+        package = MetaPackage(
+            uid=NEOFORGE_COMPONENT,
+            name="NeoForge",
+            project_url="https://neoforged.net",
+        )
+        package.recommended = recommended_versions
+        package.write(os.path.join(LAUNCHER_DIR, NEOFORGE_COMPONENT, "package.json"))
+
+
+if __name__ == "__main__":
+    main()

index.py

@@ -48,7 +48,6 @@ for package in sorted(os.listdir(LAUNCHER_DIR)):
     for filename in os.listdir(LAUNCHER_DIR + "/%s" % package):
         if filename in ignore:
             continue
-
         # parse and hash the version file
         filepath = LAUNCHER_DIR + "/%s/%s" % (package, filename)
         filehash = hash_file(hashlib.sha256, filepath)
@@ -58,7 +57,7 @@ for package in sorted(os.listdir(LAUNCHER_DIR)):
         versionEntry = MetaVersionIndexEntry.from_meta_version(
             versionFile, is_recommended, filehash
         )
-
+        
         versionList.versions.append(versionEntry)
 
     # sort the versions in descending order by time of release

meta/common/neoforge.py

@@ -0,0 +1,17 @@
+from os.path import join
+
+BASE_DIR = "neoforge"
+
+JARS_DIR = join(BASE_DIR, "jars")
+INSTALLER_INFO_DIR = join(BASE_DIR, "installer_info")
+INSTALLER_MANIFEST_DIR = join(BASE_DIR, "installer_manifests")
+VERSION_MANIFEST_DIR = join(BASE_DIR, "version_manifests")
+FILE_MANIFEST_DIR = join(BASE_DIR, "files_manifests")
+DERIVED_INDEX_FILE = join(BASE_DIR, "derived_index.json")
+
+STATIC_LEGACYINFO_FILE = join(BASE_DIR, "neoforge-legacyinfo.json")
+
+NEOFORGE_COMPONENT = "net.neoforged"
+
+FORGEWRAPPER_MAVEN = "https://github.com/ZekerZhayard/ForgeWrapper/releases/download/1.5.6/ForgeWrapper-1.5.6.jar"
+BAD_VERSIONS = [""]

meta/model/__init__.py

@@ -1,5 +1,6 @@
 import copy
 from datetime import datetime
+from pathlib import Path
 from typing import Optional, List, Dict, Any, Iterator
 
 import pydantic
@@ -146,6 +147,7 @@ class MetaBase(pydantic.BaseModel):
         )
 
     def write(self, file_path):
+        Path(file_path).parent.mkdir(parents=True, exist_ok=True)
         with open(file_path, "w") as f:
             f.write(self.json())
 

meta/model/neoforge.py

@@ -0,0 +1,264 @@
+from datetime import datetime
+from typing import Optional, List, Dict
+
+from pydantic import Field
+
+from . import MetaBase, GradleSpecifier, MojangLibrary
+from .mojang import MojangVersion
+
+
+class NeoForgeFile(MetaBase):
+    classifier: str
+    extension: str
+
+    def filename(self, long_version):
+        return "%s-%s-%s.%s" % ("forge", long_version, self.classifier, self.extension)
+
+    def url(self, long_version):
+        return "https://maven.neoforged.net/net/neoforged/forge/%s/%s" % (
+            long_version,
+            self.filename(long_version),
+        )
+
+
+class NeoForgeEntry(MetaBase):
+    long_version: str = Field(alias="longversion")
+    mc_version: str = Field(alias="mcversion")
+    version: str
+    build: int
+    branch: Optional[str]
+    latest: Optional[bool]
+    recommended: Optional[bool]
+    files: Optional[Dict[str, NeoForgeFile]]
+
+
+class NeoForgeMCVersionInfo(MetaBase):
+    latest: Optional[str]
+    recommended: Optional[str]
+    versions: List[str] = Field([])
+
+
+class DerivedNeoForgeIndex(MetaBase):
+    versions: Dict[str, NeoForgeEntry] = Field({})
+    by_mc_version: Dict[str, NeoForgeMCVersionInfo] = Field({}, alias="by_mcversion")
+
+
+class FMLLib(
+    MetaBase
+):  # old ugly stuff. Maybe merge this with Library or MojangLibrary later
+    filename: str
+    checksum: str
+    ours: bool
+
+
+class NeoForgeInstallerProfileInstallSection(MetaBase):
+    """
+    "install": {
+        "profileName": "NeoForge",
+        "target":"NeoForge8.9.0.753",
+        "path":"net.minecraftNeoForge:minecraftNeoForge:8.9.0.753",
+        "version":"NeoForge 8.9.0.753",
+        "filePath":"minecraftNeoForge-universal-1.6.1-8.9.0.753.jar",
+        "welcome":"Welcome to the simple NeoForge installer.",
+        "minecraft":"1.6.1",
+        "logo":"/big_logo.png",
+        "mirrorList": "http://files.minecraftNeoForge.net/mirror-brand.list"
+    },
+    "install": {
+        "profileName": "NeoForge",
+        "target":"1.11-NeoForge1.11-13.19.0.2141",
+        "path":"net.minecraftNeoForge:NeoForge:1.11-13.19.0.2141",
+        "version":"NeoForge 1.11-13.19.0.2141",
+        "filePath":"NeoForge-1.11-13.19.0.2141-universal.jar",
+        "welcome":"Welcome to the simple NeoForge installer.",
+        "minecraft":"1.11",
+        "mirrorList" : "http://files.minecraftNeoForge.net/mirror-brand.list",
+        "logo":"/big_logo.png",
+        "modList":"none"
+    },
+    """
+
+    profile_name: str = Field(alias="profileName")
+    target: str
+    path: GradleSpecifier
+    version: str
+    file_path: str = Field(alias="filePath")
+    welcome: str
+    minecraft: str
+    logo: str
+    mirror_list: str = Field(alias="mirrorList")
+    mod_list: Optional[str] = Field(alias="modList")
+
+
+class NeoForgeLibrary(MojangLibrary):
+    url: Optional[str]
+    server_req: Optional[bool] = Field(alias="serverreq")
+    client_req: Optional[bool] = Field(alias="clientreq")
+    checksums: Optional[List[str]]
+    comment: Optional[str]
+
+
+class NeoForgeVersionFile(MojangVersion):
+    libraries: Optional[List[NeoForgeLibrary]]  # overrides Mojang libraries
+    inherits_from: Optional[str] = Field("inheritsFrom")
+    jar: Optional[str]
+
+
+class NeoForgeOptional(MetaBase):
+    """
+    "optionals": [
+        {
+            "name": "Mercurius",
+            "client": true,
+            "server": true,
+            "default": true,
+            "inject": true,
+            "desc": "A mod that collects statistics about Minecraft and your system.<br>Useful for NeoForge to understand how Minecraft/NeoForge are used.",
+            "url": "http://www.minecraftNeoForge.net/forum/index.php?topic=43278.0",
+            "artifact": "net.minecraftNeoForge:MercuriusUpdater:1.11.2",
+            "maven": "http://maven.minecraftNeoForge.net/"
+        }
+    ]
+    """
+
+    name: Optional[str]
+    client: Optional[bool]
+    server: Optional[bool]
+    default: Optional[bool]
+    inject: Optional[bool]
+    desc: Optional[str]
+    url: Optional[str]
+    artifact: Optional[GradleSpecifier]
+    maven: Optional[str]
+
+
+class NeoForgeInstallerProfile(MetaBase):
+    install: NeoForgeInstallerProfileInstallSection
+    version_info: NeoForgeVersionFile = Field(alias="versionInfo")
+    optionals: Optional[List[NeoForgeOptional]]
+
+
+class NeoForgeLegacyInfo(MetaBase):
+    release_time: Optional[datetime] = Field(alias="releaseTime")
+    size: Optional[int]
+    sha256: Optional[str]
+    sha1: Optional[str]
+
+
+class NeoForgeLegacyInfoList(MetaBase):
+    number: Dict[str, NeoForgeLegacyInfo] = Field({})
+
+
+class DataSpec(MetaBase):
+    client: Optional[str]
+    server: Optional[str]
+
+
+class ProcessorSpec(MetaBase):
+    jar: Optional[str]
+    classpath: Optional[List[str]]
+    args: Optional[List[str]]
+    outputs: Optional[Dict[str, str]]
+    sides: Optional[List[str]]
+
+
+class NeoForgeInstallerProfileV2(MetaBase):
+    _comment: Optional[List[str]]
+    spec: Optional[int]
+    profile: Optional[str]
+    version: Optional[str]
+    icon: Optional[str]
+    json_data: Optional[str] = Field(alias="json")
+    path: Optional[GradleSpecifier]
+    logo: Optional[str]
+    minecraft: Optional[str]
+    welcome: Optional[str]
+    data: Optional[Dict[str, DataSpec]]
+    processors: Optional[List[ProcessorSpec]]
+    libraries: Optional[List[MojangLibrary]]
+    mirror_list: Optional[str] = Field(alias="mirrorList")
+    server_jar_path: Optional[str] = Field(alias="serverJarPath")
+
+
+class InstallerInfo(MetaBase):
+    sha1hash: Optional[str]
+    sha256hash: Optional[str]
+    size: Optional[int]
+
+
+# A post-processed entry constructed from the reconstructed NeoForge version index
+class NeoForgeVersion:
+    def __init__(self, entry: NeoForgeEntry):
+        self.build = entry.build
+        self.rawVersion = entry.version
+        self.mc_version = entry.mc_version
+        self.mc_version_sane = self.mc_version.replace("_pre", "-pre", 1)
+        self.branch = entry.branch
+        self.installer_filename = None
+        self.installer_url = None
+        self.universal_filename = None
+        self.universal_url = None
+        self.changelog_url = None
+        self.long_version = "%s-%s" % (self.mc_version, self.rawVersion)
+        if self.branch is not None:
+            self.long_version += "-%s" % self.branch
+
+        # this comment's whole purpose is to say this: cringe
+        for classifier, file in entry.files.items():
+            extension = file.extension
+            filename = file.filename(self.long_version)
+            url = file.url(self.long_version)
+            print(url)
+            print(self.long_version)
+            if (classifier == "installer") and (extension == "jar"):
+                self.installer_filename = filename
+                self.installer_url = url
+            if (classifier == "universal" or classifier == "client") and (
+                extension == "jar" or extension == "zip"
+            ):
+                self.universal_filename = filename
+                self.universal_url = url
+            if (classifier == "changelog") and (extension == "txt"):
+                self.changelog_url = url
+
+    def name(self):
+        return "forge %d" % self.build
+
+    def uses_installer(self):
+        if self.installer_url is None:
+            return False
+        if self.mc_version == "1.5.2":
+            return False
+        return True
+
+    def filename(self):
+        if self.uses_installer():
+            return self.installer_filename
+        return self.universal_filename
+
+    def url(self):
+        if self.uses_installer():
+            return self.installer_url
+        return self.universal_url
+
+    def is_supported(self):
+        if self.url() is None:
+            return False
+
+        foo = self.rawVersion.split(".")
+        if len(foo) < 1:
+            return False
+
+        major_version = foo[0]
+        if not major_version.isnumeric():
+            return False
+
+        # majorVersion = int(majorVersionStr)
+        # if majorVersion >= 37:
+        #    return False
+
+        return True
+
+
+def fml_libs_for_version(mc_version: str) -> List[FMLLib]:
+    return []

testNeo.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+BASEDIR=$(dirname "$0")
+cd "${BASEDIR}" || exit 1
+BASEDIR=$(pwd)
+
+set -x
+
+source config.sh
+if [ -f config/config_local.sh ]; then
+    source config/config_local.sh
+fi
+
+MODE=${MODE:-develop}
+
+BRANCH_var="BRANCH_$MODE"
+BRANCH="${!BRANCH_var}"
+
+function fail_in {
+    upstream_git reset --hard HEAD
+    exit 1
+}
+
+function fail_out {
+    launcher_git reset --hard HEAD
+    exit 1
+}
+
+function upstream_git {
+    git -C "${BASEDIR}/${UPSTREAM_DIR}" "$@"
+}
+
+function launcher_git {
+    git -C "${BASEDIR}/${LAUNCHER_DIR}" "$@"
+}
+
+# make sure we *could* push to our repo
+
+currentDate=$(date -I)
+
+upstream_git reset --hard HEAD || exit 1
+upstream_git checkout "${BRANCH}" || exit 1
+
+
+python updateNeoForge.py || fail_in
+
+launcher_git reset --hard HEAD || exit 1
+launcher_git checkout "${BRANCH}" || exit 1
+
+
+python generateNeoForge.py || fail_out
+python index.py || fail_out

update.sh

@@ -43,6 +43,7 @@ upstream_git checkout "${BRANCH}" || exit 1
 
 python updateMojang.py || fail_in
 python updateForge.py || fail_in
+python updateNeoForge.py || fail_in
 python updateFabric.py || fail_in
 python updateQuilt.py || fail_in
 python updateLiteloader.py || fail_in
@@ -64,6 +65,7 @@ launcher_git checkout "${BRANCH}" || exit 1
 
 python generateMojang.py || fail_out
 python generateForge.py || fail_out
+python generateNeoForge.py || fail_out
 python generateFabric.py || fail_out
 python generateQuilt.py || fail_out
 python generateLiteloader.py || fail_out

updateNeoForge.py

@@ -0,0 +1,338 @@
+"""
+ Get the source files necessary for generating Forge versions
+"""
+import copy
+import hashlib
+import json
+import os
+import re
+import sys
+import zipfile
+from contextlib import suppress
+from datetime import datetime
+from pathlib import Path
+from pprint import pprint
+import urllib.parse
+
+from pydantic import ValidationError
+
+from meta.common import upstream_path, ensure_upstream_dir, static_path, default_session
+from meta.common.forge import (
+    JARS_DIR,
+    INSTALLER_INFO_DIR,
+    INSTALLER_MANIFEST_DIR,
+    VERSION_MANIFEST_DIR,
+    FILE_MANIFEST_DIR,
+    BAD_VERSIONS,
+    STATIC_LEGACYINFO_FILE,
+)
+from meta.model.neoforge import (
+    NeoForgeFile,
+    NeoForgeEntry,
+    NeoForgeMCVersionInfo,
+    NeoForgeLegacyInfoList,
+    DerivedNeoForgeIndex,
+    NeoForgeVersion,
+    NeoForgeInstallerProfile,
+    NeoForgeInstallerProfileV2,
+    InstallerInfo,
+    NeoForgeLegacyInfo,
+)
+from meta.model.mojang import MojangVersion
+
+UPSTREAM_DIR = upstream_path()
+STATIC_DIR = static_path()
+
+ensure_upstream_dir(JARS_DIR)
+ensure_upstream_dir(INSTALLER_INFO_DIR)
+ensure_upstream_dir(INSTALLER_MANIFEST_DIR)
+ensure_upstream_dir(VERSION_MANIFEST_DIR)
+ensure_upstream_dir(FILE_MANIFEST_DIR)
+
+LEGACYINFO_PATH = os.path.join(STATIC_DIR, STATIC_LEGACYINFO_FILE)
+
+sess = default_session()
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def filehash(filename, hashtype, blocksize=65536):
+    hashtype = hashtype()
+    with open(filename, "rb") as f:
+        for block in iter(lambda: f.read(blocksize), b""):
+            hashtype.update(block)
+    return hashtype.hexdigest()
+
+def find_nth(haystack, needle, n):
+    start = haystack.find(needle)
+    while start >= 0 and n > 1:
+        start = haystack.find(needle, start+len(needle))
+        n -= 1
+    return start
+
+def get_single_forge_files_manifest(longversion):
+    print(f"Getting NeoForge manifest for {longversion}")
+    path_thing = UPSTREAM_DIR + "/neoforge/files_manifests/%s.json" % longversion
+    files_manifest_file = Path(path_thing)
+    from_file = False
+    if files_manifest_file.is_file():
+        with open(path_thing, "r") as f:
+            files_json = json.load(f)
+            from_file = True
+    else:
+        file_url = (
+            "https://maven.neoforged.net/api/maven/details/releases/net%2Fneoforged%2Fforge%2F" + urllib.parse.quote(longversion)
+        )
+        r = sess.get(file_url)
+        r.raise_for_status()
+        files_json = r.json()
+
+    ret_dict = dict()
+
+    for file in files_json.get("files"):
+        assert type(file) == dict
+        name = file["name"]
+        file_name, file_ext = os.path.splitext(name)
+        if file_ext in [".md5", ".sha1", ".sha256", ".sha512"]:
+            continue
+
+        classifier = file["name"][find_nth(name, "-", 3)+1:len(file_name)]
+
+        # assert len(extensionObj.items()) == 1
+        index = 0
+        count = 0
+        file_obj = NeoForgeFile(
+                classifier=classifier, extension=file_ext[1:]
+        )
+        if count == 0:
+            ret_dict[classifier] = file_obj
+            index += 1
+            count += 1
+        else:
+            print(
+                "%s: Multiple objects detected for classifier %s:"
+                % (longversion, classifier)
+            )
+            assert False
+
+            
+
+    if not from_file:
+        Path(path_thing).parent.mkdir(parents=True, exist_ok=True)
+        with open(path_thing, "w", encoding="utf-8") as f:
+            json.dump(files_json, f, sort_keys=True, indent=4)
+
+    return ret_dict
+
+
+def main():
+    # get the remote version list fragments
+    r = sess.get(
+        "https://maven.neoforged.net/api/maven/versions/releases/net%2Fneoforged%2Fforge"
+    )
+    r.raise_for_status()
+    main_json = r.json()["versions"]
+    assert type(main_json) == list
+
+    new_index = DerivedNeoForgeIndex()
+
+    version_expression = re.compile(
+        "^(?P<mc>[0-9a-zA-Z_\\.]+)-(?P<ver>[0-9\\.]+\\.(?P<build>[0-9]+))(-(?P<branch>[a-zA-Z0-9\\.]+))?$"
+    )
+
+    print("")
+    print("Processing versions:")
+    for long_version in main_json:
+        assert type(long_version) == str
+        mc_version = long_version.split("-")[0]
+        match = version_expression.match(long_version)
+        if not match:
+            pprint(long_version)
+            assert match
+        assert match.group("mc") == mc_version
+        try:
+            files = get_single_forge_files_manifest(long_version)
+        except:
+            continue
+        build = int(match.group("build"))
+        version = match.group("ver")
+        branch = match.group("branch")
+
+        is_recommended = False
+
+        entry = NeoForgeEntry(
+            long_version=long_version,
+            mc_version=mc_version,
+            version=version,
+            build=build,
+            branch=branch,
+            # NOTE: we add this later after the fact. The forge promotions file lies about these.
+            latest=False,
+            recommended=is_recommended,
+            files=files,
+        )
+        new_index.versions[long_version] = entry
+        if not new_index.by_mc_version:
+            new_index.by_mc_version = dict()
+        if mc_version not in new_index.by_mc_version:
+            new_index.by_mc_version.setdefault(mc_version, NeoForgeMCVersionInfo())
+        new_index.by_mc_version[mc_version].versions.append(long_version)
+        # NOTE: we add this later after the fact. The forge promotions file lies about these.
+        # if entry.latest:
+        # new_index.by_mc_version[mc_version].latest = long_version
+        if entry.recommended:
+            new_index.by_mc_version[mc_version].recommended = long_version
+
+    print("")
+    print("Dumping index files...")
+
+    with open(UPSTREAM_DIR + "/neoforge/maven-metadata.json", "w", encoding="utf-8") as f:
+        json.dump(main_json, f, sort_keys=True, indent=4)
+
+    new_index.write(UPSTREAM_DIR + "/neoforge/derived_index.json")
+
+    legacy_info_list = NeoForgeLegacyInfoList()
+
+    print("Grabbing installers and dumping installer profiles...")
+    # get the installer jars - if needed - and get the installer profiles out of them
+    for key, entry in new_index.versions.items():
+        eprint("Updating NeoForge %s" % key)
+        if entry.mc_version is None:
+            eprint("Skipping %d with invalid MC version" % entry.build)
+            continue
+
+        version = NeoForgeVersion(entry)
+        if version.url() is None:
+            eprint("Skipping %d with no valid files" % version.build)
+            continue
+        if version.long_version in BAD_VERSIONS:
+            eprint(f"Skipping bad version {version.long_version}")
+            continue
+
+        jar_path = os.path.join(UPSTREAM_DIR, JARS_DIR, version.filename())
+
+        if version.uses_installer():
+            installer_info_path = (
+                UPSTREAM_DIR + "/neoforge/installer_info/%s.json" % version.long_version
+            )
+            profile_path = (
+                UPSTREAM_DIR
+                + "/neoforge/installer_manifests/%s.json" % version.long_version
+            )
+            version_file_path = (
+                UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version
+            )
+
+            installer_refresh_required = not os.path.isfile(
+                profile_path
+            ) or not os.path.isfile(installer_info_path)
+
+            if installer_refresh_required:
+                # grab the installer if it's not there
+                if not os.path.isfile(jar_path):
+                    eprint("Downloading %s" % version.url())
+                    try:
+                        rfile = sess.get(version.url(), stream=True)
+                        rfile.raise_for_status()
+                        Path(jar_path).parent.mkdir(parents=True, exist_ok=True)
+                        with open(jar_path, "wb") as f:
+                            for chunk in rfile.iter_content(chunk_size=128):
+                                f.write(chunk)
+                    except Exception as e:
+                        eprint("Failed to download %s" % version.url())
+                        eprint("Error is %s" % e)
+                        continue
+
+            eprint("Processing %s" % version.url())
+            # harvestables from the installer
+            if not os.path.isfile(profile_path):
+                print(jar_path)
+                with zipfile.ZipFile(jar_path) as jar:
+                    with suppress(KeyError):
+                        with jar.open("version.json") as profile_zip_entry:
+                            version_data = profile_zip_entry.read()
+
+                            # Process: does it parse?
+                            MojangVersion.parse_raw(version_data)
+
+                            Path(version_file_path).parent.mkdir(parents=True, exist_ok=True)
+                            with open(version_file_path, "wb") as versionJsonFile:
+                                versionJsonFile.write(version_data)
+                                versionJsonFile.close()
+
+                    with jar.open("install_profile.json") as profile_zip_entry:
+                        install_profile_data = profile_zip_entry.read()
+
+                        # Process: does it parse?
+                        is_parsable = False
+                        exception = None
+                        try:
+                            NeoForgeInstallerProfile.parse_raw(install_profile_data)
+                            is_parsable = True
+                        except ValidationError as err:
+                            exception = err
+                        try:
+                            NeoForgeInstallerProfileV2.parse_raw(install_profile_data)
+                            is_parsable = True
+                        except ValidationError as err:
+                            exception = err
+
+                        if not is_parsable:
+                            if version.is_supported():
+                                raise exception
+                            else:
+                                eprint(
+                                    "Version %s is not supported and won't be generated later."
+                                    % version.long_version
+                                )
+
+                        Path(profile_path).parent.mkdir(parents=True, exist_ok=True)
+                        with open(profile_path, "wb") as profileFile:
+                            profileFile.write(install_profile_data)
+                            profileFile.close()
+
+            # installer info v1
+            if not os.path.isfile(installer_info_path):
+                installer_info = InstallerInfo()
+                installer_info.sha1hash = filehash(jar_path, hashlib.sha1)
+                installer_info.sha256hash = filehash(jar_path, hashlib.sha256)
+                installer_info.size = os.path.getsize(jar_path)
+                installer_info.write(installer_info_path)
+        else:
+            # ignore the two versions without install manifests and jar mod class files
+            # TODO: fix those versions?
+            if version.mc_version_sane == "1.6.1":
+                continue
+
+            # only gather legacy info if it's missing
+            if not os.path.isfile(LEGACYINFO_PATH):
+                # grab the jar/zip if it's not there
+                if not os.path.isfile(jar_path):
+                    rfile = sess.get(version.url(), stream=True)
+                    rfile.raise_for_status()
+                    with open(jar_path, "wb") as f:
+                        for chunk in rfile.iter_content(chunk_size=128):
+                            f.write(chunk)
+                # find the latest timestamp in the zip file
+                tstamp = datetime.fromtimestamp(0)
+                with zipfile.ZipFile(jar_path) as jar:
+                    for info in jar.infolist():
+                        tstamp_new = datetime(*info.date_time)
+                        if tstamp_new > tstamp:
+                            tstamp = tstamp_new
+                legacy_info = NeoForgeLegacyInfo()
+                legacy_info.release_time = tstamp
+                legacy_info.sha1 = filehash(jar_path, hashlib.sha1)
+                legacy_info.sha256 = filehash(jar_path, hashlib.sha256)
+                legacy_info.size = os.path.getsize(jar_path)
+                legacy_info_list.number[key] = legacy_info
+
+    # only write legacy info if it's missing
+    if not os.path.isfile(LEGACYINFO_PATH):
+        legacy_info_list.write(LEGACYINFO_PATH)
+
+
+if __name__ == "__main__":
+    main()