Merge pull request #21 from PrismLauncher/neoforge

generateForge.py

@@ -1,7 +1,7 @@
 import os
 import re
 import sys
-from distutils.version import LooseVersion
+from packaging import version as pversion
 from operator import attrgetter
 from typing import Collection
 
@@ -14,7 +14,7 @@ from meta.common.forge import (
     STATIC_LEGACYINFO_FILE,
     INSTALLER_INFO_DIR,
     BAD_VERSIONS,
-    FORGEWRAPPER_MAVEN,
+    FORGEWRAPPER_LIBRARY,
 )
 from meta.common.mojang import MINECRAFT_COMPONENT
 from meta.model import (
@@ -81,7 +81,7 @@ def should_ignore_artifact(libs: Collection[GradleSpecifier], match: GradleSpeci
             if ver.version == match.version:
                 # Everything is matched perfectly - this one will be ignored
                 return True
-            elif LooseVersion(ver.version) > LooseVersion(match.version):
+            elif pversion.parse(ver.version) > pversion.parse(match.version):
                 return True
             else:
                 # Otherwise it did not match - new version is higher and this is an upgrade
@@ -287,16 +287,7 @@ def version_from_build_system_installer(
 
     v.libraries = []
 
-    wrapper_lib = Library(
+    v.libraries.append(FORGEWRAPPER_LIBRARY)
-        name=GradleSpecifier("io.github.zekerzhayard", "ForgeWrapper", "mmc2")
-    )
-    wrapper_lib.downloads = MojangLibraryDownloads()
-    wrapper_lib.downloads.artifact = MojangArtifact(
-        url=FORGEWRAPPER_MAVEN % (wrapper_lib.name.path()),
-        sha1="4ee5f25cc9c7efbf54aff4c695da1054c1a1d7a3",
-        size=34444,
-    )
-    v.libraries.append(wrapper_lib)
 
     for upstream_lib in installer.libraries:
         forge_lib = Library.parse_obj(upstream_lib.dict())
@@ -437,7 +428,6 @@ def main():
                 v = version_from_build_system_installer(installer, profile, version)
         else:
             if version.uses_installer():
-
                 # If we do not have the Forge json, we ignore this version
                 if not os.path.isfile(profile_filepath):
                     eprint("Skipping %s with missing profile json" % key)

generateNeoForge.py

@@ -0,0 +1,199 @@
+import os
+import re
+import sys
+from operator import attrgetter
+from typing import Collection
+
+from meta.common import ensure_component_dir, launcher_path, upstream_path, static_path
+from meta.common.neoforge import (
+    NEOFORGE_COMPONENT,
+    INSTALLER_MANIFEST_DIR,
+    VERSION_MANIFEST_DIR,
+    DERIVED_INDEX_FILE,
+    INSTALLER_INFO_DIR,
+)
+from meta.common.forge import FORGEWRAPPER_LIBRARY
+from meta.common.mojang import MINECRAFT_COMPONENT
+from meta.model import (
+    MetaVersion,
+    Dependency,
+    Library,
+    GradleSpecifier,
+    MojangLibraryDownloads,
+    MojangArtifact,
+    MetaPackage,
+)
+from meta.model.neoforge import (
+    NeoForgeVersion,
+    NeoForgeInstallerProfileV2,
+    InstallerInfo,
+    DerivedNeoForgeIndex,
+)
+from meta.model.mojang import MojangVersion
+
+LAUNCHER_DIR = launcher_path()
+UPSTREAM_DIR = upstream_path()
+STATIC_DIR = static_path()
+
+ensure_component_dir(NEOFORGE_COMPONENT)
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def version_from_build_system_installer(
+    installer: MojangVersion,
+    profile: NeoForgeInstallerProfileV2,
+    version: NeoForgeVersion,
+) -> MetaVersion:
+    v = MetaVersion(name="NeoForge", version=version.rawVersion, uid=NEOFORGE_COMPONENT)
+    v.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=version.mc_version_sane)]
+    v.main_class = "io.github.zekerzhayard.forgewrapper.installer.Main"
+
+    # FIXME: Add the size and hash here
+    v.maven_files = []
+
+    # load the locally cached installer file info and use it to add the installer entry in the json
+    info = InstallerInfo.parse_file(
+        os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version.long_version}.json")
+    )
+    installer_lib = Library(
+        name=GradleSpecifier(
+            "net.neoforged", "forge", version.long_version, "installer"
+        )
+    )
+    installer_lib.downloads = MojangLibraryDownloads()
+    installer_lib.downloads.artifact = MojangArtifact(
+        url="https://maven.neoforged.net/%s" % (installer_lib.name.path()),
+        sha1=info.sha1hash,
+        size=info.size,
+    )
+    v.maven_files.append(installer_lib)
+
+    for upstream_lib in profile.libraries:
+        forge_lib = Library.parse_obj(upstream_lib.dict())
+        if forge_lib.name.is_log4j():
+            continue
+
+        if (
+            forge_lib.name.group == "net.neoforged"
+            and forge_lib.name.artifact == "forge"
+            and forge_lib.name.classifier == "universal"
+        ):
+            forge_lib.downloads.artifact.url = (
+                "https://maven.neoforged.net/%s" % forge_lib.name.path()
+            )
+        v.maven_files.append(forge_lib)
+
+    v.libraries = []
+
+    v.libraries.append(FORGEWRAPPER_LIBRARY)
+
+    for upstream_lib in installer.libraries:
+        forge_lib = Library.parse_obj(upstream_lib.dict())
+        if forge_lib.name.is_log4j():
+            continue
+
+        if forge_lib.name.group == "net.neoforged":
+            if forge_lib.name.artifact == "forge":
+                forge_lib.name.classifier = "launcher"
+                forge_lib.downloads.artifact.path = forge_lib.name.path()
+                forge_lib.downloads.artifact.url = (
+                    "https://maven.neoforged.net/%s" % forge_lib.name.path()
+                )
+                forge_lib.name = forge_lib.name
+        v.libraries.append(forge_lib)
+
+    v.release_time = installer.release_time
+    v.order = 5
+    mc_args = (
+        "--username ${auth_player_name} --version ${version_name} --gameDir ${game_directory} "
+        "--assetsDir ${assets_root} --assetIndex ${assets_index_name} --uuid ${auth_uuid} "
+        "--accessToken ${auth_access_token} --userType ${user_type} --versionType ${version_type}"
+    )
+    for arg in installer.arguments.game:
+        mc_args += f" {arg}"
+    v.minecraft_arguments = mc_args
+    return v
+
+
+def main():
+    # load the locally cached version list
+    remote_versions = DerivedNeoForgeIndex.parse_file(
+        os.path.join(UPSTREAM_DIR, DERIVED_INDEX_FILE)
+    )
+    recommended_versions = []
+
+    for key, entry in remote_versions.versions.items():
+        if entry.mc_version is None:
+            eprint("Skipping %s with invalid MC version" % key)
+            continue
+
+        version = NeoForgeVersion(entry)
+
+        if version.url() is None:
+            eprint("Skipping %s with no valid files" % key)
+            continue
+        eprint("Processing Forge %s" % version.rawVersion)
+        version_elements = version.rawVersion.split(".")
+        if len(version_elements) < 1:
+            eprint("Skipping version %s with not enough version elements" % key)
+            continue
+
+        major_version_str = version_elements[0]
+        if not major_version_str.isnumeric():
+            eprint(
+                "Skipping version %s with non-numeric major version %s"
+                % (key, major_version_str)
+            )
+            continue
+
+        if entry.recommended:
+            recommended_versions.append(version.rawVersion)
+
+        # If we do not have the corresponding Minecraft version, we ignore it
+        if not os.path.isfile(
+            os.path.join(
+                LAUNCHER_DIR, MINECRAFT_COMPONENT, f"{version.mc_version_sane}.json"
+            )
+        ):
+            eprint(
+                "Skipping %s with no corresponding Minecraft version %s"
+                % (key, version.mc_version_sane)
+            )
+            continue
+
+        # Path for new-style build system based installers
+        installer_version_filepath = os.path.join(
+            UPSTREAM_DIR, VERSION_MANIFEST_DIR, f"{version.long_version}.json"
+        )
+        profile_filepath = os.path.join(
+            UPSTREAM_DIR, INSTALLER_MANIFEST_DIR, f"{version.long_version}.json"
+        )
+
+        eprint(installer_version_filepath)
+        assert os.path.isfile(
+            installer_version_filepath
+        ), f"version {installer_version_filepath} does not have installer version manifest"
+        installer = MojangVersion.parse_file(installer_version_filepath)
+        profile = NeoForgeInstallerProfileV2.parse_file(profile_filepath)
+        v = version_from_build_system_installer(installer, profile, version)
+
+        v.write(os.path.join(LAUNCHER_DIR, NEOFORGE_COMPONENT, f"{v.version}.json"))
+
+        recommended_versions.sort()
+
+        print("Recommended versions:", recommended_versions)
+
+        package = MetaPackage(
+            uid=NEOFORGE_COMPONENT,
+            name="NeoForge",
+            project_url="https://neoforged.net",
+        )
+        package.recommended = recommended_versions
+        package.write(os.path.join(LAUNCHER_DIR, NEOFORGE_COMPONENT, "package.json"))
+
+
+if __name__ == "__main__":
+    main()

index.py

@@ -48,7 +48,6 @@ for package in sorted(os.listdir(LAUNCHER_DIR)):
     for filename in os.listdir(LAUNCHER_DIR + "/%s" % package):
         if filename in ignore:
             continue
-
         # parse and hash the version file
         filepath = LAUNCHER_DIR + "/%s/%s" % (package, filename)
         filehash = hash_file(hashlib.sha256, filepath)

meta/common/__init__.py

@@ -6,6 +6,8 @@ import requests
 from cachecontrol import CacheControl
 from cachecontrol.caches import FileCache
 
+LAUNCHER_MAVEN = "https://files.prismlauncher.org/maven/%s"
+
 
 def serialize_datetime(dt: datetime.datetime):
     if dt.tzinfo is None:

meta/common/forge.py

@@ -1,5 +1,7 @@
 from os.path import join
 
+from ..model import GradleSpecifier, make_launcher_library
+
 BASE_DIR = "forge"
 
 JARS_DIR = join(BASE_DIR, "jars")
@@ -13,5 +15,9 @@ STATIC_LEGACYINFO_FILE = join(BASE_DIR, "forge-legacyinfo.json")
 
 FORGE_COMPONENT = "net.minecraftforge"
 
-FORGEWRAPPER_MAVEN = "https://files.prismlauncher.org/maven/%s"
+FORGEWRAPPER_LIBRARY = make_launcher_library(
+    GradleSpecifier("io.github.zekerzhayard", "ForgeWrapper", "1.5.6-prism"),
+    "b059aa8c4d2508055c6ed2a2561923a5e670a5eb",
+    34860,
+)
 BAD_VERSIONS = ["1.12.2-14.23.5.2851"]

meta/common/neoforge.py

@@ -0,0 +1,14 @@
+from os.path import join
+
+from ..model import GradleSpecifier, make_launcher_library
+
+BASE_DIR = "neoforge"
+
+JARS_DIR = join(BASE_DIR, "jars")
+INSTALLER_INFO_DIR = join(BASE_DIR, "installer_info")
+INSTALLER_MANIFEST_DIR = join(BASE_DIR, "installer_manifests")
+VERSION_MANIFEST_DIR = join(BASE_DIR, "version_manifests")
+FILE_MANIFEST_DIR = join(BASE_DIR, "files_manifests")
+DERIVED_INDEX_FILE = join(BASE_DIR, "derived_index.json")
+
+NEOFORGE_COMPONENT = "net.neoforged"

meta/model/__init__.py

@@ -1,11 +1,13 @@
 import copy
 from datetime import datetime
+from pathlib import Path
 from typing import Optional, List, Dict, Any, Iterator
 
 import pydantic
 from pydantic import Field, validator
 
 from ..common import (
+    LAUNCHER_MAVEN,
     serialize_datetime,
     replace_old_launchermeta_url,
     get_all_bases,
@@ -146,6 +148,7 @@ class MetaBase(pydantic.BaseModel):
         )
 
     def write(self, file_path):
+        Path(file_path).parent.mkdir(parents=True, exist_ok=True)
         with open(file_path, "w") as f:
             f.write(self.json())
 
@@ -328,3 +331,10 @@ class MetaPackage(Versioned):
     authors: Optional[List[str]]
     description: Optional[str]
     project_url: Optional[str] = Field(alias="projectUrl")
+
+
+def make_launcher_library(
+    name: GradleSpecifier, hash: str, size: int, maven=LAUNCHER_MAVEN
+):
+    artifact = MojangArtifact(url=maven % name.path(), sha1=hash, size=size)
+    return Library(name=name, downloads=MojangLibraryDownloads(artifact=artifact))

meta/model/neoforge.py

@@ -0,0 +1,239 @@
+from datetime import datetime
+from typing import Optional, List, Dict
+
+from pydantic import Field
+
+from . import MetaBase, GradleSpecifier, MojangLibrary
+from .mojang import MojangVersion
+
+
+class NeoForgeFile(MetaBase):
+    classifier: str
+    extension: str
+
+    def filename(self, long_version):
+        return "%s-%s-%s.%s" % ("forge", long_version, self.classifier, self.extension)
+
+    def url(self, long_version):
+        return "https://maven.neoforged.net/net/neoforged/forge/%s/%s" % (
+            long_version,
+            self.filename(long_version),
+        )
+
+
+class NeoForgeEntry(MetaBase):
+    long_version: str = Field(alias="longversion")
+    mc_version: str = Field(alias="mcversion")
+    version: str
+    build: int
+    branch: Optional[str]
+    latest: Optional[bool]
+    recommended: Optional[bool]
+    files: Optional[Dict[str, NeoForgeFile]]
+
+
+class NeoForgeMCVersionInfo(MetaBase):
+    latest: Optional[str]
+    recommended: Optional[str]
+    versions: List[str] = Field([])
+
+
+class DerivedNeoForgeIndex(MetaBase):
+    versions: Dict[str, NeoForgeEntry] = Field({})
+    by_mc_version: Dict[str, NeoForgeMCVersionInfo] = Field({}, alias="by_mcversion")
+
+
+class FMLLib(
+    MetaBase
+):  # old ugly stuff. Maybe merge this with Library or MojangLibrary later
+    filename: str
+    checksum: str
+    ours: bool
+
+
+class NeoForgeInstallerProfileInstallSection(MetaBase):
+    """
+    "install": {
+        "profileName": "NeoForge",
+        "target":"NeoForge8.9.0.753",
+        "path":"net.minecraftNeoForge:minecraftNeoForge:8.9.0.753",
+        "version":"NeoForge 8.9.0.753",
+        "filePath":"minecraftNeoForge-universal-1.6.1-8.9.0.753.jar",
+        "welcome":"Welcome to the simple NeoForge installer.",
+        "minecraft":"1.6.1",
+        "logo":"/big_logo.png",
+        "mirrorList": "http://files.minecraftNeoForge.net/mirror-brand.list"
+    },
+    "install": {
+        "profileName": "NeoForge",
+        "target":"1.11-NeoForge1.11-13.19.0.2141",
+        "path":"net.minecraftNeoForge:NeoForge:1.11-13.19.0.2141",
+        "version":"NeoForge 1.11-13.19.0.2141",
+        "filePath":"NeoForge-1.11-13.19.0.2141-universal.jar",
+        "welcome":"Welcome to the simple NeoForge installer.",
+        "minecraft":"1.11",
+        "mirrorList" : "http://files.minecraftNeoForge.net/mirror-brand.list",
+        "logo":"/big_logo.png",
+        "modList":"none"
+    },
+    """
+
+    profile_name: str = Field(alias="profileName")
+    target: str
+    path: GradleSpecifier
+    version: str
+    file_path: str = Field(alias="filePath")
+    welcome: str
+    minecraft: str
+    logo: str
+    mirror_list: str = Field(alias="mirrorList")
+    mod_list: Optional[str] = Field(alias="modList")
+
+
+class NeoForgeLibrary(MojangLibrary):
+    url: Optional[str]
+    server_req: Optional[bool] = Field(alias="serverreq")
+    client_req: Optional[bool] = Field(alias="clientreq")
+    checksums: Optional[List[str]]
+    comment: Optional[str]
+
+
+class NeoForgeVersionFile(MojangVersion):
+    libraries: Optional[List[NeoForgeLibrary]]  # overrides Mojang libraries
+    inherits_from: Optional[str] = Field("inheritsFrom")
+    jar: Optional[str]
+
+
+class NeoForgeOptional(MetaBase):
+    """
+    "optionals": [
+        {
+            "name": "Mercurius",
+            "client": true,
+            "server": true,
+            "default": true,
+            "inject": true,
+            "desc": "A mod that collects statistics about Minecraft and your system.<br>Useful for NeoForge to understand how Minecraft/NeoForge are used.",
+            "url": "http://www.minecraftNeoForge.net/forum/index.php?topic=43278.0",
+            "artifact": "net.minecraftNeoForge:MercuriusUpdater:1.11.2",
+            "maven": "http://maven.minecraftNeoForge.net/"
+        }
+    ]
+    """
+
+    name: Optional[str]
+    client: Optional[bool]
+    server: Optional[bool]
+    default: Optional[bool]
+    inject: Optional[bool]
+    desc: Optional[str]
+    url: Optional[str]
+    artifact: Optional[GradleSpecifier]
+    maven: Optional[str]
+
+
+class DataSpec(MetaBase):
+    client: Optional[str]
+    server: Optional[str]
+
+
+class ProcessorSpec(MetaBase):
+    jar: Optional[str]
+    classpath: Optional[List[str]]
+    args: Optional[List[str]]
+    outputs: Optional[Dict[str, str]]
+    sides: Optional[List[str]]
+
+
+class NeoForgeInstallerProfileV2(MetaBase):
+    _comment: Optional[List[str]]
+    spec: Optional[int]
+    profile: Optional[str]
+    version: Optional[str]
+    icon: Optional[str]
+    json_data: Optional[str] = Field(alias="json")
+    path: Optional[GradleSpecifier]
+    logo: Optional[str]
+    minecraft: Optional[str]
+    welcome: Optional[str]
+    data: Optional[Dict[str, DataSpec]]
+    processors: Optional[List[ProcessorSpec]]
+    libraries: Optional[List[MojangLibrary]]
+    mirror_list: Optional[str] = Field(alias="mirrorList")
+    server_jar_path: Optional[str] = Field(alias="serverJarPath")
+
+
+class InstallerInfo(MetaBase):
+    sha1hash: Optional[str]
+    sha256hash: Optional[str]
+    size: Optional[int]
+
+
+# A post-processed entry constructed from the reconstructed NeoForge version index
+class NeoForgeVersion:
+    def __init__(self, entry: NeoForgeEntry):
+        self.build = entry.build
+        self.rawVersion = entry.version
+        self.mc_version = entry.mc_version
+        self.mc_version_sane = self.mc_version.replace("_pre", "-pre", 1)
+        self.branch = entry.branch
+        self.installer_filename = None
+        self.installer_url = None
+        self.universal_filename = None
+        self.universal_url = None
+        self.changelog_url = None
+        self.long_version = "%s-%s" % (self.mc_version, self.rawVersion)
+        if self.branch is not None:
+            self.long_version += "-%s" % self.branch
+
+        # this comment's whole purpose is to say this: cringe
+        for classifier, file in entry.files.items():
+            extension = file.extension
+            filename = file.filename(self.long_version)
+            url = file.url(self.long_version)
+            print(url)
+            print(self.long_version)
+            if (classifier == "installer") and (extension == "jar"):
+                self.installer_filename = filename
+                self.installer_url = url
+            if (classifier == "universal" or classifier == "client") and (
+                extension == "jar" or extension == "zip"
+            ):
+                self.universal_filename = filename
+                self.universal_url = url
+            if (classifier == "changelog") and (extension == "txt"):
+                self.changelog_url = url
+
+    def name(self):
+        return "neoforge %d" % self.build
+
+    def uses_installer(self):
+        return self.installer_url is not None
+
+    def filename(self):
+        if self.uses_installer():
+            return self.installer_filename
+        return self.universal_filename
+
+    def url(self):
+        if self.uses_installer():
+            return self.installer_url
+        return self.universal_url
+
+    def is_supported(self):
+        if self.url() is None:
+            return False
+
+        foo = self.rawVersion.split(".")
+        if len(foo) < 1:
+            return False
+
+        major_version = foo[0]
+        if not major_version.isnumeric():
+            return False
+
+        # majorVersion = int(majorVersionStr)
+        # if majorVersion >= 37:
+        #    return False
+
+        return True

nix/dev.nix

@@ -34,6 +34,8 @@
             requests
             packaging
             pydantic
+
+            coverage
           ]))
       ];
     };

update.sh

@@ -43,6 +43,7 @@ upstream_git checkout "${BRANCH}" || exit 1
 
 python updateMojang.py || fail_in
 python updateForge.py || fail_in
+python updateNeoForge.py || fail_in
 python updateFabric.py || fail_in
 python updateQuilt.py || fail_in
 python updateLiteloader.py || fail_in
@@ -50,6 +51,7 @@ python updateLiteloader.py || fail_in
 if [ "${DEPLOY_TO_GIT}" = true ] ; then
     upstream_git add mojang/version_manifest_v2.json mojang/versions/* || fail_in
     upstream_git add forge/*.json forge/version_manifests/*.json forge/installer_manifests/*.json forge/files_manifests/*.json forge/installer_info/*.json || fail_in
+    upstream_git add neoforge/*.json neoforge/version_manifests/*.json neoforge/installer_manifests/*.json neoforge/files_manifests/*.json neoforge/installer_info/*.json || fail_in
     upstream_git add fabric/loader-installer-json/*.json fabric/meta-v2/*.json fabric/jars/*.json || fail_in
     upstream_git add quilt/loader-installer-json/*.json quilt/meta-v3/*.json quilt/jars/*.json || fail_in
     upstream_git add liteloader/*.json || fail_in
@@ -64,6 +66,7 @@ launcher_git checkout "${BRANCH}" || exit 1
 
 python generateMojang.py || fail_out
 python generateForge.py || fail_out
+python generateNeoForge.py || fail_out
 python generateFabric.py || fail_out
 python generateQuilt.py || fail_out
 python generateLiteloader.py || fail_out
@@ -72,6 +75,7 @@ python index.py || fail_out
 if [ "${DEPLOY_TO_GIT}" = true ] ; then
     launcher_git add index.json org.lwjgl/* org.lwjgl3/* net.minecraft/* || fail_out
     launcher_git add net.minecraftforge/* || fail_out
+    launcher_git add net.neoforged/* || fail_out
     launcher_git add net.fabricmc.fabric-loader/* net.fabricmc.intermediary/* || fail_out
     launcher_git add org.quiltmc.quilt-loader/* || fail_out  # TODO: add Quilt hashed, once it is actually used
     launcher_git add com.mumfrey.liteloader/* || fail_out

updateNeoForge.py

@@ -0,0 +1,282 @@
+"""
+ Get the source files necessary for generating Forge versions
+"""
+import copy
+import hashlib
+import json
+import os
+import re
+import sys
+import zipfile
+from contextlib import suppress
+from datetime import datetime
+from pathlib import Path
+from pprint import pprint
+import urllib.parse
+
+from pydantic import ValidationError
+
+from meta.common import upstream_path, ensure_upstream_dir, static_path, default_session
+from meta.common.neoforge import (
+    JARS_DIR,
+    INSTALLER_INFO_DIR,
+    INSTALLER_MANIFEST_DIR,
+    VERSION_MANIFEST_DIR,
+    FILE_MANIFEST_DIR,
+)
+from meta.model.neoforge import (
+    NeoForgeFile,
+    NeoForgeEntry,
+    NeoForgeMCVersionInfo,
+    DerivedNeoForgeIndex,
+    NeoForgeVersion,
+    NeoForgeInstallerProfileV2,
+    InstallerInfo,
+)
+from meta.model.mojang import MojangVersion
+
+UPSTREAM_DIR = upstream_path()
+STATIC_DIR = static_path()
+
+ensure_upstream_dir(JARS_DIR)
+ensure_upstream_dir(INSTALLER_INFO_DIR)
+ensure_upstream_dir(INSTALLER_MANIFEST_DIR)
+ensure_upstream_dir(VERSION_MANIFEST_DIR)
+ensure_upstream_dir(FILE_MANIFEST_DIR)
+
+sess = default_session()
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def filehash(filename, hashtype, blocksize=65536):
+    hashtype = hashtype()
+    with open(filename, "rb") as f:
+        for block in iter(lambda: f.read(blocksize), b""):
+            hashtype.update(block)
+    return hashtype.hexdigest()
+
+
+def find_nth(haystack, needle, n):
+    start = haystack.find(needle)
+    while start >= 0 and n > 1:
+        start = haystack.find(needle, start + len(needle))
+        n -= 1
+    return start
+
+
+def get_single_forge_files_manifest(longversion):
+    print(f"Getting NeoForge manifest for {longversion}")
+    path_thing = UPSTREAM_DIR + "/neoforge/files_manifests/%s.json" % longversion
+    files_manifest_file = Path(path_thing)
+    from_file = False
+    if files_manifest_file.is_file():
+        with open(path_thing, "r") as f:
+            files_json = json.load(f)
+            from_file = True
+    else:
+        file_url = (
+            "https://maven.neoforged.net/api/maven/details/releases/net%2Fneoforged%2Fforge%2F"
+            + urllib.parse.quote(longversion)
+        )
+        r = sess.get(file_url)
+        r.raise_for_status()
+        files_json = r.json()
+
+    ret_dict = dict()
+
+    for file in files_json.get("files"):
+        assert type(file) == dict
+        name = file["name"]
+        file_name, file_ext = os.path.splitext(name)
+        if file_ext in [".md5", ".sha1", ".sha256", ".sha512"]:
+            continue
+
+        classifier = file["name"][find_nth(name, "-", 3) + 1 : len(file_name)]
+
+        # assert len(extensionObj.items()) == 1
+        file_obj = NeoForgeFile(classifier=classifier, extension=file_ext[1:])
+        ret_dict[classifier] = file_obj
+
+    if not from_file:
+        Path(path_thing).parent.mkdir(parents=True, exist_ok=True)
+        with open(path_thing, "w", encoding="utf-8") as f:
+            json.dump(files_json, f, sort_keys=True, indent=4)
+
+    return ret_dict
+
+
+def main():
+    # get the remote version list fragments
+    r = sess.get(
+        "https://maven.neoforged.net/api/maven/versions/releases/net%2Fneoforged%2Fforge"
+    )
+    r.raise_for_status()
+    main_json = r.json()["versions"]
+    assert type(main_json) == list
+
+    new_index = DerivedNeoForgeIndex()
+
+    version_expression = re.compile(
+        "^(?P<mc>[0-9a-zA-Z_\\.]+)-(?P<ver>[0-9\\.]+\\.(?P<build>[0-9]+))(-(?P<branch>[a-zA-Z0-9\\.]+))?$"
+    )
+
+    print("")
+    print("Processing versions:")
+    for long_version in main_json:
+        assert type(long_version) == str
+        mc_version = long_version.split("-")[0]
+        match = version_expression.match(long_version)
+        assert match, f"{long_version} doesn't match version regex"
+        assert match.group("mc") == mc_version
+        try:
+            files = get_single_forge_files_manifest(long_version)
+        except:
+            continue
+        build = int(match.group("build"))
+        version = match.group("ver")
+        branch = match.group("branch")
+
+        # TODO: what *is* recommended?
+        is_recommended = False
+
+        entry = NeoForgeEntry(
+            long_version=long_version,
+            mc_version=mc_version,
+            version=version,
+            build=build,
+            branch=branch,
+            # NOTE: we add this later after the fact. The forge promotions file lies about these.
+            latest=False,
+            recommended=is_recommended,
+            files=files,
+        )
+        new_index.versions[long_version] = entry
+        if not new_index.by_mc_version:
+            new_index.by_mc_version = dict()
+        if mc_version not in new_index.by_mc_version:
+            new_index.by_mc_version.setdefault(mc_version, NeoForgeMCVersionInfo())
+        new_index.by_mc_version[mc_version].versions.append(long_version)
+        # NOTE: we add this later after the fact. The forge promotions file lies about these.
+        # if entry.latest:
+        # new_index.by_mc_version[mc_version].latest = long_version
+        if entry.recommended:
+            new_index.by_mc_version[mc_version].recommended = long_version
+
+    print("")
+    print("Dumping index files...")
+
+    with open(
+        UPSTREAM_DIR + "/neoforge/maven-metadata.json", "w", encoding="utf-8"
+    ) as f:
+        json.dump(main_json, f, sort_keys=True, indent=4)
+
+    new_index.write(UPSTREAM_DIR + "/neoforge/derived_index.json")
+
+    print("Grabbing installers and dumping installer profiles...")
+    # get the installer jars - if needed - and get the installer profiles out of them
+    for key, entry in new_index.versions.items():
+        eprint("Updating NeoForge %s" % key)
+        if entry.mc_version is None:
+            eprint("Skipping %d with invalid MC version" % entry.build)
+            continue
+
+        version = NeoForgeVersion(entry)
+        if version.url() is None:
+            eprint("Skipping %d with no valid files" % version.build)
+            continue
+        if not version.uses_installer():
+            eprint(f"version {version.long_version} does not use installer")
+            continue
+
+        jar_path = os.path.join(UPSTREAM_DIR, JARS_DIR, version.filename())
+
+        installer_info_path = (
+            UPSTREAM_DIR + "/neoforge/installer_info/%s.json" % version.long_version
+        )
+        profile_path = (
+            UPSTREAM_DIR
+            + "/neoforge/installer_manifests/%s.json" % version.long_version
+        )
+        version_file_path = (
+            UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version
+        )
+
+        installer_refresh_required = not os.path.isfile(
+            profile_path
+        ) or not os.path.isfile(installer_info_path)
+
+        if installer_refresh_required:
+            # grab the installer if it's not there
+            if not os.path.isfile(jar_path):
+                eprint("Downloading %s" % version.url())
+                try:
+                    rfile = sess.get(version.url(), stream=True)
+                    rfile.raise_for_status()
+                    Path(jar_path).parent.mkdir(parents=True, exist_ok=True)
+                    with open(jar_path, "wb") as f:
+                        for chunk in rfile.iter_content(chunk_size=128):
+                            f.write(chunk)
+                except Exception as e:
+                    eprint("Failed to download %s" % version.url())
+                    eprint("Error is %s" % e)
+                    continue
+
+        eprint("Processing %s" % version.url())
+        # harvestables from the installer
+        if not os.path.isfile(profile_path):
+            print(jar_path)
+            with zipfile.ZipFile(jar_path) as jar:
+                with suppress(KeyError):
+                    with jar.open("version.json") as profile_zip_entry:
+                        version_data = profile_zip_entry.read()
+
+                        # Process: does it parse?
+                        MojangVersion.parse_raw(version_data)
+
+                        Path(version_file_path).parent.mkdir(
+                            parents=True, exist_ok=True
+                        )
+                        with open(version_file_path, "wb") as versionJsonFile:
+                            versionJsonFile.write(version_data)
+                            versionJsonFile.close()
+
+                with jar.open("install_profile.json") as profile_zip_entry:
+                    install_profile_data = profile_zip_entry.read()
+
+                    # Process: does it parse?
+                    is_parsable = False
+                    exception = None
+                    try:
+                        NeoForgeInstallerProfileV2.parse_raw(install_profile_data)
+                        is_parsable = True
+                    except ValidationError as err:
+                        exception = err
+
+                    if not is_parsable:
+                        if version.is_supported():
+                            raise exception
+                        else:
+                            eprint(
+                                "Version %s is not supported and won't be generated later."
+                                % version.long_version
+                            )
+
+                    Path(profile_path).parent.mkdir(parents=True, exist_ok=True)
+                    with open(profile_path, "wb") as profileFile:
+                        profileFile.write(install_profile_data)
+                        profileFile.close()
+
+        # installer info v1
+        if not os.path.isfile(installer_info_path):
+            installer_info = InstallerInfo()
+            installer_info.sha1hash = filehash(jar_path, hashlib.sha1)
+            installer_info.sha256hash = filehash(jar_path, hashlib.sha256)
+            installer_info.size = os.path.getsize(jar_path)
+            installer_info.write(installer_info_path)
+
+
+if __name__ == "__main__":
+    main()