meta-unmojang/meta/run/update_mojang.py

import concurrent.futures
import json
import os
import zipfile

from meta.common import upstream_path, ensure_upstream_dir, default_session
from meta.common.http import download_binary_file
from meta.common.mojang import (
    BASE_DIR,
    VERSION_MANIFEST_FILE,
    VERSIONS_DIR,
    ASSETS_DIR,
    STATIC_EXPERIMENTS_FILE,
    STATIC_OLD_SNAPSHOTS_FILE,
    JAVA_MANIFEST_FILE,
)
from meta.model.mojang import (
    MojangIndexWrap,
    MojangIndex,
    ExperimentIndex,
    ExperimentIndexWrap,
    OldSnapshotIndexWrap,
    OldSnapshotIndex,
    JavaIndex,
)

UPSTREAM_DIR = upstream_path()

ensure_upstream_dir(BASE_DIR)
ensure_upstream_dir(VERSIONS_DIR)
ensure_upstream_dir(ASSETS_DIR)

sess = default_session()


def fetch_zipped_version(path, url):
    zip_path = f"{path}.zip"
    download_binary_file(sess, zip_path, url)
    with zipfile.ZipFile(zip_path) as z:
        for info in z.infolist():
            if info.filename.endswith(".json"):
                print(f"Found {info.filename} as version json")
                version_json = json.load(z.open(info))
                break

    assert version_json

    with open(path, "w", encoding="utf-8") as f:
        json.dump(version_json, f, sort_keys=True, indent=4)

    return version_json


def fetch_modified_version(path, version):
    r = sess.get(version.url)
    r.raise_for_status()
    version_json = r.json()

    version_json["releaseTime"] = version_json["releaseTime"] + "T00:00:00+02:00"
    version_json["time"] = version_json["releaseTime"]

    downloads = {
        "client": {"url": version.jar, "sha1": version.sha1, "size": version.size}
    }

    version_json["downloads"] = downloads
    version_json["type"] = "old_snapshot"

    with open(path, "w", encoding="utf-8") as f:
        json.dump(version_json, f, sort_keys=True, indent=4)

    return version_json


def fetch_version(path, url):
    r = sess.get(url)
    r.raise_for_status()
    version_json = r.json()

    with open(path, "w", encoding="utf-8") as f:
        json.dump(version_json, f, sort_keys=True, indent=4)

    return version_json


MOJANG_JAVA_URL = "https://piston-meta.mojang.com/v1/products/java-runtime/2ec0cc96c44e5a76b9c8b7c39df7210883d12871/all.json"


def update_javas():
    r = sess.get(MOJANG_JAVA_URL)
    r.raise_for_status()

    remote_javas = JavaIndex(__root__=r.json())

    java_manifest_path = os.path.join(UPSTREAM_DIR, JAVA_MANIFEST_FILE)

    remote_javas.write(java_manifest_path)


def fetch_version_concurrent(remote_versions, x):
    version = remote_versions.versions[x]
    print(
        "Updating "
        + version.id
        + " to timestamp "
        + version.release_time.strftime("%s")
    )
    fetch_version(os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json"), version.url)


def fetch_modified_version_concurrent(old_snapshots, x):
    version = old_snapshots.versions[x]
    old_snapshots_path = os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json")

    print("Updating old snapshot " + version.id)
    if not os.path.isfile(old_snapshots_path):
        fetch_modified_version(old_snapshots_path, version)
    else:
        print("Already have old snapshot " + version.id)


def main():
    # get the remote version list
    r = sess.get("https://piston-meta.mojang.com/mc/game/version_manifest_v2.json")
    r.raise_for_status()

    remote_versions = MojangIndexWrap(MojangIndex(**r.json()))
    remote_ids = set(remote_versions.versions.keys())

    version_manifest_path = os.path.join(UPSTREAM_DIR, VERSION_MANIFEST_FILE)

    if os.path.exists(version_manifest_path):
        # get the local version list
        current_versions = MojangIndexWrap(
            MojangIndex.parse_file(version_manifest_path)
        )
        local_ids = set(current_versions.versions.keys())

        # versions not present locally but present remotely are new
        pending_ids = remote_ids.difference(local_ids)

        for x in local_ids:
            remote_version = remote_versions.versions[x]
            local_version = current_versions.versions[x]
            if remote_version.time > local_version.time:
                pending_ids.add(x)
    else:
        pending_ids = remote_ids

    with concurrent.futures.ThreadPoolExecutor() as executor:
        for x in pending_ids:
            executor.submit(fetch_version_concurrent, remote_versions, x)

    # deal with experimental snapshots separately
    if os.path.exists(STATIC_EXPERIMENTS_FILE):
        experiments = ExperimentIndexWrap(
            ExperimentIndex.parse_file(STATIC_EXPERIMENTS_FILE)
        )
        experiment_ids = set(experiments.versions.keys())

        for x in experiment_ids:
            version = experiments.versions[x]
            experiment_path = os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json")

            print("Updating experiment " + version.id)
            if not os.path.isfile(experiment_path):
                fetch_zipped_version(experiment_path, version.url)
            else:
                print("Already have experiment " + version.id)

    # deal with old snapshots
    if os.path.exists(STATIC_OLD_SNAPSHOTS_FILE):
        old_snapshots = OldSnapshotIndexWrap(
            OldSnapshotIndex.parse_file(STATIC_OLD_SNAPSHOTS_FILE)
        )
        old_snapshots_ids = set(old_snapshots.versions.keys())

        with concurrent.futures.ThreadPoolExecutor() as executor:
            for x in old_snapshots_ids:
                executor.submit(fetch_modified_version_concurrent, old_snapshots, x)

    remote_versions.index.write(version_manifest_path)

    print("Getting Mojang Java runtime manfest")
    update_javas()


if __name__ == "__main__":
    main()