From ac5171ea4411803c731ad6e5ea3d7cc8d0f49208 Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Wed, 10 May 2023 02:38:16 -0700 Subject: [PATCH 01/22] feat: fetch java metadata Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- meta/common/java.py | 10 + meta/common/mojang.py | 2 + meta/model/__init__.py | 4 +- meta/model/java.py | 439 +++++++++++++++++++++++++++++++++++++++++ meta/model/mojang.py | 42 ++++ updateJava.py | 157 +++++++++++++++ updateMojang.py | 17 ++ 7 files changed, 669 insertions(+), 2 deletions(-) create mode 100644 meta/common/java.py create mode 100644 meta/model/java.py create mode 100644 updateJava.py diff --git a/meta/common/java.py b/meta/common/java.py new file mode 100644 index 0000000..8de3c47 --- /dev/null +++ b/meta/common/java.py @@ -0,0 +1,10 @@ +from os.path import join + +BASE_DIR = "java_runtime" + +RELEASE_FILE = join(BASE_DIR, "releases.json") +ADOPTIUM_DIR = join(BASE_DIR, "adoptium") +AZUL_DIR = join(BASE_DIR, "azul") + +ADOPTIUM_VERSIONS_DIR = join(ADOPTIUM_DIR, "versions") +AZUL_VERSIONS_DIR = join(AZUL_DIR, "versions") \ No newline at end of file diff --git a/meta/common/mojang.py b/meta/common/mojang.py index dc80f44..562ae59 100644 --- a/meta/common/mojang.py +++ b/meta/common/mojang.py @@ -14,3 +14,5 @@ LIBRARY_PATCHES_FILE = join(BASE_DIR, "library-patches.json") MINECRAFT_COMPONENT = "net.minecraft" LWJGL_COMPONENT = "org.lwjgl" LWJGL3_COMPONENT = "org.lwjgl3" + +JAVA_MANIFEST_FILE = join(BASE_DIR, "java_all.json") \ No newline at end of file diff --git a/meta/model/__init__.py b/meta/model/__init__.py index 0246cdb..0428b7f 100644 --- a/meta/model/__init__.py +++ b/meta/model/__init__.py @@ -129,7 +129,7 @@ class GradleSpecifier: class MetaBase(pydantic.BaseModel): - def dict(self, **kwargs) -> Dict[str, Any]: + def dict(self, **kwargs: Any) -> Dict[str, Any]: for k in ["by_alias"]: if k in kwargs: del kwargs[k] @@ -145,7 +145,7 @@ class MetaBase(pydantic.BaseModel): exclude_none=True, sort_keys=True, by_alias=True, indent=4, **kwargs ) - def write(self, file_path): + def write(self, file_path: str): with open(file_path, "w") as f: f.write(self.json()) diff --git a/meta/model/java.py b/meta/model/java.py new file mode 100644 index 0000000..d4d2e00 --- /dev/null +++ b/meta/model/java.py @@ -0,0 +1,439 @@ +from . import ( + MetaBase, +) +from pydantic import Field +from datetime import datetime +from enum import Enum +from typing import Optional, List, Dict, Any, Iterator, Iterable, NamedTuple +from collections import namedtuple +from urllib.parse import urljoin, urlencode, urlparse, urlunparse + +# namedtuple to match the internal signature of urlunparse + + +class URLComponents(NamedTuple): + scheme: str + netloc: str + url: str + path: str + query: str + fragment: str + + +class APIQuery(MetaBase): + + def to_query(self): + set_parts: dict[str, Any] = {} + for key, value in self.dict().items(): + if value is not None: + if isinstance(value, Enum): + set_parts[key] = value.value + elif isinstance(value, list): + if len(value) > 0: + set_parts[key] = value + elif isinstance(value, datetime): + set_parts[key] = value.isoformat() + else: + set_parts[key] = value + return urlencode(set_parts, doseq=True) + + +class AdoptiumJvmImpl(Enum): + Hostspot = "hotspot" + + +class AdoptiumVendor(Enum): + Eclipse = "eclipse" + + +class AdoptiumArchitecture(Enum): + X64 = "x64" + X86 = "x86" + X32 = "x32" + Ppc64 = "ppc64" + Ppc64le = "ppc64le" + S390x = "s390x" + Aarch64 = "aarch64" + Arm = "arm" + Sparcv9 = "sparcv9" + Riscv64 = "riscv64" + + +class AdoptiumReleaseType(Enum): + GenralAccess = "ga" + EarlyAccess = "ea" + + +class AdoptiumSortMethod(Enum): + Default = "DEFAULT" + Date = "DATE" + + +class AdoptiumSortOrder(Enum): + Asc = "ASC" + Desc = "DESC" + + +class AdoptiumImageType(Enum): + Jdk = "jdk" + Jre = "jre" + Testimage = "testimage" + Debugimage = "debugimage" + Staticlibs = "staticlibs" + Sources = "sources" + Sbom = "sbom" + + +class AdoptiumHeapSize(Enum): + Normal = "normal" + Large = "large" + + +class AdoptiumProject(Enum): + Jdk = "jdk" + Valhalla = "valhalla" + Metropolis = "metropolis" + Jfr = "jfr" + Shenandoah = "shenandoah" + + +class AdoptiumCLib(Enum): + Musl = "musl" + Glibc = "Glibc" + + +class AdoptiumOs(Enum): + Linux = "linux" + Windows = "windows" + Mac = "mac" + Solaris = "solaris" + Aix = "aix" + AlpineLinux = "alpine-linux" + + +ADOPTIUM_API_BASE = " https://api.adoptium.net" +ADOPTIUM_API_FEATURE_RELEASES = f"{ADOPTIUM_API_BASE}/v3/assets/feature_releases/{{feature_version}}/{{release_type}}" +# ?image_type={{image_type}}&heap_size={{heap_size}}&project={{project}}&vender={{vender}}&page_size={{page_size}}&page={{page}}&sort_method={{sort_method}}&sort_order={{sort_order}} +ADOPTIUM_API_AVAILABLE_RELEASES = f"{ADOPTIUM_API_BASE}/v3/info/available_releases" + + +class AdoptiumAPIFeatureReleasesQuery(APIQuery): + architecture: Optional[AdoptiumArchitecture] = None + before: Optional[datetime] = None + c_lib: Optional[AdoptiumCLib] = None + heap_size: Optional[AdoptiumHeapSize] = AdoptiumHeapSize.Normal + image_type: Optional[AdoptiumImageType] = None + jvm_impl: Optional[AdoptiumJvmImpl] = None + os: Optional[AdoptiumOs] = None + page_size: int = 10 + page: int = 0 + project: Optional[AdoptiumProject] = AdoptiumProject.Jdk + sort_method: Optional[AdoptiumSortMethod] = AdoptiumSortMethod.Default + sort_order: Optional[AdoptiumSortOrder] = AdoptiumSortOrder.Desc + vender: Optional[AdoptiumVendor] = AdoptiumVendor.Eclipse + + +def adoptiumAPIFeatureReleases( + feature: int, + release_type: AdoptiumReleaseType = AdoptiumReleaseType.GenralAccess, + query: AdoptiumAPIFeatureReleasesQuery = AdoptiumAPIFeatureReleasesQuery() +): + url = urlparse(ADOPTIUM_API_FEATURE_RELEASES.format( + feature_version=feature, + release_type=release_type.value, + )) + return urlunparse(url._replace(query=query.to_query())) + + +class AdoptiumAvailableReleases(MetaBase): + available_releases: list[int] + available_lts_releases: list[int] + most_recent_lts: Optional[int] + most_recent_feature_release: Optional[int] + most_recent_feature_version: Optional[int] + tip_version: Optional[int] + + +class AdoptiumFile(MetaBase): + name: str + link: str + size: Optional[int] + + +class AdoptiumPackage(AdoptiumFile): + checksum: Optional[str] + checksum_link: Optional[str] + signature_link: Optional[str] + download_count: Optional[int] + metadata_link: Optional[str] + + +class AdoptiumBinary(MetaBase): + os: str + architecture: AdoptiumArchitecture + image_type: AdoptiumImageType + c_lib: Optional[AdoptiumCLib] + jvm_impl: AdoptiumJvmImpl + package: Optional[AdoptiumPackage] + installer: Optional[AdoptiumPackage] + heap_size: AdoptiumHeapSize + download_count: Optional[int] + updated_at: datetime + scm_ref: Optional[str] + project: AdoptiumProject + + +class AdoptiumVersion(MetaBase): + major: Optional[int] + minor: Optional[int] + security: Optional[int] + patch: Optional[int] + pre: Optional[str] + adopt_build_number: Optional[int] + semver: str + openjdk_version: str + build: Optional[int] + optional: Optional[str] + + +class AdoptiumRelease(MetaBase): + release_id: str = Field(alias="id") + release_link: str + release_name: str + timestamp: datetime + updated_at: datetime + binaries: list[AdoptiumBinary] + download_count: Optional[int] + release_type: str + vendor: AdoptiumVendor + version_data: AdoptiumVersion + source: Optional[AdoptiumFile] + release_notes: Optional[AdoptiumFile] + + +class AdoptiumReleasesWrap(MetaBase): + releases: list[AdoptiumRelease] + + +class AzulProduct(Enum): + Zulu = "zulu" + + +class AzulAvailabilityType(Enum): + SA = "SA" + CA = "CA" + NV = "NV" + _LA = "LA" + + +class AzulJavaPackageType(Enum): + Jdk = "jdk" + Jre = "jre" + + +class AzulReleaseType(Enum): + CPU = "CPU" + PSU = "PSU" + LU = "LU" + + +class AzulOs(Enum): + Linux = "linux" + Macos = "macos" + Qnx = "qnx" + Windows = "windows" + Solaris = "solaris" + + +class AzulLibCType(Enum): + Glibc = "glibc" + Uclibc = "uclibc" + Musl = "musl" + + +class AzulCPUGen(Enum): + V5 = "v5" + V6kV6kz = "v6k_v6kz" + V6t2 = "v6t2" + V7 = "v7" + V8 = "v8" + + +class AzulArch(Enum): + Arm = "arm" + X86 = "x86" + Mips = "mips" + Ppc = "ppc" + Sparcv9 = "sparcv9" + Sparc = "sparc" + + +class AzulHwBitness(Enum): + X32 = 32 + X64 = 64 + + +class AzulAbi(Enum): + HardFloat = "hard_float" + SoftFloat = "soft_float" + Spe = "spe" + Any = "any" + + +class AzulArchiveType(Enum): + Deb = "deb" + Rpm = "rpm" + Dmg = "dmg" + Targz = "tar.gz" + Zip = "zip" + Cab = "cab" + Msi = "msi" + + +class AzulReleaseStatus(Enum): + Eval = "eval" + Ea = "ea" + Ga = "ga" + Both = "both" + + +class AzulSupportTerm(Enum): + Sts = "sts" + Mts = "mts" + Lts = "lts" + + +class AzulCertifications(Enum): + Tck = "tck" + _Aqavit = "aqavit" + none = "none" + + +class AzulSignatureType(Enum): + Openpgp = "openpgp" + + +class AzulOsQueryParam(Enum): + Macos = "macos" + Windows = "windows" + Linux = "linux" + LinuxMusl = "linux-musl" + LinuxGlibc = "linux-glibc" + Qnx = "qnx" + Solaris = "solaris" + + +class AzulArchQueryParam(Enum): + X86 = "x86" + X64 = "x64" + Amd64 = "amd64" + I686 = "i686" + Arm = "arm" + Aarch64 = "aarch64" + Aarch32 = "aarch32" + Aarch32sf = "aarch32sf" + Aarch32hf = "aarch32hf" + Ppc = "ppc" + Ppc64 = "ppc64" + Ppc64hf = "ppc64hf" + Ppc32 = "ppc32" + Ppc32spe = "ppc32spe" + Ppc32hf = "ppc32hf" + Sparc = "sparc" + Sparc32 = "sparc32" + Sparcv9 = "sparcv9" + Sparcv9_64 = "sparcv9-64" + + +AZUL_API_BASE = "https://api.azul.com/metadata/v1" +AZUL_API_PACKAGES = f"{AZUL_API_BASE}/zulu/packages/" +AZUL_API_PACKAGE_DETAIL = f"{AZUL_API_BASE}/zulu/packages/{{package_uuid}}" + + +class AzulApiPackagesQuery(APIQuery): + java_version: Optional[str] = None + os: Optional[AzulOsQueryParam] = None + arch: Optional[AzulArchQueryParam] = None + archive_type: Optional[AzulArchiveType] = None + java_package_type: Optional[AzulJavaPackageType] = None + javafx_bundled: Optional[bool] = None + crac_supported: Optional[bool] = None + support_term: Optional[AzulSupportTerm] = None + release_type: Optional[AzulReleaseType] = None + latest: Optional[bool] = None + distro_version: Optional[str] = None + java_package_features: list[str] = [] + release_status: Optional[AzulReleaseStatus] = None + availability_types: list[AzulAvailabilityType] = None + certifications: list[AzulCertifications] = None + include_fields: list[str] = [] + page: int = 0 + page_size: int = 100 + + +def azulApiPackages(query: AzulApiPackagesQuery = AzulApiPackagesQuery()): + url = urlparse(AZUL_API_PACKAGES) + return urlunparse(url._replace(query=query.to_query())) + + +def azulApiPackageDetail(package_uuid: str): + return AZUL_API_PACKAGE_DETAIL.format(package_uuid=package_uuid) + + +class ZuluSignatureDetail(MetaBase): + type: AzulSignatureType + url: str + details: dict[str, Any] + signature_index: int + signature: str + + +class ZuluPackageDetail(MetaBase): + package_uuid: str + name: Optional[str] + md5_hash: Optional[str] + sha256_hash: Optional[str] + build_date: datetime + last_modified: datetime + download_url: str + product: AzulProduct + availability_type: AzulAvailabilityType + java_version: list[int] + openjdk_build_number: Optional[int] + java_package_type: AzulJavaPackageType + javafx_bundled: bool + release_type: AzulReleaseType + os: AzulOs + lib_c_type: Optional[AzulLibCType] + cpu_gen: Optional[list[AzulCPUGen]] + arch: AzulArch + hw_bitness: AzulHwBitness + abi: AzulAbi + archive_type: AzulArchiveType + release_status: AzulReleaseStatus + support_term: AzulSupportTerm + certifications: Optional[list[AzulCertifications]] + latest: Optional[bool] + size: int + distro_version: list[int] + signatures: list[ZuluSignatureDetail] + + +class ZuluPackageList(MetaBase): + package_uuid: str + name: Optional[str] + java_version: list[int] + openjdk_build_number: Optional[int] + latest: Optional[bool] + download_url: str + product: Optional[AzulProduct] + distro_version: list[int] + availability_type: Optional[AzulAvailabilityType] + + +class ZuluPackagesListWrap(MetaBase): + packages: list[ZuluPackageList] + + +class ZuluPackagesDetailListWrap(MetaBase): + packages: list[ZuluPackageDetail] diff --git a/meta/model/mojang.py b/meta/model/mojang.py index 2e35634..ce178ef 100644 --- a/meta/model/mojang.py +++ b/meta/model/mojang.py @@ -190,6 +190,48 @@ class JavaVersion(MetaBase): major_version: int = Field(8, alias="majorVersion") +class JavaIndexAvailability(MetaBase): + group: int + progress: int + + +class JavaIndexManifest(MetaBase): + sha1: str + size: int + url: str + + +class JavaIndexVersion(MetaBase): + name: str + released: datetime + + +class JavaRuntime(MetaBase): + availability: JavaIndexAvailability + manifest: JavaIndexManifest + version: JavaIndexVersion + + +class JavaIndexEntry(MetaBase): + java_runtime_alpha: list[JavaRuntime] = Field(alias="java-runtime-alpha") + java_runtime_beta: list[JavaRuntime] = Field(alias="java-runtime-beta") + java_runtime_gamma: list[JavaRuntime] = Field(alias="java-runtime-gamma") + jre_legacy: list[JavaRuntime] = Field(alias="jre-legacy") + minecraft_java_exe: list[JavaRuntime] = Field(alias="minecraft-java-exe") + + +class JavaIndex(MetaBase): + gamecore: JavaIndexEntry + linux: JavaIndexEntry + linux_i386: JavaIndexEntry = Field(alias="linux-i386") + mac_os: JavaIndexEntry = Field(alias="mac-os") + mac_os_arm64: JavaIndexEntry = Field(alias="mac-os-arm64") + windows_arm64: JavaIndexEntry = Field(alias="windows-arm64") + windows_x64: JavaIndexEntry = Field(alias="windows-x64") + windows_x86: JavaIndexEntry = Field(alias="windows-x86") + + + class MojangVersion(MetaBase): @validator("minimum_launcher_version") def validate_minimum_launcher_version(cls, v): diff --git a/updateJava.py b/updateJava.py new file mode 100644 index 0000000..4eb6ffc --- /dev/null +++ b/updateJava.py @@ -0,0 +1,157 @@ +import json +import os +import zipfile + +from meta.common import upstream_path, ensure_upstream_dir, static_path, default_session +from meta.common.java import ( + BASE_DIR, + ADOPTIUM_DIR, + AZUL_DIR, + ADOPTIUM_VERSIONS_DIR, + AZUL_VERSIONS_DIR, +) +from meta.model.java import ( + ADOPTIUM_API_AVAILABLE_RELEASES, + adoptiumAPIFeatureReleases, + AdoptiumImageType, + AdoptiumAPIFeatureReleasesQuery, + AdoptiumAvailableReleases, + AdoptiumRelease, + AdoptiumReleasesWrap, + azulApiPackages, + AzulApiPackagesQuery, + ZuluPackageList, + ZuluPackagesListWrap, + AzulArchiveType, + AzulReleaseStatus, + AzulAvailabilityType, + AzulJavaPackageType, + azulApiPackageDetail, + ZuluPackageDetail, + ZuluPackagesDetailListWrap, +) + +UPSTREAM_DIR = upstream_path() +STATIC_DIR = static_path() + +ensure_upstream_dir(BASE_DIR) +ensure_upstream_dir(ADOPTIUM_DIR) +ensure_upstream_dir(AZUL_DIR) +ensure_upstream_dir(ADOPTIUM_VERSIONS_DIR) +ensure_upstream_dir(AZUL_VERSIONS_DIR) + + +sess = default_session() + + +def main(): + print("Getting Adoptium Release Manifests ") + r = sess.get(ADOPTIUM_API_AVAILABLE_RELEASES) + r.raise_for_status() + + available = AdoptiumAvailableReleases(**r.json()) + + available_releases_file = os.path.join( + UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json") + available.write(available_releases_file) + + for feature in available.available_releases: + print("Getting Manifests for Adoptium feature release:", feature) + page = 0 + page_size = 10 + + releases_for_feature: list[AdoptiumRelease] = [] + + while True: + query = AdoptiumAPIFeatureReleasesQuery( + image_type=AdoptiumImageType.Jre, page_size=page_size, page=page) + api_call = adoptiumAPIFeatureReleases(feature, query=query) + print("Fetching Page:", page, api_call) + r_rls = sess.get(api_call) + if r_rls.status_code == 404: + break + else: + r_rls.raise_for_status() + + releases = list(AdoptiumRelease(**rls) for rls in r_rls.json()) + releases_for_feature.extend(releases) + + if len(r_rls.json()) < page_size: + break + page += 1 + + print("Total Adoptium releases for feature:", len(releases_for_feature)) + releases = AdoptiumReleasesWrap(releases=releases_for_feature) + feature_file = os.path.join( + UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, "{}.json".format(feature)) + releases.write(feature_file) + + print("Getting Azul Release Manifests") + zulu_packages: list[ZuluPackageList] = [] + page = 1 + page_size = 100 + while True: + + query = AzulApiPackagesQuery( + archive_type=AzulArchiveType.Zip, + release_status=AzulReleaseStatus.Ga, + availability_types=[AzulAvailabilityType.CA], + java_package_type=AzulJavaPackageType.Jre, + javafx_bundled=False, + page=page, + page_size=page_size) + api_call = azulApiPackages(query=query) + + print("Processing Page:", page, api_call) + + r = sess.get(api_call) + if r.status_code == 404: + break + else: + r.raise_for_status() + + packages = list(ZuluPackageList(**pkg) for pkg in r.json()) + zulu_packages.extend(packages) + if len(packages) < page_size: + break + page += 1 + + print("Total Azul Packages:", len(zulu_packages)) + packages = ZuluPackagesListWrap(packages=zulu_packages) + azul_manifest_file = os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") + packages.write(azul_manifest_file) + + azul_major_versions: dict[int, ZuluPackagesListWrap] = {} + + for pkg in packages.packages: + + major_version = pkg.java_version[0] + if major_version not in azul_major_versions: + azul_major_versions[major_version] = ZuluPackagesListWrap( + packages=[]) + + azul_major_versions[major_version].packages.append(pkg) + + pkg_file = os.path.join( + UPSTREAM_DIR, AZUL_VERSIONS_DIR, "{}.json".format(pkg.package_uuid)) + if os.path.exists(pkg_file) and os.path.isfile(pkg_file): + pass + else: + + api_call = azulApiPackageDetail(pkg.package_uuid) + print("Fetching Azul package manifest:", pkg.package_uuid) + r_pkg = sess.get(api_call) + r_pkg.raise_for_status() + + pkg_detail = ZuluPackageDetail(**r_pkg.json()) + pkg_detail.write(pkg_file) + + for major in azul_major_versions: + major_file = os.path.join( + UPSTREAM_DIR, AZUL_VERSIONS_DIR, "{}.json".format(major)) + azul_major_versions[major].write(major_file) + + +if __name__ == "__main__": + main() + # diff --git a/updateMojang.py b/updateMojang.py index 8d3a229..91ca558 100755 --- a/updateMojang.py +++ b/updateMojang.py @@ -11,6 +11,7 @@ from meta.common.mojang import ( ASSETS_DIR, STATIC_EXPERIMENTS_FILE, STATIC_OLD_SNAPSHOTS_FILE, + JAVA_MANIFEST_FILE, ) from meta.model.mojang import ( MojangIndexWrap, @@ -19,6 +20,7 @@ from meta.model.mojang import ( ExperimentIndexWrap, OldSnapshotIndexWrap, OldSnapshotIndex, + JavaIndex, ) UPSTREAM_DIR = upstream_path() @@ -81,6 +83,19 @@ def fetch_version(path, url): return version_json +MOJANG_JAVA_URL = "https://piston-meta.mojang.com/v1/products/java-runtime/2ec0cc96c44e5a76b9c8b7c39df7210883d12871/all.json" + +def update_javas(): + r = sess.get(MOJANG_JAVA_URL) + r.raise_for_status() + + remote_javas = JavaIndex(**r.json()) + + java_manifest_path = os.path.join(UPSTREAM_DIR, JAVA_MANIFEST_FILE) + + remote_javas.write(java_manifest_path) + + def main(): # get the remote version list r = sess.get("https://piston-meta.mojang.com/mc/game/version_manifest_v2.json") @@ -160,6 +175,8 @@ def main(): remote_versions.index.write(version_manifest_path) + print("Getting Mojang Java runtime manfest") + update_javas() if __name__ == "__main__": main() From 651ff8356ce125c6ee41da5edd9e981492fa7d1a Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Wed, 10 May 2023 20:55:45 -0700 Subject: [PATCH 02/22] feat: generate java componate Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- generateJava.py | 277 +++++++++++++++++++++++++++++++++++++++++++ meta/common/java.py | 4 +- meta/model/enum.py | 32 +++++ meta/model/java.py | 207 ++++++++++++++++++++++++-------- meta/model/mojang.py | 64 ++++++---- updateJava.py | 41 +++---- updateMojang.py | 2 +- 7 files changed, 533 insertions(+), 94 deletions(-) create mode 100644 generateJava.py create mode 100644 meta/model/enum.py diff --git a/generateJava.py b/generateJava.py new file mode 100644 index 0000000..11a449a --- /dev/null +++ b/generateJava.py @@ -0,0 +1,277 @@ +import copy +import hashlib +import os +from collections import defaultdict, namedtuple +from operator import attrgetter +from pprint import pprint +from packaging import version as pversion +from typing import Optional, List + +from meta.common import ensure_component_dir, launcher_path, upstream_path, static_path + +from meta.common.java import ( + JAVA_COMPONENT, + ADOPTIUM_DIR, + ADOPTIUM_VERSIONS_DIR, + AZUL_DIR, + AZUL_VERSIONS_DIR +) +from meta.model.java import ( + JavaRuntimeOS, + JavaRuntimeMap, + JavaRuntimeMeta, + JavaVersionMeta, + JavaChecksumMeta, + JavaChecksumType, + JavaRuntimeDownloadType, + AdoptiumAvailableReleases, + AdoptiumReleases, + AdoptiumRelease, + AdoptiumBinary, + ZuluPackages, + ZuluPackageDetail, + AzulArch, +) + +from meta.common.mojang import ( + + JAVA_MANIFEST_FILE, +) + +from meta.model.mojang import ( + JavaIndex, + MojangJavaComponent, + MojangJavaOsName, + MojangJavaRuntime, +) + +LAUNCHER_DIR = launcher_path() +UPSTREAM_DIR = upstream_path() +STATIC_DIR = static_path() + +ensure_component_dir(JAVA_COMPONENT) + +MOJANG_JAVA_OS_NAMES = [ + "gamecore", + "linux", + "linux-i386", + "mac-os", + "mac-os-arm64", + "windows-arm64", + "windows-x64", + "windows-x86", +] + +MOJANG_OS_ARCHITECTURES = [ + "x64", + "x86", + "arm64", + "arm32", +] + +MOJANG_OS_ARCHITECTURE_TRANSLATIONS = { + 64: "x64", + 32: "x86", + "x32": "x86", + "i386": "x86", + "aarch64": "arm64", + "x86_64": "x64", + "arm": "arm32" +} + + +def translate_arch(arch: str | int): + if isinstance(arch, str): + arch = arch.lower() + if arch in MOJANG_OS_ARCHITECTURES: + return arch + elif arch in MOJANG_OS_ARCHITECTURE_TRANSLATIONS: + return MOJANG_OS_ARCHITECTURE_TRANSLATIONS[arch] + else: + return None + +MOJANG_OS_NAMES = [ + "mac-os", + "linux", + "windows", +] + +MOJANG_OS_TRANSLATIONS = { + "osx": "mac-os", + "mac": "mac-os", + "macos": "mac-os", +} + +def translate_os(os: str): + os = os.lower() + if os in MOJANG_OS_NAMES: + return os + elif os in MOJANG_OS_TRANSLATIONS: + return MOJANG_OS_TRANSLATIONS[os] + else: + return None + + +def mojang_os_to_java_os(mojang_os: MojangJavaOsName) -> JavaRuntimeOS: + if mojang_os == MojangJavaOsName.Linux: + return JavaRuntimeOS.LinuxX64 + elif mojang_os == MojangJavaOsName.Linuxi386: + return JavaRuntimeOS.LinuxX86 + elif mojang_os == MojangJavaOsName.MacOs: + return JavaRuntimeOS.MacOsX64 + elif mojang_os == MojangJavaOsName.MacOSArm64: + return JavaRuntimeOS.MacOsArm64 + elif mojang_os == MojangJavaOsName.WindowsArm64: + return JavaRuntimeOS.WindowsArm64 + elif mojang_os == MojangJavaOsName.WindowsX64: + return JavaRuntimeOS.WindowsX64 + elif mojang_os == MojangJavaOsName.WindowsX86: + return JavaRuntimeOS.WindowsX86 + else: + return JavaRuntimeOS.Unknown + + +def mojang_runtime_to_java_runtime(mojang_runtime: MojangJavaRuntime) -> JavaRuntimeMeta: + return JavaRuntimeMeta( + name=f"mojang_jre_{mojang_runtime.version.name}", + vender="mojang", + url=mojang_runtime.manifest.url, + release_time=mojang_runtime.version.released, + checksum=JavaChecksumMeta( + type=JavaChecksumType.Sha1, + hash=mojang_runtime.manifest.sha1), + recomended=True, + download_type=JavaRuntimeDownloadType.Manifest) + +def adoptium_release_binary_to_java_runtime(rls: AdoptiumRelease, binary: AdoptiumBinary) -> JavaRuntimeMeta: + version = JavaVersionMeta( + major=rls.version_data.major, + minor=rls.version_data.minor, + security=rls.version_data.security, + build=rls.version_data.build + ) + rls_name = f"{rls.vendor}_temurin_{binary.image_type}{version}" + return JavaRuntimeMeta( + name=rls_name, + vender=rls.vendor, + url=binary.package.link, + release_time=rls.timestamp, + checksum=JavaChecksumMeta( + type=JavaChecksumType.Sha256, + hash=binary.package.checksum), + recomended=False, + download_type=JavaRuntimeDownloadType.Archive + ) + +def azul_package_to_java_runtime(pkg: ZuluPackageDetail) -> JavaRuntimeMeta: + version_parts = copy.copy(pkg.java_version) + while len(version_parts) < 4: + version_parts.append(None) + + version = JavaVersionMeta( + major=version_parts[0], + minor=version_parts[1], + security=version_parts[2], + build=version_parts[3] + ) + + rls_name = f"azul_{pkg.product}_{pkg.java_package_type}{version}" + + return JavaRuntimeMeta( + name=rls_name, + vender="azul", + url=pkg.download_url, + release_time=pkg.build_date, + checksum=JavaChecksumMeta( + type=JavaChecksumType.Sha256, + hash=pkg.sha256_hash), + recomended=False, + download_type=JavaRuntimeDownloadType.Archive + ) + +def main(): + + javas: dict[int, JavaRuntimeMap] = {} + + def ensure_javamap(major: int): + if major not in javas: + javas[major] = JavaRuntimeMap() + + def add_java_runtime(runtime: JavaRuntimeMeta, major: int, java_os: JavaRuntimeOS): + ensure_javamap(major) + print(f"Regestering runtime: {runtime.name} for Java {major} {java_os}") + javas[major][java_os].append(runtime) + + print("Processing Mojang Javas") + mojang_java_manifest = JavaIndex.parse_file( + os.path.join(UPSTREAM_DIR, JAVA_MANIFEST_FILE) + ) + for mojang_os_name in mojang_java_manifest: + if mojang_os_name == MojangJavaOsName.Gamecore: + continue + java_os = mojang_os_to_java_os(mojang_os_name) + for comp in mojang_java_manifest[mojang_os_name]: + mojang_runtimes = mojang_java_manifest[mojang_os_name][comp] + for mojang_runtime in mojang_runtimes: + if comp == MojangJavaComponent.JreLegacy: + major = 8 + else: + major = int(mojang_runtime.version.name.partition('.')[0]) + runtime = mojang_runtime_to_java_runtime(mojang_runtime) + add_java_runtime(runtime, major, java_os) + + print("Processing Adoptium Releases") + adoptium_available_releases = AdoptiumAvailableReleases.parse_file( + os.path.join(UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json") + ) + for major in adoptium_available_releases.available_releases: + adoptium_releases = AdoptiumReleases.parse_file( + os.path.join(UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, f"java{major}.json") + ) + for rls in adoptium_releases: + for binary in rls.binaries: + if binary.package is None: + continue + binary_arch = translate_arch(str(binary.architecture)) + binary_os = translate_os(str(binary.os)) + if binary_arch is None or binary_os is None: + print(f"Ignoring release for {binary.os} {binary.architecture}") + continue + + java_os = JavaRuntimeOS(f"{binary_os}-{binary_arch}") + runtime = adoptium_release_binary_to_java_runtime(rls, binary) + add_java_runtime(runtime, major, java_os) + + print("Processing Azul Packages") + azul_packages = ZuluPackages.parse_file( + os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") + ) + for pkg in azul_packages: + pkg_detail = ZuluPackageDetail.parse_file( + os.path.join(UPSTREAM_DIR, AZUL_VERSIONS_DIR, f"{pkg.package_uuid}.json") + ) + major = pkg_detail.java_version[0] + pkg_os = translate_os(str(pkg_detail.os)) + if pkg_detail.arch == AzulArch.Arm: + pkg_arch = translate_arch(f"{pkg_detail.arch}{pkg_detail.hw_bitness}") + elif pkg_detail.arch == AzulArch.X86: + pkg_arch = translate_arch(int(pkg_detail.hw_bitness)) + else: + pkg_arch = None + if pkg_arch is None or pkg_os is None: + print(f"Ignoring release for {pkg_detail.os} {pkg_detail.arch}_{pkg_detail.hw_bitness}") + continue + + java_os = JavaRuntimeOS(f"{pkg_os}-{pkg_arch}") + runtime = azul_package_to_java_runtime(pkg_detail) + add_java_runtime(runtime, major, java_os) + + for major, runtimes in javas.items(): + for java_os in runtimes: + print(f"Total runtimes for Java {major} {java_os}:", len(runtimes[java_os])) + runtimes_file = os.path.join(LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") + runtimes.write(runtimes_file) + + +if __name__ == "__main__": + main() diff --git a/meta/common/java.py b/meta/common/java.py index 8de3c47..541d2fb 100644 --- a/meta/common/java.py +++ b/meta/common/java.py @@ -7,4 +7,6 @@ ADOPTIUM_DIR = join(BASE_DIR, "adoptium") AZUL_DIR = join(BASE_DIR, "azul") ADOPTIUM_VERSIONS_DIR = join(ADOPTIUM_DIR, "versions") -AZUL_VERSIONS_DIR = join(AZUL_DIR, "versions") \ No newline at end of file +AZUL_VERSIONS_DIR = join(AZUL_DIR, "versions") + +JAVA_COMPONENT = "net.minecraft.java" \ No newline at end of file diff --git a/meta/model/enum.py b/meta/model/enum.py new file mode 100644 index 0000000..08dda4b --- /dev/null +++ b/meta/model/enum.py @@ -0,0 +1,32 @@ +import enum + + +class StrEnum(str, enum.Enum): + """ + StrEnum is a Python ``enum.Enum`` that inherits from ``str``. The default + ``auto()`` behavior uses the member name as its value. + + Example usage:: + + class Example(StrEnum): + UPPER_CASE = auto() + lower_case = auto() + MixedCase = auto() + + assert Example.UPPER_CASE == "UPPER_CASE" + assert Example.lower_case == "lower_case" + assert Example.MixedCase == "MixedCase" + """ + + def __new__(cls, value, *args, **kwargs): + if not isinstance(value, (str, enum.auto)): + raise TypeError( + f"Values of StrEnums must be strings: {value!r} is a {type(value)}" + ) + return super().__new__(cls, value, *args, **kwargs) + + def __str__(self): + return str(self.value) + + def _generate_next_value_(name, *_): + return name diff --git a/meta/model/java.py b/meta/model/java.py index d4d2e00..a7907d9 100644 --- a/meta/model/java.py +++ b/meta/model/java.py @@ -3,7 +3,8 @@ from . import ( ) from pydantic import Field from datetime import datetime -from enum import Enum +from enum import IntEnum, Enum +from .enum import StrEnum from typing import Optional, List, Dict, Any, Iterator, Iterable, NamedTuple from collections import namedtuple from urllib.parse import urljoin, urlencode, urlparse, urlunparse @@ -11,6 +12,75 @@ from urllib.parse import urljoin, urlencode, urlparse, urlunparse # namedtuple to match the internal signature of urlunparse +class JavaRuntimeOS(StrEnum): + MacOsX64 = "mac-os-x64" + MacOsX86 = "mac-os-x86" # rare + MacOsArm64 = "mac-os-arm64" + # MacOsArm32 = "mac-os-arm32" # doesn't exsist + LinuxX64 = "linux-x64" + LinuxX86 = "linux-x86" + LinuxArm64 = "linux-arm64" + LinuxArm32 = "linux-arm32" + WindowsX64 = "windows-x64" + WindowsX86 = "windows-x86" + WindowsArm64 = "windows-arm64" + WindowsArm32 = "windows-arm32" + Unknown = "unknown" + + +class JavaRuntimeDownloadType(StrEnum): + Manifest = "manifest" + Archive = "archive" + + +class JavaVersionMeta(MetaBase): + major: int + minor: int + security: int + build: Optional[int] + + def __str__(self): + ver = f"{self.major}.{self.minor}.{self.security}" + if self.build is not None: + ver = f"{ver}+{self.build}" + return ver + + +class JavaChecksumType(StrEnum): + Sha1 = "sha1" + Sha256 = "sha256" + + +class JavaChecksumMeta(MetaBase): + type: JavaChecksumType + hash: str + + +class JavaRuntimeMeta(MetaBase): + name: str + vender: str + url: str + release_time: datetime = Field(alias="releaseTime") + checksum: Optional[JavaChecksumMeta] + recomended: bool + download_type: JavaRuntimeDownloadType = Field(alias="downloadType") + + +class JavaRuntimeMap(MetaBase): + __root__: dict[JavaRuntimeOS, list[JavaRuntimeMeta]] = { + os: [] for os in JavaRuntimeOS if os != JavaRuntimeOS.Unknown + } + + def __iter__(self) -> Iterator[JavaRuntimeOS]: + return iter(self.__root__) + + def __getitem__(self, item) -> list[JavaRuntimeMeta]: + return self.__root__[item] + + def __len__(self): + return len(self.__root__) + + class URLComponents(NamedTuple): scheme: str netloc: str @@ -21,7 +91,6 @@ class URLComponents(NamedTuple): class APIQuery(MetaBase): - def to_query(self): set_parts: dict[str, Any] = {} for key, value in self.dict().items(): @@ -38,15 +107,15 @@ class APIQuery(MetaBase): return urlencode(set_parts, doseq=True) -class AdoptiumJvmImpl(Enum): +class AdoptiumJvmImpl(StrEnum): Hostspot = "hotspot" -class AdoptiumVendor(Enum): +class AdoptiumVendor(StrEnum): Eclipse = "eclipse" -class AdoptiumArchitecture(Enum): +class AdoptiumArchitecture(StrEnum): X64 = "x64" X86 = "x86" X32 = "x32" @@ -59,22 +128,22 @@ class AdoptiumArchitecture(Enum): Riscv64 = "riscv64" -class AdoptiumReleaseType(Enum): +class AdoptiumReleaseType(StrEnum): GenralAccess = "ga" EarlyAccess = "ea" -class AdoptiumSortMethod(Enum): +class AdoptiumSortMethod(StrEnum): Default = "DEFAULT" Date = "DATE" -class AdoptiumSortOrder(Enum): +class AdoptiumSortOrder(StrEnum): Asc = "ASC" Desc = "DESC" -class AdoptiumImageType(Enum): +class AdoptiumImageType(StrEnum): Jdk = "jdk" Jre = "jre" Testimage = "testimage" @@ -84,12 +153,12 @@ class AdoptiumImageType(Enum): Sbom = "sbom" -class AdoptiumHeapSize(Enum): +class AdoptiumHeapSize(StrEnum): Normal = "normal" Large = "large" -class AdoptiumProject(Enum): +class AdoptiumProject(StrEnum): Jdk = "jdk" Valhalla = "valhalla" Metropolis = "metropolis" @@ -97,12 +166,12 @@ class AdoptiumProject(Enum): Shenandoah = "shenandoah" -class AdoptiumCLib(Enum): +class AdoptiumCLib(StrEnum): Musl = "musl" - Glibc = "Glibc" + Glibc = "glibc" -class AdoptiumOs(Enum): +class AdoptiumOs(StrEnum): Linux = "linux" Windows = "windows" Mac = "mac" @@ -128,20 +197,22 @@ class AdoptiumAPIFeatureReleasesQuery(APIQuery): page_size: int = 10 page: int = 0 project: Optional[AdoptiumProject] = AdoptiumProject.Jdk - sort_method: Optional[AdoptiumSortMethod] = AdoptiumSortMethod.Default - sort_order: Optional[AdoptiumSortOrder] = AdoptiumSortOrder.Desc - vender: Optional[AdoptiumVendor] = AdoptiumVendor.Eclipse + sort_method: Optional[AdoptiumSortMethod] = AdoptiumSortMethod.Default + sort_order: Optional[AdoptiumSortOrder] = AdoptiumSortOrder.Desc + vender: Optional[AdoptiumVendor] = AdoptiumVendor.Eclipse -def adoptiumAPIFeatureReleases( - feature: int, - release_type: AdoptiumReleaseType = AdoptiumReleaseType.GenralAccess, - query: AdoptiumAPIFeatureReleasesQuery = AdoptiumAPIFeatureReleasesQuery() +def adoptiumAPIFeatureReleasesUrl( + feature: int, + release_type: AdoptiumReleaseType = AdoptiumReleaseType.GenralAccess, + query: AdoptiumAPIFeatureReleasesQuery = AdoptiumAPIFeatureReleasesQuery(), ): - url = urlparse(ADOPTIUM_API_FEATURE_RELEASES.format( - feature_version=feature, - release_type=release_type.value, - )) + url = urlparse( + ADOPTIUM_API_FEATURE_RELEASES.format( + feature_version=feature, + release_type=release_type.value, + ) + ) return urlunparse(url._replace(query=query.to_query())) @@ -211,33 +282,42 @@ class AdoptiumRelease(MetaBase): release_notes: Optional[AdoptiumFile] -class AdoptiumReleasesWrap(MetaBase): - releases: list[AdoptiumRelease] +class AdoptiumReleases(MetaBase): + __root__: list[AdoptiumRelease] + + def __iter__(self) -> Iterator[AdoptiumRelease]: + return iter(self.__root__) + + def __getitem__(self, item) -> AdoptiumRelease: + return self.__root__[item] + + def append(self, rls: AdoptiumRelease): + self.__root__.append(rls) -class AzulProduct(Enum): +class AzulProduct(StrEnum): Zulu = "zulu" -class AzulAvailabilityType(Enum): +class AzulAvailabilityType(StrEnum): SA = "SA" CA = "CA" NV = "NV" _LA = "LA" -class AzulJavaPackageType(Enum): +class AzulJavaPackageType(StrEnum): Jdk = "jdk" Jre = "jre" -class AzulReleaseType(Enum): +class AzulReleaseType(StrEnum): CPU = "CPU" PSU = "PSU" LU = "LU" -class AzulOs(Enum): +class AzulOs(StrEnum): Linux = "linux" Macos = "macos" Qnx = "qnx" @@ -245,13 +325,13 @@ class AzulOs(Enum): Solaris = "solaris" -class AzulLibCType(Enum): +class AzulLibCType(StrEnum): Glibc = "glibc" Uclibc = "uclibc" Musl = "musl" -class AzulCPUGen(Enum): +class AzulCPUGen(StrEnum): V5 = "v5" V6kV6kz = "v6k_v6kz" V6t2 = "v6t2" @@ -259,7 +339,7 @@ class AzulCPUGen(Enum): V8 = "v8" -class AzulArch(Enum): +class AzulArch(StrEnum): Arm = "arm" X86 = "x86" Mips = "mips" @@ -268,19 +348,19 @@ class AzulArch(Enum): Sparc = "sparc" -class AzulHwBitness(Enum): +class AzulHwBitness(IntEnum): X32 = 32 X64 = 64 -class AzulAbi(Enum): +class AzulAbi(StrEnum): HardFloat = "hard_float" SoftFloat = "soft_float" Spe = "spe" Any = "any" -class AzulArchiveType(Enum): +class AzulArchiveType(StrEnum): Deb = "deb" Rpm = "rpm" Dmg = "dmg" @@ -290,30 +370,30 @@ class AzulArchiveType(Enum): Msi = "msi" -class AzulReleaseStatus(Enum): +class AzulReleaseStatus(StrEnum): Eval = "eval" Ea = "ea" Ga = "ga" Both = "both" -class AzulSupportTerm(Enum): +class AzulSupportTerm(StrEnum): Sts = "sts" Mts = "mts" Lts = "lts" -class AzulCertifications(Enum): +class AzulCertifications(StrEnum): Tck = "tck" _Aqavit = "aqavit" none = "none" -class AzulSignatureType(Enum): +class AzulSignatureType(StrEnum): Openpgp = "openpgp" -class AzulOsQueryParam(Enum): +class AzulOsQueryParam(StrEnum): Macos = "macos" Windows = "windows" Linux = "linux" @@ -323,7 +403,7 @@ class AzulOsQueryParam(Enum): Solaris = "solaris" -class AzulArchQueryParam(Enum): +class AzulArchQueryParam(StrEnum): X86 = "x86" X64 = "x64" Amd64 = "amd64" @@ -371,12 +451,12 @@ class AzulApiPackagesQuery(APIQuery): page_size: int = 100 -def azulApiPackages(query: AzulApiPackagesQuery = AzulApiPackagesQuery()): +def azulApiPackagesUrl(query: AzulApiPackagesQuery = AzulApiPackagesQuery()): url = urlparse(AZUL_API_PACKAGES) return urlunparse(url._replace(query=query.to_query())) -def azulApiPackageDetail(package_uuid: str): +def azulApiPackageDetailUrl(package_uuid: str): return AZUL_API_PACKAGE_DETAIL.format(package_uuid=package_uuid) @@ -431,9 +511,36 @@ class ZuluPackageList(MetaBase): availability_type: Optional[AzulAvailabilityType] -class ZuluPackagesListWrap(MetaBase): - packages: list[ZuluPackageList] +class ZuluPackages(MetaBase): + __root__: list[ZuluPackageList] + + def __iter__(self) -> Iterator[ZuluPackageList]: + return iter(self.__root__) + + def __getitem__(self, item) -> ZuluPackageList: + return self.__root__[item] + + def append(self, pkg: ZuluPackageList): + self.__root__.append(pkg) -class ZuluPackagesDetailListWrap(MetaBase): - packages: list[ZuluPackageDetail] +class ZuluPackagesDetail(MetaBase): + __root__: list[ZuluPackageDetail] + + def __iter__(self) -> Iterator[ZuluPackageDetail]: + return iter(self.__root__) + + def __getitem__(self, item) -> ZuluPackageDetail: + return self.__root__[item] + + def append(self, pkg: ZuluPackageDetail): + self.__root__.append(pkg) + + +MOJANG_OS_NAMES = ["mac-os", "linux", "windows"] + +MOJANG_OS_ARCHITECTURES = [ + "x64" "x86", + "arm64", + "arm32", +] diff --git a/meta/model/mojang.py b/meta/model/mojang.py index ce178ef..2e43899 100644 --- a/meta/model/mojang.py +++ b/meta/model/mojang.py @@ -1,5 +1,6 @@ from datetime import datetime from typing import Optional, List, Dict, Any, Iterator +from .enum import StrEnum from pydantic import validator, Field @@ -185,51 +186,70 @@ class MojangLogging(MetaBase): type: str +class MojangJavaComponent(StrEnum): + JreLegacy = "jre-legacy" + Alpha = "java-runtime-alpha" + Beta = "java-runtime-beta" + Gamma = "java-runtime-gamma" + Exe = "minecraft-java-exe" + + class JavaVersion(MetaBase): - component: str = "jre-legacy" + component: MojangJavaComponent = MojangJavaComponent.JreLegacy major_version: int = Field(8, alias="majorVersion") -class JavaIndexAvailability(MetaBase): +class MojangJavaIndexAvailability(MetaBase): group: int progress: int -class JavaIndexManifest(MetaBase): +class MojangJavaIndexManifest(MetaBase): sha1: str size: int url: str -class JavaIndexVersion(MetaBase): +class MojangJavaIndexVersion(MetaBase): name: str released: datetime -class JavaRuntime(MetaBase): - availability: JavaIndexAvailability - manifest: JavaIndexManifest - version: JavaIndexVersion +class MojangJavaRuntime(MetaBase): + availability: MojangJavaIndexAvailability + manifest: MojangJavaIndexManifest + version: MojangJavaIndexVersion -class JavaIndexEntry(MetaBase): - java_runtime_alpha: list[JavaRuntime] = Field(alias="java-runtime-alpha") - java_runtime_beta: list[JavaRuntime] = Field(alias="java-runtime-beta") - java_runtime_gamma: list[JavaRuntime] = Field(alias="java-runtime-gamma") - jre_legacy: list[JavaRuntime] = Field(alias="jre-legacy") - minecraft_java_exe: list[JavaRuntime] = Field(alias="minecraft-java-exe") +class MojangJavaIndexEntry(MetaBase): + __root__: dict[MojangJavaComponent, list[MojangJavaRuntime]] + + def __iter__(self) -> Iterator[MojangJavaComponent]: + return iter(self.__root__) + + def __getitem__(self, item) -> list[MojangJavaRuntime]: + return self.__root__[item] + + +class MojangJavaOsName(StrEnum): + Gamecore = "gamecore" + Linux = "linux" + Linuxi386 = "linux-i386" + MacOs = "mac-os" + MacOSArm64 = "mac-os-arm64" + WindowsArm64 = "windows-arm64" + WindowsX64 = "windows-x64" + WindowsX86 = "windows-x86" class JavaIndex(MetaBase): - gamecore: JavaIndexEntry - linux: JavaIndexEntry - linux_i386: JavaIndexEntry = Field(alias="linux-i386") - mac_os: JavaIndexEntry = Field(alias="mac-os") - mac_os_arm64: JavaIndexEntry = Field(alias="mac-os-arm64") - windows_arm64: JavaIndexEntry = Field(alias="windows-arm64") - windows_x64: JavaIndexEntry = Field(alias="windows-x64") - windows_x86: JavaIndexEntry = Field(alias="windows-x86") + __root__: dict[MojangJavaOsName, MojangJavaIndexEntry] + def __iter__(self) -> Iterator[MojangJavaOsName]: + return iter(self.__root__) + + def __getitem__(self, item) -> MojangJavaIndexEntry: + return self.__root__[item] class MojangVersion(MetaBase): diff --git a/updateJava.py b/updateJava.py index 4eb6ffc..ece9769 100644 --- a/updateJava.py +++ b/updateJava.py @@ -12,23 +12,23 @@ from meta.common.java import ( ) from meta.model.java import ( ADOPTIUM_API_AVAILABLE_RELEASES, - adoptiumAPIFeatureReleases, + adoptiumAPIFeatureReleasesUrl, AdoptiumImageType, AdoptiumAPIFeatureReleasesQuery, AdoptiumAvailableReleases, AdoptiumRelease, - AdoptiumReleasesWrap, - azulApiPackages, + AdoptiumReleases, + azulApiPackagesUrl, AzulApiPackagesQuery, ZuluPackageList, - ZuluPackagesListWrap, + ZuluPackages, AzulArchiveType, AzulReleaseStatus, AzulAvailabilityType, AzulJavaPackageType, - azulApiPackageDetail, + azulApiPackageDetailUrl, ZuluPackageDetail, - ZuluPackagesDetailListWrap, + ZuluPackagesDetail, ) UPSTREAM_DIR = upstream_path() @@ -65,7 +65,7 @@ def main(): while True: query = AdoptiumAPIFeatureReleasesQuery( image_type=AdoptiumImageType.Jre, page_size=page_size, page=page) - api_call = adoptiumAPIFeatureReleases(feature, query=query) + api_call = adoptiumAPIFeatureReleasesUrl(feature, query=query) print("Fetching Page:", page, api_call) r_rls = sess.get(api_call) if r_rls.status_code == 404: @@ -81,9 +81,9 @@ def main(): page += 1 print("Total Adoptium releases for feature:", len(releases_for_feature)) - releases = AdoptiumReleasesWrap(releases=releases_for_feature) + releases = AdoptiumReleases(__root__=releases_for_feature) feature_file = os.path.join( - UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, "{}.json".format(feature)) + UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, f"java{feature}.json") releases.write(feature_file) print("Getting Azul Release Manifests") @@ -100,7 +100,7 @@ def main(): javafx_bundled=False, page=page, page_size=page_size) - api_call = azulApiPackages(query=query) + api_call = azulApiPackagesUrl(query=query) print("Processing Page:", page, api_call) @@ -117,38 +117,39 @@ def main(): page += 1 print("Total Azul Packages:", len(zulu_packages)) - packages = ZuluPackagesListWrap(packages=zulu_packages) + packages = ZuluPackages(__root__=zulu_packages) azul_manifest_file = os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") packages.write(azul_manifest_file) - azul_major_versions: dict[int, ZuluPackagesListWrap] = {} + azul_major_versions: dict[int, ZuluPackages] = {} - for pkg in packages.packages: + for pkg in packages: major_version = pkg.java_version[0] if major_version not in azul_major_versions: - azul_major_versions[major_version] = ZuluPackagesListWrap( - packages=[]) + azul_major_versions[major_version] = ZuluPackagesDetail(__root__=[]) - azul_major_versions[major_version].packages.append(pkg) + pkg_file = os.path.join( - UPSTREAM_DIR, AZUL_VERSIONS_DIR, "{}.json".format(pkg.package_uuid)) + UPSTREAM_DIR, AZUL_VERSIONS_DIR, f"{pkg.package_uuid}.json") if os.path.exists(pkg_file) and os.path.isfile(pkg_file): - pass + pkg_detail = ZuluPackageDetail.parse_file(pkg_file) + azul_major_versions[major_version].append(pkg_detail) else: - api_call = azulApiPackageDetail(pkg.package_uuid) + api_call = azulApiPackageDetailUrl(pkg.package_uuid) print("Fetching Azul package manifest:", pkg.package_uuid) r_pkg = sess.get(api_call) r_pkg.raise_for_status() pkg_detail = ZuluPackageDetail(**r_pkg.json()) pkg_detail.write(pkg_file) + azul_major_versions[major_version].append(pkg_detail) for major in azul_major_versions: major_file = os.path.join( - UPSTREAM_DIR, AZUL_VERSIONS_DIR, "{}.json".format(major)) + UPSTREAM_DIR, AZUL_VERSIONS_DIR, f"java{major}.json") azul_major_versions[major].write(major_file) diff --git a/updateMojang.py b/updateMojang.py index 91ca558..03a77df 100755 --- a/updateMojang.py +++ b/updateMojang.py @@ -89,7 +89,7 @@ def update_javas(): r = sess.get(MOJANG_JAVA_URL) r.raise_for_status() - remote_javas = JavaIndex(**r.json()) + remote_javas = JavaIndex(__root__=r.json()) java_manifest_path = os.path.join(UPSTREAM_DIR, JAVA_MANIFEST_FILE) From 030c5e70de38dc40994d14707018ec46700787e7 Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Wed, 10 May 2023 21:03:19 -0700 Subject: [PATCH 03/22] update update.sh Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- update.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/update.sh b/update.sh index 9286c80..573a333 100755 --- a/update.sh +++ b/update.sh @@ -46,13 +46,15 @@ python updateForge.py || fail_in python updateFabric.py || fail_in python updateQuilt.py || fail_in python updateLiteloader.py || fail_in +python updateJava.py || fail_in if [ "${DEPLOY_TO_GIT}" = true ] ; then - upstream_git add mojang/version_manifest_v2.json mojang/versions/* || fail_in + upstream_git add mojang/version_manifest_v2.json mojang/java_all.json mojang/versions/* || fail_in upstream_git add forge/*.json forge/version_manifests/*.json forge/installer_manifests/*.json forge/files_manifests/*.json forge/installer_info/*.json || fail_in upstream_git add fabric/loader-installer-json/*.json fabric/meta-v2/*.json fabric/jars/*.json || fail_in upstream_git add quilt/loader-installer-json/*.json quilt/meta-v3/*.json quilt/jars/*.json || fail_in upstream_git add liteloader/*.json || fail_in + upstream_git add java_runtime/adoptium/available_releases.json java_runtime/adoptium/versions/*.json java_runtime/azul/packages.json java_runtime/azul/versions/*.json || fail_in if ! upstream_git diff --cached --exit-code ; then upstream_git commit -a -m "Update ${currentDate}" || fail_in upstream_git push || exit 1 @@ -67,6 +69,7 @@ python generateForge.py || fail_out python generateFabric.py || fail_out python generateQuilt.py || fail_out python generateLiteloader.py || fail_out +python generageJava.py || fail_out python index.py || fail_out if [ "${DEPLOY_TO_GIT}" = true ] ; then @@ -75,6 +78,7 @@ if [ "${DEPLOY_TO_GIT}" = true ] ; then launcher_git add net.fabricmc.fabric-loader/* net.fabricmc.intermediary/* || fail_out launcher_git add org.quiltmc.quilt-loader/* || fail_out # TODO: add Quilt hashed, once it is actually used launcher_git add com.mumfrey.liteloader/* || fail_out + launcher_git add net.minecraft.java/* || fail_out if ! launcher_git diff --cached --exit-code ; then launcher_git commit -a -m "Update ${currentDate}" || fail_out From 1b8d9d408c71453ccd5afc130f7b1da8df254dbc Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Thu, 11 May 2023 02:10:12 -0700 Subject: [PATCH 04/22] fix: ensure there is a recomended version Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- generateJava.py | 166 +++++++++++++++++++++++++++++++++++---------- meta/model/java.py | 20 +++++- 2 files changed, 146 insertions(+), 40 deletions(-) diff --git a/generateJava.py b/generateJava.py index 11a449a..85041bc 100644 --- a/generateJava.py +++ b/generateJava.py @@ -14,7 +14,7 @@ from meta.common.java import ( ADOPTIUM_DIR, ADOPTIUM_VERSIONS_DIR, AZUL_DIR, - AZUL_VERSIONS_DIR + AZUL_VERSIONS_DIR, ) from meta.model.java import ( JavaRuntimeOS, @@ -34,7 +34,6 @@ from meta.model.java import ( ) from meta.common.mojang import ( - JAVA_MANIFEST_FILE, ) @@ -64,7 +63,7 @@ MOJANG_JAVA_OS_NAMES = [ MOJANG_OS_ARCHITECTURES = [ "x64", - "x86", + "x86", "arm64", "arm32", ] @@ -76,7 +75,7 @@ MOJANG_OS_ARCHITECTURE_TRANSLATIONS = { "i386": "x86", "aarch64": "arm64", "x86_64": "x64", - "arm": "arm32" + "arm": "arm32", } @@ -90,6 +89,7 @@ def translate_arch(arch: str | int): else: return None + MOJANG_OS_NAMES = [ "mac-os", "linux", @@ -102,6 +102,7 @@ MOJANG_OS_TRANSLATIONS = { "macos": "mac-os", } + def translate_os(os: str): os = os.lower() if os in MOJANG_OS_NAMES: @@ -131,24 +132,49 @@ def mojang_os_to_java_os(mojang_os: MojangJavaOsName) -> JavaRuntimeOS: return JavaRuntimeOS.Unknown -def mojang_runtime_to_java_runtime(mojang_runtime: MojangJavaRuntime) -> JavaRuntimeMeta: +def mojang_runtime_to_java_runtime( + mojang_runtime: MojangJavaRuntime, +) -> JavaRuntimeMeta: + major, _, security = mojang_runtime.version.name.partition("u") + if major and security: + version_parts = [int(major), 0, int(security)] + else: + version_parts = [int(part) for part in mojang_runtime.version.name.split(".")] + + while len(version_parts) < 3: + version_parts.append(0) + if len(version_parts) < 4: + version_parts.append(None) + + version = JavaVersionMeta( + major=version_parts[0], + minor=version_parts[1], + security=version_parts[2], + build=version_parts[3], + name=mojang_runtime.version.name, + ) return JavaRuntimeMeta( name=f"mojang_jre_{mojang_runtime.version.name}", vender="mojang", url=mojang_runtime.manifest.url, release_time=mojang_runtime.version.released, checksum=JavaChecksumMeta( - type=JavaChecksumType.Sha1, - hash=mojang_runtime.manifest.sha1), + type=JavaChecksumType.Sha1, hash=mojang_runtime.manifest.sha1 + ), recomended=True, - download_type=JavaRuntimeDownloadType.Manifest) + download_type=JavaRuntimeDownloadType.Manifest, + version=version, + ) -def adoptium_release_binary_to_java_runtime(rls: AdoptiumRelease, binary: AdoptiumBinary) -> JavaRuntimeMeta: + +def adoptium_release_binary_to_java_runtime( + rls: AdoptiumRelease, binary: AdoptiumBinary +) -> JavaRuntimeMeta: version = JavaVersionMeta( major=rls.version_data.major, minor=rls.version_data.minor, security=rls.version_data.security, - build=rls.version_data.build + build=rls.version_data.build, ) rls_name = f"{rls.vendor}_temurin_{binary.image_type}{version}" return JavaRuntimeMeta( @@ -157,69 +183,131 @@ def adoptium_release_binary_to_java_runtime(rls: AdoptiumRelease, binary: Adopti url=binary.package.link, release_time=rls.timestamp, checksum=JavaChecksumMeta( - type=JavaChecksumType.Sha256, - hash=binary.package.checksum), + type=JavaChecksumType.Sha256, hash=binary.package.checksum + ), recomended=False, - download_type=JavaRuntimeDownloadType.Archive + download_type=JavaRuntimeDownloadType.Archive, + version=version, ) + def azul_package_to_java_runtime(pkg: ZuluPackageDetail) -> JavaRuntimeMeta: version_parts = copy.copy(pkg.java_version) while len(version_parts) < 4: version_parts.append(None) - + version = JavaVersionMeta( major=version_parts[0], minor=version_parts[1], security=version_parts[2], - build=version_parts[3] + build=version_parts[3], ) - + rls_name = f"azul_{pkg.product}_{pkg.java_package_type}{version}" - + return JavaRuntimeMeta( name=rls_name, vender="azul", url=pkg.download_url, release_time=pkg.build_date, - checksum=JavaChecksumMeta( - type=JavaChecksumType.Sha256, - hash=pkg.sha256_hash), + checksum=JavaChecksumMeta(type=JavaChecksumType.Sha256, hash=pkg.sha256_hash), recomended=False, - download_type=JavaRuntimeDownloadType.Archive + download_type=JavaRuntimeDownloadType.Archive, + version=version, ) + +PREFERED_VENDER_ORDER = ["mojang", "eclipse", "azul"] + +__PREFERED_VENDER_ORDER = list(reversed(PREFERED_VENDER_ORDER)) + + +def vender_priority(vender: str) -> int: + """Get a numeric priority for a given vendor + + Args: + vendor (str): the vender to check + + Returns: + int: how preferable the vender is, the higher the better + """ + if vender not in PREFERED_VENDER_ORDER: + return -1 + return __PREFERED_VENDER_ORDER.index(vender) + + +def ensure_one_recomended(runtimes: list[JavaRuntimeMeta]): + if len(runtimes) < 1: + return # can't do anything + + recomended: Optional[JavaRuntimeMeta] = None + found_first = False + need_resort = False + for runtime in runtimes: + if runtime.recomended: + if not found_first: + recomended = runtime + else: + runtime.recomended = False + need_resort = True + + if recomended and not need_resort: + print("Recomending", recomended.name) + return # we have one recomended already + + if recomended is None: + recomended = runtimes[0] + + def better_java_runtime(runtime: JavaRuntimeMeta): + if vender_priority(runtime.vender) < vender_priority(recomended.vender): + return False + if runtime.version < recomended.version: + return False + if runtime.release_time < recomended.release_time: + return False + return True + + for runtime in runtimes: + if better_java_runtime(runtime): + recomended.recomended = False + recomended = runtime + recomended.recomended = True + + print("Recomending", recomended.name) + + def main(): - javas: dict[int, JavaRuntimeMap] = {} - + def ensure_javamap(major: int): if major not in javas: javas[major] = JavaRuntimeMap() - + def add_java_runtime(runtime: JavaRuntimeMeta, major: int, java_os: JavaRuntimeOS): ensure_javamap(major) print(f"Regestering runtime: {runtime.name} for Java {major} {java_os}") javas[major][java_os].append(runtime) - + print("Processing Mojang Javas") mojang_java_manifest = JavaIndex.parse_file( os.path.join(UPSTREAM_DIR, JAVA_MANIFEST_FILE) ) for mojang_os_name in mojang_java_manifest: if mojang_os_name == MojangJavaOsName.Gamecore: - continue + continue # empty java_os = mojang_os_to_java_os(mojang_os_name) for comp in mojang_java_manifest[mojang_os_name]: + if comp == MojangJavaComponent.Exe: + continue # doesn't appear to be used and not marked with a full verison so I don't trust it mojang_runtimes = mojang_java_manifest[mojang_os_name][comp] for mojang_runtime in mojang_runtimes: if comp == MojangJavaComponent.JreLegacy: major = 8 else: - major = int(mojang_runtime.version.name.partition('.')[0]) + major = int(mojang_runtime.version.name.partition(".")[0]) runtime = mojang_runtime_to_java_runtime(mojang_runtime) add_java_runtime(runtime, major, java_os) - + print("Processing Adoptium Releases") adoptium_available_releases = AdoptiumAvailableReleases.parse_file( os.path.join(UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json") @@ -237,11 +325,11 @@ def main(): if binary_arch is None or binary_os is None: print(f"Ignoring release for {binary.os} {binary.architecture}") continue - + java_os = JavaRuntimeOS(f"{binary_os}-{binary_arch}") runtime = adoptium_release_binary_to_java_runtime(rls, binary) add_java_runtime(runtime, major, java_os) - + print("Processing Azul Packages") azul_packages = ZuluPackages.parse_file( os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") @@ -250,28 +338,32 @@ def main(): pkg_detail = ZuluPackageDetail.parse_file( os.path.join(UPSTREAM_DIR, AZUL_VERSIONS_DIR, f"{pkg.package_uuid}.json") ) - major = pkg_detail.java_version[0] + major = pkg_detail.java_version[0] pkg_os = translate_os(str(pkg_detail.os)) - if pkg_detail.arch == AzulArch.Arm: + if pkg_detail.arch == AzulArch.Arm: pkg_arch = translate_arch(f"{pkg_detail.arch}{pkg_detail.hw_bitness}") - elif pkg_detail.arch == AzulArch.X86: + elif pkg_detail.arch == AzulArch.X86: pkg_arch = translate_arch(int(pkg_detail.hw_bitness)) else: pkg_arch = None if pkg_arch is None or pkg_os is None: - print(f"Ignoring release for {pkg_detail.os} {pkg_detail.arch}_{pkg_detail.hw_bitness}") + print( + f"Ignoring release for {pkg_detail.os} {pkg_detail.arch}_{pkg_detail.hw_bitness}" + ) continue - + java_os = JavaRuntimeOS(f"{pkg_os}-{pkg_arch}") runtime = azul_package_to_java_runtime(pkg_detail) add_java_runtime(runtime, major, java_os) - + for major, runtimes in javas.items(): for java_os in runtimes: print(f"Total runtimes for Java {major} {java_os}:", len(runtimes[java_os])) + ensure_one_recomended(runtimes[java_os]) + runtimes_file = os.path.join(LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") runtimes.write(runtimes_file) - + if __name__ == "__main__": main() diff --git a/meta/model/java.py b/meta/model/java.py index a7907d9..ca45085 100644 --- a/meta/model/java.py +++ b/meta/model/java.py @@ -8,7 +8,7 @@ from .enum import StrEnum from typing import Optional, List, Dict, Any, Iterator, Iterable, NamedTuple from collections import namedtuple from urllib.parse import urljoin, urlencode, urlparse, urlunparse - +from functools import total_ordering # namedtuple to match the internal signature of urlunparse @@ -32,18 +32,31 @@ class JavaRuntimeDownloadType(StrEnum): Manifest = "manifest" Archive = "archive" - +@total_ordering class JavaVersionMeta(MetaBase): major: int minor: int security: int - build: Optional[int] + build: Optional[int] = None + name: Optional[str] = None def __str__(self): ver = f"{self.major}.{self.minor}.{self.security}" if self.build is not None: ver = f"{ver}+{self.build}" return ver + + def to_tuple(self): + build = 0 + if self.build is not None: + build = self.build + return (self.major, self.minor, self.security, build) + + def __eq__(self, other: 'JavaVersionMeta'): + return (self.to_tuple() == other.to_tuple()) + + def __lt__(self, other: 'JavaVersionMeta'): + return (self.to_tuple() < other.to_tuple()) class JavaChecksumType(StrEnum): @@ -64,6 +77,7 @@ class JavaRuntimeMeta(MetaBase): checksum: Optional[JavaChecksumMeta] recomended: bool download_type: JavaRuntimeDownloadType = Field(alias="downloadType") + version: JavaVersionMeta class JavaRuntimeMap(MetaBase): From c911fbd7748aaae30242543cf43e9008400c8798 Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Thu, 11 May 2023 02:20:57 -0700 Subject: [PATCH 05/22] fix: spelling Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- generateJava.py | 62 +++++++++++++++++++++++----------------------- meta/model/java.py | 8 +++--- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/generateJava.py b/generateJava.py index 85041bc..ac9b7b9 100644 --- a/generateJava.py +++ b/generateJava.py @@ -155,13 +155,13 @@ def mojang_runtime_to_java_runtime( ) return JavaRuntimeMeta( name=f"mojang_jre_{mojang_runtime.version.name}", - vender="mojang", + vendor="mojang", url=mojang_runtime.manifest.url, release_time=mojang_runtime.version.released, checksum=JavaChecksumMeta( type=JavaChecksumType.Sha1, hash=mojang_runtime.manifest.sha1 ), - recomended=True, + recommended=True, download_type=JavaRuntimeDownloadType.Manifest, version=version, ) @@ -179,13 +179,13 @@ def adoptium_release_binary_to_java_runtime( rls_name = f"{rls.vendor}_temurin_{binary.image_type}{version}" return JavaRuntimeMeta( name=rls_name, - vender=rls.vendor, + vendor=rls.vendor, url=binary.package.link, release_time=rls.timestamp, checksum=JavaChecksumMeta( type=JavaChecksumType.Sha256, hash=binary.package.checksum ), - recomended=False, + recommended=False, download_type=JavaRuntimeDownloadType.Archive, version=version, ) @@ -207,73 +207,73 @@ def azul_package_to_java_runtime(pkg: ZuluPackageDetail) -> JavaRuntimeMeta: return JavaRuntimeMeta( name=rls_name, - vender="azul", + vendor="azul", url=pkg.download_url, release_time=pkg.build_date, checksum=JavaChecksumMeta(type=JavaChecksumType.Sha256, hash=pkg.sha256_hash), - recomended=False, + recommended=False, download_type=JavaRuntimeDownloadType.Archive, version=version, ) -PREFERED_VENDER_ORDER = ["mojang", "eclipse", "azul"] +PREFERED_vendor_ORDER = ["mojang", "eclipse", "azul"] -__PREFERED_VENDER_ORDER = list(reversed(PREFERED_VENDER_ORDER)) +__PREFERED_vendor_ORDER = list(reversed(PREFERED_vendor_ORDER)) -def vender_priority(vender: str) -> int: +def vendor_priority(vendor: str) -> int: """Get a numeric priority for a given vendor Args: - vendor (str): the vender to check + vendor (str): the vendor to check Returns: - int: how preferable the vender is, the higher the better + int: how preferable the vendor is, the higher the better """ - if vender not in PREFERED_VENDER_ORDER: + if vendor not in PREFERED_vendor_ORDER: return -1 - return __PREFERED_VENDER_ORDER.index(vender) + return __PREFERED_vendor_ORDER.index(vendor) -def ensure_one_recomended(runtimes: list[JavaRuntimeMeta]): +def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]): if len(runtimes) < 1: return # can't do anything - recomended: Optional[JavaRuntimeMeta] = None + recommended: Optional[JavaRuntimeMeta] = None found_first = False need_resort = False for runtime in runtimes: - if runtime.recomended: + if runtime.recommended: if not found_first: - recomended = runtime + recommended = runtime else: - runtime.recomended = False + runtime.recommended = False need_resort = True - if recomended and not need_resort: - print("Recomending", recomended.name) - return # we have one recomended already + if recommended and not need_resort: + print("Recommending", recommended.name) + return # we have one recommended already - if recomended is None: - recomended = runtimes[0] + if recommended is None: + recommended = runtimes[0] def better_java_runtime(runtime: JavaRuntimeMeta): - if vender_priority(runtime.vender) < vender_priority(recomended.vender): + if vendor_priority(runtime.vendor) < vendor_priority(recommended.vendor): return False - if runtime.version < recomended.version: + if runtime.version < recommended.version: return False - if runtime.release_time < recomended.release_time: + if runtime.release_time < recommended.release_time: return False return True for runtime in runtimes: if better_java_runtime(runtime): - recomended.recomended = False - recomended = runtime - recomended.recomended = True + recommended.recommended = False + recommended = runtime + recommended.recommended = True - print("Recomending", recomended.name) + print("Recommending", recommended.name) def main(): @@ -359,7 +359,7 @@ def main(): for major, runtimes in javas.items(): for java_os in runtimes: print(f"Total runtimes for Java {major} {java_os}:", len(runtimes[java_os])) - ensure_one_recomended(runtimes[java_os]) + ensure_one_recommended(runtimes[java_os]) runtimes_file = os.path.join(LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") runtimes.write(runtimes_file) diff --git a/meta/model/java.py b/meta/model/java.py index ca45085..7c5da18 100644 --- a/meta/model/java.py +++ b/meta/model/java.py @@ -71,11 +71,11 @@ class JavaChecksumMeta(MetaBase): class JavaRuntimeMeta(MetaBase): name: str - vender: str + vendor: str url: str release_time: datetime = Field(alias="releaseTime") checksum: Optional[JavaChecksumMeta] - recomended: bool + recommended: bool download_type: JavaRuntimeDownloadType = Field(alias="downloadType") version: JavaVersionMeta @@ -196,7 +196,7 @@ class AdoptiumOs(StrEnum): ADOPTIUM_API_BASE = " https://api.adoptium.net" ADOPTIUM_API_FEATURE_RELEASES = f"{ADOPTIUM_API_BASE}/v3/assets/feature_releases/{{feature_version}}/{{release_type}}" -# ?image_type={{image_type}}&heap_size={{heap_size}}&project={{project}}&vender={{vender}}&page_size={{page_size}}&page={{page}}&sort_method={{sort_method}}&sort_order={{sort_order}} +# ?image_type={{image_type}}&heap_size={{heap_size}}&project={{project}}&vendor={{vendor}}&page_size={{page_size}}&page={{page}}&sort_method={{sort_method}}&sort_order={{sort_order}} ADOPTIUM_API_AVAILABLE_RELEASES = f"{ADOPTIUM_API_BASE}/v3/info/available_releases" @@ -213,7 +213,7 @@ class AdoptiumAPIFeatureReleasesQuery(APIQuery): project: Optional[AdoptiumProject] = AdoptiumProject.Jdk sort_method: Optional[AdoptiumSortMethod] = AdoptiumSortMethod.Default sort_order: Optional[AdoptiumSortOrder] = AdoptiumSortOrder.Desc - vender: Optional[AdoptiumVendor] = AdoptiumVendor.Eclipse + vendor: Optional[AdoptiumVendor] = AdoptiumVendor.Eclipse def adoptiumAPIFeatureReleasesUrl( From cbe8419d800e78274c6ce9635e326fa60789724f Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Thu, 11 May 2023 02:25:39 -0700 Subject: [PATCH 06/22] fix: varcase Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- generateJava.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/generateJava.py b/generateJava.py index ac9b7b9..9828957 100644 --- a/generateJava.py +++ b/generateJava.py @@ -217,9 +217,9 @@ def azul_package_to_java_runtime(pkg: ZuluPackageDetail) -> JavaRuntimeMeta: ) -PREFERED_vendor_ORDER = ["mojang", "eclipse", "azul"] +PREFERED_VENDOR_ORDER = ["mojang", "eclipse", "azul"] -__PREFERED_vendor_ORDER = list(reversed(PREFERED_vendor_ORDER)) +__PREFERED_VENDOR_ORDER = list(reversed(PREFERED_VENDOR_ORDER)) def vendor_priority(vendor: str) -> int: @@ -231,9 +231,9 @@ def vendor_priority(vendor: str) -> int: Returns: int: how preferable the vendor is, the higher the better """ - if vendor not in PREFERED_vendor_ORDER: + if vendor not in PREFERED_VENDOR_ORDER: return -1 - return __PREFERED_vendor_ORDER.index(vendor) + return __PREFERED_VENDOR_ORDER.index(vendor) def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]): From acc6a4cd95d06327d93ca44578a998f5dac89c17 Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Sat, 13 May 2023 16:20:50 -0700 Subject: [PATCH 07/22] fix: ensure JDK options are included (some platform vendor combos don't offer jre) Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- generateJava.py | 105 ++++++++++++++++++++++++++-------------- meta/common/__init__.py | 6 +-- meta/model/java.py | 41 +++++++++------- updateJava.py | 33 +++++++++---- 4 files changed, 120 insertions(+), 65 deletions(-) diff --git a/generateJava.py b/generateJava.py index 9828957..5e3420b 100644 --- a/generateJava.py +++ b/generateJava.py @@ -1,11 +1,6 @@ import copy -import hashlib import os -from collections import defaultdict, namedtuple -from operator import attrgetter -from pprint import pprint -from packaging import version as pversion -from typing import Optional, List +from typing import Optional from meta.common import ensure_component_dir, launcher_path, upstream_path, static_path @@ -21,6 +16,7 @@ from meta.model.java import ( JavaRuntimeMap, JavaRuntimeMeta, JavaVersionMeta, + JavaPackageType, JavaChecksumMeta, JavaChecksumType, JavaRuntimeDownloadType, @@ -143,26 +139,29 @@ def mojang_runtime_to_java_runtime( while len(version_parts) < 3: version_parts.append(0) - if len(version_parts) < 4: - version_parts.append(None) + + build = None + if len(version_parts) >= 4: + build = version_parts[3] version = JavaVersionMeta( major=version_parts[0], minor=version_parts[1], security=version_parts[2], - build=version_parts[3], + build=build, name=mojang_runtime.version.name, ) return JavaRuntimeMeta( name=f"mojang_jre_{mojang_runtime.version.name}", vendor="mojang", url=mojang_runtime.manifest.url, - release_time=mojang_runtime.version.released, + releaseTime=mojang_runtime.version.released, checksum=JavaChecksumMeta( type=JavaChecksumType.Sha1, hash=mojang_runtime.manifest.sha1 ), recommended=True, - download_type=JavaRuntimeDownloadType.Manifest, + downloadType=JavaRuntimeDownloadType.Manifest, + packageType=JavaPackageType.Jre, version=version, ) @@ -170,10 +169,20 @@ def mojang_runtime_to_java_runtime( def adoptium_release_binary_to_java_runtime( rls: AdoptiumRelease, binary: AdoptiumBinary ) -> JavaRuntimeMeta: + assert binary.package is not None + + checksum = None + if binary.package.checksum is not None: + checksum = JavaChecksumMeta( + type=JavaChecksumType.Sha256, hash=binary.package.checksum + ) + + pkg_type = JavaPackageType(str(binary.image_type)) + version = JavaVersionMeta( - major=rls.version_data.major, - minor=rls.version_data.minor, - security=rls.version_data.security, + major=rls.version_data.major if rls.version_data.major is not None else 0, + minor=rls.version_data.minor if rls.version_data.minor is not None else 0, + security=rls.version_data.security if rls.version_data.security is not None else 0, build=rls.version_data.build, ) rls_name = f"{rls.vendor}_temurin_{binary.image_type}{version}" @@ -181,38 +190,51 @@ def adoptium_release_binary_to_java_runtime( name=rls_name, vendor=rls.vendor, url=binary.package.link, - release_time=rls.timestamp, - checksum=JavaChecksumMeta( - type=JavaChecksumType.Sha256, hash=binary.package.checksum - ), + releaseTime=rls.timestamp, + checksum=checksum, recommended=False, - download_type=JavaRuntimeDownloadType.Archive, + downloadType=JavaRuntimeDownloadType.Archive, + packageType=pkg_type, version=version, ) def azul_package_to_java_runtime(pkg: ZuluPackageDetail) -> JavaRuntimeMeta: version_parts = copy.copy(pkg.java_version) - while len(version_parts) < 4: - version_parts.append(None) + + build = None + while len(version_parts) < 3: + version_parts.append(0) + + if len(version_parts) >= 4: + build = version_parts[3] version = JavaVersionMeta( major=version_parts[0], minor=version_parts[1], security=version_parts[2], - build=version_parts[3], + build=build, ) + pkg_type = JavaPackageType(str(pkg.java_package_type)) + rls_name = f"azul_{pkg.product}_{pkg.java_package_type}{version}" + checksum = None + if pkg.sha256_hash is not None: + checksum = JavaChecksumMeta( + type=JavaChecksumType.Sha256, hash=pkg.sha256_hash + ) + return JavaRuntimeMeta( name=rls_name, vendor="azul", url=pkg.download_url, - release_time=pkg.build_date, - checksum=JavaChecksumMeta(type=JavaChecksumType.Sha256, hash=pkg.sha256_hash), + releaseTime=pkg.build_date, + checksum=checksum, recommended=False, - download_type=JavaRuntimeDownloadType.Archive, + downloadType=JavaRuntimeDownloadType.Archive, + packageType=pkg_type, version=version, ) @@ -236,9 +258,17 @@ def vendor_priority(vendor: str) -> int: return __PREFERED_VENDOR_ORDER.index(vendor) -def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]): +def pkg_type_priority(pkg_type: JavaPackageType) -> int: + if pkg_type == JavaPackageType.Jre: + return 2 + elif pkg_type == JavaPackageType.Jdk: + return 1 + else: + return -1 + +def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]) -> Optional[JavaRuntimeMeta]: if len(runtimes) < 1: - return # can't do anything + return None# can't do anything recommended: Optional[JavaRuntimeMeta] = None found_first = False @@ -252,15 +282,18 @@ def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]): need_resort = True if recommended and not need_resort: - print("Recommending", recommended.name) - return # we have one recommended already + return recommended # we have one recommended already if recommended is None: recommended = runtimes[0] + def better_java_runtime(runtime: JavaRuntimeMeta): + assert recommended is not None if vendor_priority(runtime.vendor) < vendor_priority(recommended.vendor): return False + if pkg_type_priority(runtime.package_type) < pkg_type_priority(recommended.package_type): + return False if runtime.version < recommended.version: return False if runtime.release_time < recommended.release_time: @@ -273,7 +306,7 @@ def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]): recommended = runtime recommended.recommended = True - print("Recommending", recommended.name) + return recommended def main(): @@ -316,7 +349,7 @@ def main(): adoptium_releases = AdoptiumReleases.parse_file( os.path.join(UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, f"java{major}.json") ) - for rls in adoptium_releases: + for _, rls in adoptium_releases: for binary in rls.binaries: if binary.package is None: continue @@ -334,7 +367,7 @@ def main(): azul_packages = ZuluPackages.parse_file( os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") ) - for pkg in azul_packages: + for _, pkg in azul_packages: pkg_detail = ZuluPackageDetail.parse_file( os.path.join(UPSTREAM_DIR, AZUL_VERSIONS_DIR, f"{pkg.package_uuid}.json") ) @@ -357,9 +390,11 @@ def main(): add_java_runtime(runtime, major, java_os) for major, runtimes in javas.items(): - for java_os in runtimes: - print(f"Total runtimes for Java {major} {java_os}:", len(runtimes[java_os])) - ensure_one_recommended(runtimes[java_os]) + for java_os, runtime_list in runtimes: + print(f"Total runtimes for Java {major} {java_os}:", len(runtime_list)) + rec = ensure_one_recommended(runtime_list) + if rec is not None: + print(f"Recomending {rec.name} for Java {major} {java_os}") runtimes_file = os.path.join(LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") runtimes.write(runtimes_file) diff --git a/meta/common/__init__.py b/meta/common/__init__.py index 7a6514b..f799cea 100644 --- a/meta/common/__init__.py +++ b/meta/common/__init__.py @@ -32,13 +32,13 @@ def static_path(): return "static" -def ensure_upstream_dir(path): +def ensure_upstream_dir(path: str): path = os.path.join(upstream_path(), path) if not os.path.exists(path): os.makedirs(path) -def ensure_component_dir(component_id): +def ensure_component_dir(component_id: str): path = os.path.join(launcher_path(), component_id) if not os.path.exists(path): os.makedirs(path) @@ -48,7 +48,7 @@ def transform_maven_key(maven_key: str): return maven_key.replace(":", ".") -def replace_old_launchermeta_url(url): +def replace_old_launchermeta_url(url: str): o = urlparse(url) if o.netloc == "launchermeta.mojang.com": return o._replace(netloc="piston-meta.mojang.com").geturl() diff --git a/meta/model/java.py b/meta/model/java.py index 7c5da18..d234b6c 100644 --- a/meta/model/java.py +++ b/meta/model/java.py @@ -5,9 +5,8 @@ from pydantic import Field from datetime import datetime from enum import IntEnum, Enum from .enum import StrEnum -from typing import Optional, List, Dict, Any, Iterator, Iterable, NamedTuple -from collections import namedtuple -from urllib.parse import urljoin, urlencode, urlparse, urlunparse +from typing import Optional, Any, NamedTuple, Generator +from urllib.parse import urlencode, urlparse, urlunparse from functools import total_ordering # namedtuple to match the internal signature of urlunparse @@ -52,7 +51,7 @@ class JavaVersionMeta(MetaBase): build = self.build return (self.major, self.minor, self.security, build) - def __eq__(self, other: 'JavaVersionMeta'): + def __eq__(self, other: Any): return (self.to_tuple() == other.to_tuple()) def __lt__(self, other: 'JavaVersionMeta'): @@ -69,6 +68,10 @@ class JavaChecksumMeta(MetaBase): hash: str +class JavaPackageType(StrEnum): + Jre = "jre" + Jdk = "jdk" + class JavaRuntimeMeta(MetaBase): name: str vendor: str @@ -77,6 +80,7 @@ class JavaRuntimeMeta(MetaBase): checksum: Optional[JavaChecksumMeta] recommended: bool download_type: JavaRuntimeDownloadType = Field(alias="downloadType") + package_type: JavaPackageType = Field(alias="packageType") version: JavaVersionMeta @@ -85,10 +89,10 @@ class JavaRuntimeMap(MetaBase): os: [] for os in JavaRuntimeOS if os != JavaRuntimeOS.Unknown } - def __iter__(self) -> Iterator[JavaRuntimeOS]: - return iter(self.__root__) + def __iter__(self) -> Generator[tuple[str, list[JavaRuntimeMeta]], None, None]: + yield from ((str(os), runtime) for os, runtime in self.__root__.items()) - def __getitem__(self, item) -> list[JavaRuntimeMeta]: + def __getitem__(self, item:JavaRuntimeOS) -> list[JavaRuntimeMeta]: return self.__root__[item] def __len__(self): @@ -299,10 +303,11 @@ class AdoptiumRelease(MetaBase): class AdoptiumReleases(MetaBase): __root__: list[AdoptiumRelease] - def __iter__(self) -> Iterator[AdoptiumRelease]: - return iter(self.__root__) + def __iter__(self) -> Generator[tuple[str, AdoptiumRelease], None, None]: + yield from ((str(i), val) for i, val in enumerate(self.__root__)) + - def __getitem__(self, item) -> AdoptiumRelease: + def __getitem__(self, item: int) -> AdoptiumRelease: return self.__root__[item] def append(self, rls: AdoptiumRelease): @@ -458,8 +463,8 @@ class AzulApiPackagesQuery(APIQuery): distro_version: Optional[str] = None java_package_features: list[str] = [] release_status: Optional[AzulReleaseStatus] = None - availability_types: list[AzulAvailabilityType] = None - certifications: list[AzulCertifications] = None + availability_types: list[AzulAvailabilityType] = [] + certifications: list[AzulCertifications] = [] include_fields: list[str] = [] page: int = 0 page_size: int = 100 @@ -528,10 +533,10 @@ class ZuluPackageList(MetaBase): class ZuluPackages(MetaBase): __root__: list[ZuluPackageList] - def __iter__(self) -> Iterator[ZuluPackageList]: - return iter(self.__root__) + def __iter__(self) -> Generator[tuple[str, ZuluPackageList], None, None]: + yield from ((str(i), val) for i, val in enumerate(self.__root__)) - def __getitem__(self, item) -> ZuluPackageList: + def __getitem__(self, item: int) -> ZuluPackageList: return self.__root__[item] def append(self, pkg: ZuluPackageList): @@ -541,10 +546,10 @@ class ZuluPackages(MetaBase): class ZuluPackagesDetail(MetaBase): __root__: list[ZuluPackageDetail] - def __iter__(self) -> Iterator[ZuluPackageDetail]: - return iter(self.__root__) + def __iter__(self) -> Generator[tuple[str, ZuluPackageDetail], None, None]: + yield from ((str(i), val) for i, val in enumerate(self.__root__)) - def __getitem__(self, item) -> ZuluPackageDetail: + def __getitem__(self, item: int) -> ZuluPackageDetail: return self.__root__[item] def append(self, pkg: ZuluPackageDetail): diff --git a/updateJava.py b/updateJava.py index ece9769..4d174c9 100644 --- a/updateJava.py +++ b/updateJava.py @@ -1,6 +1,4 @@ -import json import os -import zipfile from meta.common import upstream_path, ensure_upstream_dir, static_path, default_session from meta.common.java import ( @@ -25,7 +23,6 @@ from meta.model.java import ( AzulArchiveType, AzulReleaseStatus, AzulAvailabilityType, - AzulJavaPackageType, azulApiPackageDetailUrl, ZuluPackageDetail, ZuluPackagesDetail, @@ -57,16 +54,35 @@ def main(): for feature in available.available_releases: print("Getting Manifests for Adoptium feature release:", feature) - page = 0 + page_size = 10 releases_for_feature: list[AdoptiumRelease] = [] - + page = 0 while True: query = AdoptiumAPIFeatureReleasesQuery( image_type=AdoptiumImageType.Jre, page_size=page_size, page=page) api_call = adoptiumAPIFeatureReleasesUrl(feature, query=query) - print("Fetching Page:", page, api_call) + print("Fetching JRE Page:", page, api_call) + r_rls = sess.get(api_call) + if r_rls.status_code == 404: + break + else: + r_rls.raise_for_status() + + releases = list(AdoptiumRelease(**rls) for rls in r_rls.json()) + releases_for_feature.extend(releases) + + if len(r_rls.json()) < page_size: + break + page += 1 + + page = 0 + while True: + query = AdoptiumAPIFeatureReleasesQuery( + image_type=AdoptiumImageType.Jdk, page_size=page_size, page=page) + api_call = adoptiumAPIFeatureReleasesUrl(feature, query=query) + print("Fetching JDK Page:", page, api_call) r_rls = sess.get(api_call) if r_rls.status_code == 404: break @@ -96,7 +112,6 @@ def main(): archive_type=AzulArchiveType.Zip, release_status=AzulReleaseStatus.Ga, availability_types=[AzulAvailabilityType.CA], - java_package_type=AzulJavaPackageType.Jre, javafx_bundled=False, page=page, page_size=page_size) @@ -121,9 +136,9 @@ def main(): azul_manifest_file = os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") packages.write(azul_manifest_file) - azul_major_versions: dict[int, ZuluPackages] = {} + azul_major_versions: dict[int, ZuluPackagesDetail] = {} - for pkg in packages: + for _, pkg in packages: major_version = pkg.java_version[0] if major_version not in azul_major_versions: From 19b046d235d7599ae4f75c3a7ed5715ecf51d02f Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Sat, 13 May 2023 20:06:22 -0700 Subject: [PATCH 08/22] don't track azul java verisons older than java 8 Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- generateJava.py | 43 +++++++++++++++++++++++++++---------------- meta/model/java.py | 29 +++++++++++++++-------------- updateJava.py | 17 ++++++++--------- 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/generateJava.py b/generateJava.py index 5e3420b..b32b786 100644 --- a/generateJava.py +++ b/generateJava.py @@ -24,7 +24,7 @@ from meta.model.java import ( AdoptiumReleases, AdoptiumRelease, AdoptiumBinary, - ZuluPackages, + ZuluPackageList, ZuluPackageDetail, AzulArch, ) @@ -135,7 +135,8 @@ def mojang_runtime_to_java_runtime( if major and security: version_parts = [int(major), 0, int(security)] else: - version_parts = [int(part) for part in mojang_runtime.version.name.split(".")] + version_parts = [int(part) + for part in mojang_runtime.version.name.split(".")] while len(version_parts) < 3: version_parts.append(0) @@ -170,7 +171,7 @@ def adoptium_release_binary_to_java_runtime( rls: AdoptiumRelease, binary: AdoptiumBinary ) -> JavaRuntimeMeta: assert binary.package is not None - + checksum = None if binary.package.checksum is not None: checksum = JavaChecksumMeta( @@ -178,7 +179,7 @@ def adoptium_release_binary_to_java_runtime( ) pkg_type = JavaPackageType(str(binary.image_type)) - + version = JavaVersionMeta( major=rls.version_data.major if rls.version_data.major is not None else 0, minor=rls.version_data.minor if rls.version_data.minor is not None else 0, @@ -205,7 +206,7 @@ def azul_package_to_java_runtime(pkg: ZuluPackageDetail) -> JavaRuntimeMeta: build = None while len(version_parts) < 3: version_parts.append(0) - + if len(version_parts) >= 4: build = version_parts[3] @@ -266,9 +267,10 @@ def pkg_type_priority(pkg_type: JavaPackageType) -> int: else: return -1 + def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]) -> Optional[JavaRuntimeMeta]: if len(runtimes) < 1: - return None# can't do anything + return None # can't do anything recommended: Optional[JavaRuntimeMeta] = None found_first = False @@ -282,12 +284,11 @@ def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]) -> Optional[JavaRunt need_resort = True if recommended and not need_resort: - return recommended # we have one recommended already + return recommended # we have one recommended already if recommended is None: recommended = runtimes[0] - def better_java_runtime(runtime: JavaRuntimeMeta): assert recommended is not None if vendor_priority(runtime.vendor) < vendor_priority(recommended.vendor): @@ -318,7 +319,8 @@ def main(): def add_java_runtime(runtime: JavaRuntimeMeta, major: int, java_os: JavaRuntimeOS): ensure_javamap(major) - print(f"Regestering runtime: {runtime.name} for Java {major} {java_os}") + print( + f"Regestering runtime: {runtime.name} for Java {major} {java_os}") javas[major][java_os].append(runtime) print("Processing Mojang Javas") @@ -347,7 +349,8 @@ def main(): ) for major in adoptium_available_releases.available_releases: adoptium_releases = AdoptiumReleases.parse_file( - os.path.join(UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, f"java{major}.json") + os.path.join(UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, + f"java{major}.json") ) for _, rls in adoptium_releases: for binary in rls.binaries: @@ -356,7 +359,8 @@ def main(): binary_arch = translate_arch(str(binary.architecture)) binary_os = translate_os(str(binary.os)) if binary_arch is None or binary_os is None: - print(f"Ignoring release for {binary.os} {binary.architecture}") + print( + f"Ignoring release for {binary.os} {binary.architecture}") continue java_os = JavaRuntimeOS(f"{binary_os}-{binary_arch}") @@ -364,17 +368,22 @@ def main(): add_java_runtime(runtime, major, java_os) print("Processing Azul Packages") - azul_packages = ZuluPackages.parse_file( + azul_packages = ZuluPackageList.parse_file( os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") ) for _, pkg in azul_packages: pkg_detail = ZuluPackageDetail.parse_file( - os.path.join(UPSTREAM_DIR, AZUL_VERSIONS_DIR, f"{pkg.package_uuid}.json") + os.path.join(UPSTREAM_DIR, AZUL_VERSIONS_DIR, + f"{pkg.package_uuid}.json") ) major = pkg_detail.java_version[0] + if major < 8: + continue # we will never need java versions less than 8 + pkg_os = translate_os(str(pkg_detail.os)) if pkg_detail.arch == AzulArch.Arm: - pkg_arch = translate_arch(f"{pkg_detail.arch}{pkg_detail.hw_bitness}") + pkg_arch = translate_arch( + f"{pkg_detail.arch}{pkg_detail.hw_bitness}") elif pkg_detail.arch == AzulArch.X86: pkg_arch = translate_arch(int(pkg_detail.hw_bitness)) else: @@ -391,12 +400,14 @@ def main(): for major, runtimes in javas.items(): for java_os, runtime_list in runtimes: - print(f"Total runtimes for Java {major} {java_os}:", len(runtime_list)) + print(f"Total runtimes for Java {major} {java_os}:", len( + runtime_list)) rec = ensure_one_recommended(runtime_list) if rec is not None: print(f"Recomending {rec.name} for Java {major} {java_os}") - runtimes_file = os.path.join(LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") + runtimes_file = os.path.join( + LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") runtimes.write(runtimes_file) diff --git a/meta/model/java.py b/meta/model/java.py index d234b6c..ac224fe 100644 --- a/meta/model/java.py +++ b/meta/model/java.py @@ -13,7 +13,7 @@ from functools import total_ordering class JavaRuntimeOS(StrEnum): MacOsX64 = "mac-os-x64" - MacOsX86 = "mac-os-x86" # rare + MacOsX86 = "mac-os-x86" # rare MacOsArm64 = "mac-os-arm64" # MacOsArm32 = "mac-os-arm32" # doesn't exsist LinuxX64 = "linux-x64" @@ -31,6 +31,7 @@ class JavaRuntimeDownloadType(StrEnum): Manifest = "manifest" Archive = "archive" + @total_ordering class JavaVersionMeta(MetaBase): major: int @@ -44,16 +45,16 @@ class JavaVersionMeta(MetaBase): if self.build is not None: ver = f"{ver}+{self.build}" return ver - + def to_tuple(self): build = 0 if self.build is not None: build = self.build return (self.major, self.minor, self.security, build) - + def __eq__(self, other: Any): return (self.to_tuple() == other.to_tuple()) - + def __lt__(self, other: 'JavaVersionMeta'): return (self.to_tuple() < other.to_tuple()) @@ -72,6 +73,7 @@ class JavaPackageType(StrEnum): Jre = "jre" Jdk = "jdk" + class JavaRuntimeMeta(MetaBase): name: str vendor: str @@ -92,7 +94,7 @@ class JavaRuntimeMap(MetaBase): def __iter__(self) -> Generator[tuple[str, list[JavaRuntimeMeta]], None, None]: yield from ((str(os), runtime) for os, runtime in self.__root__.items()) - def __getitem__(self, item:JavaRuntimeOS) -> list[JavaRuntimeMeta]: + def __getitem__(self, item: JavaRuntimeOS) -> list[JavaRuntimeMeta]: return self.__root__[item] def __len__(self): @@ -303,9 +305,8 @@ class AdoptiumRelease(MetaBase): class AdoptiumReleases(MetaBase): __root__: list[AdoptiumRelease] - def __iter__(self) -> Generator[tuple[str, AdoptiumRelease], None, None]: + def __iter__(self) -> Generator[tuple[str, AdoptiumRelease], None, None]: yield from ((str(i), val) for i, val in enumerate(self.__root__)) - def __getitem__(self, item: int) -> AdoptiumRelease: return self.__root__[item] @@ -518,7 +519,7 @@ class ZuluPackageDetail(MetaBase): signatures: list[ZuluSignatureDetail] -class ZuluPackageList(MetaBase): +class ZuluPackage(MetaBase): package_uuid: str name: Optional[str] java_version: list[int] @@ -530,23 +531,23 @@ class ZuluPackageList(MetaBase): availability_type: Optional[AzulAvailabilityType] -class ZuluPackages(MetaBase): - __root__: list[ZuluPackageList] +class ZuluPackageList(MetaBase): + __root__: list[ZuluPackage] - def __iter__(self) -> Generator[tuple[str, ZuluPackageList], None, None]: + def __iter__(self) -> Generator[tuple[str, ZuluPackage], None, None]: yield from ((str(i), val) for i, val in enumerate(self.__root__)) - def __getitem__(self, item: int) -> ZuluPackageList: + def __getitem__(self, item: int) -> ZuluPackage: return self.__root__[item] - def append(self, pkg: ZuluPackageList): + def append(self, pkg: ZuluPackage): self.__root__.append(pkg) class ZuluPackagesDetail(MetaBase): __root__: list[ZuluPackageDetail] - def __iter__(self) -> Generator[tuple[str, ZuluPackageDetail], None, None]: + def __iter__(self) -> Generator[tuple[str, ZuluPackageDetail], None, None]: yield from ((str(i), val) for i, val in enumerate(self.__root__)) def __getitem__(self, item: int) -> ZuluPackageDetail: diff --git a/updateJava.py b/updateJava.py index 4d174c9..70862b9 100644 --- a/updateJava.py +++ b/updateJava.py @@ -18,8 +18,8 @@ from meta.model.java import ( AdoptiumReleases, azulApiPackagesUrl, AzulApiPackagesQuery, + ZuluPackage, ZuluPackageList, - ZuluPackages, AzulArchiveType, AzulReleaseStatus, AzulAvailabilityType, @@ -54,7 +54,7 @@ def main(): for feature in available.available_releases: print("Getting Manifests for Adoptium feature release:", feature) - + page_size = 10 releases_for_feature: list[AdoptiumRelease] = [] @@ -76,7 +76,7 @@ def main(): if len(r_rls.json()) < page_size: break page += 1 - + page = 0 while True: query = AdoptiumAPIFeatureReleasesQuery( @@ -103,7 +103,7 @@ def main(): releases.write(feature_file) print("Getting Azul Release Manifests") - zulu_packages: list[ZuluPackageList] = [] + zulu_packages: list[ZuluPackage] = [] page = 1 page_size = 100 while True: @@ -125,14 +125,14 @@ def main(): else: r.raise_for_status() - packages = list(ZuluPackageList(**pkg) for pkg in r.json()) + packages = list(ZuluPackage(**pkg) for pkg in r.json()) zulu_packages.extend(packages) if len(packages) < page_size: break page += 1 print("Total Azul Packages:", len(zulu_packages)) - packages = ZuluPackages(__root__=zulu_packages) + packages = ZuluPackageList(__root__=zulu_packages) azul_manifest_file = os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") packages.write(azul_manifest_file) @@ -142,9 +142,8 @@ def main(): major_version = pkg.java_version[0] if major_version not in azul_major_versions: - azul_major_versions[major_version] = ZuluPackagesDetail(__root__=[]) - - + azul_major_versions[major_version] = ZuluPackagesDetail( + __root__=[]) pkg_file = os.path.join( UPSTREAM_DIR, AZUL_VERSIONS_DIR, f"{pkg.package_uuid}.json") From 95223b83cbb6c726e5422b8803befc771afe36fa Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Sat, 13 May 2023 20:44:26 -0700 Subject: [PATCH 09/22] cleanup: types Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- generateJava.py | 56 +++++++++++++++++------------------------ meta/common/__init__.py | 11 ++++---- meta/model/__init__.py | 33 +++++++++++++----------- meta/model/java.py | 2 +- 4 files changed, 48 insertions(+), 54 deletions(-) diff --git a/generateJava.py b/generateJava.py index b32b786..b5f0934 100644 --- a/generateJava.py +++ b/generateJava.py @@ -46,17 +46,6 @@ STATIC_DIR = static_path() ensure_component_dir(JAVA_COMPONENT) -MOJANG_JAVA_OS_NAMES = [ - "gamecore", - "linux", - "linux-i386", - "mac-os", - "mac-os-arm64", - "windows-arm64", - "windows-x64", - "windows-x86", -] - MOJANG_OS_ARCHITECTURES = [ "x64", "x86", @@ -110,22 +99,23 @@ def translate_os(os: str): def mojang_os_to_java_os(mojang_os: MojangJavaOsName) -> JavaRuntimeOS: - if mojang_os == MojangJavaOsName.Linux: - return JavaRuntimeOS.LinuxX64 - elif mojang_os == MojangJavaOsName.Linuxi386: - return JavaRuntimeOS.LinuxX86 - elif mojang_os == MojangJavaOsName.MacOs: - return JavaRuntimeOS.MacOsX64 - elif mojang_os == MojangJavaOsName.MacOSArm64: - return JavaRuntimeOS.MacOsArm64 - elif mojang_os == MojangJavaOsName.WindowsArm64: - return JavaRuntimeOS.WindowsArm64 - elif mojang_os == MojangJavaOsName.WindowsX64: - return JavaRuntimeOS.WindowsX64 - elif mojang_os == MojangJavaOsName.WindowsX86: - return JavaRuntimeOS.WindowsX86 - else: - return JavaRuntimeOS.Unknown + match mojang_os: + case MojangJavaOsName.Linux: + return JavaRuntimeOS.LinuxX64 + case MojangJavaOsName.Linuxi386: + return JavaRuntimeOS.LinuxX86 + case MojangJavaOsName.MacOs: + return JavaRuntimeOS.MacOsX64 + case MojangJavaOsName.MacOSArm64: + return JavaRuntimeOS.MacOsArm64 + case MojangJavaOsName.WindowsArm64: + return JavaRuntimeOS.WindowsArm64 + case MojangJavaOsName.WindowsX64: + return JavaRuntimeOS.WindowsX64 + case MojangJavaOsName.WindowsX86: + return JavaRuntimeOS.WindowsX86 + case _: + return JavaRuntimeOS.Unknown def mojang_runtime_to_java_runtime( @@ -260,12 +250,12 @@ def vendor_priority(vendor: str) -> int: def pkg_type_priority(pkg_type: JavaPackageType) -> int: - if pkg_type == JavaPackageType.Jre: - return 2 - elif pkg_type == JavaPackageType.Jdk: - return 1 - else: - return -1 + match pkg_type: + case JavaPackageType.Jre: + return 2 + case JavaPackageType.Jdk: + return 1 + return -1 def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]) -> Optional[JavaRuntimeMeta]: diff --git a/meta/common/__init__.py b/meta/common/__init__.py index f799cea..8f6a829 100644 --- a/meta/common/__init__.py +++ b/meta/common/__init__.py @@ -1,10 +1,11 @@ import os import datetime from urllib.parse import urlparse +from typing import Any, Optional import requests -from cachecontrol import CacheControl -from cachecontrol.caches import FileCache +from cachecontrol import CacheControl # type: ignore +from cachecontrol.caches import FileCache # type: ignore def serialize_datetime(dt: datetime.datetime): @@ -56,7 +57,7 @@ def replace_old_launchermeta_url(url: str): return url -def get_all_bases(cls, bases=None): +def get_all_bases(cls: type, bases: Optional[list[type]] = None): bases = bases or [] bases.append(cls) for c in cls.__bases__: @@ -64,10 +65,10 @@ def get_all_bases(cls, bases=None): return tuple(bases) -def merge_dict(base: dict, overlay: dict): +def merge_dict(base: dict[Any, Any], overlay: dict[Any, Any]): for k, v in base.items(): if isinstance(v, dict): - merge_dict(v, overlay.setdefault(k, {})) + merge_dict(v, overlay.setdefault(k, {})) # type: ignore else: if k not in overlay: overlay[k] = v diff --git a/meta/model/__init__.py b/meta/model/__init__.py index 0428b7f..efb6adf 100644 --- a/meta/model/__init__.py +++ b/meta/model/__init__.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import Optional, List, Dict, Any, Iterator import pydantic -from pydantic import Field, validator +from pydantic import Field, validator # type: ignore from ..common import ( serialize_datetime, @@ -85,13 +85,16 @@ class GradleSpecifier: def is_log4j(self): return self.group == "org.apache.logging.log4j" - def __eq__(self, other): - return str(self) == str(other) + def __eq__(self, other: Any): + if isinstance(other, GradleSpecifier): + return str(self) == str(other) + else: + return False - def __lt__(self, other): + def __lt__(self, other: 'GradleSpecifier'): return str(self) < str(other) - def __gt__(self, other): + def __gt__(self, other: 'GradleSpecifier'): return str(self) > str(other) def __hash__(self): @@ -120,7 +123,7 @@ class GradleSpecifier: return cls(group, artifact, version, classifier, extension) @classmethod - def validate(cls, v): + def validate(cls, v: 'str | GradleSpecifier'): if isinstance(v, cls): return v if isinstance(v, str): @@ -149,7 +152,7 @@ class MetaBase(pydantic.BaseModel): with open(file_path, "w") as f: f.write(self.json()) - def merge(self, other): + def merge(self, other: 'MetaBase'): """ Merge other object with self. - Concatenates lists @@ -173,14 +176,14 @@ class MetaBase(pydantic.BaseModel): elif isinstance(ours, set): ours |= theirs elif isinstance(ours, dict): - result = merge_dict(ours, copy.deepcopy(theirs)) + result = merge_dict(ours, copy.deepcopy(theirs)) # type: ignore setattr(self, key, result) elif MetaBase in get_all_bases(field.type_): ours.merge(theirs) else: setattr(self, key, theirs) - def __hash__(self): + def __hash__(self): #type: ignore return hash(self.json()) class Config: @@ -191,7 +194,7 @@ class MetaBase(pydantic.BaseModel): class Versioned(MetaBase): @validator("format_version") - def format_version_must_be_supported(cls, v): + def format_version_must_be_supported(cls, v: int): assert v <= META_FORMAT_VERSION return v @@ -206,7 +209,7 @@ class MojangArtifactBase(MetaBase): class MojangAssets(MojangArtifactBase): @validator("url") - def validate_url(cls, v): + def validate_url(cls, v: str): return replace_old_launchermeta_url(v) id: str @@ -242,7 +245,7 @@ class MojangLibraryDownloads(MetaBase): class OSRule(MetaBase): @validator("name") - def name_must_be_os(cls, v): + def name_must_be_os(cls, v: str): assert v in [ "osx", "linux", @@ -260,7 +263,7 @@ class OSRule(MetaBase): class MojangRule(MetaBase): @validator("action") - def action_must_be_allow_disallow(cls, v): + def action_must_be_allow_disallow(cls, v: str): assert v in ["allow", "disallow"] return v @@ -271,10 +274,10 @@ class MojangRule(MetaBase): class MojangRules(MetaBase): __root__: List[MojangRule] - def __iter__(self) -> Iterator[MojangRule]: + def __iter__(self) -> Iterator[MojangRule]: #type: ignore return iter(self.__root__) - def __getitem__(self, item) -> MojangRule: + def __getitem__(self, item: int) -> MojangRule: return self.__root__[item] diff --git a/meta/model/java.py b/meta/model/java.py index ac224fe..6a335df 100644 --- a/meta/model/java.py +++ b/meta/model/java.py @@ -118,7 +118,7 @@ class APIQuery(MetaBase): if isinstance(value, Enum): set_parts[key] = value.value elif isinstance(value, list): - if len(value) > 0: + if len(value) > 0: #type: ignore set_parts[key] = value elif isinstance(value, datetime): set_parts[key] = value.isoformat() From cc881a82d894c92be6809b453c6d7d2ccf28d512 Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Fri, 16 Jun 2023 21:14:58 -0700 Subject: [PATCH 10/22] write java `package.json` Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- generateJava.py | 14 ++++++++++++-- meta/model/java.py | 4 ++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/generateJava.py b/generateJava.py index b5f0934..470146e 100644 --- a/generateJava.py +++ b/generateJava.py @@ -11,9 +11,11 @@ from meta.common.java import ( AZUL_DIR, AZUL_VERSIONS_DIR, ) +from meta.model import MetaPackage from meta.model.java import ( JavaRuntimeOS, JavaRuntimeMap, + JavaRuntimeVersion, JavaRuntimeMeta, JavaVersionMeta, JavaPackageType, @@ -396,9 +398,17 @@ def main(): if rec is not None: print(f"Recomending {rec.name} for Java {major} {java_os}") - runtimes_file = os.path.join( + version_file = os.path.join( LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") - runtimes.write(runtimes_file) + java_version = JavaRuntimeVersion(name = f"Java {major}", uid = JAVA_COMPONENT, version = f"java{major}", runtimes = runtimes) + java_version.write(version_file) + + package = MetaPackage( + uid = JAVA_COMPONENT, + name = "Java Runtimes", + recommended = ["java8", "java17"] + ) + package.write(os.path.josn(LAUNCHER_DIR, JAVA_COMPONENT, "package.json")) if __name__ == "__main__": diff --git a/meta/model/java.py b/meta/model/java.py index 6a335df..d232196 100644 --- a/meta/model/java.py +++ b/meta/model/java.py @@ -1,5 +1,7 @@ from . import ( MetaBase, + MetaVersion, + Versioned, ) from pydantic import Field from datetime import datetime @@ -100,6 +102,8 @@ class JavaRuntimeMap(MetaBase): def __len__(self): return len(self.__root__) +class JavaRuntimeVersion(MetaVersion): + runtimes: JavaRuntimeMap class URLComponents(NamedTuple): scheme: str From 939233e42c1fc18ad3f395a54cf18fdfb0f54583 Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Fri, 16 Jun 2023 22:40:54 -0700 Subject: [PATCH 11/22] feat: index java verisons Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- generateJava.py | 23 +++++++++++++++++++++-- updateMojang.py | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/generateJava.py b/generateJava.py index 470146e..2b3f218 100644 --- a/generateJava.py +++ b/generateJava.py @@ -1,6 +1,8 @@ import copy +import datetime import os from typing import Optional +from functools import reduce from meta.common import ensure_component_dir, launcher_path, upstream_path, static_path @@ -398,9 +400,26 @@ def main(): if rec is not None: print(f"Recomending {rec.name} for Java {major} {java_os}") + def newest_timestamp(a: datetime.datetime | None, b: datetime.datetime): + if a is None or a < b: + return b + return a + version_file = os.path.join( LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") - java_version = JavaRuntimeVersion(name = f"Java {major}", uid = JAVA_COMPONENT, version = f"java{major}", runtimes = runtimes) + java_version = JavaRuntimeVersion( + name = f"Java {major}", + uid = JAVA_COMPONENT, + version = f"java{major}", + releaseTime=reduce( + newest_timestamp, + (runtime.release_time + for _, runtime_list in runtimes + for runtime in runtime_list + ), + None + ), + runtimes = runtimes) java_version.write(version_file) package = MetaPackage( @@ -408,7 +427,7 @@ def main(): name = "Java Runtimes", recommended = ["java8", "java17"] ) - package.write(os.path.josn(LAUNCHER_DIR, JAVA_COMPONENT, "package.json")) + package.write(os.path.join(LAUNCHER_DIR, JAVA_COMPONENT, "package.json")) if __name__ == "__main__": diff --git a/updateMojang.py b/updateMojang.py index 03a77df..97b8f29 100755 --- a/updateMojang.py +++ b/updateMojang.py @@ -130,7 +130,7 @@ def main(): "Updating " + version.id + " to timestamp " - + version.release_time.strftime("%s") + + version.release_time.isoformat() ) fetch_version( os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json"), version.url From 6def1eece6c5399335f3189844f3bb1420664d98 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Tue, 16 Jan 2024 17:13:57 +0200 Subject: [PATCH 12/22] Added new java enum Signed-off-by: Trial97 --- meta/model/mojang.py | 1 + update.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/meta/model/mojang.py b/meta/model/mojang.py index efab4a5..48a5e5a 100644 --- a/meta/model/mojang.py +++ b/meta/model/mojang.py @@ -198,6 +198,7 @@ class MojangJavaComponent(StrEnum): Alpha = "java-runtime-alpha" Beta = "java-runtime-beta" Gamma = "java-runtime-gamma" + GammaSnapshot = "java-runtime-gamma-snapshot" Exe = "minecraft-java-exe" diff --git a/update.sh b/update.sh index 001a25d..777a8f0 100755 --- a/update.sh +++ b/update.sh @@ -72,7 +72,7 @@ python generateNeoForge.py || fail_out python generateFabric.py || fail_out python generateQuilt.py || fail_out python generateLiteloader.py || fail_out -python generageJava.py || fail_out +python generateJava.py || fail_out python index.py || fail_out if [ "${DEPLOY_TO_GIT}" = true ] ; then From abc32095d8fd61ab71c23c4710d41e5755fd9f5a Mon Sep 17 00:00:00 2001 From: Trial97 Date: Thu, 25 Jan 2024 18:50:17 +0200 Subject: [PATCH 13/22] Changed mojang java names to match meta Signed-off-by: Trial97 --- generateJava.py | 153 +++++++++++++++++++++-------------------- meta/model/__init__.py | 15 ++-- meta/model/mojang.py | 6 ++ 3 files changed, 91 insertions(+), 83 deletions(-) diff --git a/generateJava.py b/generateJava.py index 2b3f218..6a0bfc1 100644 --- a/generateJava.py +++ b/generateJava.py @@ -124,13 +124,13 @@ def mojang_os_to_java_os(mojang_os: MojangJavaOsName) -> JavaRuntimeOS: def mojang_runtime_to_java_runtime( mojang_runtime: MojangJavaRuntime, + mojang_component: MojangJavaComponent, ) -> JavaRuntimeMeta: major, _, security = mojang_runtime.version.name.partition("u") if major and security: version_parts = [int(major), 0, int(security)] else: - version_parts = [int(part) - for part in mojang_runtime.version.name.split(".")] + version_parts = [int(part) for part in mojang_runtime.version.name.split(".")] while len(version_parts) < 3: version_parts.append(0) @@ -147,7 +147,7 @@ def mojang_runtime_to_java_runtime( name=mojang_runtime.version.name, ) return JavaRuntimeMeta( - name=f"mojang_jre_{mojang_runtime.version.name}", + name=mojang_component, vendor="mojang", url=mojang_runtime.manifest.url, releaseTime=mojang_runtime.version.released, @@ -177,7 +177,9 @@ def adoptium_release_binary_to_java_runtime( version = JavaVersionMeta( major=rls.version_data.major if rls.version_data.major is not None else 0, minor=rls.version_data.minor if rls.version_data.minor is not None else 0, - security=rls.version_data.security if rls.version_data.security is not None else 0, + security=rls.version_data.security + if rls.version_data.security is not None + else 0, build=rls.version_data.build, ) rls_name = f"{rls.vendor}_temurin_{binary.image_type}{version}" @@ -217,9 +219,7 @@ def azul_package_to_java_runtime(pkg: ZuluPackageDetail) -> JavaRuntimeMeta: checksum = None if pkg.sha256_hash is not None: - checksum = JavaChecksumMeta( - type=JavaChecksumType.Sha256, hash=pkg.sha256_hash - ) + checksum = JavaChecksumMeta(type=JavaChecksumType.Sha256, hash=pkg.sha256_hash) return JavaRuntimeMeta( name=rls_name, @@ -262,7 +262,9 @@ def pkg_type_priority(pkg_type: JavaPackageType) -> int: return -1 -def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]) -> Optional[JavaRuntimeMeta]: +def ensure_one_recommended( + runtimes: list[JavaRuntimeMeta], +) -> Optional[JavaRuntimeMeta]: if len(runtimes) < 1: return None # can't do anything @@ -287,7 +289,9 @@ def ensure_one_recommended(runtimes: list[JavaRuntimeMeta]) -> Optional[JavaRunt assert recommended is not None if vendor_priority(runtime.vendor) < vendor_priority(recommended.vendor): return False - if pkg_type_priority(runtime.package_type) < pkg_type_priority(recommended.package_type): + if pkg_type_priority(runtime.package_type) < pkg_type_priority( + recommended.package_type + ): return False if runtime.version < recommended.version: return False @@ -313,8 +317,7 @@ def main(): def add_java_runtime(runtime: JavaRuntimeMeta, major: int, java_os: JavaRuntimeOS): ensure_javamap(major) - print( - f"Regestering runtime: {runtime.name} for Java {major} {java_os}") + print(f"Regestering runtime: {runtime.name} for Java {major} {java_os}") javas[major][java_os].append(runtime) print("Processing Mojang Javas") @@ -334,68 +337,67 @@ def main(): major = 8 else: major = int(mojang_runtime.version.name.partition(".")[0]) - runtime = mojang_runtime_to_java_runtime(mojang_runtime) + runtime = mojang_runtime_to_java_runtime(mojang_runtime, comp) add_java_runtime(runtime, major, java_os) print("Processing Adoptium Releases") - adoptium_available_releases = AdoptiumAvailableReleases.parse_file( - os.path.join(UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json") - ) - for major in adoptium_available_releases.available_releases: - adoptium_releases = AdoptiumReleases.parse_file( - os.path.join(UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, - f"java{major}.json") + adoptium_path = os.path.join(UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json") + if os.path.exists(adoptium_path): + adoptium_available_releases = AdoptiumAvailableReleases.parse_file( + adoptium_path ) - for _, rls in adoptium_releases: - for binary in rls.binaries: - if binary.package is None: - continue - binary_arch = translate_arch(str(binary.architecture)) - binary_os = translate_os(str(binary.os)) - if binary_arch is None or binary_os is None: - print( - f"Ignoring release for {binary.os} {binary.architecture}") - continue + for major in adoptium_available_releases.available_releases: + adoptium_releases = AdoptiumReleases.parse_file( + os.path.join(UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, f"java{major}.json") + ) + for _, rls in adoptium_releases: + for binary in rls.binaries: + if binary.package is None: + continue + binary_arch = translate_arch(str(binary.architecture)) + binary_os = translate_os(str(binary.os)) + if binary_arch is None or binary_os is None: + print(f"Ignoring release for {binary.os} {binary.architecture}") + continue - java_os = JavaRuntimeOS(f"{binary_os}-{binary_arch}") - runtime = adoptium_release_binary_to_java_runtime(rls, binary) - add_java_runtime(runtime, major, java_os) + java_os = JavaRuntimeOS(f"{binary_os}-{binary_arch}") + runtime = adoptium_release_binary_to_java_runtime(rls, binary) + add_java_runtime(runtime, major, java_os) print("Processing Azul Packages") - azul_packages = ZuluPackageList.parse_file( - os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") - ) - for _, pkg in azul_packages: - pkg_detail = ZuluPackageDetail.parse_file( - os.path.join(UPSTREAM_DIR, AZUL_VERSIONS_DIR, - f"{pkg.package_uuid}.json") - ) - major = pkg_detail.java_version[0] - if major < 8: - continue # we will never need java versions less than 8 - - pkg_os = translate_os(str(pkg_detail.os)) - if pkg_detail.arch == AzulArch.Arm: - pkg_arch = translate_arch( - f"{pkg_detail.arch}{pkg_detail.hw_bitness}") - elif pkg_detail.arch == AzulArch.X86: - pkg_arch = translate_arch(int(pkg_detail.hw_bitness)) - else: - pkg_arch = None - if pkg_arch is None or pkg_os is None: - print( - f"Ignoring release for {pkg_detail.os} {pkg_detail.arch}_{pkg_detail.hw_bitness}" + azul_path = os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") + if os.path.exists(azul_path): + azul_packages = ZuluPackageList.parse_file(azul_path) + for _, pkg in azul_packages: + pkg_detail = ZuluPackageDetail.parse_file( + os.path.join( + UPSTREAM_DIR, AZUL_VERSIONS_DIR, f"{pkg.package_uuid}.json" + ) ) - continue + major = pkg_detail.java_version[0] + if major < 8: + continue # we will never need java versions less than 8 - java_os = JavaRuntimeOS(f"{pkg_os}-{pkg_arch}") - runtime = azul_package_to_java_runtime(pkg_detail) - add_java_runtime(runtime, major, java_os) + pkg_os = translate_os(str(pkg_detail.os)) + if pkg_detail.arch == AzulArch.Arm: + pkg_arch = translate_arch(f"{pkg_detail.arch}{pkg_detail.hw_bitness}") + elif pkg_detail.arch == AzulArch.X86: + pkg_arch = translate_arch(int(pkg_detail.hw_bitness)) + else: + pkg_arch = None + if pkg_arch is None or pkg_os is None: + print( + f"Ignoring release for {pkg_detail.os} {pkg_detail.arch}_{pkg_detail.hw_bitness}" + ) + continue + + java_os = JavaRuntimeOS(f"{pkg_os}-{pkg_arch}") + runtime = azul_package_to_java_runtime(pkg_detail) + add_java_runtime(runtime, major, java_os) for major, runtimes in javas.items(): for java_os, runtime_list in runtimes: - print(f"Total runtimes for Java {major} {java_os}:", len( - runtime_list)) + print(f"Total runtimes for Java {major} {java_os}:", len(runtime_list)) rec = ensure_one_recommended(runtime_list) if rec is not None: print(f"Recomending {rec.name} for Java {major} {java_os}") @@ -405,27 +407,26 @@ def main(): return b return a - version_file = os.path.join( - LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") + version_file = os.path.join(LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") java_version = JavaRuntimeVersion( - name = f"Java {major}", - uid = JAVA_COMPONENT, - version = f"java{major}", + name=f"Java {major}", + uid=JAVA_COMPONENT, + version=f"java{major}", releaseTime=reduce( - newest_timestamp, - (runtime.release_time - for _, runtime_list in runtimes - for runtime in runtime_list - ), - None + newest_timestamp, + ( + runtime.release_time + for _, runtime_list in runtimes + for runtime in runtime_list ), - runtimes = runtimes) + None, + ), + runtimes=runtimes, + ) java_version.write(version_file) package = MetaPackage( - uid = JAVA_COMPONENT, - name = "Java Runtimes", - recommended = ["java8", "java17"] + uid=JAVA_COMPONENT, name="Java Runtimes", recommended=["java8", "java17"] ) package.write(os.path.join(LAUNCHER_DIR, JAVA_COMPONENT, "package.json")) diff --git a/meta/model/__init__.py b/meta/model/__init__.py index 733f78d..3d06f99 100644 --- a/meta/model/__init__.py +++ b/meta/model/__init__.py @@ -93,10 +93,10 @@ class GradleSpecifier: else: return False - def __lt__(self, other: 'GradleSpecifier'): + def __lt__(self, other: "GradleSpecifier"): return str(self) < str(other) - def __gt__(self, other: 'GradleSpecifier'): + def __gt__(self, other: "GradleSpecifier"): return str(self) > str(other) def __hash__(self): @@ -125,7 +125,7 @@ class GradleSpecifier: return cls(group, artifact, version, classifier, extension) @classmethod - def validate(cls, v: 'str | GradleSpecifier'): + def validate(cls, v: "str | GradleSpecifier"): if isinstance(v, cls): return v if isinstance(v, str): @@ -155,7 +155,7 @@ class MetaBase(pydantic.BaseModel): with open(file_path, "w") as f: f.write(self.json()) - def merge(self, other: 'MetaBase'): + def merge(self, other: "MetaBase"): """ Merge other object with self. - Concatenates lists @@ -179,14 +179,14 @@ class MetaBase(pydantic.BaseModel): elif isinstance(ours, set): ours |= theirs elif isinstance(ours, dict): - result = merge_dict(ours, copy.deepcopy(theirs)) # type: ignore + result = merge_dict(ours, copy.deepcopy(theirs)) # type: ignore setattr(self, key, result) elif MetaBase in get_all_bases(field.type_): ours.merge(theirs) else: setattr(self, key, theirs) - def __hash__(self): #type: ignore + def __hash__(self): # type: ignore return hash(self.json()) class Config: @@ -277,7 +277,7 @@ class MojangRule(MetaBase): class MojangRules(MetaBase): __root__: List[MojangRule] - def __iter__(self) -> Iterator[MojangRule]: #type: ignore + def __iter__(self) -> Iterator[MojangRule]: # type: ignore return iter(self.__root__) def __getitem__(self, item: int) -> MojangRule: @@ -319,6 +319,7 @@ class MetaVersion(Versioned): minecraft_arguments: Optional[str] = Field(alias="minecraftArguments") release_time: Optional[datetime] = Field(alias="releaseTime") compatible_java_majors: Optional[List[int]] = Field(alias="compatibleJavaMajors") + compatible_java_name: Optional[str] = Field(alias="compatibleJavaName") additional_traits: Optional[List[str]] = Field(alias="+traits") additional_tweakers: Optional[List[str]] = Field(alias="+tweakers") additional_jvm_args: Optional[List[str]] = Field(alias="+jvmArgs") diff --git a/meta/model/mojang.py b/meta/model/mojang.py index 48a5e5a..749ce7e 100644 --- a/meta/model/mojang.py +++ b/meta/model/mojang.py @@ -18,6 +18,9 @@ from . import ( SUPPORTED_LAUNCHER_VERSION = 21 SUPPORTED_COMPLIANCE_LEVEL = 1 DEFAULT_JAVA_MAJOR = 8 # By default, we should recommend Java 8 if we don't know better +DEFAULT_JAVA_NAME = ( + "jre-legacy" # By default, we should recommend Java 8 if we don't know better +) COMPATIBLE_JAVA_MAPPINGS = {16: [17]} """ @@ -318,10 +321,12 @@ class MojangVersion(MetaBase): raise Exception(f"Unsupported compliance level {self.compliance_level}") major = DEFAULT_JAVA_MAJOR + javaName = DEFAULT_JAVA_NAME if ( self.javaVersion is not None ): # some versions don't have this. TODO: maybe maintain manual overrides major = self.javaVersion.major_version + javaName = self.javaVersion.component compatible_java_majors = [major] if ( @@ -343,6 +348,7 @@ class MojangVersion(MetaBase): release_time=self.release_time, type=new_type, compatible_java_majors=compatible_java_majors, + compatible_java_name=javaName, additional_traits=addn_traits, main_jar=main_jar, ) From 97f8750ed72957c26b13146e3c149f2cb43599e9 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Tue, 19 Mar 2024 22:20:11 +0200 Subject: [PATCH 14/22] Split java source Signed-off-by: Trial97 --- generateJava.py | 153 ++++++++++++++++--------------------------- meta/common/java.py | 4 +- meta/model/java.py | 28 +++----- meta/model/mojang.py | 1 + 4 files changed, 67 insertions(+), 119 deletions(-) diff --git a/generateJava.py b/generateJava.py index 6a0bfc1..b9042f1 100644 --- a/generateJava.py +++ b/generateJava.py @@ -7,7 +7,9 @@ from functools import reduce from meta.common import ensure_component_dir, launcher_path, upstream_path, static_path from meta.common.java import ( - JAVA_COMPONENT, + JAVA_MINECRAFT_COMPONENT, + JAVA_ADOPTIUM_COMPONENT, + JAVA_AZUL_COMPONENT, ADOPTIUM_DIR, ADOPTIUM_VERSIONS_DIR, AZUL_DIR, @@ -16,7 +18,6 @@ from meta.common.java import ( from meta.model import MetaPackage from meta.model.java import ( JavaRuntimeOS, - JavaRuntimeMap, JavaRuntimeVersion, JavaRuntimeMeta, JavaVersionMeta, @@ -48,7 +49,6 @@ LAUNCHER_DIR = launcher_path() UPSTREAM_DIR = upstream_path() STATIC_DIR = static_path() -ensure_component_dir(JAVA_COMPONENT) MOJANG_OS_ARCHITECTURES = [ "x64", @@ -125,6 +125,7 @@ def mojang_os_to_java_os(mojang_os: MojangJavaOsName) -> JavaRuntimeOS: def mojang_runtime_to_java_runtime( mojang_runtime: MojangJavaRuntime, mojang_component: MojangJavaComponent, + runtime_os: JavaRuntimeOS, ) -> JavaRuntimeMeta: major, _, security = mojang_runtime.version.name.partition("u") if major and security: @@ -154,15 +155,17 @@ def mojang_runtime_to_java_runtime( checksum=JavaChecksumMeta( type=JavaChecksumType.Sha1, hash=mojang_runtime.manifest.sha1 ), - recommended=True, downloadType=JavaRuntimeDownloadType.Manifest, packageType=JavaPackageType.Jre, version=version, + runtime_os=runtime_os, ) def adoptium_release_binary_to_java_runtime( - rls: AdoptiumRelease, binary: AdoptiumBinary + rls: AdoptiumRelease, + binary: AdoptiumBinary, + runtime_os: JavaRuntimeOS, ) -> JavaRuntimeMeta: assert binary.package is not None @@ -189,14 +192,16 @@ def adoptium_release_binary_to_java_runtime( url=binary.package.link, releaseTime=rls.timestamp, checksum=checksum, - recommended=False, downloadType=JavaRuntimeDownloadType.Archive, packageType=pkg_type, version=version, + runtime_os=runtime_os, ) -def azul_package_to_java_runtime(pkg: ZuluPackageDetail) -> JavaRuntimeMeta: +def azul_package_to_java_runtime( + pkg: ZuluPackageDetail, runtime_os: JavaRuntimeOS +) -> JavaRuntimeMeta: version_parts = copy.copy(pkg.java_version) build = None @@ -227,10 +232,10 @@ def azul_package_to_java_runtime(pkg: ZuluPackageDetail) -> JavaRuntimeMeta: url=pkg.download_url, releaseTime=pkg.build_date, checksum=checksum, - recommended=False, downloadType=JavaRuntimeDownloadType.Archive, packageType=pkg_type, version=version, + runtime_os=runtime_os, ) @@ -262,63 +267,41 @@ def pkg_type_priority(pkg_type: JavaPackageType) -> int: return -1 -def ensure_one_recommended( - runtimes: list[JavaRuntimeMeta], -) -> Optional[JavaRuntimeMeta]: - if len(runtimes) < 1: - return None # can't do anything +def writeJavas(javas: dict[int, list[JavaRuntimeMeta]], uid: str): + ensure_component_dir(uid) + for major, runtimes in javas.items(): - recommended: Optional[JavaRuntimeMeta] = None - found_first = False - need_resort = False - for runtime in runtimes: - if runtime.recommended: - if not found_first: - recommended = runtime - else: - runtime.recommended = False - need_resort = True + def newest_timestamp(a: datetime.datetime | None, b: datetime.datetime): + if a is None or a < b: + return b + return a - if recommended and not need_resort: - return recommended # we have one recommended already + version_file = os.path.join(LAUNCHER_DIR, uid, f"java{major}.json") + java_version = JavaRuntimeVersion( + name=f"Java {major}", + uid=uid, + version=f"java{major}", + releaseTime=reduce( + newest_timestamp, + (runtime.release_time for runtime in runtimes), + None, + ), + runtimes=runtimes, + ) + java_version.write(version_file) - if recommended is None: - recommended = runtimes[0] - - def better_java_runtime(runtime: JavaRuntimeMeta): - assert recommended is not None - if vendor_priority(runtime.vendor) < vendor_priority(recommended.vendor): - return False - if pkg_type_priority(runtime.package_type) < pkg_type_priority( - recommended.package_type - ): - return False - if runtime.version < recommended.version: - return False - if runtime.release_time < recommended.release_time: - return False - return True - - for runtime in runtimes: - if better_java_runtime(runtime): - recommended.recommended = False - recommended = runtime - recommended.recommended = True - - return recommended + package = MetaPackage(uid=uid, name="Java Runtimes", recommended=[]) + package.write(os.path.join(LAUNCHER_DIR, uid, "package.json")) def main(): - javas: dict[int, JavaRuntimeMap] = {} + javas: dict[int, list[JavaRuntimeMeta]] = {} - def ensure_javamap(major: int): + def add_java_runtime(runtime: JavaRuntimeMeta, major: int): if major not in javas: - javas[major] = JavaRuntimeMap() - - def add_java_runtime(runtime: JavaRuntimeMeta, major: int, java_os: JavaRuntimeOS): - ensure_javamap(major) - print(f"Regestering runtime: {runtime.name} for Java {major} {java_os}") - javas[major][java_os].append(runtime) + javas[major] = list[JavaRuntimeMeta]() + print(f"Regestering runtime: {runtime.name} for Java {major}") + javas[major].append(runtime) print("Processing Mojang Javas") mojang_java_manifest = JavaIndex.parse_file( @@ -337,8 +320,11 @@ def main(): major = 8 else: major = int(mojang_runtime.version.name.partition(".")[0]) - runtime = mojang_runtime_to_java_runtime(mojang_runtime, comp) - add_java_runtime(runtime, major, java_os) + runtime = mojang_runtime_to_java_runtime(mojang_runtime, comp, java_os) + add_java_runtime(runtime, major) + + writeJavas(javas=javas, uid=JAVA_MINECRAFT_COMPONENT) + javas = {} print("Processing Adoptium Releases") adoptium_path = os.path.join(UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json") @@ -361,9 +347,13 @@ def main(): continue java_os = JavaRuntimeOS(f"{binary_os}-{binary_arch}") - runtime = adoptium_release_binary_to_java_runtime(rls, binary) - add_java_runtime(runtime, major, java_os) + runtime = adoptium_release_binary_to_java_runtime( + rls, binary, java_os + ) + add_java_runtime(runtime, major) + writeJavas(javas=javas, uid=JAVA_ADOPTIUM_COMPONENT) + javas = {} print("Processing Azul Packages") azul_path = os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") if os.path.exists(azul_path): @@ -392,43 +382,10 @@ def main(): continue java_os = JavaRuntimeOS(f"{pkg_os}-{pkg_arch}") - runtime = azul_package_to_java_runtime(pkg_detail) - add_java_runtime(runtime, major, java_os) - - for major, runtimes in javas.items(): - for java_os, runtime_list in runtimes: - print(f"Total runtimes for Java {major} {java_os}:", len(runtime_list)) - rec = ensure_one_recommended(runtime_list) - if rec is not None: - print(f"Recomending {rec.name} for Java {major} {java_os}") - - def newest_timestamp(a: datetime.datetime | None, b: datetime.datetime): - if a is None or a < b: - return b - return a - - version_file = os.path.join(LAUNCHER_DIR, JAVA_COMPONENT, f"java{major}.json") - java_version = JavaRuntimeVersion( - name=f"Java {major}", - uid=JAVA_COMPONENT, - version=f"java{major}", - releaseTime=reduce( - newest_timestamp, - ( - runtime.release_time - for _, runtime_list in runtimes - for runtime in runtime_list - ), - None, - ), - runtimes=runtimes, - ) - java_version.write(version_file) - - package = MetaPackage( - uid=JAVA_COMPONENT, name="Java Runtimes", recommended=["java8", "java17"] - ) - package.write(os.path.join(LAUNCHER_DIR, JAVA_COMPONENT, "package.json")) + runtime = azul_package_to_java_runtime(pkg_detail, java_os) + add_java_runtime(runtime, major) + writeJavas(javas=javas, uid=JAVA_AZUL_COMPONENT) + javas = {} if __name__ == "__main__": diff --git a/meta/common/java.py b/meta/common/java.py index 541d2fb..ec13b2a 100644 --- a/meta/common/java.py +++ b/meta/common/java.py @@ -9,4 +9,6 @@ AZUL_DIR = join(BASE_DIR, "azul") ADOPTIUM_VERSIONS_DIR = join(ADOPTIUM_DIR, "versions") AZUL_VERSIONS_DIR = join(AZUL_DIR, "versions") -JAVA_COMPONENT = "net.minecraft.java" \ No newline at end of file +JAVA_MINECRAFT_COMPONENT = "net.minecraft.java" +JAVA_ADOPTIUM_COMPONENT = "net.adoptium.java" +JAVA_AZUL_COMPONENT = "com.azul.java" diff --git a/meta/model/java.py b/meta/model/java.py index d232196..e24932e 100644 --- a/meta/model/java.py +++ b/meta/model/java.py @@ -10,6 +10,7 @@ from .enum import StrEnum from typing import Optional, Any, NamedTuple, Generator from urllib.parse import urlencode, urlparse, urlunparse from functools import total_ordering + # namedtuple to match the internal signature of urlunparse @@ -55,10 +56,10 @@ class JavaVersionMeta(MetaBase): return (self.major, self.minor, self.security, build) def __eq__(self, other: Any): - return (self.to_tuple() == other.to_tuple()) + return self.to_tuple() == other.to_tuple() - def __lt__(self, other: 'JavaVersionMeta'): - return (self.to_tuple() < other.to_tuple()) + def __lt__(self, other: "JavaVersionMeta"): + return self.to_tuple() < other.to_tuple() class JavaChecksumType(StrEnum): @@ -82,28 +83,15 @@ class JavaRuntimeMeta(MetaBase): url: str release_time: datetime = Field(alias="releaseTime") checksum: Optional[JavaChecksumMeta] - recommended: bool download_type: JavaRuntimeDownloadType = Field(alias="downloadType") package_type: JavaPackageType = Field(alias="packageType") version: JavaVersionMeta + runtime_os: JavaRuntimeOS = Field(alias="runtimeOS") -class JavaRuntimeMap(MetaBase): - __root__: dict[JavaRuntimeOS, list[JavaRuntimeMeta]] = { - os: [] for os in JavaRuntimeOS if os != JavaRuntimeOS.Unknown - } - - def __iter__(self) -> Generator[tuple[str, list[JavaRuntimeMeta]], None, None]: - yield from ((str(os), runtime) for os, runtime in self.__root__.items()) - - def __getitem__(self, item: JavaRuntimeOS) -> list[JavaRuntimeMeta]: - return self.__root__[item] - - def __len__(self): - return len(self.__root__) - class JavaRuntimeVersion(MetaVersion): - runtimes: JavaRuntimeMap + runtimes: list[JavaRuntimeMeta] + class URLComponents(NamedTuple): scheme: str @@ -122,7 +110,7 @@ class APIQuery(MetaBase): if isinstance(value, Enum): set_parts[key] = value.value elif isinstance(value, list): - if len(value) > 0: #type: ignore + if len(value) > 0: # type: ignore set_parts[key] = value elif isinstance(value, datetime): set_parts[key] = value.isoformat() diff --git a/meta/model/mojang.py b/meta/model/mojang.py index 749ce7e..1ad31d6 100644 --- a/meta/model/mojang.py +++ b/meta/model/mojang.py @@ -203,6 +203,7 @@ class MojangJavaComponent(StrEnum): Gamma = "java-runtime-gamma" GammaSnapshot = "java-runtime-gamma-snapshot" Exe = "minecraft-java-exe" + Delta = "java-runtime-delta" class JavaVersion(MetaBase): From b49bec7a5c34fd97726d9ec69fdd67ac2886a022 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Wed, 20 Mar 2024 17:40:46 +0200 Subject: [PATCH 15/22] Removed jdk Signed-off-by: Trial97 --- generateJava.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/generateJava.py b/generateJava.py index b9042f1..1eabda2 100644 --- a/generateJava.py +++ b/generateJava.py @@ -28,9 +28,11 @@ from meta.model.java import ( AdoptiumAvailableReleases, AdoptiumReleases, AdoptiumRelease, + AdoptiumImageType, AdoptiumBinary, ZuluPackageList, ZuluPackageDetail, + AzulJavaPackageType, AzulArch, ) @@ -338,7 +340,10 @@ def main(): ) for _, rls in adoptium_releases: for binary in rls.binaries: - if binary.package is None: + if ( + binary.package is None + or binary.image_type is not AdoptiumImageType.Jre + ): continue binary_arch = translate_arch(str(binary.architecture)) binary_os = translate_os(str(binary.os)) @@ -365,7 +370,7 @@ def main(): ) ) major = pkg_detail.java_version[0] - if major < 8: + if major < 8 or pkg_detail.java_package_type is not AzulJavaPackageType.Jre: continue # we will never need java versions less than 8 pkg_os = translate_os(str(pkg_detail.os)) From d996dc2a20c8664b03259cead7c888fd85d9848c Mon Sep 17 00:00:00 2001 From: Trial97 Date: Thu, 21 Mar 2024 20:49:58 +0200 Subject: [PATCH 16/22] Force sort after major Signed-off-by: Trial97 --- generateJava.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/generateJava.py b/generateJava.py index 1eabda2..f584692 100644 --- a/generateJava.py +++ b/generateJava.py @@ -270,24 +270,35 @@ def pkg_type_priority(pkg_type: JavaPackageType) -> int: def writeJavas(javas: dict[int, list[JavaRuntimeMeta]], uid: str): + def oldest_timestamp(a: datetime.datetime | None, b: datetime.datetime): + if a is None or a > b: + return b + return a + ensure_component_dir(uid) + + # small hack to sort the versions after major + javas = dict(sorted(javas.items(), key=lambda item: item[0])) + timestamps: dict[int, datetime.datetime | None] = {} + prevDate: datetime.datetime | None = None for major, runtimes in javas.items(): + releaseTime = reduce( + oldest_timestamp, + (runtime.release_time for runtime in runtimes), + None, + ) + if prevDate is not None and releaseTime < prevDate: + releaseTime = prevDate + datetime.timedelta(seconds=1) + prevDate = releaseTime + timestamps[major] = releaseTime - def newest_timestamp(a: datetime.datetime | None, b: datetime.datetime): - if a is None or a < b: - return b - return a - + for major, runtimes in javas.items(): version_file = os.path.join(LAUNCHER_DIR, uid, f"java{major}.json") java_version = JavaRuntimeVersion( name=f"Java {major}", uid=uid, version=f"java{major}", - releaseTime=reduce( - newest_timestamp, - (runtime.release_time for runtime in runtimes), - None, - ), + releaseTime=timestamps.get(major), runtimes=runtimes, ) java_version.write(version_file) From 60a3f5f194974399cb1fb20b5414c7ddcaa69419 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Thu, 9 May 2024 10:03:00 +0300 Subject: [PATCH 17/22] Map alpha and beta to gamma in case of arm os Signed-off-by: Trial97 --- meta/run/generate_java.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/meta/run/generate_java.py b/meta/run/generate_java.py index 95803f2..110d871 100644 --- a/meta/run/generate_java.py +++ b/meta/run/generate_java.py @@ -327,6 +327,15 @@ def main(): if comp == MojangJavaComponent.Exe: continue # doesn't appear to be used and not marked with a full verison so I don't trust it mojang_runtimes = mojang_java_manifest[mojang_os_name][comp] + if ( + len(mojang_runtimes) == 0 + and mojang_os_name + in [MojangJavaOsName.WindowsArm64, MojangJavaOsName.MacOSArm64] + and comp in [MojangJavaComponent.Alpha, MojangJavaComponent.Beta] + ): + mojang_runtimes = mojang_java_manifest[mojang_os_name][ + MojangJavaComponent.Gamma + ] for mojang_runtime in mojang_runtimes: if comp == MojangJavaComponent.JreLegacy: major = 8 From adef8d673c93b5a1fdfcffa05aeb01af94f466d8 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Fri, 10 May 2024 23:58:50 +0300 Subject: [PATCH 18/22] Mapped linux-arm variants to mojang provider Signed-off-by: Trial97 --- meta/run/generate_java.py | 181 ++++++++++++++++++++++++++------------ 1 file changed, 123 insertions(+), 58 deletions(-) diff --git a/meta/run/generate_java.py b/meta/run/generate_java.py index 110d871..fd32308 100644 --- a/meta/run/generate_java.py +++ b/meta/run/generate_java.py @@ -123,6 +123,26 @@ def mojang_os_to_java_os(mojang_os: MojangJavaOsName) -> JavaRuntimeOS: return JavaRuntimeOS.Unknown +def mojang_component_to_major(mojang_component: MojangJavaComponent) -> int: + match mojang_component: + case MojangJavaComponent.JreLegacy: + return 8 + case MojangJavaComponent.Alpha: + return 17 + case MojangJavaComponent.Beta: + return 17 + case MojangJavaComponent.Gamma: + return 17 + case MojangJavaComponent.GammaSnapshot: + return 17 + case MojangJavaComponent.Exe: + return 0 + case MojangJavaComponent.Delta: + return 21 + case _: + return 0 + + def mojang_runtime_to_java_runtime( mojang_runtime: MojangJavaRuntime, mojang_component: MojangJavaComponent, @@ -240,34 +260,6 @@ def azul_package_to_java_runtime( ) -PREFERED_VENDOR_ORDER = ["mojang", "eclipse", "azul"] - -__PREFERED_VENDOR_ORDER = list(reversed(PREFERED_VENDOR_ORDER)) - - -def vendor_priority(vendor: str) -> int: - """Get a numeric priority for a given vendor - - Args: - vendor (str): the vendor to check - - Returns: - int: how preferable the vendor is, the higher the better - """ - if vendor not in PREFERED_VENDOR_ORDER: - return -1 - return __PREFERED_VENDOR_ORDER.index(vendor) - - -def pkg_type_priority(pkg_type: JavaPackageType) -> int: - match pkg_type: - case JavaPackageType.Jre: - return 2 - case JavaPackageType.Jdk: - return 1 - return -1 - - def writeJavas(javas: dict[int, list[JavaRuntimeMeta]], uid: str): def oldest_timestamp(a: datetime.datetime | None, b: datetime.datetime): if a is None or a > b: @@ -308,6 +300,7 @@ def writeJavas(javas: dict[int, list[JavaRuntimeMeta]], uid: str): def main(): javas: dict[int, list[JavaRuntimeMeta]] = {} + extra_mojang_javas: dict[int, list[JavaRuntimeMeta]] = {} def add_java_runtime(runtime: JavaRuntimeMeta, major: int): if major not in javas: @@ -315,37 +308,27 @@ def main(): print(f"Regestering runtime: {runtime.name} for Java {major}") javas[major].append(runtime) - print("Processing Mojang Javas") - mojang_java_manifest = JavaIndex.parse_file( - os.path.join(UPSTREAM_DIR, JAVA_MANIFEST_FILE) - ) - for mojang_os_name in mojang_java_manifest: - if mojang_os_name == MojangJavaOsName.Gamecore: - continue # empty - java_os = mojang_os_to_java_os(mojang_os_name) - for comp in mojang_java_manifest[mojang_os_name]: - if comp == MojangJavaComponent.Exe: - continue # doesn't appear to be used and not marked with a full verison so I don't trust it - mojang_runtimes = mojang_java_manifest[mojang_os_name][comp] - if ( - len(mojang_runtimes) == 0 - and mojang_os_name - in [MojangJavaOsName.WindowsArm64, MojangJavaOsName.MacOSArm64] - and comp in [MojangJavaComponent.Alpha, MojangJavaComponent.Beta] - ): - mojang_runtimes = mojang_java_manifest[mojang_os_name][ - MojangJavaComponent.Gamma + # only add specific versions to the list + if ( + ( + runtime.runtime_os + in [JavaRuntimeOS.MacOsArm64, JavaRuntimeOS.WindowsArm64] + and major == 8 + ) + or ( + runtime.runtime_os + in [ + JavaRuntimeOS.WindowsArm32, + JavaRuntimeOS.LinuxArm32, + JavaRuntimeOS.LinuxArm64, ] - for mojang_runtime in mojang_runtimes: - if comp == MojangJavaComponent.JreLegacy: - major = 8 - else: - major = int(mojang_runtime.version.name.partition(".")[0]) - runtime = mojang_runtime_to_java_runtime(mojang_runtime, comp, java_os) - add_java_runtime(runtime, major) - - writeJavas(javas=javas, uid=JAVA_MINECRAFT_COMPONENT) - javas = {} + and major in [8, 17, 21] + ) + or (runtime.runtime_os == JavaRuntimeOS.LinuxX86 and major in [17, 21]) + ): + if major not in extra_mojang_javas: + extra_mojang_javas[major] = list[JavaRuntimeMeta]() + extra_mojang_javas[major].append(runtime) print("Processing Adoptium Releases") adoptium_path = os.path.join(UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json") @@ -411,6 +394,88 @@ def main(): writeJavas(javas=javas, uid=JAVA_AZUL_COMPONENT) javas = {} + # constructs the missing mojang javas based on adoptium or azul + def get_mojang_extra_java( + mojang_component: MojangJavaComponent, java_os: JavaRuntimeOS + ) -> JavaRuntimeMeta | None: + java_major = mojang_component_to_major(mojang_component) + if not java_major in extra_mojang_javas: + return None + posible_javas = list( + filter(lambda x: x.runtime_os == java_os, extra_mojang_javas[java_major]) + ) + if len(posible_javas) == 0: + return None + prefered_vendor = list(filter(lambda x: x.vendor != "azul", posible_javas)) + if len(prefered_vendor) == 0: + prefered_vendor = posible_javas + prefered_vendor.sort(key=lambda x: x.version, reverse=True) + runtime = prefered_vendor[0] + runtime.name = mojang_component + return runtime + + print("Processing Mojang Javas") + mojang_java_manifest = JavaIndex.parse_file( + os.path.join(UPSTREAM_DIR, JAVA_MANIFEST_FILE) + ) + for mojang_os_name in mojang_java_manifest: + if mojang_os_name == MojangJavaOsName.Gamecore: + continue # empty + java_os = mojang_os_to_java_os(mojang_os_name) + for comp in mojang_java_manifest[mojang_os_name]: + if comp == MojangJavaComponent.Exe: + continue # doesn't appear to be used and not marked with a full verison so I don't trust it + mojang_runtimes = mojang_java_manifest[mojang_os_name][comp] + if len(mojang_runtimes) == 0: + if mojang_os_name in [ + MojangJavaOsName.WindowsArm64, + MojangJavaOsName.MacOSArm64, + ]: + if comp in [MojangJavaComponent.Alpha, MojangJavaComponent.Beta]: + mojang_runtimes = mojang_java_manifest[mojang_os_name][ + MojangJavaComponent.Gamma + ] + elif ( + comp == MojangJavaComponent.JreLegacy + ): # arm version of win and mac is missing the legacy java + runtime = get_mojang_extra_java(comp, java_os) + if runtime != None: + add_java_runtime(runtime, mojang_component_to_major(comp)) + if ( + mojang_os_name == MojangJavaOsName.Linuxi386 + and comp != MojangJavaComponent.JreLegacy + ): # the linux x86 is missing all but legacy + runtime = get_mojang_extra_java(comp, java_os) + if runtime != None: + add_java_runtime(runtime, mojang_component_to_major(comp)) + for mojang_runtime in mojang_runtimes: + if comp == MojangJavaComponent.JreLegacy: + major = 8 + else: + major = int(mojang_runtime.version.name.partition(".")[0]) + runtime = mojang_runtime_to_java_runtime(mojang_runtime, comp, java_os) + add_java_runtime(runtime, major) + # mojang doesn't provide any versions for the following systems so borrow info from adoptium/azul + for java_os in [ + JavaRuntimeOS.WindowsArm32, + JavaRuntimeOS.LinuxArm32, + JavaRuntimeOS.LinuxArm64, + ]: + for comp in [ + MojangJavaComponent.JreLegacy, + MojangJavaComponent.Alpha, + MojangJavaComponent.Beta, + MojangJavaComponent.Gamma, + MojangJavaComponent.GammaSnapshot, + MojangJavaComponent.Delta, + ]: + runtime = get_mojang_extra_java(comp, java_os) + if runtime != None: + add_java_runtime(runtime, mojang_component_to_major(comp)) + + writeJavas(javas=javas, uid=JAVA_MINECRAFT_COMPONENT) + javas = {} + if __name__ == "__main__": main() From c550feabe16b61c767cca55c83cb8541924a98bc Mon Sep 17 00:00:00 2001 From: Trial97 Date: Tue, 21 May 2024 16:06:37 +0300 Subject: [PATCH 19/22] add java folders to launcher git Signed-off-by: Trial97 --- update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.sh b/update.sh index b50838d..8e0fe38 100755 --- a/update.sh +++ b/update.sh @@ -74,7 +74,7 @@ if [ "${DEPLOY_TO_GIT}" = true ]; then launcher_git add net.fabricmc.fabric-loader/* net.fabricmc.intermediary/* || fail_out launcher_git add org.quiltmc.quilt-loader/* || fail_out # TODO: add Quilt hashed, once it is actually used launcher_git add com.mumfrey.liteloader/* || fail_out - launcher_git add net.minecraft.java/* || fail_out + launcher_git add net.minecraft.java/* net.adoptium.java/* com.azul.java/* || fail_out if ! launcher_git diff --cached --exit-code; then launcher_git commit -a -m "Update ${currentDate}" || fail_out From 641cbc658884ca50a62bdcf7a1878782e38b7b76 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Sun, 9 Jun 2024 18:57:26 +0300 Subject: [PATCH 20/22] update tool.poetry.scripts Signed-off-by: Trial97 --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index ca55558..10cf5dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,12 +16,14 @@ generateLiteloader = "meta.run.generate_liteloader:main" generateMojang = "meta.run.generate_mojang:main" generateNeoForge = "meta.run.generate_neoforge:main" generateQuilt = "meta.run.generate_quilt:main" +generateJava = "meta.run.generate_java:main" updateFabric = "meta.run.update_fabric:main" updateForge = "meta.run.update_forge:main" updateLiteloader = "meta.run.update_liteloader:main" updateMojang = "meta.run.update_mojang:main" updateNeoForge = "meta.run.update_neoforge:main" updateQuilt = "meta.run.update_quilt:main" +updateJava = "meta.run.update_java:main" index = "meta.run.index:main" [tool.poetry.dependencies] From e233614c5414272409f523b9c3506f828a957fd9 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Thu, 13 Jun 2024 19:00:41 +0300 Subject: [PATCH 21/22] remove jdk Signed-off-by: Trial97 --- meta/run/update_java.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/meta/run/update_java.py b/meta/run/update_java.py index dc9e159..b77bb1d 100644 --- a/meta/run/update_java.py +++ b/meta/run/update_java.py @@ -23,6 +23,7 @@ from meta.model.java import ( AzulArchiveType, AzulReleaseStatus, AzulAvailabilityType, + AzulJavaPackageType, azulApiPackageDetailUrl, ZuluPackageDetail, ZuluPackagesDetail, @@ -78,26 +79,6 @@ def main(): break page += 1 - page = 0 - while True: - query = AdoptiumAPIFeatureReleasesQuery( - image_type=AdoptiumImageType.Jdk, page_size=page_size, page=page - ) - api_call = adoptiumAPIFeatureReleasesUrl(feature, query=query) - print("Fetching JDK Page:", page, api_call) - r_rls = sess.get(api_call) - if r_rls.status_code == 404: - break - else: - r_rls.raise_for_status() - - releases = list(AdoptiumRelease(**rls) for rls in r_rls.json()) - releases_for_feature.extend(releases) - - if len(r_rls.json()) < page_size: - break - page += 1 - print("Total Adoptium releases for feature:", len(releases_for_feature)) releases = AdoptiumReleases(__root__=releases_for_feature) feature_file = os.path.join( @@ -115,6 +96,7 @@ def main(): archive_type=AzulArchiveType.Zip, release_status=AzulReleaseStatus.Ga, availability_types=[AzulAvailabilityType.CA], + java_package_type=AzulJavaPackageType.Jre, javafx_bundled=False, page=page, page_size=page_size, From 6706cb5258e023bdada0cb4de0291bc96e113132 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Thu, 13 Jun 2024 19:58:16 +0300 Subject: [PATCH 22/22] use only latest for azul Signed-off-by: Trial97 --- meta/run/update_java.py | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/run/update_java.py b/meta/run/update_java.py index b77bb1d..326fb5f 100644 --- a/meta/run/update_java.py +++ b/meta/run/update_java.py @@ -98,6 +98,7 @@ def main(): availability_types=[AzulAvailabilityType.CA], java_package_type=AzulJavaPackageType.Jre, javafx_bundled=False, + latest=True, page=page, page_size=page_size, )