From d8ab80eab96c0823bd8ef958a741a59d3842f28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Fri, 22 Mar 2019 01:27:53 +0100 Subject: [PATCH] Update for new forge version indexes --- forgeutil.py | 82 ++++++++++++ generateForge2.py | 183 ++++++++++++++++++++++++++ index.py | 2 +- jsonobject/containers.py | 4 +- update.sh | 4 +- updateForge2.py | 277 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 547 insertions(+), 5 deletions(-) create mode 100755 generateForge2.py create mode 100755 updateForge2.py diff --git a/forgeutil.py b/forgeutil.py index 52cce83..0d3034a 100644 --- a/forgeutil.py +++ b/forgeutil.py @@ -54,6 +54,88 @@ class ForgeVersion: else: return self.universal_url +# A post-processed entry constructed from the reconstructed Forge version index +class ForgeVersion2: + def __init__(self, entry): + self.build = entry.build + self.rawVersion = entry.version + self.mcversion = entry.mcversion + self.mcversion_sane = self.mcversion.replace("_pre", "-pre", 1) + self.branch = entry.branch + self.installer_filename = None + self.installer_url = None + self.universal_filename = None + self.universal_url = None + self.changelog_url = None + self.longVersion = "%s-%s" % (self.mcversion, self.rawVersion) + if self.branch != None: + self.longVersion = self.longVersion + "-%s" % (self.branch) + for classifier, fileentry in entry.files.items(): + extension = fileentry.extension + checksum = fileentry.hash + filename = fileentry.filename(self.longVersion) + url = fileentry.url(self.longVersion) + if classifier == "installer": + self.installer_filename = filename + self.installer_url = url + if classifier == "universal" or classifier == "client": + self.universal_filename = filename + self.universal_url = url + if classifier == "changelog": + self.changelog_url = url + + def name(self): + return "Forge %d" % (self.build) + + def usesInstaller(self): + if self.installer_url == None: + return False + if self.mcversion == "1.5.2": + return False + return True + + def filename(self): + if self.usesInstaller(): + return self.installer_filename + else: + return self.universal_filename + + def url(self): + if self.usesInstaller(): + return self.installer_url + else: + return self.universal_url + +class NewForgeFile(JsonObject): + classifier = StringProperty(required=True) + hash = StringProperty(required=True) + extension = StringProperty(required=True) + + def filename(self, longversion): + return "%s-%s-%s.%s" % ("forge", longversion, self.classifier, self.extension) + + def url(self, longversion): + return "https://files.minecraftforge.net/maven/net/minecraftforge/forge/%s/%s" % (longversion, self.filename(longversion)) + +class NewForgeEntry(JsonObject): + longversion = StringProperty(required=True) + mcversion = StringProperty(required=True) + version = StringProperty(required=True) + build = IntegerProperty(required=True) + branch = StringProperty() + latest = BooleanProperty() + recommended = BooleanProperty() + files = DictProperty(NewForgeFile) + +class ForgeMcVersionInfo(JsonObject): + latest = StringProperty() + recommended = StringProperty() + versions = ListProperty(StringProperty()) + +class NewForgeIndex(JsonObject): + versions = DictProperty(NewForgeEntry) + by_mcversion = DictProperty(ForgeMcVersionInfo) + # A raw entry from the main Forge version index class ForgeEntry(JsonObject): branch = StringProperty() diff --git a/generateForge2.py b/generateForge2.py new file mode 100755 index 0000000..ed6fe6b --- /dev/null +++ b/generateForge2.py @@ -0,0 +1,183 @@ +#!/usr/bin/python3 +from __future__ import print_function +import sys +import os +import re +from metautil import * +from forgeutil import * +from jsonobject import * +from distutils.version import LooseVersion + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +# Contruct a set of libraries out of a Minecraft version file, for filtering. +mcVersionCache = {} +def loadMcVersionFilter(version): + if version in mcVersionCache: + return mcVersionCache[version] + libSet = set() + with open("multimc/net.minecraft/%s.json" % version, 'r', encoding='utf-8') as mcFile: + mcVersion = MultiMCVersionFile(json.load(mcFile)) + for lib in mcVersion.libraries: + libSet.add(lib.name) + mcVersionCache[version] = libSet + return libSet + +''' +Match a library coordinate to a set of library coordinates. + * Block those that pass completely. + * For others, block those with lower versions than in the set. +''' +def shouldIgnoreArtifact(libSet, match): + for ver in libSet: + if ver.group == match.group and ver.artifact == match.artifact and ver.classifier == match.classifier: + if ver.version == match.version: + # Everything is matched perfectly - this one will be ignored + return True + else: + # We say the lib matches (is the same) also when the new version is lower than the old one + if LooseVersion(ver.version) > LooseVersion(match.version): + # eprint ("Lower version on %s:%s:%s: OLD=%s NEW=%s" % (ver.group, ver.artifact, ver.classifier, ver.version, match.version)) + return True + # Otherwise it did not match - new version is higher and this is an upgrade + return False + # No match found in the set - we need to keep this + return False + +def versionFromProfile(profile, version): + result = MultiMCVersionFile({"name":"Forge", "version":version.rawVersion, "uid":"net.minecraftforge" }) + mcversion = profile.install.minecraft + result.requires = [DependencyEntry(uid='net.minecraft', equals=mcversion)] + result.mainClass = profile.versionInfo.mainClass + args = profile.versionInfo.minecraftArguments + tweakers = [] + expression = re.compile("--tweakClass ([a-zA-Z0-9\\.]+)") + match = expression.search(args) + while match != None: + tweakers.append(match.group(1)); + args = args[:match.start()] + args[match.end():] + match = expression.search(args); + if len(tweakers) > 0: + args = args.strip() + result.addTweakers = tweakers; + # result.minecraftArguments = args + result.releaseTime = profile.versionInfo.time + libs = [] + mcFilter = loadMcVersionFilter(mcversion) + for forgeLib in profile.versionInfo.libraries: + if forgeLib.name.isLwjgl(): + continue + if shouldIgnoreArtifact(mcFilter, forgeLib.name): + continue + fixedName = forgeLib.name + if fixedName.group == "net.minecraftforge": + if fixedName.artifact == "minecraftforge": + fixedName.artifact = "forge" + fixedName.classifier = "universal" + fixedName.version = "%s-%s" % (mcversion, fixedName.version) + elif fixedName.artifact == "forge": + fixedName.classifier = "universal" + ourLib = MultiMCLibrary(name=fixedName) + ourLib.url = forgeLib.url + if forgeLib.checksums and len(forgeLib.checksums) == 2: + ourLib.mmcHint = "forge-pack-xz" + libs.append(ourLib) + result.libraries = libs + result.order = 5 + return result + +def versionFromLegacy(version, legacyinfo : ForgeLegacyInfo): + result = MultiMCVersionFile({"name":"Forge", "version":version.rawVersion, "uid":"net.minecraftforge" }) + mcversion = version.mcversion_sane + result.requires = [DependencyEntry(uid='net.minecraft', equals=mcversion)] + result.releaseTime = legacyinfo.releaseTime + result.order = 5 + if mcversion in fmlLibsMapping: + result.addTraits = ["legacyFML"] + url = version.url() + classifier = None + if "universal" in url: + classifier = "universal" + else: + classifier = "client" + coord = GradleSpecifier("net.minecraftforge:forge:%s:%s" % (version.longVersion,classifier)) + mainmod = MultiMCLibrary(name = coord) + mainmod.downloads = MojangLibraryDownloads() + mainmod.downloads.artifact = MojangArtifact() + mainmod.downloads.artifact.path = None + mainmod.downloads.artifact.url = version.url() + mainmod.downloads.artifact.sha1 = legacyinfo.sha1 + mainmod.downloads.artifact.size = legacyinfo.size + result.jarMods = [mainmod] + return result + +# load the locally cached version list +with open("upstream/forge/derived_index.json", 'r', encoding='utf-8') as f: + main_json = json.load(f) + remoteVersionlist = NewForgeIndex(main_json) + +recommendedVersions = [] + +tsPath = "static/forge-legacyinfo.json" + +legacyinfolist = None +with open(tsPath, 'r', encoding='utf-8') as tsFile: + legacyinfolist = ForgeLegacyInfoList(json.load(tsFile)) + +for id, entry in remoteVersionlist.versions.items(): + if entry.mcversion == None: + eprint ("Skipping %s with invalid MC version" % id) + continue + + version = ForgeVersion2(entry) + if version.url() == None: + eprint ("Skipping %s with no valid files" % id) + continue + + if entry.recommended: + recommendedVersions.append(version.rawVersion) + + # If we do not have the corresponding Minecraft version, we ignore it + if not os.path.isfile("multimc/net.minecraft/%s.json" % version.mcversion_sane): + eprint ("Skipping %s with no corresponding Minecraft version %s" % (id, version.mcversion_sane)) + continue + + if version.mcversion_sane.startswith('1.13') or version.mcversion_sane.startswith('1.14'): + eprint ("Skipping %s with unsupported Minecraft version %s" % (id, version.mcversion_sane)) + continue + + outVersion = None + + if version.usesInstaller(): + profileFilepath = "upstream/forge/installer_manifests/%s.json" % version.longVersion + # If we do not have the Forge json, we ignore this version + if not os.path.isfile(profileFilepath): + eprint ("Skipping %s with missing profile json" % id) + continue + with open(profileFilepath, 'r', encoding='utf-8') as profileFile: + profile = ForgeInstallerProfile(json.load(profileFile)) + outVersion = versionFromProfile(profile, version) + else: + # Generate json for legacy here + if version.mcversion_sane == "1.6.1": + continue + build = version.build + if not str(build).encode('utf-8').decode('utf8') in legacyinfolist.number: + eprint("Legacy build %d is missing in legacy info. Ignoring." % build) + continue + + outVersion = versionFromLegacy(version, legacyinfolist.number[build]) + + outFilepath = "multimc/net.minecraftforge/%s.json" % outVersion.version + with open(outFilepath, 'w') as outfile: + json.dump(outVersion.to_json(), outfile, sort_keys=True, indent=4) + +recommendedVersions.sort() + +print ('Recommended versions:', recommendedVersions) + +sharedData = MultiMCSharedPackageData(uid = 'net.minecraftforge', name = "Forge") +sharedData.projectUrl = 'http://www.minecraftforge.net/forum/' +sharedData.recommended = recommendedVersions +sharedData.write() diff --git a/index.py b/index.py index 6491f40..726b8f0 100755 --- a/index.py +++ b/index.py @@ -22,7 +22,7 @@ ignore = set(["index.json", "package.json", ".git"]) packages = MultiMCPackageIndex() # walk thorugh all the package folders -for package in os.listdir('multimc'): +for package in sorted(os.listdir('multimc')): if package in ignore: continue diff --git a/jsonobject/containers.py b/jsonobject/containers.py index f34550a..14dc5bd 100644 --- a/jsonobject/containers.py +++ b/jsonobject/containers.py @@ -136,7 +136,7 @@ class JsonDict(SimpleDict): def __setitem__(self, key, value): if isinstance(key, int): - key = unicode(key) + key = str(key) wrapped, unwrapped = self.__unwrap(key, value) self._obj[key] = unwrapped @@ -148,7 +148,7 @@ class JsonDict(SimpleDict): def __getitem__(self, key): if isinstance(key, int): - key = unicode(key) + key = str(key) return super(JsonDict, self).__getitem__(key) diff --git a/update.sh b/update.sh index 329cfc2..ddddc4b 100755 --- a/update.sh +++ b/update.sh @@ -39,7 +39,7 @@ git checkout ${BRANCH} || exit 1 cd "${BASEDIR}" ./updateMojang.py || fail_in -#./updateForge.py || fail_in +./updateForge2.py || fail_in ./updateLiteloader.py || fail_in cd "${BASEDIR}/${UPSTREAM_DIR}" @@ -56,7 +56,7 @@ git checkout ${BRANCH} || exit 1 cd "${BASEDIR}" ./generateMojang.py || fail_out -#./generateForge.py || fail_out +./generateForge2.py || fail_out ./generateLiteloader.py || fail_out ./index.py || fail_out diff --git a/updateForge2.py b/updateForge2.py new file mode 100755 index 0000000..07afbf4 --- /dev/null +++ b/updateForge2.py @@ -0,0 +1,277 @@ +#!/usr/bin/python3 +''' + Get the source files necessary for generating Forge versions +''' +from __future__ import print_function +import sys + +import requests +from cachecontrol import CacheControl +from cachecontrol.caches import FileCache + +import json +import copy +import re +import zipfile +from metautil import * +from jsonobject import * +from forgeutil import * +import os.path +import datetime +import hashlib +from pathlib import Path + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +def filehash(filename, hashtype, blocksize=65536): + hash = hashtype() + with open(filename, "rb") as f: + for block in iter(lambda: f.read(blocksize), b""): + hash.update(block) + return hash.hexdigest() + +forever_cache = FileCache('http_cache', forever=True) +sess = CacheControl(requests.Session(), forever_cache) + +# get the remote version list fragments +r = sess.get('https://files.minecraftforge.net/maven/net/minecraftforge/forge/maven-metadata.json') +r.raise_for_status() +main_json = r.json() +assert type(main_json) == dict + +r = sess.get('https://files.minecraftforge.net/maven/net/minecraftforge/forge/promotions_slim.json') +r.raise_for_status() +promotions_json = r.json() +assert type(promotions_json) == dict + +promotedKeyExpression = re.compile("((?P[0-9\\.]+)-)?(?P(latest)|(recommended))(-(?P[a-zA-Z0-9\\.]+))?") + +recommendedSet = set() + +newIndex = NewForgeIndex() + +# FIXME: does not fully validate that the file has not changed format +# NOTE: For some insane reason, the format of the versions here is special. It having a branch at the end means it affects that particular branch +# We don't care about Forge having branches. +# Therefore we only use the short version part for later identification and filter out the branch-specific promotions (among other errors). +print("Processing promotions:") +for promoKey, shortversion in promotions_json.get('promos').items(): + match = promotedKeyExpression.match(promoKey) + if not match: + print('Skipping promotion %s, the key did not parse:' % promoKey) + pprint(promoKey) + assert match + if not match.group('mc'): + print('Skipping promotion %s, because it has no Minecraft version.' % promoKey) + continue + if match.group('branch'): + print('Skipping promotion %s, because it on a branch only.' % promoKey) + continue + elif match.group('promotion') == 'recommended': + recommendedSet.add(shortversion) + print ('%s added to recommended set' % shortversion) + elif match.group('promotion') == 'latest': + pass + else: + assert False + +versionExpression = re.compile("^(?P[0-9a-zA-Z_\\.]+)-(?P[0-9\\.]+\\.(?P[0-9]+))(-(?P[a-zA-Z0-9\\.]+))?$") + +def getSingleForgeFilesManifest(longversion): + files_manifest_file = Path("upstream/forge/files_manifests/%s.json" % longversion) + from_file = False + if files_manifest_file.is_file(): + with open(files_manifest_file, 'r') as f: + files_json=json.load(f) + from_file = True + else: + fileUrl = 'https://files.minecraftforge.net/maven/net/minecraftforge/forge/%s/meta.json' % longversion + r = sess.get(fileUrl) + r.raise_for_status() + files_json = r.json() + + retDict = dict() + + for classifier, extensionObj in files_json.get('classifiers').items(): + assert type(classifier) == str + assert type(extensionObj) == dict + + # assert len(extensionObj.items()) == 1 + index = 0 + count = 0 + while index < len(extensionObj.items()): + mutableCopy = copy.deepcopy(extensionObj) + extension, hash = mutableCopy.popitem() + assert type(classifier) == str + assert type(hash) == str + processedHash = re.sub(r"\W", "", hash) + if not len(processedHash) == 32: + print('%s: Skipping invalid hash for extension %s:' % (longversion, extension)) + pprint(extensionObj) + index = index + 1 + continue + + fileObj = NewForgeFile( + classifier=classifier, + hash=processedHash, + extension=extension + ) + if count == 0: + retDict[classifier] = fileObj + index = index + 1 + count = count + 1 + else: + print('%s: Multiple objects detected for classifier %s:' % (longversion, classifier)) + pprint(extensionObj) + assert False + + if not from_file: + with open(files_manifest_file, 'w', encoding='utf-8') as f: + json.dump(files_json, f, sort_keys=True, indent=4) + + return retDict + + +print("") +print("Processing versions:") +for mcversion, value in main_json.items(): + assert type(mcversion) == str + assert type(value) == list + for longversion in value: + assert type(longversion) == str + match = versionExpression.match(longversion) + if not match: + pprint(longversion) + assert match + assert match.group('mc') == mcversion + + files = getSingleForgeFilesManifest(longversion) + + build = int(match.group('build')) + version = match.group('ver') + branch = match.group('branch') + + isRecommended = (version in recommendedSet) + + entry = NewForgeEntry( + longversion=longversion, + mcversion=mcversion, + version=version, + build=build, + branch=branch, + # NOTE: we add this later after the fact. The forge promotions file lies about these. + latest=False, + recommended=isRecommended, + files=files + ) + newIndex.versions[longversion] = entry + if not newIndex.by_mcversion: + newIndex.by_mcversion = dict() + if not mcversion in newIndex.by_mcversion: + newIndex.by_mcversion.setdefault(mcversion, ForgeMcVersionInfo()) + newIndex.by_mcversion[mcversion].versions.append(longversion) + # NOTE: we add this later after the fact. The forge promotions file lies about these. + #if entry.latest: + #newIndex.by_mcversion[mcversion].latest = longversion + if entry.recommended: + newIndex.by_mcversion[mcversion].recommended = longversion + +print("") +print("Post processing promotions and adding missing 'latest':") +for mcversion, info in newIndex.by_mcversion.items(): + latestVersion = info.versions[-1] + info.latest = latestVersion + newIndex.versions[latestVersion].latest = True + print("Added %s as latest for %s" % (latestVersion, mcversion)) + +print("") +print("Dumping index files...") + +with open("upstream/forge/maven-metadata.json", 'w', encoding='utf-8') as f: + json.dump(main_json, f, sort_keys=True, indent=4) + +with open("upstream/forge/promotions_slim.json", 'w', encoding='utf-8') as f: + json.dump(promotions_json, f, sort_keys=True, indent=4) + +with open("upstream/forge/derived_index.json", 'w', encoding='utf-8') as f: + json.dump(newIndex.to_json(), f, sort_keys=True, indent=4) + +versions = [] +legacyinfolist = ForgeLegacyInfoList() +tsPath = "static/forge-legacyinfo.json" + +print("Grabbing installers and dumping installer profiles...") +# get the installer jars - if needed - and get the installer profiles out of them +for id, entry in newIndex.versions.items(): + if entry.mcversion == None: + eprint ("Skipping %d with invalid MC version" % entry.build) + continue + + version = ForgeVersion2(entry) + if version.url() == None: + eprint ("Skipping %d with no valid files" % version.build) + continue + + jarFilepath = "upstream/forge/jars/%s" % version.filename() + + if version.usesInstaller(): + profileFilepath = "upstream/forge/installer_manifests/%s.json" % version.longVersion + versionJsonFilepath = "upstream/forge/version_manifests/%s.json" % version.longVersion + print(version.name()) + if not os.path.isfile(profileFilepath): + # grab the installer if it's not there + if not os.path.isfile(jarFilepath): + eprint ("Downloading %s" % version.url()) + rfile = sess.get(version.url(), stream=True) + rfile.raise_for_status() + with open(jarFilepath, 'wb') as f: + for chunk in rfile.iter_content(chunk_size=128): + f.write(chunk) + print(jarFilepath) + with zipfile.ZipFile(jarFilepath, 'r') as jar: + with jar.open('install_profile.json', 'r') as profileZipEntry: + with open(profileFilepath, 'wb') as profileFile: + profileFile.write(profileZipEntry.read()) + profileFile.close() + profileZipEntry.close() + with jar.open('version.json', 'r') as profileZipEntry: + with open(versionJsonFilepath, 'wb') as versionJsonFile: + versionJsonFile.write(profileZipEntry.read()) + versionJsonFile.close() + profileZipEntry.close() + else: + pass + # ignore the two versions without install manifests and jar mod class files + # TODO: fix those versions? + if version.mcversion_sane == "1.6.1": + continue + + # only gather legacy info if it's missing + if not os.path.isfile(tsPath): + # grab the jar/zip if it's not there + if not os.path.isfile(jarFilepath): + rfile = sess.get(version.url(), stream=True) + rfile.raise_for_status() + with open(jarFilepath, 'wb') as f: + for chunk in rfile.iter_content(chunk_size=128): + f.write(chunk) + # find the latest timestamp in the zip file + tstamp = datetime.datetime.fromtimestamp(0) + with zipfile.ZipFile(jarFilepath, 'r') as jar: + allinfo = jar.infolist() + for info in allinfo: + tstampNew = datetime.datetime(*info.date_time) + if tstampNew > tstamp: + tstamp = tstampNew + legacyInfo = ForgeLegacyInfo() + legacyInfo.releaseTime = tstamp + legacyInfo.sha1 = filehash(jarFilepath, hashlib.sha1) + legacyInfo.sha256 = filehash(jarFilepath, hashlib.sha256) + legacyInfo.size = os.path.getsize(jarFilepath) + legacyinfolist.number[id] = legacyInfo + +# only write legacy info if it's missing +if not os.path.isfile(tsPath): + with open(tsPath, 'w') as outfile: + json.dump(legacyinfolist.to_json(), outfile, sort_keys=True, indent=4)