From 6777305c000f997183646adef2d38ae10e603ebc Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Tue, 5 Apr 2022 15:05:33 +0200 Subject: [PATCH] refactor: move index to pydantic models --- index.py | 61 +++++++++++++++++---------------------------- meta/model/index.py | 46 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 38 deletions(-) create mode 100644 meta/model/index.py diff --git a/index.py b/index.py index b03e665..a967836 100755 --- a/index.py +++ b/index.py @@ -1,41 +1,41 @@ import hashlib -from operator import itemgetter +import os +from operator import itemgetter, attrgetter -from meta.metautil import * from meta.common import polymc_path +from meta.model import MetaVersion, MetaPackage +from meta.model.index import MetaPackageIndex, MetaVersionIndex, MetaVersionIndexEntry, MetaPackageIndexEntry PMC_DIR = polymc_path() # take the hash type (like hashlib.md5) and filename, return hex string of hash -def HashFile(hash, fname): - hash_instance = hash() - with open(fname, "rb") as f: +def hash_file(hash_fn, file_name): + hash_instance = hash_fn() + with open(file_name, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): hash_instance.update(chunk) return hash_instance.hexdigest() # ignore these files when indexing versions -ignore = set(["index.json", "package.json", ".git", ".github"]) +ignore = {"index.json", "package.json", ".git", ".github"} # initialize output structures - package list level -packages = PolyMCPackageIndex() +packages = MetaPackageIndex() # walk thorugh all the package folders for package in sorted(os.listdir(PMC_DIR)): if package in ignore: continue - sharedData = readSharedPackageData(package) + sharedData = MetaPackage.parse_file(os.path.join(PMC_DIR, package, "package.json")) recommendedVersions = set() if sharedData.recommended: recommendedVersions = set(sharedData.recommended) # initialize output structures - version list level - versionList = PolyMCVersionIndex() - versionList.uid = package - versionList.name = sharedData.name + versionList = MetaVersionIndex(uid=package, name=sharedData.name) # walk through all the versions of the package for filename in os.listdir(PMC_DIR + "/%s" % (package)): @@ -44,42 +44,27 @@ for package in sorted(os.listdir(PMC_DIR)): # parse and hash the version file filepath = PMC_DIR + "/%s/%s" % (package, filename) - filehash = HashFile(hashlib.sha256, filepath) - versionFile = None - with open(filepath) as json_file: - versionFile = PolyMCVersionFile(json.load(json_file)) + filehash = hash_file(hashlib.sha256, filepath) + versionFile = MetaVersion.parse_file(filepath) + is_recommended = versionFile.version in recommendedVersions + + versionEntry = MetaVersionIndexEntry.from_meta_version(versionFile, is_recommended, filehash) - # pull information from the version file - versionEntry = PolyMCVersionIndexEntry() - if versionFile.version in recommendedVersions: - versionEntry.recommended = True - versionEntry.version = versionFile.version - versionEntry.type = versionFile.type - versionEntry.releaseTime = versionFile.releaseTime - versionEntry.sha256 = filehash - versionEntry.requires = versionFile.requires - versionEntry.conflicts = versionFile.conflicts - versionEntry.volatile = versionFile.volatile versionList.versions.append(versionEntry) # sort the versions in descending order by time of release - versionList.versions = sorted(versionList.versions, key=itemgetter('releaseTime'), reverse=True) + versionList.versions = sorted(versionList.versions, key=attrgetter('release_time'), reverse=True) # write the version index for the package outFilePath = PMC_DIR + "/%s/index.json" % (package) - with open(outFilePath, 'w') as outfile: - json.dump(versionList.to_json(), outfile, sort_keys=True, indent=4) + versionList.write(outFilePath) # insert entry into the package index - packageEntry = PolyMCPackageIndexEntry( - { - "uid": package, - "name": sharedData.name, - "sha256": HashFile(hashlib.sha256, outFilePath) - } + packageEntry = MetaPackageIndexEntry( + uid=package, + name=sharedData.name, + sha256=hash_file(hashlib.sha256, outFilePath) ) packages.packages.append(packageEntry) -# write the repository package index -with open(PMC_DIR + "/index.json", 'w') as outfile: - json.dump(packages.to_json(), outfile, sort_keys=True, indent=4) +packages.write(os.path.join(PMC_DIR, "index.json")) diff --git a/meta/model/index.py b/meta/model/index.py new file mode 100644 index 0000000..f7cdc36 --- /dev/null +++ b/meta/model/index.py @@ -0,0 +1,46 @@ +from datetime import datetime +from typing import Optional, List + +from pydantic import Field + +from meta.model import Dependency, MetaBase, Versioned, MetaVersion + + +class MetaVersionIndexEntry(MetaBase): + version: str + type: Optional[str] + release_time: datetime = Field(alias="releaseTime") + requires: Optional[List[Dependency]] + conflicts: Optional[List[Dependency]] + recommended: Optional[bool] + volatile: Optional[bool] + sha256: str + + @classmethod + def from_meta_version(cls, v: MetaVersion, recommended: bool, sha256: str): + return cls( + version=v.version, + type=v.type, + release_time=v.release_time, + requires=v.requires, + conflicts=v.conflicts, + recommended=recommended, + volatile=v.volatile, + sha256=sha256 + ) + + +class MetaVersionIndex(Versioned): + name: str + uid: str + versions: List[MetaVersionIndexEntry] = Field([]) + + +class MetaPackageIndexEntry(MetaBase): + name: str + uid: str + sha256: str + + +class MetaPackageIndex(Versioned): + packages: List[MetaPackageIndexEntry] = Field([])