diff --git a/.github/workflows/detektAnalysis.yml b/.github/workflows/detektAnalysis.yml index 8be40728e7..14414a6829 100644 --- a/.github/workflows/detektAnalysis.yml +++ b/.github/workflows/detektAnalysis.yml @@ -29,7 +29,7 @@ jobs: id: setup_detekt uses: peter-murray/setup-detekt@v2 with: - detekt_version: '1.23.1' + detekt_version: '1.23.4' - name: Detekt errors run: detekt-cli --parallel --config detekt/config/detekt-errors.yml diff --git a/android/assets/jsons/translations/template.properties b/android/assets/jsons/translations/template.properties index 8fe9f01c91..aa438ac0e6 100644 --- a/android/assets/jsons/translations/template.properties +++ b/android/assets/jsons/translations/template.properties @@ -1789,7 +1789,9 @@ Download [modName] = Update [modName] = Could not download mod list = Download mod from URL = -Please enter the mod repository -or- archive zip -or- branch url: = +Please enter the mod repository -or- archive zip -or- branch -or- release url: = +That is not a valid ZIP file = +Invalid Mod archive structure = Invalid link! = Paste from clipboard = Download = diff --git a/core/src/com/unciv/logic/GameInfo.kt b/core/src/com/unciv/logic/GameInfo.kt index 2a1fed890a..19dde70785 100644 --- a/core/src/com/unciv/logic/GameInfo.kt +++ b/core/src/com/unciv/logic/GameInfo.kt @@ -39,7 +39,7 @@ import com.unciv.models.ruleset.unique.UniqueType import com.unciv.models.translations.tr import com.unciv.ui.audio.MusicMood import com.unciv.ui.audio.MusicTrackChooserFlags -import com.unciv.ui.screens.pickerscreens.Github.repoNameToFolderName +import com.unciv.logic.github.Github.repoNameToFolderName import com.unciv.ui.screens.savescreens.Gzip import com.unciv.ui.screens.worldscreen.status.NextTurnProgress import com.unciv.utils.DebugUtils diff --git a/core/src/com/unciv/logic/github/Github.kt b/core/src/com/unciv/logic/github/Github.kt new file mode 100644 index 0000000000..3c9e2e932d --- /dev/null +++ b/core/src/com/unciv/logic/github/Github.kt @@ -0,0 +1,356 @@ +package com.unciv.logic.github + +import com.badlogic.gdx.Files +import com.badlogic.gdx.files.FileHandle +import com.badlogic.gdx.graphics.Pixmap +import com.unciv.json.fromJsonFile +import com.unciv.json.json +import com.unciv.logic.BackwardCompatibility.updateDeprecations +import com.unciv.logic.UncivShowableException +import com.unciv.logic.github.Github.download +import com.unciv.logic.github.Github.downloadAndExtract +import com.unciv.logic.github.Github.tryGetGithubReposWithTopic +import com.unciv.logic.github.GithubAPI.getUrlForTreeQuery +import com.unciv.models.ruleset.ModOptions +import com.unciv.utils.Log +import java.io.BufferedReader +import java.io.FileFilter +import java.io.InputStream +import java.io.InputStreamReader +import java.net.HttpURLConnection +import java.net.URL +import java.nio.ByteBuffer +import java.util.zip.ZipException + + +/** + * Utility managing Github access (except the link in WorldScreenCommunityPopup) + * + * Singleton - RateLimit is shared app-wide and has local variables, and is not tested for thread safety. + * Therefore, additional effort is required should [tryGetGithubReposWithTopic] ever be called non-sequentially. + * [download] and [downloadAndExtract] should be thread-safe as they are self-contained. + * They do not join in the [RateLimit] handling because Github doc suggests each API + * has a separate limit (and I found none for cloning via a zip). + */ +object Github { + private const val contentDispositionHeader = "Content-Disposition" + private const val attachmentDispositionPrefix = "attachment;filename=" + + // Consider merging this with the Dropbox function + /** + * Helper opens am url and accesses its input stream, logging errors to the console + * @param url String representing a [URL] to download. + * @param action Optional callback that will be executed between opening the connection and + * accessing its data - passes the [connection][HttpURLConnection] and allows e.g. reading the response headers. + * @return The [InputStream] if successful, `null` otherwise. + */ + fun download(url: String, action: (HttpURLConnection) -> Unit = {}): InputStream? { + try { + // Problem type 1 - opening the URL connection + with(URL(url).openConnection() as HttpURLConnection) + { + action(this) + // Problem type 2 - getting the information + try { + return inputStream + } catch (ex: Exception) { + // No error handling, just log the message. + // NOTE that this 'read error stream' CAN ALSO fail, but will be caught by the big try/catch + val reader = BufferedReader(InputStreamReader(errorStream, Charsets.UTF_8)) + Log.error("Message from GitHub: %s", reader.readText()) + throw ex + } + } + } catch (ex: Exception) { + Log.error("Exception during GitHub download", ex) + return null + } + } + + /** + * Download a mod and extract, deleting any pre-existing version. + * @param folderFileHandle Destination handle of mods folder - also controls Android internal/external + * @author **Warning**: This took a long time to get just right, so if you're changing this, ***TEST IT THOROUGHLY*** on _both_ Desktop _and_ Phone + * @return FileHandle for the downloaded Mod's folder or null if download failed + */ + fun downloadAndExtract( + repo: GithubAPI.Repo, + folderFileHandle: FileHandle + ): FileHandle? { + var modNameFromFileName = repo.name + + val defaultBranch = repo.default_branch + val zipUrl: String + val tempName: String + if (repo.direct_zip_url.isEmpty()) { + val gitRepoUrl = repo.html_url + // Initiate download - the helper returns null when it fails + zipUrl = GithubAPI.getUrlForBranchZip(gitRepoUrl, defaultBranch) + + // Get a mod-specific temp file name + tempName = "temp-" + gitRepoUrl.hashCode().toString(16) + } else { + zipUrl = repo.direct_zip_url + tempName = "temp-" + repo.toString().hashCode().toString(16) + } + val inputStream = download(zipUrl) { + val disposition = it.getHeaderField(contentDispositionHeader) + if (disposition.startsWith(attachmentDispositionPrefix)) + modNameFromFileName = disposition.removePrefix(attachmentDispositionPrefix) + .removeSuffix(".zip").replace('.', ' ') + // We could check Content-Type=[application/x-zip-compressed] here, but the ZipFile will catch that anyway. Would save some bandwidth, however. + } ?: return null + + // Download to temporary zip + val tempZipFileHandle = folderFileHandle.child("$tempName.zip") + tempZipFileHandle.write(inputStream, false) + + // prepare temp unpacking folder + val unzipDestination = tempZipFileHandle.sibling(tempName) // folder, not file + // prevent mixing new content with old - hopefully there will never be cadavers of our tempZip stuff + if (unzipDestination.exists()) + if (unzipDestination.isDirectory) unzipDestination.deleteDirectory() else unzipDestination.delete() + + try { + Zip.extractFolder(tempZipFileHandle, unzipDestination) + } catch (ex: ZipException) { + throw UncivShowableException("That is not a valid ZIP file", ex) + } + + val (innerFolder, modName) = resolveZipStructure(unzipDestination, modNameFromFileName) + + // modName can be "$repoName-$defaultBranch" + val finalDestinationName = modName.replace("-$defaultBranch", "").repoNameToFolderName() + // finalDestinationName is now the mod name as we display it. Folder name needs to be identical. + val finalDestination = folderFileHandle.child(finalDestinationName) + + // prevent mixing new content with old + var tempBackup: FileHandle? = null + if (finalDestination.exists()) { + tempBackup = finalDestination.sibling("$finalDestinationName.updating") + finalDestination.renameOrMove(tempBackup) + } + + // Move temp unpacked content to their final place + finalDestination.mkdirs() // If we don't create this as a directory, it will think this is a file and nothing will work. + // The move will reset the last modified time (recursively, at least on Linux) + // This sort will guarantee the desktop launcher will not re-pack textures and overwrite the atlas as delivered by the mod + for (innerFileOrFolder in innerFolder.list() + .sortedBy { file -> file.extension() == "atlas" } ) { + innerFileOrFolder.renameOrMove(finalDestination) + } + + // clean up + tempZipFileHandle.delete() + unzipDestination.deleteDirectory() + if (tempBackup != null) + if (tempBackup.isDirectory) tempBackup.deleteDirectory() else tempBackup.delete() + + return finalDestination + } + + private fun isValidModFolder(dir: FileHandle): Boolean { + val goodFolders = listOf("Images", "jsons", "maps", "music", "sounds", "Images\\..*") + .map { Regex(it, RegexOption.IGNORE_CASE) } + val goodFiles = listOf(".*\\.atlas", ".*\\.png", "preview.jpg", ".*\\.md", "Atlases.json", ".nomedia", "license") + .map { Regex(it, RegexOption.IGNORE_CASE) } + var good = 0 + var bad = 0 + for (file in dir.list()) { + val goodList = if (file.isDirectory) goodFolders else goodFiles + if (goodList.any { file.name().matches(it) }) good++ else bad++ + } + return good > 0 && good > bad + } + + /** Check whether the unpacked zip contains a subfolder with mod content or is already the mod. + * If there's a subfolder we'll assume it is the mod name, optionally suffixed with branch or release-tag name like github does. + * @return Pair: actual mod content folder to name (subfolder name or [defaultModName]) + */ + private fun resolveZipStructure(dir: FileHandle, defaultModName: String): Pair { + if (isValidModFolder(dir)) + return dir to defaultModName + val subdirs = dir.list(FileFilter { it.isDirectory }) // See detekt/#6822 - a direct lambda-to-SAM with typed `it` fails detektAnalysis + if (subdirs.size == 1 && isValidModFolder(subdirs[0])) + return subdirs[0] to subdirs[0].name() + throw UncivShowableException("Invalid Mod archive structure") + } + + private fun FileHandle.renameOrMove(dest: FileHandle) { + // Gdx tries a java rename for Absolute and External, but NOT for Local - rectify that + if (type() == Files.FileType.Local) { + // See #5346: renameTo 'unpacks' a source folder if the destination exists and is an + // empty folder, dropping the name of the source entirely. + // Safer to ask for a move to the not-yet-existing full resulting path instead. + if (isDirectory) + if (file().renameTo(dest.child(name()).file())) return + else + if (file().renameTo(dest.file())) return + } + moveTo(dest) + } + + /** + * Query GitHub for repositories marked "unciv-mod" + * @param amountPerPage Number of search results to return for this request. + * @param page The "page" number, starting at 1. + * @return Parsed [RepoSearch][GithubAPI.RepoSearch] json on success, `null` on failure. + * @see Github API doc + */ + fun tryGetGithubReposWithTopic(amountPerPage:Int, page:Int, searchRequest: String = ""): GithubAPI.RepoSearch? { + val link = GithubAPI.getUrlForModListing(searchRequest, amountPerPage, page) + var retries = 2 + while (retries > 0) { + retries-- + // obey rate limit + if (RateLimit.waitForLimit()) return null + // try download + val inputStream = download(link) { + if (it.responseCode == 403 || it.responseCode == 200 && page == 1 && retries == 1) { + // Pass the response headers to the rate limit handler so it can process the rate limit headers + RateLimit.notifyHttpResponse(it) + retries++ // An extra retry so the 403 is ignored in the retry count + } + } ?: continue + return json().fromJson(GithubAPI.RepoSearch::class.java, inputStream.bufferedReader().readText()) + } + return null + } + + /** Get a Pixmap from a "preview" png or jpg file at the root of the repo, falling back to the + * repo owner's avatar [avatarUrl]. The file content url is constructed from [modUrl] and [defaultBranch] + * by replacing the host with `raw.githubusercontent.com`. + */ + fun tryGetPreviewImage(modUrl: String, defaultBranch: String, avatarUrl: String?): Pixmap? { + // Side note: github repos also have a "Social Preview" optionally assignable on the repo's + // settings page, but that info is inaccessible using the v3 API anonymously. The easiest way + // to get it would be to query the the repo's frontend page (modUrl), and parse out + // `head/meta[property=og:image]/@content`, which is one extra spurious roundtrip and a + // non-trivial waste of bandwidth. + // Thus we ask for a "preview" file as part of the repo contents instead. + val fileLocation = GithubAPI.getUrlForPreview(modUrl, defaultBranch) + try { + val file = download("$fileLocation.jpg") + ?: download("$fileLocation.png") + // Note: avatar urls look like: https://avatars.githubusercontent.com/u/?v=4 + // So the image format is only recognizable from the response "Content-Type" header + // or by looking for magic markers in the bits - which the Pixmap constructor below does. + ?: avatarUrl?.let { download(it) } + ?: return null + val byteArray = file.readBytes() + val buffer = ByteBuffer.allocateDirect(byteArray.size).put(byteArray).position(0) + return Pixmap(buffer) + } catch (_: Throwable) { + return null + } + } + + /** Queries github for a tree and calculates the sum of the blob sizes. + * @return -1 on failure, else size rounded to kB + * @see Github API "Get a tree" + */ + fun getRepoSize(repo: GithubAPI.Repo): Int { + val link = repo.getUrlForTreeQuery() + + var retries = 2 + while (retries > 0) { + retries-- + // obey rate limit + if (RateLimit.waitForLimit()) return -1 + // try download + val inputStream = download(link) { + if (it.responseCode == 403 || it.responseCode == 200 && retries == 1) { + // Pass the response headers to the rate limit handler so it can process the rate limit headers + RateLimit.notifyHttpResponse(it) + retries++ // An extra retry so the 403 is ignored in the retry count + } + } ?: continue + + val tree = json().fromJson(GithubAPI.Tree::class.java, inputStream.bufferedReader().readText()) + if (tree.truncated) return -1 // unlikely: >100k blobs or blob > 7MB + + var totalSizeBytes = 0L + for (file in tree.tree) + totalSizeBytes += file.size + + // overflow unlikely: >2TB + return ((totalSizeBytes + 512) / 1024).toInt() + } + return -1 + } + + /** + * Query GitHub for topics named "unciv-mod*" + * @return Parsed [TopicSearchResponse][GithubAPI.TopicSearchResponse] json on success, `null` on failure. + */ + fun tryGetGithubTopics(): GithubAPI.TopicSearchResponse? { + val link = GithubAPI.urlToQueryModTopics + var retries = 2 + while (retries > 0) { + retries-- + // obey rate limit + if (RateLimit.waitForLimit()) return null + // try download + val inputStream = download(link) { + if (it.responseCode == 403 || it.responseCode == 200 && retries == 1) { + // Pass the response headers to the rate limit handler so it can process the rate limit headers + RateLimit.notifyHttpResponse(it) + retries++ // An extra retry so the 403 is ignored in the retry count + } + } ?: continue + return json().fromJson(GithubAPI.TopicSearchResponse::class.java, inputStream.bufferedReader().readText()) + } + return null + } + + /** Rewrite modOptions file for a mod we just installed to include metadata we got from the GitHub api + * + * (called on background thread) + */ + fun rewriteModOptions(repo: GithubAPI.Repo, modFolder: FileHandle) { + val modOptionsFile = modFolder.child("jsons/ModOptions.json") + val modOptions = if (modOptionsFile.exists()) json().fromJsonFile(ModOptions::class.java, modOptionsFile) else ModOptions() + + // If this is false we didn't get github repo info, do a defensive merge so the Repo.parseUrl or download + // code can decide defaults but leave any meaningful field of a zip-included ModOptions alone. + val overwriteAlways = repo.direct_zip_url.isEmpty() + + if (overwriteAlways || modOptions.modUrl.isEmpty()) modOptions.modUrl = repo.html_url + if (overwriteAlways || modOptions.defaultBranch == "master" && repo.default_branch.isNotEmpty()) + modOptions.defaultBranch = repo.default_branch + if (overwriteAlways || modOptions.lastUpdated.isEmpty()) modOptions.lastUpdated = repo.pushed_at + if (overwriteAlways || modOptions.author.isEmpty()) modOptions.author = repo.owner.login + if (overwriteAlways || modOptions.modSize == 0) modOptions.modSize = repo.size + if (overwriteAlways || modOptions.topics.isEmpty()) modOptions.topics = repo.topics + + modOptions.updateDeprecations() + json().toJson(modOptions, modOptionsFile) + } + + private const val outerBlankReplacement = '=' + // Github disallows **any** special chars and replaces them with '-' - so use something ascii the + // OS accepts but still is recognizable as non-original, to avoid confusion + + /** Convert a [Repo][[GithubAPI.Repo] name to a local name for both display and folder name + * + * Replaces '-' with blanks but ensures no leading or trailing blanks. + * As mad modders know no limits, trailing "-" did indeed happen, causing things to break due to trailing blanks on a folder name. + * As "test-" and "test" are different allowed repository names, trimmed blanks are replaced with one equals sign per side. + * @param onlyOuterBlanks If `true` ignores inner dashes - only start and end are treated. Useful when modders have manually created local folder names using dashes. + */ + fun String.repoNameToFolderName(onlyOuterBlanks: Boolean = false): String { + var result = if (onlyOuterBlanks) this else replace('-', ' ') + if (result.endsWith(' ')) result = result.trimEnd() + outerBlankReplacement + if (result.startsWith(' ')) result = outerBlankReplacement + result.trimStart() + return result + } + + /** Inverse of [repoNameToFolderName] */ + // As of this writing, only used for loadMissingMods + fun String.folderNameToRepoName(): String { + var result = replace(' ', '-') + if (result.endsWith(outerBlankReplacement)) result = result.trimEnd(outerBlankReplacement) + '-' + if (result.startsWith(outerBlankReplacement)) result = '-' + result.trimStart(outerBlankReplacement) + return result + } +} diff --git a/core/src/com/unciv/logic/github/GithubAPI.kt b/core/src/com/unciv/logic/github/GithubAPI.kt new file mode 100644 index 0000000000..32e416a23e --- /dev/null +++ b/core/src/com/unciv/logic/github/GithubAPI.kt @@ -0,0 +1,242 @@ +package com.unciv.logic.github + +import com.unciv.json.json + +/** + * "Namespace" collects all Github API structural knowledge + * - Response schema + * - Query URL builders + */ +@Suppress("PropertyName") // We're declaring an external API schema +object GithubAPI { + // region URL formatters + + /** Format a download URL for a branch archive */ + // URL format see: https://docs.github.com/en/repositories/working-with-files/using-files/downloading-source-code-archives#source-code-archive-urls + // Note: https://api.github.com/repos/owner/mod/zipball would be an alternative. Its response is a redirect, but our lib follows that and delivers the zip just fine. + // Problems with the latter: Internal zip structure different, finalDestinationName would need a patch. Plus, normal URL escaping for owner/reponame does not work. + internal fun getUrlForBranchZip(gitRepoUrl: String, branch: String) = "$gitRepoUrl/archive/refs/heads/$branch.zip" + + /** Format a URL to query for Mod repos by topic */ + internal fun getUrlForModListing(searchRequest: String, amountPerPage: Int, page: Int) = + // Add + if needed to separate the query text from its parameters + "https://api.github.com/search/repositories?q=$searchRequest${ if (searchRequest.isEmpty()) "" else "+" }%20topic:unciv-mod%20fork:true&sort:stars&per_page=$amountPerPage&page=$page" + + /** Format URL to fetch one specific [Repo] metadata from the API */ + private fun getUrlForSingleRepoQuery(owner: String, repoName: String) = "https://api.github.com/repos/$owner/$repoName" + + /** Format a download URL for a release archive */ + private fun Repo.getUrlForReleaseZip() = "$html_url/archive/refs/tags/$release_tag.zip" + + /** Format a URL to query a repo tree - to calculate actual size */ + internal fun Repo.getUrlForTreeQuery() = + "https://api.github.com/repos/$full_name/git/trees/$default_branch?recursive=true" + + /** Format a URL to fetch a preview image - without extension */ + internal fun getUrlForPreview(modUrl: String, branch: String) = "$modUrl/$branch/preview" + .replace("github.com", "raw.githubusercontent.com") + + /** A query returning all known topics staring with "unciv-mod" and having at least two uses */ + // `+repositories:>1` means ignore unused or practically unused topics + internal const val urlToQueryModTopics = "https://api.github.com/search/topics?q=unciv-mod+repositories:%3E1&sort=name&order=asc" + + //endregion + //region responses + + /** + * Parsed Github repo search response + * @property total_count Total number of hits for the search (ignoring paging window) + * @property incomplete_results A flag set by github to indicate search was incomplete (never seen it on) + * @property items Array of [repositories][Repo] + * @see Github API doc + */ + class RepoSearch { + @Suppress("MemberVisibilityCanBePrivate") + var total_count = 0 + var incomplete_results = false + var items = ArrayList() + } + + /** Part of [RepoSearch] in Github API response - one repository entry in [items][RepoSearch.items] */ + class Repo { + + /** Unlike the rest of this class, this is not part of the API but added by us locally + * to track whether [getRepoSize][Github.getRepoSize] has been run successfully for this repo */ + var hasUpdatedSize = false + + /** Not part of the github schema: Explicit final zip download URL for non-github or release downloads */ + var direct_zip_url = "" + /** Not part of the github schema: release tag, for debugging (DL via direct_zip_url) */ + var release_tag = "" + + var name = "" + var full_name = "" + var description: String? = null + var owner = RepoOwner() + var stargazers_count = 0 + var default_branch = "" + var html_url = "" + var pushed_at = "" // don't use updated_at - see https://github.com/yairm210/Unciv/issues/6106 + var size = 0 + var topics = mutableListOf() + //var stargazers_url = "" + //var homepage: String? = null // might use instead of go to repo? + //var has_wiki = false // a wiki could mean proper documentation for the mod? + + /** String representation to be used for logging */ + override fun toString() = name.ifEmpty { direct_zip_url } + + companion object { + /** Create a [Repo] metadata instance from a [url], supporting various formats + * from a repository landing page url to a free non-github zip download. + * + * @see GithubAPI.parseUrl + * @return `null` for invalid links or any other failures + */ + fun parseUrl(url: String): Repo? = Repo().parseUrl(url) + + /** Query Github API for [owner]'s [repoName] repository metadata */ + fun query(owner: String, repoName: String): Repo? { + val response = Github.download(getUrlForSingleRepoQuery(owner, repoName)) + ?: return null + val repoString = response.bufferedReader().readText() + return json().fromJson(Repo::class.java, repoString) + } + } + } + + /** Part of [Repo] in Github API response */ + class RepoOwner { + var login = "" + var avatar_url: String? = null + } + + /** Topic search response */ + class TopicSearchResponse { + // Commented out: Github returns them, but we're not interested +// var total_count = 0 +// var incomplete_results = false + var items = ArrayList() + class Topic { + var name = "" + var display_name: String? = null // Would need to be curated, which is alottawork +// var featured = false +// var curated = false + var created_at = "" // iso datetime with "Z" timezone + var updated_at = "" // iso datetime with "Z" timezone + } + } + + /** Class to receive a github API "Get a tree" response parsed as json */ + // Parts of the response we ignore are commented out + internal class Tree { + //val sha = "" + //val url = "" + + class TreeFile { + //val path = "" + //val mode = 0 + //val type = "" // blob / tree + //val sha = "" + //val url = "" + var size: Long = 0L + } + + @Suppress("MemberNameEqualsClassName") + var tree = ArrayList() + var truncated = false + } + + //endregion + + //region Flexible URL parsing + /** + * Initialize `this` with an url, extracting all possible fields from it + * (html_url, author, repoName, branchName). + * + * Allow url formats: + * * Basic repo url: + * https://github.com/author/repoName + * * or complete 'zip' url from github's code->download zip menu: + * https://github.com/author/repoName/archive/refs/heads/branchName.zip + * * or the branch url same as one navigates to on github through the "branches" menu: + * https://github.com/author/repoName/tree/branchName + * * or release tag + * https://github.com/author/repoName/releases/tag/tagname + * https://github.com/author/repoName/archive/refs/tags/tagname.zip + * + * In the case of the basic repo url, an [API query](https://docs.github.com/en/rest/repos/repos#get-a-repository) is sent to determine the default branch. + * Other url forms will not go online. + * + * @return a new Repo instance for the 'Basic repo url' case, otherwise `this`, modified, to allow chaining, `null` for invalid links or any other failures + * @see Github API Repository Code Samples + */ + private fun Repo.parseUrl(url: String): Repo? { + fun processMatch(matchResult: MatchResult): Repo { + html_url = matchResult.groups[1]!!.value + owner.login = matchResult.groups[2]!!.value + name = matchResult.groups[3]!!.value + default_branch = matchResult.groups[4]!!.value + return this + } + + html_url = url + default_branch = "master" + val matchZip = Regex("""^(.*/(.*)/(.*))/archive/(?:.*/)?heads/([^.]+).zip$""").matchEntire(url) + if (matchZip != null && matchZip.groups.size > 4) + return processMatch(matchZip) + + val matchBranch = Regex("""^(.*/(.*)/(.*))/tree/([^/]+)$""").matchEntire(url) + if (matchBranch != null && matchBranch.groups.size > 4) + return processMatch(matchBranch) + + // Releases and tags - + // TODO Query for latest release and save as Mod Version? + // https://docs.github.com/en/rest/releases/releases#get-the-latest-release + // TODO Query a specific release for its name attribute - the page will link the tag + // https://docs.github.com/en/rest/releases/releases#get-a-release-by-tag-name + + val matchTagArchive = Regex("""^(.*/(.*)/(.*))/archive/(?:.*/)?tags/([^.]+).zip$""").matchEntire(url) + if (matchTagArchive != null && matchTagArchive.groups.size > 4) { + processMatch(matchTagArchive) + release_tag = default_branch + // leave default_branch even if it's actually a tag not a branch name + // so the suffix of the inner first level folder inside the zip can be removed later + direct_zip_url = url + return this + } + val matchTagPage = Regex("""^(.*/(.*)/(.*))/releases/(?:.*/)?tag/([^.]+)$""").matchEntire(url) + if (matchTagPage != null && matchTagPage.groups.size > 4) { + processMatch(matchTagPage) + release_tag = default_branch + direct_zip_url = getUrlForReleaseZip() + return this + } + + val matchRepo = Regex("""^.*//.*/(.+)/(.+)/?$""").matchEntire(url) + if (matchRepo != null && matchRepo.groups.size > 2) { + // Query API if we got the 'https://github.com/author/repoName' URL format to get the correct default branch + val repo = Repo.query(matchRepo.groups[1]!!.value, matchRepo.groups[2]!!.value) + if (repo != null) return repo + } + + // Only complain about invalid link if it isn't a http protocol (to think about: android document protocol? file protocol?) + if (!url.startsWith("http://") && !url.startsWith("https://")) + return null + + // From here, we'll always return success and treat the url as direct-downloadable zip. + // The Repo instance will be a pseudo-repo not corresponding to an actual github repo. + html_url = "" + direct_zip_url = url + owner.login = "-unknown-" + default_branch = "master" // only used to remove this suffix should the zip contain a inner folder + // But see if we can extract a file name from the url + // Will use filename from response headers, if content-disposition is sent, for the Mod name instead, done in downloadAndExtract + val matchAnyZip = Regex("""^.*//(?:.*/)*([^/]+\.zip)$""").matchEntire(url) + if (matchAnyZip != null && matchAnyZip.groups.size > 1) + name = matchAnyZip.groups[1]!!.value + full_name = name + return this + } + //endregion +} diff --git a/core/src/com/unciv/logic/github/RateLimit.kt b/core/src/com/unciv/logic/github/RateLimit.kt new file mode 100644 index 0000000000..2eff8d483b --- /dev/null +++ b/core/src/com/unciv/logic/github/RateLimit.kt @@ -0,0 +1,90 @@ +package com.unciv.logic.github + +import com.unciv.utils.debug +import java.net.HttpURLConnection + +/** + * Implements the ability wo work with GitHub's rate limit, recognize blocks from previous attempts, wait and retry. + * @see Github API doc + */ +object RateLimit { + private const val maxRequestsPerInterval = 10 + private const val intervalInMilliSeconds = 60000L + private const val maxWaitLoop = 3 + + private var account = 0 // used requests + private var firstRequest = 0L // timestamp window start (java epoch millisecond) + + /* + Github rate limits do not use sliding windows - you (if anonymous) get one window + which starts with the first request (if a window is not already active) + and ends 60s later, and a budget of 10 requests in that window. Once it expires, + everything is forgotten and the process starts from scratch + */ + + private val millis: Long + get() = System.currentTimeMillis() + + /** calculate required wait in ms + * @return Estimated number of milliseconds to wait for the rate limit window to expire + */ + private fun getWaitLength() + = (firstRequest + intervalInMilliSeconds - millis) + + /** Maintain and check a rate-limit + * @return **true** if rate-limited, **false** if another request is allowed + */ + private fun isLimitReached(): Boolean { + val now = millis + val elapsed = if (firstRequest == 0L) intervalInMilliSeconds else now - firstRequest + if (elapsed >= intervalInMilliSeconds) { + firstRequest = now + account = 1 + return false + } + if (account >= maxRequestsPerInterval) return true + account++ + return false + } + + /** If rate limit in effect, sleep long enough to allow next request. + * + * @return **true** if waiting did not clear isLimitReached() (can only happen if the clock is broken), + * or the wait has been interrupted by Thread.interrupt() + * **false** if we were below the limit or slept long enough to drop out of it. + */ + fun waitForLimit(): Boolean { + var loopCount = 0 + while (isLimitReached()) { + val waitLength = getWaitLength() + try { + Thread.sleep(waitLength) + } catch ( ex: InterruptedException ) { + return true + } + if (++loopCount >= maxWaitLoop) return true + } + return false + } + + /** http responses should be passed to this so the actual rate limit window can be evaluated and used. + * The very first response and all 403 ones are good candidates if they can be expected to contain GitHub's rate limit headers. + * + * @see Github API doc + */ + fun notifyHttpResponse(response: HttpURLConnection) { + if (response.responseMessage != "rate limit exceeded" && response.responseCode != 200) return + + fun getHeaderLong(name: String, default: Long = 0L) = + response.headerFields[name]?.get(0)?.toLongOrNull() ?: default + val limit = getHeaderLong("X-RateLimit-Limit", maxRequestsPerInterval.toLong()).toInt() + val remaining = getHeaderLong("X-RateLimit-Remaining").toInt() + val reset = getHeaderLong("X-RateLimit-Reset") + + if (limit != maxRequestsPerInterval) + debug("GitHub API Limit reported via http (%s) not equal assumed value (%s)", limit, maxRequestsPerInterval) + account = maxRequestsPerInterval - remaining + if (reset == 0L) return + firstRequest = (reset + 1L) * 1000L - intervalInMilliSeconds + } +} diff --git a/core/src/com/unciv/logic/github/Zip.kt b/core/src/com/unciv/logic/github/Zip.kt new file mode 100644 index 0000000000..928efa58fd --- /dev/null +++ b/core/src/com/unciv/logic/github/Zip.kt @@ -0,0 +1,88 @@ +package com.unciv.logic.github + +import com.badlogic.gdx.files.FileHandle +import com.unciv.utils.debug +import java.io.BufferedInputStream +import java.io.BufferedOutputStream +import java.io.FileOutputStream +import java.io.InputStream +import java.util.zip.ZipEntry +import java.util.zip.ZipFile + +/** Utility - extract Zip archives + * @see [Zip.extractFolder] + */ +object Zip { + private const val bufferSize = 2048 + + /** + * Extract one Zip file recursively (nested Zip files are extracted in turn). + * + * The source Zip is not deleted, but successfully extracted nested ones are. + * + * **Warning**: Extracting into a non-empty destination folder will merge contents. Existing + * files also included in the archive will be partially overwritten, when the new data is shorter + * than the old you will get _mixed contents!_ + * + * @param zipFile The Zip file to extract + * @param unzipDestination The folder to extract into, preferably empty (not enforced). + */ + fun extractFolder(zipFile: FileHandle, unzipDestination: FileHandle) { + // I went through a lot of similar answers that didn't work until I got to this gem by NeilMonday + // (with mild changes to fit the FileHandles) + // https://stackoverflow.com/questions/981578/how-to-unzip-files-recursively-in-java + + debug("Extracting %s to %s", zipFile, unzipDestination) + // establish buffer for writing file + val data = ByteArray(bufferSize) + + fun streamCopy(fromStream: InputStream, toHandle: FileHandle) { + val inputStream = BufferedInputStream(fromStream) + var currentByte: Int + + // write the current file to disk + val fos = FileOutputStream(toHandle.file()) + val dest = BufferedOutputStream(fos, bufferSize) + + // read and write until last byte is encountered + while (inputStream.read(data, 0, bufferSize).also { currentByte = it } != -1) { + dest.write(data, 0, currentByte) + } + dest.flush() + dest.close() + inputStream.close() + } + + val file = zipFile.file() + val zip = ZipFile(file) + //unzipDestination.mkdirs() + val zipFileEntries = zip.entries() + + // Process each entry + while (zipFileEntries.hasMoreElements()) { + // grab a zip file entry + val entry = zipFileEntries.nextElement() as ZipEntry + val currentEntry = entry.name + val destFile = unzipDestination.child(currentEntry) + val destinationParent = destFile.parent() + + // create the parent directory structure if needed + destinationParent.mkdirs() + if (!entry.isDirectory) { + streamCopy ( zip.getInputStream(entry), destFile) + } + // The new file has a current last modification time + // and not the one stored in the archive - we could: + // 'destFile.file().setLastModified(entry.time)' + // but later handling will throw these away anyway, + // and GitHub sets all timestamps to the download time. + + if (currentEntry.endsWith(".zip")) { + // found a zip file, try to open + extractFolder(destFile, destinationParent) + destFile.delete() + } + } + zip.close() // Needed so we can delete the zip file later + } +} diff --git a/core/src/com/unciv/logic/map/tile/TileStatFunctions.kt b/core/src/com/unciv/logic/map/tile/TileStatFunctions.kt index 14ddc8fc44..38565a6585 100644 --- a/core/src/com/unciv/logic/map/tile/TileStatFunctions.kt +++ b/core/src/com/unciv/logic/map/tile/TileStatFunctions.kt @@ -30,9 +30,7 @@ class TileStatFunctions(val tile: Tile) { fun getTileStats(city: City?, observingCiv: Civilization?, localUniqueCache: LocalUniqueCache = LocalUniqueCache(false) ): Stats { - val stats = Stats() - for (statsToAdd in getTileStatsBreakdown(city, observingCiv, localUniqueCache)) - stats.add(statsToAdd.second) + val stats = getTileStatsBreakdown(city, observingCiv, localUniqueCache).toStats() for ((stat, value) in getTilePercentageStats(observingCiv, city, localUniqueCache)) { stats[stat] *= value.toPercent() diff --git a/core/src/com/unciv/models/metadata/ModCategories.kt b/core/src/com/unciv/models/metadata/ModCategories.kt index 21b7997ac7..39770f6594 100644 --- a/core/src/com/unciv/models/metadata/ModCategories.kt +++ b/core/src/com/unciv/models/metadata/ModCategories.kt @@ -2,8 +2,9 @@ package com.unciv.models.metadata import com.badlogic.gdx.Gdx import com.unciv.json.json +import com.unciv.logic.github.GithubAPI import com.unciv.ui.components.widgets.TranslatedSelectBox -import com.unciv.ui.screens.pickerscreens.Github +import com.unciv.logic.github.Github class ModCategories : ArrayList() { @@ -21,12 +22,12 @@ class ModCategories : ArrayList() { constructor() : this("", "", false, "", "") - constructor(topic: Github.TopicSearchResponse.Topic) : + constructor(topic: GithubAPI.TopicSearchResponse.Topic) : this(labelSuggestion(topic), topic.name, true, topic.created_at, topic.updated_at) companion object { val All = Category("All mods", "unciv-mod", false, "", "") - fun labelSuggestion(topic: Github.TopicSearchResponse.Topic) = + fun labelSuggestion(topic: GithubAPI.TopicSearchResponse.Topic) = topic.display_name?.takeUnless { it.isBlank() } ?: topic.name.removePrefix("unciv-mod-").replaceFirstChar(Char::titlecase) } diff --git a/core/src/com/unciv/ui/screens/modmanager/ModInfoAndActionPane.kt b/core/src/com/unciv/ui/screens/modmanager/ModInfoAndActionPane.kt index 24e22c00c8..b6f56122cd 100644 --- a/core/src/com/unciv/ui/screens/modmanager/ModInfoAndActionPane.kt +++ b/core/src/com/unciv/ui/screens/modmanager/ModInfoAndActionPane.kt @@ -4,6 +4,7 @@ import com.badlogic.gdx.Gdx import com.badlogic.gdx.graphics.Texture import com.badlogic.gdx.scenes.scene2d.ui.Image import com.badlogic.gdx.scenes.scene2d.ui.Table +import com.unciv.logic.github.GithubAPI import com.unciv.models.metadata.BaseRuleset import com.unciv.models.ruleset.Ruleset import com.unciv.models.ruleset.validation.ModCompatibility @@ -14,7 +15,7 @@ import com.unciv.ui.components.extensions.toCheckBox import com.unciv.ui.components.extensions.toLabel import com.unciv.ui.components.extensions.toTextButton import com.unciv.ui.components.input.onClick -import com.unciv.ui.screens.pickerscreens.Github +import com.unciv.logic.github.Github import com.unciv.utils.Concurrency import kotlin.math.max @@ -34,7 +35,7 @@ internal class ModInfoAndActionPane : Table() { /** Recreate the information part of the right-hand column * @param repo: the repository instance as received from the GitHub api */ - fun update(repo: Github.Repo) { + fun update(repo: GithubAPI.Repo) { isBuiltin = false enableVisualCheckBox = false update( diff --git a/core/src/com/unciv/ui/screens/modmanager/ModManagementScreen.kt b/core/src/com/unciv/ui/screens/modmanager/ModManagementScreen.kt index 1d2ca7e457..375cad6ce8 100644 --- a/core/src/com/unciv/ui/screens/modmanager/ModManagementScreen.kt +++ b/core/src/com/unciv/ui/screens/modmanager/ModManagementScreen.kt @@ -13,6 +13,8 @@ import com.badlogic.gdx.utils.SerializationException import com.unciv.UncivGame import com.unciv.json.fromJsonFile import com.unciv.json.json +import com.unciv.logic.UncivShowableException +import com.unciv.logic.github.GithubAPI import com.unciv.models.ruleset.Ruleset import com.unciv.models.ruleset.RulesetCache import com.unciv.models.tilesets.TileSetCache @@ -41,8 +43,8 @@ import com.unciv.ui.screens.basescreen.BaseScreen import com.unciv.ui.screens.basescreen.RecreateOnResize import com.unciv.ui.screens.mainmenuscreen.MainMenuScreen import com.unciv.ui.screens.modmanager.ModManagementOptions.SortType -import com.unciv.ui.screens.pickerscreens.Github -import com.unciv.ui.screens.pickerscreens.Github.repoNameToFolderName +import com.unciv.logic.github.Github +import com.unciv.logic.github.Github.repoNameToFolderName import com.unciv.ui.screens.pickerscreens.PickerScreen import com.unciv.utils.Concurrency import com.unciv.utils.Log @@ -93,7 +95,7 @@ class ModManagementScreen private constructor( private var lastSelectedButton: ModDecoratedButton? = null private var lastSyncMarkedButton: ModDecoratedButton? = null - private var selectedMod: Github.Repo? = null + private var selectedMod: GithubAPI.Repo? = null private val modDescriptionLabel: WrappableLabel @@ -233,7 +235,7 @@ class ModManagementScreen private constructor( */ private fun tryDownloadPage(pageNum: Int) { runningSearchJob = Concurrency.run("GitHubSearch") { - val repoSearch: Github.RepoSearch + val repoSearch: GithubAPI.RepoSearch try { repoSearch = Github.tryGetGithubReposWithTopic(amountPerPage, pageNum)!! } catch (ex: Exception) { @@ -254,7 +256,7 @@ class ModManagementScreen private constructor( } } - private fun addModInfoFromRepoSearch(repoSearch: Github.RepoSearch, pageNum: Int) { + private fun addModInfoFromRepoSearch(repoSearch: GithubAPI.RepoSearch, pageNum: Int) { // clear and remove last cell if it is the "..." indicator val lastCell = onlineModsTable.cells.lastOrNull() if (lastCell != null && lastCell.actor is Label && (lastCell.actor as Label).text.toString() == "...") { @@ -363,7 +365,7 @@ class ModManagementScreen private constructor( val downloadButton = "Download mod from URL".toTextButton() downloadButton.onClick { val popup = Popup(this) - popup.addGoodSizedLabel("Please enter the mod repository -or- archive zip -or- branch url:").row() + popup.addGoodSizedLabel("Please enter the mod repository -or- archive zip -or- branch -or- release url:").row() val textField = UncivTextField.create("").apply { maxLength = 666 } popup.add(textField).width(stage.width / 2).row() val pasteLinkButton = "Paste from clipboard".toTextButton() @@ -375,7 +377,7 @@ class ModManagementScreen private constructor( actualDownloadButton.onClick { actualDownloadButton.setText("Downloading...".tr()) actualDownloadButton.disable() - val repo = Github.Repo().parseUrl(textField.text) + val repo = GithubAPI.Repo.parseUrl(textField.text) if (repo == null) { ToastPopup("«RED»{Invalid link!}«»", this@ModManagementScreen) .apply { isVisible = true } @@ -392,7 +394,7 @@ class ModManagementScreen private constructor( } /** Used as onClick handler for the online Mod list buttons */ - private fun onlineButtonAction(repo: Github.Repo, button: ModDecoratedButton) { + private fun onlineButtonAction(repo: GithubAPI.Repo, button: ModDecoratedButton) { syncOnlineSelected(repo.name, button) showModDescription(repo.name) @@ -435,7 +437,7 @@ class ModManagementScreen private constructor( } /** Download and install a mod in the background, called both from the right-bottom button and the URL entry popup */ - private fun downloadMod(repo: Github.Repo, postAction: () -> Unit = {}) { + private fun downloadMod(repo: GithubAPI.Repo, postAction: () -> Unit = {}) { Concurrency.run("DownloadMod") { // to avoid ANRs - we've learnt our lesson from previous download-related actions try { val modFolder = Github.downloadAndExtract( @@ -456,8 +458,14 @@ class ModManagementScreen private constructor( unMarkUpdatedMod(repoName) postAction() } + } catch (ex: UncivShowableException) { + Log.error("Could not download $repo", ex) + launchOnGLThread { + ToastPopup(ex.message, this@ModManagementScreen) + postAction() + } } catch (ex: Exception) { - Log.error("Could not download ${repo.name}", ex) + Log.error("Could not download $repo", ex) launchOnGLThread { ToastPopup("Could not download [${repo.name}]", this@ModManagementScreen) postAction() diff --git a/core/src/com/unciv/ui/screens/modmanager/ModUIData.kt b/core/src/com/unciv/ui/screens/modmanager/ModUIData.kt index e6ff5f32eb..620f858ede 100644 --- a/core/src/com/unciv/ui/screens/modmanager/ModUIData.kt +++ b/core/src/com/unciv/ui/screens/modmanager/ModUIData.kt @@ -1,12 +1,9 @@ package com.unciv.ui.screens.modmanager -import com.badlogic.gdx.scenes.scene2d.ui.TextButton +import com.unciv.logic.github.GithubAPI import com.unciv.models.metadata.ModCategories import com.unciv.models.ruleset.Ruleset import com.unciv.models.translations.tr -import com.unciv.ui.components.extensions.toLabel -import com.unciv.ui.components.extensions.toTextButton -import com.unciv.ui.screens.pickerscreens.Github /** Helper class holds combined mod info for ModManagementScreen, used for both installed and online lists. * @@ -18,7 +15,7 @@ internal class ModUIData private constructor( val name: String, val description: String, val ruleset: Ruleset?, - val repo: Github.Repo?, + val repo: GithubAPI.Repo?, var isVisual: Boolean = false, var hasUpdate: Boolean = false ) { @@ -30,7 +27,7 @@ internal class ModUIData private constructor( ruleset, null, isVisual = isVisual ) - constructor(repo: Github.Repo, isUpdated: Boolean): this ( + constructor(repo: GithubAPI.Repo, isUpdated: Boolean): this ( repo.name, (repo.description ?: "-{No description provided}-".tr()) + "\n" + "[${repo.stargazers_count}]✯".tr(), diff --git a/core/src/com/unciv/ui/screens/pickerscreens/GitHub.kt b/core/src/com/unciv/ui/screens/pickerscreens/GitHub.kt deleted file mode 100644 index 255db4f34e..0000000000 --- a/core/src/com/unciv/ui/screens/pickerscreens/GitHub.kt +++ /dev/null @@ -1,603 +0,0 @@ -package com.unciv.ui.screens.pickerscreens - -import com.badlogic.gdx.Files -import com.badlogic.gdx.files.FileHandle -import com.badlogic.gdx.graphics.Pixmap -import com.unciv.json.fromJsonFile -import com.unciv.json.json -import com.unciv.logic.BackwardCompatibility.updateDeprecations -import com.unciv.models.ruleset.ModOptions -import com.unciv.ui.screens.pickerscreens.Github.RateLimit -import com.unciv.ui.screens.pickerscreens.Github.download -import com.unciv.ui.screens.pickerscreens.Github.downloadAndExtract -import com.unciv.ui.screens.pickerscreens.Github.tryGetGithubReposWithTopic -import com.unciv.utils.Log -import com.unciv.utils.debug -import java.io.BufferedInputStream -import java.io.BufferedOutputStream -import java.io.BufferedReader -import java.io.FileOutputStream -import java.io.InputStream -import java.io.InputStreamReader -import java.net.HttpURLConnection -import java.net.URL -import java.nio.ByteBuffer -import java.util.zip.ZipEntry -import java.util.zip.ZipFile - - -/** - * Utility managing Github access (except the link in WorldScreenCommunityPopup) - * - * Singleton - RateLimit is shared app-wide and has local variables, and is not tested for thread safety. - * Therefore, additional effort is required should [tryGetGithubReposWithTopic] ever be called non-sequentially. - * [download] and [downloadAndExtract] should be thread-safe as they are self-contained. - * They do not join in the [RateLimit] handling because Github doc suggests each API - * has a separate limit (and I found none for cloning via a zip). - */ -object Github { - - // Consider merging this with the Dropbox function - /** - * Helper opens am url and accesses its input stream, logging errors to the console - * @param url String representing a [URL] to download. - * @param action Optional callback that will be executed between opening the connection and - * accessing its data - passes the [connection][HttpURLConnection] and allows e.g. reading the response headers. - * @return The [InputStream] if successful, `null` otherwise. - */ - fun download(url: String, action: (HttpURLConnection) -> Unit = {}): InputStream? { - try { - // Problem type 1 - opening the URL connection - with(URL(url).openConnection() as HttpURLConnection) - { - action(this) - // Problem type 2 - getting the information - try { - return inputStream - } catch (ex: Exception) { - // No error handling, just log the message. - // NOTE that this 'read error stream' CAN ALSO fail, but will be caught by the big try/catch - val reader = BufferedReader(InputStreamReader(errorStream, Charsets.UTF_8)) - Log.error("Message from GitHub: %s", reader.readText()) - throw ex - } - } - } catch (ex: Exception) { - Log.error("Exception during GitHub download", ex) - return null - } - } - - /** - * Download a mod and extract, deleting any pre-existing version. - * @param folderFileHandle Destination handle of mods folder - also controls Android internal/external - * @author **Warning**: This took a long time to get just right, so if you're changing this, ***TEST IT THOROUGHLY*** on _both_ Desktop _and_ Phone - * @return FileHandle for the downloaded Mod's folder or null if download failed - */ - fun downloadAndExtract( - repo: Repo, - folderFileHandle: FileHandle - ): FileHandle? { - val defaultBranch = repo.default_branch - val gitRepoUrl = repo.html_url - // Initiate download - the helper returns null when it fails - // URL format see: https://docs.github.com/en/repositories/working-with-files/using-files/downloading-source-code-archives#source-code-archive-urls - // Note: https://api.github.com/repos/owner/mod/zipball would be an alternative. Its response is a redirect, but our lib follows that and delivers the zip just fine. - // Problems with the latter: Internal zip structure different, finalDestinationName would need a patch. Plus, normal URL escaping for owner/reponame does not work. - val zipUrl = "$gitRepoUrl/archive/refs/heads/$defaultBranch.zip" - val inputStream = download(zipUrl) ?: return null - - // Get a mod-specific temp file name - val tempName = "temp-" + gitRepoUrl.hashCode().toString(16) - - // Download to temporary zip - val tempZipFileHandle = folderFileHandle.child("$tempName.zip") - tempZipFileHandle.write(inputStream, false) - - // prepare temp unpacking folder - val unzipDestination = tempZipFileHandle.sibling(tempName) // folder, not file - // prevent mixing new content with old - hopefully there will never be cadavers of our tempZip stuff - if (unzipDestination.exists()) - if (unzipDestination.isDirectory) unzipDestination.deleteDirectory() else unzipDestination.delete() - - Zip.extractFolder(tempZipFileHandle, unzipDestination) - - val innerFolder = unzipDestination.list().first() - // innerFolder should now be "$tempName/$repoName-$defaultBranch/" - use this to get mod name - val finalDestinationName = innerFolder.name().replace("-$defaultBranch", "").repoNameToFolderName() - // finalDestinationName is now the mod name as we display it. Folder name needs to be identical. - val finalDestination = folderFileHandle.child(finalDestinationName) - - // prevent mixing new content with old - var tempBackup: FileHandle? = null - if (finalDestination.exists()) { - tempBackup = finalDestination.sibling("$finalDestinationName.updating") - finalDestination.renameOrMove(tempBackup) - } - - // Move temp unpacked content to their final place - finalDestination.mkdirs() // If we don't create this as a directory, it will think this is a file and nothing will work. - // The move will reset the last modified time (recursively, at least on Linux) - // This sort will guarantee the desktop launcher will not re-pack textures and overwrite the atlas as delivered by the mod - for (innerFileOrFolder in innerFolder.list() - .sortedBy { file -> file.extension() == "atlas" } ) { - innerFileOrFolder.renameOrMove(finalDestination) - } - - // clean up - tempZipFileHandle.delete() - unzipDestination.deleteDirectory() - if (tempBackup != null) - if (tempBackup.isDirectory) tempBackup.deleteDirectory() else tempBackup.delete() - - return finalDestination - } - - private fun FileHandle.renameOrMove(dest: FileHandle) { - // Gdx tries a java rename for Absolute and External, but NOT for Local - rectify that - if (type() == Files.FileType.Local) { - // See #5346: renameTo 'unpacks' a source folder if the destination exists and is an - // empty folder, dropping the name of the source entirely. - // Safer to ask for a move to the not-yet-existing full resulting path instead. - if (isDirectory) - if (file().renameTo(dest.child(name()).file())) return - else - if (file().renameTo(dest.file())) return - } - moveTo(dest) - } - - /** - * Implements the ability wo work with GitHub's rate limit, recognize blocks from previous attempts, wait and retry. - */ - object RateLimit { - // https://docs.github.com/en/rest/reference/search#rate-limit - private const val maxRequestsPerInterval = 10 - private const val intervalInMilliSeconds = 60000L - private const val maxWaitLoop = 3 - - private var account = 0 // used requests - private var firstRequest = 0L // timestamp window start (java epoch millisecond) - - /* - Github rate limits do not use sliding windows - you (if anonymous) get one window - which starts with the first request (if a window is not already active) - and ends 60s later, and a budget of 10 requests in that window. Once it expires, - everything is forgotten and the process starts from scratch - */ - - private val millis: Long - get() = System.currentTimeMillis() - - /** calculate required wait in ms - * @return Estimated number of milliseconds to wait for the rate limit window to expire - */ - private fun getWaitLength() - = (firstRequest + intervalInMilliSeconds - millis) - - /** Maintain and check a rate-limit - * @return **true** if rate-limited, **false** if another request is allowed - */ - private fun isLimitReached(): Boolean { - val now = millis - val elapsed = if (firstRequest == 0L) intervalInMilliSeconds else now - firstRequest - if (elapsed >= intervalInMilliSeconds) { - firstRequest = now - account = 1 - return false - } - if (account >= maxRequestsPerInterval) return true - account++ - return false - } - - /** If rate limit in effect, sleep long enough to allow next request. - * - * @return **true** if waiting did not clear isLimitReached() (can only happen if the clock is broken), - * or the wait has been interrupted by Thread.interrupt() - * **false** if we were below the limit or slept long enough to drop out of it. - */ - fun waitForLimit(): Boolean { - var loopCount = 0 - while (isLimitReached()) { - val waitLength = getWaitLength() - try { - Thread.sleep(waitLength) - } catch ( ex: InterruptedException ) { - return true - } - if (++loopCount >= maxWaitLoop) return true - } - return false - } - - /** http responses should be passed to this so the actual rate limit window can be evaluated and used. - * The very first response and all 403 ones are good candidates if they can be expected to contain GitHub's rate limit headers. - * - * see: https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting - */ - fun notifyHttpResponse(response: HttpURLConnection) { - if (response.responseMessage != "rate limit exceeded" && response.responseCode != 200) return - - fun getHeaderLong(name: String, default: Long = 0L) = - response.headerFields[name]?.get(0)?.toLongOrNull() ?: default - val limit = getHeaderLong("X-RateLimit-Limit", maxRequestsPerInterval.toLong()).toInt() - val remaining = getHeaderLong("X-RateLimit-Remaining").toInt() - val reset = getHeaderLong("X-RateLimit-Reset") - - if (limit != maxRequestsPerInterval) - debug("GitHub API Limit reported via http (%s) not equal assumed value (%s)", limit, maxRequestsPerInterval) - account = maxRequestsPerInterval - remaining - if (reset == 0L) return - firstRequest = (reset + 1L) * 1000L - intervalInMilliSeconds - } - } - - /** - * Query GitHub for repositories marked "unciv-mod" - * @param amountPerPage Number of search results to return for this request. - * @param page The "page" number, starting at 1. - * @return Parsed [RepoSearch] json on success, `null` on failure. - * @see Github API doc - */ - fun tryGetGithubReposWithTopic(amountPerPage:Int, page:Int, searchRequest: String = ""): RepoSearch? { - // Add + here to separate the query text from its parameters - val searchText = if (searchRequest != "") "$searchRequest+" else "" - val link = "https://api.github.com/search/repositories?q=${searchText}%20topic:unciv-mod%20fork:true&sort:stars&per_page=$amountPerPage&page=$page" - var retries = 2 - while (retries > 0) { - retries-- - // obey rate limit - if (RateLimit.waitForLimit()) return null - // try download - val inputStream = download(link) { - if (it.responseCode == 403 || it.responseCode == 200 && page == 1 && retries == 1) { - // Pass the response headers to the rate limit handler so it can process the rate limit headers - RateLimit.notifyHttpResponse(it) - retries++ // An extra retry so the 403 is ignored in the retry count - } - } ?: continue - return json().fromJson(RepoSearch::class.java, inputStream.bufferedReader().readText()) - } - return null - } - - /** Get a Pixmap from a "preview" png or jpg file at the root of the repo, falling back to the - * repo owner's avatar [avatarUrl]. The file content url is constructed from [modUrl] and [defaultBranch] - * by replacing the host with `raw.githubusercontent.com`. - */ - fun tryGetPreviewImage(modUrl: String, defaultBranch: String, avatarUrl: String?): Pixmap? { - // Side note: github repos also have a "Social Preview" optionally assignable on the repo's - // settings page, but that info is inaccessible using the v3 API anonymously. The easiest way - // to get it would be to query the the repo's frontend page (modUrl), and parse out - // `head/meta[property=og:image]/@content`, which is one extra spurious roundtrip and a - // non-trivial waste of bandwidth. - // Thus we ask for a "preview" file as part of the repo contents instead. - val fileLocation = "$modUrl/$defaultBranch/preview" - .replace("github.com", "raw.githubusercontent.com") - try { - val file = download("$fileLocation.jpg") - ?: download("$fileLocation.png") - // Note: avatar urls look like: https://avatars.githubusercontent.com/u/?v=4 - // So the image format is only recognizable from the response "Content-Type" header - // or by looking for magic markers in the bits - which the Pixmap constructor below does. - ?: avatarUrl?.let { download(it) } - ?: return null - val byteArray = file.readBytes() - val buffer = ByteBuffer.allocateDirect(byteArray.size).put(byteArray).position(0) - return Pixmap(buffer) - } catch (_: Throwable) { - return null - } - } - - /** Class to receive a github API "Get a tree" response parsed as json */ - // Parts of the response we ignore are commented out - private class Tree { - //val sha = "" - //val url = "" - - class TreeFile { - //val path = "" - //val mode = 0 - //val type = "" // blob / tree - //val sha = "" - //val url = "" - var size: Long = 0L - } - - @Suppress("MemberNameEqualsClassName") - var tree = ArrayList() - var truncated = false - } - - /** Queries github for a tree and calculates the sum of the blob sizes. - * @return -1 on failure, else size rounded to kB - */ - fun getRepoSize(repo: Repo): Int { - // See https://docs.github.com/en/rest/git/trees#get-a-tree - val link = "https://api.github.com/repos/${repo.full_name}/git/trees/${repo.default_branch}?recursive=true" - - var retries = 2 - while (retries > 0) { - retries-- - // obey rate limit - if (RateLimit.waitForLimit()) return -1 - // try download - val inputStream = download(link) { - if (it.responseCode == 403 || it.responseCode == 200 && retries == 1) { - // Pass the response headers to the rate limit handler so it can process the rate limit headers - RateLimit.notifyHttpResponse(it) - retries++ // An extra retry so the 403 is ignored in the retry count - } - } ?: continue - - val tree = json().fromJson(Tree::class.java, inputStream.bufferedReader().readText()) - if (tree.truncated) return -1 // unlikely: >100k blobs or blob > 7MB - - var totalSizeBytes = 0L - for (file in tree.tree) - totalSizeBytes += file.size - - // overflow unlikely: >2TB - return ((totalSizeBytes + 512) / 1024).toInt() - } - return -1 - } - - /** - * Parsed GitHub repo search response - * @property total_count Total number of hits for the search (ignoring paging window) - * @property incomplete_results A flag set by github to indicate search was incomplete (never seen it on) - * @property items Array of [repositories][Repo] - * @see Github API doc - */ - @Suppress("PropertyName") - class RepoSearch { - @Suppress("MemberVisibilityCanBePrivate") - var total_count = 0 - var incomplete_results = false - var items = ArrayList() - } - - /** Part of [RepoSearch] in Github API response - one repository entry in [items][RepoSearch.items] */ - @Suppress("PropertyName") - class Repo { - - /** Unlike the rest of this class, this is not part of the API but added by us locally - * to track whether [getRepoSize] has been run successfully for this repo */ - var hasUpdatedSize = false - - var name = "" - var full_name = "" - var description: String? = null - var owner = RepoOwner() - var stargazers_count = 0 - var default_branch = "" - var html_url = "" - var pushed_at = "" // don't use updated_at - see https://github.com/yairm210/Unciv/issues/6106 - var size = 0 - var topics = mutableListOf() - //var stargazers_url = "" - //var homepage: String? = null // might use instead of go to repo? - //var has_wiki = false // a wiki could mean proper documentation for the mod? - - /** - * Initialize `this` with an url, extracting all possible fields from it - * (html_url, author, repoName, branchName). - * - * Allow url formats: - * * Basic repo url: - * https://github.com/author/repoName - * * or complete 'zip' url from github's code->download zip menu: - * https://github.com/author/repoName/archive/refs/heads/branchName.zip - * * or the branch url same as one navigates to on github through the "branches" menu: - * https://github.com/author/repoName/tree/branchName - * - * In the case of the basic repo url, an API query is sent to determine the default branch. - * Other url forms will not go online. - * - * @return `this` to allow chaining, `null` for invalid links or any other failures - */ - fun parseUrl(url: String): Repo? { - fun processMatch(matchResult: MatchResult): Repo { - html_url = matchResult.groups[1]!!.value - owner.login = matchResult.groups[2]!!.value - name = matchResult.groups[3]!!.value - default_branch = matchResult.groups[4]!!.value - return this - } - - html_url = url - default_branch = "master" - val matchZip = Regex("""^(.*/(.*)/(.*))/archive/(?:.*/)?([^.]+).zip$""").matchEntire(url) - if (matchZip != null && matchZip.groups.size > 4) - return processMatch(matchZip) - - val matchBranch = Regex("""^(.*/(.*)/(.*))/tree/([^/]+)$""").matchEntire(url) - if (matchBranch != null && matchBranch.groups.size > 4) - return processMatch(matchBranch) - - val matchRepo = Regex("""^.*//.*/(.+)/(.+)/?$""").matchEntire(url) - if (matchRepo != null && matchRepo.groups.size > 2) { - // Query API if we got the first URL format to get the correct default branch - val response = download("https://api.github.com/repos/${matchRepo.groups[1]!!.value}/${matchRepo.groups[2]!!.value}") - ?: return null - val repoString = response.bufferedReader().readText() - return json().fromJson(Repo::class.java, repoString) - } - return null - } - } - - /** Part of [Repo] in Github API response */ - @Suppress("PropertyName") - class RepoOwner { - var login = "" - var avatar_url: String? = null - } - - /** - * Query GitHub for topics named "unciv-mod*" - * @return Parsed [TopicSearchResponse] json on success, `null` on failure. - */ - fun tryGetGithubTopics(): TopicSearchResponse? { - // `+repositories:>1` means ignore unused or practically unused topics - val link = "https://api.github.com/search/topics?q=unciv-mod+repositories:%3E1&sort=name&order=asc" - var retries = 2 - while (retries > 0) { - retries-- - // obey rate limit - if (RateLimit.waitForLimit()) return null - // try download - val inputStream = download(link) { - if (it.responseCode == 403 || it.responseCode == 200 && retries == 1) { - // Pass the response headers to the rate limit handler so it can process the rate limit headers - RateLimit.notifyHttpResponse(it) - retries++ // An extra retry so the 403 is ignored in the retry count - } - } ?: continue - return json().fromJson(TopicSearchResponse::class.java, inputStream.bufferedReader().readText()) - } - return null - } - - /** Topic search response */ - @Suppress("PropertyName") - class TopicSearchResponse { - // Commented out: Github returns them, but we're not interested -// var total_count = 0 -// var incomplete_results = false - var items = ArrayList() - class Topic { - var name = "" - var display_name: String? = null // Would need to be curated, which is alottawork -// var featured = false -// var curated = false - var created_at = "" // iso datetime with "Z" timezone - var updated_at = "" // iso datetime with "Z" timezone - } - } - - /** Rewrite modOptions file for a mod we just installed to include metadata we got from the GitHub api - * - * (called on background thread) - */ - fun rewriteModOptions(repo: Repo, modFolder: FileHandle) { - val modOptionsFile = modFolder.child("jsons/ModOptions.json") - val modOptions = if (modOptionsFile.exists()) json().fromJsonFile(ModOptions::class.java, modOptionsFile) else ModOptions() - modOptions.modUrl = repo.html_url - modOptions.defaultBranch = repo.default_branch - modOptions.lastUpdated = repo.pushed_at - modOptions.author = repo.owner.login - modOptions.modSize = repo.size - modOptions.topics = repo.topics - modOptions.updateDeprecations() - json().toJson(modOptions, modOptionsFile) - } - - private const val outerBlankReplacement = '=' - // Github disallows **any** special chars and replaces them with '-' - so use something ascii the - // OS accepts but still is recognizable as non-original, to avoid confusion - - /** Convert a [Repo] name to a local name for both display and folder name - * - * Replaces '-' with blanks but ensures no leading or trailing blanks. - * As mad modders know no limits, trailing "-" did indeed happen, causing things to break due to trailing blanks on a folder name. - * As "test-" and "test" are different allowed repository names, trimmed blanks are replaced with one equals sign per side. - * @param onlyOuterBlanks If `true` ignores inner dashes - only start and end are treated. Useful when modders have manually created local folder names using dashes. - */ - fun String.repoNameToFolderName(onlyOuterBlanks: Boolean = false): String { - var result = if (onlyOuterBlanks) this else replace('-', ' ') - if (result.endsWith(' ')) result = result.trimEnd() + outerBlankReplacement - if (result.startsWith(' ')) result = outerBlankReplacement + result.trimStart() - return result - } - - /** Inverse of [repoNameToFolderName] */ - // As of this writing, only used for loadMissingMods - fun String.folderNameToRepoName(): String { - var result = replace(' ', '-') - if (result.endsWith(outerBlankReplacement)) result = result.trimEnd(outerBlankReplacement) + '-' - if (result.startsWith(outerBlankReplacement)) result = '-' + result.trimStart(outerBlankReplacement) - return result - } -} - -/** Utility - extract Zip archives - * @see [Zip.extractFolder] - */ -object Zip { - private const val bufferSize = 2048 - - /** - * Extract one Zip file recursively (nested Zip files are extracted in turn). - * - * The source Zip is not deleted, but successfully extracted nested ones are. - * - * **Warning**: Extracting into a non-empty destination folder will merge contents. Existing - * files also included in the archive will be partially overwritten, when the new data is shorter - * than the old you will get _mixed contents!_ - * - * @param zipFile The Zip file to extract - * @param unzipDestination The folder to extract into, preferably empty (not enforced). - */ - fun extractFolder(zipFile: FileHandle, unzipDestination: FileHandle) { - // I went through a lot of similar answers that didn't work until I got to this gem by NeilMonday - // (with mild changes to fit the FileHandles) - // https://stackoverflow.com/questions/981578/how-to-unzip-files-recursively-in-java - - debug("Extracting %s to %s", zipFile, unzipDestination) - // establish buffer for writing file - val data = ByteArray(bufferSize) - - fun streamCopy(fromStream: InputStream, toHandle: FileHandle) { - val inputStream = BufferedInputStream(fromStream) - var currentByte: Int - - // write the current file to disk - val fos = FileOutputStream(toHandle.file()) - val dest = BufferedOutputStream(fos, bufferSize) - - // read and write until last byte is encountered - while (inputStream.read(data, 0, bufferSize).also { currentByte = it } != -1) { - dest.write(data, 0, currentByte) - } - dest.flush() - dest.close() - inputStream.close() - } - - val file = zipFile.file() - val zip = ZipFile(file) - //unzipDestination.mkdirs() - val zipFileEntries = zip.entries() - - // Process each entry - while (zipFileEntries.hasMoreElements()) { - // grab a zip file entry - val entry = zipFileEntries.nextElement() as ZipEntry - val currentEntry = entry.name - val destFile = unzipDestination.child(currentEntry) - val destinationParent = destFile.parent() - - // create the parent directory structure if needed - destinationParent.mkdirs() - if (!entry.isDirectory) { - streamCopy ( zip.getInputStream(entry), destFile) - } - // The new file has a current last modification time - // and not the one stored in the archive - we could: - // 'destFile.file().setLastModified(entry.time)' - // but later handling will throw these away anyway, - // and GitHub sets all timestamps to the download time. - - if (currentEntry.endsWith(".zip")) { - // found a zip file, try to open - extractFolder(destFile, destinationParent) - destFile.delete() - } - } - zip.close() // Needed so we can delete the zip file later - } -} diff --git a/core/src/com/unciv/ui/screens/savescreens/LoadGameScreen.kt b/core/src/com/unciv/ui/screens/savescreens/LoadGameScreen.kt index c382e9e08e..5405a85830 100644 --- a/core/src/com/unciv/ui/screens/savescreens/LoadGameScreen.kt +++ b/core/src/com/unciv/ui/screens/savescreens/LoadGameScreen.kt @@ -27,8 +27,8 @@ import com.unciv.ui.components.input.onClick import com.unciv.ui.popups.LoadingPopup import com.unciv.ui.popups.Popup import com.unciv.ui.popups.ToastPopup -import com.unciv.ui.screens.pickerscreens.Github -import com.unciv.ui.screens.pickerscreens.Github.folderNameToRepoName +import com.unciv.logic.github.Github +import com.unciv.logic.github.Github.folderNameToRepoName import com.unciv.utils.Concurrency import com.unciv.utils.Log import com.unciv.utils.launchOnGLThread