AnarchistGamers/Unciv
1
0
Fork 0
mirror of https://github.com/yairm210/Unciv.git synced 2025-08-03 20:48:49 -04:00
Code Issues Packages Projects Releases Wiki Activity

Download mod releases or any mod zip (#10881)

Browse Source 
* Allow almost any mod zip download source

* Linting

* Add basic "Release page section" link capability, more linting

* Refactor: Move Github to logic

* Refactor: Split Github into files

* Refactor: centralizing all API knowledge

* Try bumping detekt version to fix false complaint

* Attempt to trick detekt to not complain about `it`

* Better overload-ambiguity-solving + detekt-compatible SAM use
This commit is contained in:
SomeTroglodyte 2024-01-07 15:07:15 +01:00 committed by GitHub
parent 68786d7603
commit 50a6e5bbdb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 812 additions and 632 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/detektAnalysis.yml vendored
View File

@ -29,7 +29,7 @@ jobs:
id: setup_detekt
uses: peter-murray/setup-detekt@v2
with:
detekt_version: '1.23.1'
detekt_version: '1.23.4'
- name: Detekt errors
run: detekt-cli --parallel --config detekt/config/detekt-errors.yml

4
android/assets/jsons/translations/template.properties
View File

@ -1789,7 +1789,9 @@ Download [modName] =
Update [modName] =
Could not download mod list =
Download mod from URL =
Please enter the mod repository -or- archive zip -or- branch url: =
Please enter the mod repository -or- archive zip -or- branch -or- release url: =
That is not a valid ZIP file =
Invalid Mod archive structure =
Invalid link! =
Paste from clipboard =
Download =

2
core/src/com/unciv/logic/GameInfo.kt
View File

@ -39,7 +39,7 @@ import com.unciv.models.ruleset.unique.UniqueType
import com.unciv.models.translations.tr
import com.unciv.ui.audio.MusicMood
import com.unciv.ui.audio.MusicTrackChooserFlags
import com.unciv.ui.screens.pickerscreens.Github.repoNameToFolderName
import com.unciv.logic.github.Github.repoNameToFolderName
import com.unciv.ui.screens.savescreens.Gzip
import com.unciv.ui.screens.worldscreen.status.NextTurnProgress
import com.unciv.utils.DebugUtils

356
core/src/com/unciv/logic/github/Github.kt Normal file
View File

@ -0,0 +1,356 @@
package com.unciv.logic.github
import com.badlogic.gdx.Files
import com.badlogic.gdx.files.FileHandle
import com.badlogic.gdx.graphics.Pixmap
import com.unciv.json.fromJsonFile
import com.unciv.json.json
import com.unciv.logic.BackwardCompatibility.updateDeprecations
import com.unciv.logic.UncivShowableException
import com.unciv.logic.github.Github.download
import com.unciv.logic.github.Github.downloadAndExtract
import com.unciv.logic.github.Github.tryGetGithubReposWithTopic
import com.unciv.logic.github.GithubAPI.getUrlForTreeQuery
import com.unciv.models.ruleset.ModOptions
import com.unciv.utils.Log
import java.io.BufferedReader
import java.io.FileFilter
import java.io.InputStream
import java.io.InputStreamReader
import java.net.HttpURLConnection
import java.net.URL
import java.nio.ByteBuffer
import java.util.zip.ZipException
/**
* Utility managing Github access (except the link in WorldScreenCommunityPopup)
*
* Singleton - RateLimit is shared app-wide and has local variables, and is not tested for thread safety.
* Therefore, additional effort is required should [tryGetGithubReposWithTopic] ever be called non-sequentially.
* [download] and [downloadAndExtract] should be thread-safe as they are self-contained.
* They do not join in the [RateLimit] handling because Github doc suggests each API
* has a separate limit (and I found none for cloning via a zip).
*/
object Github {
private const val contentDispositionHeader = "Content-Disposition"
private const val attachmentDispositionPrefix = "attachment;filename="
// Consider merging this with the Dropbox function
/**
* Helper opens am url and accesses its input stream, logging errors to the console
* @param url String representing a [URL] to download.
* @param action Optional callback that will be executed between opening the connection and
* accessing its data - passes the [connection][HttpURLConnection] and allows e.g. reading the response headers.
* @return The [InputStream] if successful, `null` otherwise.
*/
fun download(url: String, action: (HttpURLConnection) -> Unit = {}): InputStream? {
try {
// Problem type 1 - opening the URL connection
with(URL(url).openConnection() as HttpURLConnection)
{
action(this)
// Problem type 2 - getting the information
try {
return inputStream
} catch (ex: Exception) {
// No error handling, just log the message.
// NOTE that this 'read error stream' CAN ALSO fail, but will be caught by the big try/catch
val reader = BufferedReader(InputStreamReader(errorStream, Charsets.UTF_8))
Log.error("Message from GitHub: %s", reader.readText())
throw ex
}
}
} catch (ex: Exception) {
Log.error("Exception during GitHub download", ex)
return null
}
}
/**
* Download a mod and extract, deleting any pre-existing version.
* @param folderFileHandle Destination handle of mods folder - also controls Android internal/external
* @author **Warning**: This took a long time to get just right, so if you're changing this, ***TEST IT THOROUGHLY*** on _both_ Desktop _and_ Phone
* @return FileHandle for the downloaded Mod's folder or null if download failed
*/
fun downloadAndExtract(
repo: GithubAPI.Repo,
folderFileHandle: FileHandle
): FileHandle? {
var modNameFromFileName = repo.name
val defaultBranch = repo.default_branch
val zipUrl: String
val tempName: String
if (repo.direct_zip_url.isEmpty()) {
val gitRepoUrl = repo.html_url
// Initiate download - the helper returns null when it fails
zipUrl = GithubAPI.getUrlForBranchZip(gitRepoUrl, defaultBranch)
// Get a mod-specific temp file name
tempName = "temp-" + gitRepoUrl.hashCode().toString(16)
} else {
zipUrl = repo.direct_zip_url
tempName = "temp-" + repo.toString().hashCode().toString(16)
}
val inputStream = download(zipUrl) {
val disposition = it.getHeaderField(contentDispositionHeader)
if (disposition.startsWith(attachmentDispositionPrefix))
modNameFromFileName = disposition.removePrefix(attachmentDispositionPrefix)
.removeSuffix(".zip").replace('.', ' ')
// We could check Content-Type=[application/x-zip-compressed] here, but the ZipFile will catch that anyway. Would save some bandwidth, however.
} ?: return null
// Download to temporary zip
val tempZipFileHandle = folderFileHandle.child("$tempName.zip")
tempZipFileHandle.write(inputStream, false)
// prepare temp unpacking folder
val unzipDestination = tempZipFileHandle.sibling(tempName) // folder, not file
// prevent mixing new content with old - hopefully there will never be cadavers of our tempZip stuff
if (unzipDestination.exists())
if (unzipDestination.isDirectory) unzipDestination.deleteDirectory() else unzipDestination.delete()
try {
Zip.extractFolder(tempZipFileHandle, unzipDestination)
} catch (ex: ZipException) {
throw UncivShowableException("That is not a valid ZIP file", ex)
}
val (innerFolder, modName) = resolveZipStructure(unzipDestination, modNameFromFileName)
// modName can be "$repoName-$defaultBranch"
val finalDestinationName = modName.replace("-$defaultBranch", "").repoNameToFolderName()
// finalDestinationName is now the mod name as we display it. Folder name needs to be identical.
val finalDestination = folderFileHandle.child(finalDestinationName)
// prevent mixing new content with old
var tempBackup: FileHandle? = null
if (finalDestination.exists()) {
tempBackup = finalDestination.sibling("$finalDestinationName.updating")
finalDestination.renameOrMove(tempBackup)
}
// Move temp unpacked content to their final place
finalDestination.mkdirs() // If we don't create this as a directory, it will think this is a file and nothing will work.
// The move will reset the last modified time (recursively, at least on Linux)
// This sort will guarantee the desktop launcher will not re-pack textures and overwrite the atlas as delivered by the mod
for (innerFileOrFolder in innerFolder.list()
.sortedBy { file -> file.extension() == "atlas" } ) {
innerFileOrFolder.renameOrMove(finalDestination)
}
// clean up
tempZipFileHandle.delete()
unzipDestination.deleteDirectory()
if (tempBackup != null)
if (tempBackup.isDirectory) tempBackup.deleteDirectory() else tempBackup.delete()
return finalDestination
}
private fun isValidModFolder(dir: FileHandle): Boolean {
val goodFolders = listOf("Images", "jsons", "maps", "music", "sounds", "Images\\..*")
.map { Regex(it, RegexOption.IGNORE_CASE) }
val goodFiles = listOf(".*\\.atlas", ".*\\.png", "preview.jpg", ".*\\.md", "Atlases.json", ".nomedia", "license")
.map { Regex(it, RegexOption.IGNORE_CASE) }
var good = 0
var bad = 0
for (file in dir.list()) {
val goodList = if (file.isDirectory) goodFolders else goodFiles
if (goodList.any { file.name().matches(it) }) good++ else bad++
}
return good > 0 && good > bad
}
/** Check whether the unpacked zip contains a subfolder with mod content or is already the mod.
* If there's a subfolder we'll assume it is the mod name, optionally suffixed with branch or release-tag name like github does.
* @return Pair: actual mod content folder to name (subfolder name or [defaultModName])
*/
private fun resolveZipStructure(dir: FileHandle, defaultModName: String): Pair<FileHandle, String> {
if (isValidModFolder(dir))
return dir to defaultModName
val subdirs = dir.list(FileFilter { it.isDirectory }) // See detekt/#6822 - a direct lambda-to-SAM with typed `it` fails detektAnalysis
if (subdirs.size == 1 && isValidModFolder(subdirs[0]))
return subdirs[0] to subdirs[0].name()
throw UncivShowableException("Invalid Mod archive structure")
}
private fun FileHandle.renameOrMove(dest: FileHandle) {
// Gdx tries a java rename for Absolute and External, but NOT for Local - rectify that
if (type() == Files.FileType.Local) {
// See #5346: renameTo 'unpacks' a source folder if the destination exists and is an
// empty folder, dropping the name of the source entirely.
// Safer to ask for a move to the not-yet-existing full resulting path instead.
if (isDirectory)
if (file().renameTo(dest.child(name()).file())) return
else
if (file().renameTo(dest.file())) return
}
moveTo(dest)
}
/**
* Query GitHub for repositories marked "unciv-mod"
* @param amountPerPage Number of search results to return for this request.
* @param page The "page" number, starting at 1.
* @return Parsed [RepoSearch][GithubAPI.RepoSearch] json on success, `null` on failure.
* @see <a href="https://docs.github.com/en/rest/reference/search#search-repositories">Github API doc</a>
*/
fun tryGetGithubReposWithTopic(amountPerPage:Int, page:Int, searchRequest: String = ""): GithubAPI.RepoSearch? {
val link = GithubAPI.getUrlForModListing(searchRequest, amountPerPage, page)
var retries = 2
while (retries > 0) {
retries--
// obey rate limit
if (RateLimit.waitForLimit()) return null
// try download
val inputStream = download(link) {
if (it.responseCode == 403 || it.responseCode == 200 && page == 1 && retries == 1) {
// Pass the response headers to the rate limit handler so it can process the rate limit headers
RateLimit.notifyHttpResponse(it)
retries++ // An extra retry so the 403 is ignored in the retry count
}
} ?: continue
return json().fromJson(GithubAPI.RepoSearch::class.java, inputStream.bufferedReader().readText())
}
return null
}
/** Get a Pixmap from a "preview" png or jpg file at the root of the repo, falling back to the
* repo owner's avatar [avatarUrl]. The file content url is constructed from [modUrl] and [defaultBranch]
* by replacing the host with `raw.githubusercontent.com`.
*/
fun tryGetPreviewImage(modUrl: String, defaultBranch: String, avatarUrl: String?): Pixmap? {
// Side note: github repos also have a "Social Preview" optionally assignable on the repo's
// settings page, but that info is inaccessible using the v3 API anonymously. The easiest way
// to get it would be to query the the repo's frontend page (modUrl), and parse out
// `head/meta[property=og:image]/@content`, which is one extra spurious roundtrip and a
// non-trivial waste of bandwidth.
// Thus we ask for a "preview" file as part of the repo contents instead.
val fileLocation = GithubAPI.getUrlForPreview(modUrl, defaultBranch)
try {
val file = download("$fileLocation.jpg")
?: download("$fileLocation.png")
// Note: avatar urls look like: https://avatars.githubusercontent.com/u/<number>?v=4
// So the image format is only recognizable from the response "Content-Type" header
// or by looking for magic markers in the bits - which the Pixmap constructor below does.
?: avatarUrl?.let { download(it) }
?: return null
val byteArray = file.readBytes()
val buffer = ByteBuffer.allocateDirect(byteArray.size).put(byteArray).position(0)
return Pixmap(buffer)
} catch (_: Throwable) {
return null
}
}
/** Queries github for a tree and calculates the sum of the blob sizes.
* @return -1 on failure, else size rounded to kB
* @see <a href="https://docs.github.com/en/rest/git/trees#get-a-tree">Github API "Get a tree"</a>
*/
fun getRepoSize(repo: GithubAPI.Repo): Int {
val link = repo.getUrlForTreeQuery()
var retries = 2
while (retries > 0) {
retries--
// obey rate limit
if (RateLimit.waitForLimit()) return -1
// try download
val inputStream = download(link) {
if (it.responseCode == 403 || it.responseCode == 200 && retries == 1) {
// Pass the response headers to the rate limit handler so it can process the rate limit headers
RateLimit.notifyHttpResponse(it)
retries++ // An extra retry so the 403 is ignored in the retry count
}
} ?: continue
val tree = json().fromJson(GithubAPI.Tree::class.java, inputStream.bufferedReader().readText())
if (tree.truncated) return -1 // unlikely: >100k blobs or blob > 7MB
var totalSizeBytes = 0L
for (file in tree.tree)
totalSizeBytes += file.size
// overflow unlikely: >2TB
return ((totalSizeBytes + 512) / 1024).toInt()
}
return -1
}
/**
* Query GitHub for topics named "unciv-mod*"
* @return Parsed [TopicSearchResponse][GithubAPI.TopicSearchResponse] json on success, `null` on failure.
*/
fun tryGetGithubTopics(): GithubAPI.TopicSearchResponse? {
val link = GithubAPI.urlToQueryModTopics
var retries = 2
while (retries > 0) {
retries--
// obey rate limit
if (RateLimit.waitForLimit()) return null
// try download
val inputStream = download(link) {
if (it.responseCode == 403 || it.responseCode == 200 && retries == 1) {
// Pass the response headers to the rate limit handler so it can process the rate limit headers
RateLimit.notifyHttpResponse(it)
retries++ // An extra retry so the 403 is ignored in the retry count
}
} ?: continue
return json().fromJson(GithubAPI.TopicSearchResponse::class.java, inputStream.bufferedReader().readText())
}
return null
}
/** Rewrite modOptions file for a mod we just installed to include metadata we got from the GitHub api
*
* (called on background thread)
*/
fun rewriteModOptions(repo: GithubAPI.Repo, modFolder: FileHandle) {
val modOptionsFile = modFolder.child("jsons/ModOptions.json")
val modOptions = if (modOptionsFile.exists()) json().fromJsonFile(ModOptions::class.java, modOptionsFile) else ModOptions()
// If this is false we didn't get github repo info, do a defensive merge so the Repo.parseUrl or download
// code can decide defaults but leave any meaningful field of a zip-included ModOptions alone.
val overwriteAlways = repo.direct_zip_url.isEmpty()
if (overwriteAlways || modOptions.modUrl.isEmpty()) modOptions.modUrl = repo.html_url
if (overwriteAlways || modOptions.defaultBranch == "master" && repo.default_branch.isNotEmpty())
modOptions.defaultBranch = repo.default_branch
if (overwriteAlways || modOptions.lastUpdated.isEmpty()) modOptions.lastUpdated = repo.pushed_at
if (overwriteAlways || modOptions.author.isEmpty()) modOptions.author = repo.owner.login
if (overwriteAlways || modOptions.modSize == 0) modOptions.modSize = repo.size
if (overwriteAlways || modOptions.topics.isEmpty()) modOptions.topics = repo.topics
modOptions.updateDeprecations()
json().toJson(modOptions, modOptionsFile)
}
private const val outerBlankReplacement = '='
// Github disallows **any** special chars and replaces them with '-' - so use something ascii the
// OS accepts but still is recognizable as non-original, to avoid confusion
/** Convert a [Repo][[GithubAPI.Repo] name to a local name for both display and folder name
*
* Replaces '-' with blanks but ensures no leading or trailing blanks.
* As mad modders know no limits, trailing "-" did indeed happen, causing things to break due to trailing blanks on a folder name.
* As "test-" and "test" are different allowed repository names, trimmed blanks are replaced with one equals sign per side.
* @param onlyOuterBlanks If `true` ignores inner dashes - only start and end are treated. Useful when modders have manually created local folder names using dashes.
*/
fun String.repoNameToFolderName(onlyOuterBlanks: Boolean = false): String {
var result = if (onlyOuterBlanks) this else replace('-', ' ')
if (result.endsWith(' ')) result = result.trimEnd() + outerBlankReplacement
if (result.startsWith(' ')) result = outerBlankReplacement + result.trimStart()
return result
}
/** Inverse of [repoNameToFolderName] */
// As of this writing, only used for loadMissingMods
fun String.folderNameToRepoName(): String {
var result = replace(' ', '-')
if (result.endsWith(outerBlankReplacement)) result = result.trimEnd(outerBlankReplacement) + '-'
if (result.startsWith(outerBlankReplacement)) result = '-' + result.trimStart(outerBlankReplacement)
return result
}
}

242
core/src/com/unciv/logic/github/GithubAPI.kt Normal file
View File

@ -0,0 +1,242 @@
package com.unciv.logic.github
import com.unciv.json.json
/**
* "Namespace" collects all Github API structural knowledge
* - Response schema
* - Query URL builders
*/
@Suppress("PropertyName") // We're declaring an external API schema
object GithubAPI {
// region URL formatters
/** Format a download URL for a branch archive */
// URL format see: https://docs.github.com/en/repositories/working-with-files/using-files/downloading-source-code-archives#source-code-archive-urls
// Note: https://api.github.com/repos/owner/mod/zipball would be an alternative. Its response is a redirect, but our lib follows that and delivers the zip just fine.
// Problems with the latter: Internal zip structure different, finalDestinationName would need a patch. Plus, normal URL escaping for owner/reponame does not work.
internal fun getUrlForBranchZip(gitRepoUrl: String, branch: String) = "$gitRepoUrl/archive/refs/heads/$branch.zip"
/** Format a URL to query for Mod repos by topic */
internal fun getUrlForModListing(searchRequest: String, amountPerPage: Int, page: Int) =
// Add + if needed to separate the query text from its parameters
"https://api.github.com/search/repositories?q=$searchRequest${ if (searchRequest.isEmpty()) "" else "+" }%20topic:unciv-mod%20fork:true&sort:stars&per_page=$amountPerPage&page=$page"
/** Format URL to fetch one specific [Repo] metadata from the API */
private fun getUrlForSingleRepoQuery(owner: String, repoName: String) = "https://api.github.com/repos/$owner/$repoName"
/** Format a download URL for a release archive */
private fun Repo.getUrlForReleaseZip() = "$html_url/archive/refs/tags/$release_tag.zip"
/** Format a URL to query a repo tree - to calculate actual size */
internal fun Repo.getUrlForTreeQuery() =
"https://api.github.com/repos/$full_name/git/trees/$default_branch?recursive=true"
/** Format a URL to fetch a preview image - without extension */
internal fun getUrlForPreview(modUrl: String, branch: String) = "$modUrl/$branch/preview"
.replace("github.com", "raw.githubusercontent.com")
/** A query returning all known topics staring with "unciv-mod" and having at least two uses */
// `+repositories:>1` means ignore unused or practically unused topics
internal const val urlToQueryModTopics = "https://api.github.com/search/topics?q=unciv-mod+repositories:%3E1&sort=name&order=asc"
//endregion
//region responses
/**
* Parsed Github repo search response
* @property total_count Total number of hits for the search (ignoring paging window)
* @property incomplete_results A flag set by github to indicate search was incomplete (never seen it on)
* @property items Array of [repositories][Repo]
* @see <a href="https://docs.github.com/en/rest/reference/search#search-repositories--code-samples">Github API doc</a>
*/
class RepoSearch {
@Suppress("MemberVisibilityCanBePrivate")
var total_count = 0
var incomplete_results = false
var items = ArrayList<Repo>()
}
/** Part of [RepoSearch] in Github API response - one repository entry in [items][RepoSearch.items] */
class Repo {
/** Unlike the rest of this class, this is not part of the API but added by us locally
* to track whether [getRepoSize][Github.getRepoSize] has been run successfully for this repo */
var hasUpdatedSize = false
/** Not part of the github schema: Explicit final zip download URL for non-github or release downloads */
var direct_zip_url = ""
/** Not part of the github schema: release tag, for debugging (DL via direct_zip_url) */
var release_tag = ""
var name = ""
var full_name = ""
var description: String? = null
var owner = RepoOwner()
var stargazers_count = 0
var default_branch = ""
var html_url = ""
var pushed_at = "" // don't use updated_at - see https://github.com/yairm210/Unciv/issues/6106
var size = 0
var topics = mutableListOf<String>()
//var stargazers_url = ""
//var homepage: String? = null // might use instead of go to repo?
//var has_wiki = false // a wiki could mean proper documentation for the mod?
/** String representation to be used for logging */
override fun toString() = name.ifEmpty { direct_zip_url }
companion object {
/** Create a [Repo] metadata instance from a [url], supporting various formats
* from a repository landing page url to a free non-github zip download.
*
* @see GithubAPI.parseUrl
* @return `null` for invalid links or any other failures
*/
fun parseUrl(url: String): Repo? = Repo().parseUrl(url)
/** Query Github API for [owner]'s [repoName] repository metadata */
fun query(owner: String, repoName: String): Repo? {
val response = Github.download(getUrlForSingleRepoQuery(owner, repoName))
?: return null
val repoString = response.bufferedReader().readText()
return json().fromJson(Repo::class.java, repoString)
}
}
}
/** Part of [Repo] in Github API response */
class RepoOwner {
var login = ""
var avatar_url: String? = null
}
/** Topic search response */
class TopicSearchResponse {
// Commented out: Github returns them, but we're not interested
// var total_count = 0
// var incomplete_results = false
var items = ArrayList<Topic>()
class Topic {
var name = ""
var display_name: String? = null // Would need to be curated, which is alottawork
// var featured = false
// var curated = false
var created_at = "" // iso datetime with "Z" timezone
var updated_at = "" // iso datetime with "Z" timezone
}
}
/** Class to receive a github API "Get a tree" response parsed as json */
// Parts of the response we ignore are commented out
internal class Tree {
//val sha = ""
//val url = ""
class TreeFile {
//val path = ""
//val mode = 0
//val type = "" // blob / tree
//val sha = ""
//val url = ""
var size: Long = 0L
}
@Suppress("MemberNameEqualsClassName")
var tree = ArrayList<TreeFile>()
var truncated = false
}
//endregion
//region Flexible URL parsing
/**
* Initialize `this` with an url, extracting all possible fields from it
* (html_url, author, repoName, branchName).
*
* Allow url formats:
* * Basic repo url:
* https://github.com/author/repoName
* * or complete 'zip' url from github's code->download zip menu:
* https://github.com/author/repoName/archive/refs/heads/branchName.zip
* * or the branch url same as one navigates to on github through the "branches" menu:
* https://github.com/author/repoName/tree/branchName
* * or release tag
* https://github.com/author/repoName/releases/tag/tagname
* https://github.com/author/repoName/archive/refs/tags/tagname.zip
*
* In the case of the basic repo url, an [API query](https://docs.github.com/en/rest/repos/repos#get-a-repository) is sent to determine the default branch.
* Other url forms will not go online.
*
* @return a new Repo instance for the 'Basic repo url' case, otherwise `this`, modified, to allow chaining, `null` for invalid links or any other failures
* @see <a href="https://docs.github.com/en/rest/repos/repos#get-a-repository--code-samples">Github API Repository Code Samples</a>
*/
private fun Repo.parseUrl(url: String): Repo? {
fun processMatch(matchResult: MatchResult): Repo {
html_url = matchResult.groups[1]!!.value
owner.login = matchResult.groups[2]!!.value
name = matchResult.groups[3]!!.value
default_branch = matchResult.groups[4]!!.value
return this
}
html_url = url
default_branch = "master"
val matchZip = Regex("""^(.*/(.*)/(.*))/archive/(?:.*/)?heads/([^.]+).zip$""").matchEntire(url)
if (matchZip != null && matchZip.groups.size > 4)
return processMatch(matchZip)
val matchBranch = Regex("""^(.*/(.*)/(.*))/tree/([^/]+)$""").matchEntire(url)
if (matchBranch != null && matchBranch.groups.size > 4)
return processMatch(matchBranch)
// Releases and tags -
// TODO Query for latest release and save as Mod Version?
// https://docs.github.com/en/rest/releases/releases#get-the-latest-release
// TODO Query a specific release for its name attribute - the page will link the tag
// https://docs.github.com/en/rest/releases/releases#get-a-release-by-tag-name
val matchTagArchive = Regex("""^(.*/(.*)/(.*))/archive/(?:.*/)?tags/([^.]+).zip$""").matchEntire(url)
if (matchTagArchive != null && matchTagArchive.groups.size > 4) {
processMatch(matchTagArchive)
release_tag = default_branch
// leave default_branch even if it's actually a tag not a branch name
// so the suffix of the inner first level folder inside the zip can be removed later
direct_zip_url = url
return this
}
val matchTagPage = Regex("""^(.*/(.*)/(.*))/releases/(?:.*/)?tag/([^.]+)$""").matchEntire(url)
if (matchTagPage != null && matchTagPage.groups.size > 4) {
processMatch(matchTagPage)
release_tag = default_branch
direct_zip_url = getUrlForReleaseZip()
return this
}
val matchRepo = Regex("""^.*//.*/(.+)/(.+)/?$""").matchEntire(url)
if (matchRepo != null && matchRepo.groups.size > 2) {
// Query API if we got the 'https://github.com/author/repoName' URL format to get the correct default branch
val repo = Repo.query(matchRepo.groups[1]!!.value, matchRepo.groups[2]!!.value)
if (repo != null) return repo
}
// Only complain about invalid link if it isn't a http protocol (to think about: android document protocol? file protocol?)
if (!url.startsWith("http://") && !url.startsWith("https://"))
return null
// From here, we'll always return success and treat the url as direct-downloadable zip.
// The Repo instance will be a pseudo-repo not corresponding to an actual github repo.
html_url = ""
direct_zip_url = url
owner.login = "-unknown-"
default_branch = "master" // only used to remove this suffix should the zip contain a inner folder
// But see if we can extract a file name from the url
// Will use filename from response headers, if content-disposition is sent, for the Mod name instead, done in downloadAndExtract
val matchAnyZip = Regex("""^.*//(?:.*/)*([^/]+\.zip)$""").matchEntire(url)
if (matchAnyZip != null && matchAnyZip.groups.size > 1)
name = matchAnyZip.groups[1]!!.value
full_name = name
return this
}
//endregion
}

90
core/src/com/unciv/logic/github/RateLimit.kt Normal file
View File

@ -0,0 +1,90 @@
package com.unciv.logic.github
import com.unciv.utils.debug
import java.net.HttpURLConnection
/**
* Implements the ability wo work with GitHub's rate limit, recognize blocks from previous attempts, wait and retry.
* @see <a href="https://docs.github.com/en/rest/reference/search#rate-limit">Github API doc</a>
*/
object RateLimit {
private const val maxRequestsPerInterval = 10
private const val intervalInMilliSeconds = 60000L
private const val maxWaitLoop = 3
private var account = 0 // used requests
private var firstRequest = 0L // timestamp window start (java epoch millisecond)
/*
Github rate limits do not use sliding windows - you (if anonymous) get one window
which starts with the first request (if a window is not already active)
and ends 60s later, and a budget of 10 requests in that window. Once it expires,
everything is forgotten and the process starts from scratch
*/
private val millis: Long
get() = System.currentTimeMillis()
/** calculate required wait in ms
* @return Estimated number of milliseconds to wait for the rate limit window to expire
*/
private fun getWaitLength()
= (firstRequest + intervalInMilliSeconds - millis)
/** Maintain and check a rate-limit
* @return **true** if rate-limited, **false** if another request is allowed
*/
private fun isLimitReached(): Boolean {
val now = millis
val elapsed = if (firstRequest == 0L) intervalInMilliSeconds else now - firstRequest
if (elapsed >= intervalInMilliSeconds) {
firstRequest = now
account = 1
return false
}
if (account >= maxRequestsPerInterval) return true
account++
return false
}
/** If rate limit in effect, sleep long enough to allow next request.
*
* @return **true** if waiting did not clear isLimitReached() (can only happen if the clock is broken),
* or the wait has been interrupted by Thread.interrupt()
* **false** if we were below the limit or slept long enough to drop out of it.
*/
fun waitForLimit(): Boolean {
var loopCount = 0
while (isLimitReached()) {
val waitLength = getWaitLength()
try {
Thread.sleep(waitLength)
} catch ( ex: InterruptedException ) {
return true
}
if (++loopCount >= maxWaitLoop) return true
}
return false
}
/** http responses should be passed to this so the actual rate limit window can be evaluated and used.
* The very first response and all 403 ones are good candidates if they can be expected to contain GitHub's rate limit headers.
*
* @see <a href="https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting">Github API doc</a>
*/
fun notifyHttpResponse(response: HttpURLConnection) {
if (response.responseMessage != "rate limit exceeded" && response.responseCode != 200) return
fun getHeaderLong(name: String, default: Long = 0L) =
response.headerFields[name]?.get(0)?.toLongOrNull() ?: default
val limit = getHeaderLong("X-RateLimit-Limit", maxRequestsPerInterval.toLong()).toInt()
val remaining = getHeaderLong("X-RateLimit-Remaining").toInt()
val reset = getHeaderLong("X-RateLimit-Reset")
if (limit != maxRequestsPerInterval)
debug("GitHub API Limit reported via http (%s) not equal assumed value (%s)", limit, maxRequestsPerInterval)
account = maxRequestsPerInterval - remaining
if (reset == 0L) return
firstRequest = (reset + 1L) * 1000L - intervalInMilliSeconds
}
}

88
core/src/com/unciv/logic/github/Zip.kt Normal file
View File

@ -0,0 +1,88 @@
package com.unciv.logic.github
import com.badlogic.gdx.files.FileHandle
import com.unciv.utils.debug
import java.io.BufferedInputStream
import java.io.BufferedOutputStream
import java.io.FileOutputStream
import java.io.InputStream
import java.util.zip.ZipEntry
import java.util.zip.ZipFile
/** Utility - extract Zip archives
* @see [Zip.extractFolder]
*/
object Zip {
private const val bufferSize = 2048
/**
* Extract one Zip file recursively (nested Zip files are extracted in turn).
*
* The source Zip is not deleted, but successfully extracted nested ones are.
*
* **Warning**: Extracting into a non-empty destination folder will merge contents. Existing
* files also included in the archive will be partially overwritten, when the new data is shorter
* than the old you will get _mixed contents!_
*
* @param zipFile The Zip file to extract
* @param unzipDestination The folder to extract into, preferably empty (not enforced).
*/
fun extractFolder(zipFile: FileHandle, unzipDestination: FileHandle) {
// I went through a lot of similar answers that didn't work until I got to this gem by NeilMonday
// (with mild changes to fit the FileHandles)
// https://stackoverflow.com/questions/981578/how-to-unzip-files-recursively-in-java
debug("Extracting %s to %s", zipFile, unzipDestination)
// establish buffer for writing file
val data = ByteArray(bufferSize)
fun streamCopy(fromStream: InputStream, toHandle: FileHandle) {
val inputStream = BufferedInputStream(fromStream)
var currentByte: Int
// write the current file to disk
val fos = FileOutputStream(toHandle.file())
val dest = BufferedOutputStream(fos, bufferSize)
// read and write until last byte is encountered
while (inputStream.read(data, 0, bufferSize).also { currentByte = it } != -1) {
dest.write(data, 0, currentByte)
}
dest.flush()
dest.close()
inputStream.close()
}
val file = zipFile.file()
val zip = ZipFile(file)
//unzipDestination.mkdirs()
val zipFileEntries = zip.entries()
// Process each entry
while (zipFileEntries.hasMoreElements()) {
// grab a zip file entry
val entry = zipFileEntries.nextElement() as ZipEntry
val currentEntry = entry.name
val destFile = unzipDestination.child(currentEntry)
val destinationParent = destFile.parent()
// create the parent directory structure if needed
destinationParent.mkdirs()
if (!entry.isDirectory) {
streamCopy ( zip.getInputStream(entry), destFile)
}
// The new file has a current last modification time
// and not the one stored in the archive - we could:
// 'destFile.file().setLastModified(entry.time)'
// but later handling will throw these away anyway,
// and GitHub sets all timestamps to the download time.
if (currentEntry.endsWith(".zip")) {
// found a zip file, try to open
extractFolder(destFile, destinationParent)
destFile.delete()
}
}
zip.close() // Needed so we can delete the zip file later
}
}

4
core/src/com/unciv/logic/map/tile/TileStatFunctions.kt
View File

@ -30,9 +30,7 @@ class TileStatFunctions(val tile: Tile) {
fun getTileStats(city: City?, observingCiv: Civilization?,
localUniqueCache: LocalUniqueCache = LocalUniqueCache(false)
): Stats {
val stats = Stats()
for (statsToAdd in getTileStatsBreakdown(city, observingCiv, localUniqueCache))
stats.add(statsToAdd.second)
val stats = getTileStatsBreakdown(city, observingCiv, localUniqueCache).toStats()
for ((stat, value) in getTilePercentageStats(observingCiv, city, localUniqueCache)) {
stats[stat] *= value.toPercent()

7
core/src/com/unciv/models/metadata/ModCategories.kt
View File

@ -2,8 +2,9 @@ package com.unciv.models.metadata
import com.badlogic.gdx.Gdx
import com.unciv.json.json
import com.unciv.logic.github.GithubAPI
import com.unciv.ui.components.widgets.TranslatedSelectBox
import com.unciv.ui.screens.pickerscreens.Github
import com.unciv.logic.github.Github
class ModCategories : ArrayList<ModCategories.Category>() {
@ -21,12 +22,12 @@ class ModCategories : ArrayList<ModCategories.Category>() {
constructor() :
this("", "", false, "", "")
constructor(topic: Github.TopicSearchResponse.Topic) :
constructor(topic: GithubAPI.TopicSearchResponse.Topic) :
this(labelSuggestion(topic), topic.name, true, topic.created_at, topic.updated_at)
companion object {
val All = Category("All mods", "unciv-mod", false, "", "")
fun labelSuggestion(topic: Github.TopicSearchResponse.Topic) =
fun labelSuggestion(topic: GithubAPI.TopicSearchResponse.Topic) =
topic.display_name?.takeUnless { it.isBlank() }
?: topic.name.removePrefix("unciv-mod-").replaceFirstChar(Char::titlecase)
}

5
core/src/com/unciv/ui/screens/modmanager/ModInfoAndActionPane.kt
View File

@ -4,6 +4,7 @@ import com.badlogic.gdx.Gdx
import com.badlogic.gdx.graphics.Texture
import com.badlogic.gdx.scenes.scene2d.ui.Image
import com.badlogic.gdx.scenes.scene2d.ui.Table
import com.unciv.logic.github.GithubAPI
import com.unciv.models.metadata.BaseRuleset
import com.unciv.models.ruleset.Ruleset
import com.unciv.models.ruleset.validation.ModCompatibility
@ -14,7 +15,7 @@ import com.unciv.ui.components.extensions.toCheckBox
import com.unciv.ui.components.extensions.toLabel
import com.unciv.ui.components.extensions.toTextButton
import com.unciv.ui.components.input.onClick
import com.unciv.ui.screens.pickerscreens.Github
import com.unciv.logic.github.Github
import com.unciv.utils.Concurrency
import kotlin.math.max
@ -34,7 +35,7 @@ internal class ModInfoAndActionPane : Table() {
/** Recreate the information part of the right-hand column
* @param repo: the repository instance as received from the GitHub api
*/
fun update(repo: Github.Repo) {
fun update(repo: GithubAPI.Repo) {
isBuiltin = false
enableVisualCheckBox = false
update(

28
core/src/com/unciv/ui/screens/modmanager/ModManagementScreen.kt
View File

@ -13,6 +13,8 @@ import com.badlogic.gdx.utils.SerializationException
import com.unciv.UncivGame
import com.unciv.json.fromJsonFile
import com.unciv.json.json
import com.unciv.logic.UncivShowableException
import com.unciv.logic.github.GithubAPI
import com.unciv.models.ruleset.Ruleset
import com.unciv.models.ruleset.RulesetCache
import com.unciv.models.tilesets.TileSetCache
@ -41,8 +43,8 @@ import com.unciv.ui.screens.basescreen.BaseScreen
import com.unciv.ui.screens.basescreen.RecreateOnResize
import com.unciv.ui.screens.mainmenuscreen.MainMenuScreen
import com.unciv.ui.screens.modmanager.ModManagementOptions.SortType
import com.unciv.ui.screens.pickerscreens.Github
import com.unciv.ui.screens.pickerscreens.Github.repoNameToFolderName
import com.unciv.logic.github.Github
import com.unciv.logic.github.Github.repoNameToFolderName
import com.unciv.ui.screens.pickerscreens.PickerScreen
import com.unciv.utils.Concurrency
import com.unciv.utils.Log
@ -93,7 +95,7 @@ class ModManagementScreen private constructor(
private var lastSelectedButton: ModDecoratedButton? = null
private var lastSyncMarkedButton: ModDecoratedButton? = null
private var selectedMod: Github.Repo? = null
private var selectedMod: GithubAPI.Repo? = null
private val modDescriptionLabel: WrappableLabel
@ -233,7 +235,7 @@ class ModManagementScreen private constructor(
*/
private fun tryDownloadPage(pageNum: Int) {
runningSearchJob = Concurrency.run("GitHubSearch") {
val repoSearch: Github.RepoSearch
val repoSearch: GithubAPI.RepoSearch
try {
repoSearch = Github.tryGetGithubReposWithTopic(amountPerPage, pageNum)!!
} catch (ex: Exception) {
@ -254,7 +256,7 @@ class ModManagementScreen private constructor(
}
}
private fun addModInfoFromRepoSearch(repoSearch: Github.RepoSearch, pageNum: Int) {
private fun addModInfoFromRepoSearch(repoSearch: GithubAPI.RepoSearch, pageNum: Int) {
// clear and remove last cell if it is the "..." indicator
val lastCell = onlineModsTable.cells.lastOrNull()
if (lastCell != null && lastCell.actor is Label && (lastCell.actor as Label).text.toString() == "...") {
@ -363,7 +365,7 @@ class ModManagementScreen private constructor(
val downloadButton = "Download mod from URL".toTextButton()
downloadButton.onClick {
val popup = Popup(this)
popup.addGoodSizedLabel("Please enter the mod repository -or- archive zip -or- branch url:").row()
popup.addGoodSizedLabel("Please enter the mod repository -or- archive zip -or- branch -or- release url:").row()
val textField = UncivTextField.create("").apply { maxLength = 666 }
popup.add(textField).width(stage.width / 2).row()
val pasteLinkButton = "Paste from clipboard".toTextButton()
@ -375,7 +377,7 @@ class ModManagementScreen private constructor(
actualDownloadButton.onClick {
actualDownloadButton.setText("Downloading...".tr())
actualDownloadButton.disable()
val repo = Github.Repo().parseUrl(textField.text)
val repo = GithubAPI.Repo.parseUrl(textField.text)
if (repo == null) {
ToastPopup("«RED»{Invalid link!}«»", this@ModManagementScreen)
.apply { isVisible = true }
@ -392,7 +394,7 @@ class ModManagementScreen private constructor(
}
/** Used as onClick handler for the online Mod list buttons */
private fun onlineButtonAction(repo: Github.Repo, button: ModDecoratedButton) {
private fun onlineButtonAction(repo: GithubAPI.Repo, button: ModDecoratedButton) {
syncOnlineSelected(repo.name, button)
showModDescription(repo.name)
@ -435,7 +437,7 @@ class ModManagementScreen private constructor(
}
/** Download and install a mod in the background, called both from the right-bottom button and the URL entry popup */
private fun downloadMod(repo: Github.Repo, postAction: () -> Unit = {}) {
private fun downloadMod(repo: GithubAPI.Repo, postAction: () -> Unit = {}) {
Concurrency.run("DownloadMod") { // to avoid ANRs - we've learnt our lesson from previous download-related actions
try {
val modFolder = Github.downloadAndExtract(
@ -456,8 +458,14 @@ class ModManagementScreen private constructor(
unMarkUpdatedMod(repoName)
postAction()
}
} catch (ex: UncivShowableException) {
Log.error("Could not download $repo", ex)
launchOnGLThread {
ToastPopup(ex.message, this@ModManagementScreen)
postAction()
}
} catch (ex: Exception) {
Log.error("Could not download ${repo.name}", ex)
Log.error("Could not download $repo", ex)
launchOnGLThread {
ToastPopup("Could not download [${repo.name}]", this@ModManagementScreen)
postAction()

9
core/src/com/unciv/ui/screens/modmanager/ModUIData.kt
View File

@ -1,12 +1,9 @@
package com.unciv.ui.screens.modmanager
import com.badlogic.gdx.scenes.scene2d.ui.TextButton
import com.unciv.logic.github.GithubAPI
import com.unciv.models.metadata.ModCategories
import com.unciv.models.ruleset.Ruleset
import com.unciv.models.translations.tr
import com.unciv.ui.components.extensions.toLabel
import com.unciv.ui.components.extensions.toTextButton
import com.unciv.ui.screens.pickerscreens.Github
/** Helper class holds combined mod info for ModManagementScreen, used for both installed and online lists.
*
@ -18,7 +15,7 @@ internal class ModUIData private constructor(
val name: String,
val description: String,
val ruleset: Ruleset?,
val repo: Github.Repo?,
val repo: GithubAPI.Repo?,
var isVisual: Boolean = false,
var hasUpdate: Boolean = false
) {
@ -30,7 +27,7 @@ internal class ModUIData private constructor(
ruleset, null, isVisual = isVisual
)
constructor(repo: Github.Repo, isUpdated: Boolean): this (
constructor(repo: GithubAPI.Repo, isUpdated: Boolean): this (
repo.name,
(repo.description ?: "-{No description provided}-".tr()) +
"\n" + "[${repo.stargazers_count}]✯".tr(),

603
core/src/com/unciv/ui/screens/pickerscreens/GitHub.kt
View File

@ -1,603 +0,0 @@
package com.unciv.ui.screens.pickerscreens
import com.badlogic.gdx.Files
import com.badlogic.gdx.files.FileHandle
import com.badlogic.gdx.graphics.Pixmap
import com.unciv.json.fromJsonFile
import com.unciv.json.json
import com.unciv.logic.BackwardCompatibility.updateDeprecations
import com.unciv.models.ruleset.ModOptions
import com.unciv.ui.screens.pickerscreens.Github.RateLimit
import com.unciv.ui.screens.pickerscreens.Github.download
import com.unciv.ui.screens.pickerscreens.Github.downloadAndExtract
import com.unciv.ui.screens.pickerscreens.Github.tryGetGithubReposWithTopic
import com.unciv.utils.Log
import com.unciv.utils.debug
import java.io.BufferedInputStream
import java.io.BufferedOutputStream
import java.io.BufferedReader
import java.io.FileOutputStream
import java.io.InputStream
import java.io.InputStreamReader
import java.net.HttpURLConnection
import java.net.URL
import java.nio.ByteBuffer
import java.util.zip.ZipEntry
import java.util.zip.ZipFile
/**
* Utility managing Github access (except the link in WorldScreenCommunityPopup)
*
* Singleton - RateLimit is shared app-wide and has local variables, and is not tested for thread safety.
* Therefore, additional effort is required should [tryGetGithubReposWithTopic] ever be called non-sequentially.
* [download] and [downloadAndExtract] should be thread-safe as they are self-contained.
* They do not join in the [RateLimit] handling because Github doc suggests each API
* has a separate limit (and I found none for cloning via a zip).
*/
object Github {
// Consider merging this with the Dropbox function
/**
* Helper opens am url and accesses its input stream, logging errors to the console
* @param url String representing a [URL] to download.
* @param action Optional callback that will be executed between opening the connection and
* accessing its data - passes the [connection][HttpURLConnection] and allows e.g. reading the response headers.
* @return The [InputStream] if successful, `null` otherwise.
*/
fun download(url: String, action: (HttpURLConnection) -> Unit = {}): InputStream? {
try {
// Problem type 1 - opening the URL connection
with(URL(url).openConnection() as HttpURLConnection)
{
action(this)
// Problem type 2 - getting the information
try {
return inputStream
} catch (ex: Exception) {
// No error handling, just log the message.
// NOTE that this 'read error stream' CAN ALSO fail, but will be caught by the big try/catch
val reader = BufferedReader(InputStreamReader(errorStream, Charsets.UTF_8))
Log.error("Message from GitHub: %s", reader.readText())
throw ex
}
}
} catch (ex: Exception) {
Log.error("Exception during GitHub download", ex)
return null
}
}
/**
* Download a mod and extract, deleting any pre-existing version.
* @param folderFileHandle Destination handle of mods folder - also controls Android internal/external
* @author **Warning**: This took a long time to get just right, so if you're changing this, ***TEST IT THOROUGHLY*** on _both_ Desktop _and_ Phone
* @return FileHandle for the downloaded Mod's folder or null if download failed
*/
fun downloadAndExtract(
repo: Repo,
folderFileHandle: FileHandle
): FileHandle? {
val defaultBranch = repo.default_branch
val gitRepoUrl = repo.html_url
// Initiate download - the helper returns null when it fails
// URL format see: https://docs.github.com/en/repositories/working-with-files/using-files/downloading-source-code-archives#source-code-archive-urls
// Note: https://api.github.com/repos/owner/mod/zipball would be an alternative. Its response is a redirect, but our lib follows that and delivers the zip just fine.
// Problems with the latter: Internal zip structure different, finalDestinationName would need a patch. Plus, normal URL escaping for owner/reponame does not work.
val zipUrl = "$gitRepoUrl/archive/refs/heads/$defaultBranch.zip"
val inputStream = download(zipUrl) ?: return null
// Get a mod-specific temp file name
val tempName = "temp-" + gitRepoUrl.hashCode().toString(16)
// Download to temporary zip
val tempZipFileHandle = folderFileHandle.child("$tempName.zip")
tempZipFileHandle.write(inputStream, false)
// prepare temp unpacking folder
val unzipDestination = tempZipFileHandle.sibling(tempName) // folder, not file
// prevent mixing new content with old - hopefully there will never be cadavers of our tempZip stuff
if (unzipDestination.exists())
if (unzipDestination.isDirectory) unzipDestination.deleteDirectory() else unzipDestination.delete()
Zip.extractFolder(tempZipFileHandle, unzipDestination)
val innerFolder = unzipDestination.list().first()
// innerFolder should now be "$tempName/$repoName-$defaultBranch/" - use this to get mod name
val finalDestinationName = innerFolder.name().replace("-$defaultBranch", "").repoNameToFolderName()
// finalDestinationName is now the mod name as we display it. Folder name needs to be identical.
val finalDestination = folderFileHandle.child(finalDestinationName)
// prevent mixing new content with old
var tempBackup: FileHandle? = null
if (finalDestination.exists()) {
tempBackup = finalDestination.sibling("$finalDestinationName.updating")
finalDestination.renameOrMove(tempBackup)
}
// Move temp unpacked content to their final place
finalDestination.mkdirs() // If we don't create this as a directory, it will think this is a file and nothing will work.
// The move will reset the last modified time (recursively, at least on Linux)
// This sort will guarantee the desktop launcher will not re-pack textures and overwrite the atlas as delivered by the mod
for (innerFileOrFolder in innerFolder.list()
.sortedBy { file -> file.extension() == "atlas" } ) {
innerFileOrFolder.renameOrMove(finalDestination)
}
// clean up
tempZipFileHandle.delete()
unzipDestination.deleteDirectory()
if (tempBackup != null)
if (tempBackup.isDirectory) tempBackup.deleteDirectory() else tempBackup.delete()
return finalDestination
}
private fun FileHandle.renameOrMove(dest: FileHandle) {
// Gdx tries a java rename for Absolute and External, but NOT for Local - rectify that
if (type() == Files.FileType.Local) {
// See #5346: renameTo 'unpacks' a source folder if the destination exists and is an
// empty folder, dropping the name of the source entirely.
// Safer to ask for a move to the not-yet-existing full resulting path instead.
if (isDirectory)
if (file().renameTo(dest.child(name()).file())) return
else
if (file().renameTo(dest.file())) return
}
moveTo(dest)
}
/**
* Implements the ability wo work with GitHub's rate limit, recognize blocks from previous attempts, wait and retry.
*/
object RateLimit {
// https://docs.github.com/en/rest/reference/search#rate-limit
private const val maxRequestsPerInterval = 10
private const val intervalInMilliSeconds = 60000L
private const val maxWaitLoop = 3
private var account = 0 // used requests
private var firstRequest = 0L // timestamp window start (java epoch millisecond)
/*
Github rate limits do not use sliding windows - you (if anonymous) get one window
which starts with the first request (if a window is not already active)
and ends 60s later, and a budget of 10 requests in that window. Once it expires,
everything is forgotten and the process starts from scratch
*/
private val millis: Long
get() = System.currentTimeMillis()
/** calculate required wait in ms
* @return Estimated number of milliseconds to wait for the rate limit window to expire
*/
private fun getWaitLength()
= (firstRequest + intervalInMilliSeconds - millis)
/** Maintain and check a rate-limit
* @return **true** if rate-limited, **false** if another request is allowed
*/
private fun isLimitReached(): Boolean {
val now = millis
val elapsed = if (firstRequest == 0L) intervalInMilliSeconds else now - firstRequest
if (elapsed >= intervalInMilliSeconds) {
firstRequest = now
account = 1
return false
}
if (account >= maxRequestsPerInterval) return true
account++
return false
}
/** If rate limit in effect, sleep long enough to allow next request.
*
* @return **true** if waiting did not clear isLimitReached() (can only happen if the clock is broken),
* or the wait has been interrupted by Thread.interrupt()
* **false** if we were below the limit or slept long enough to drop out of it.
*/
fun waitForLimit(): Boolean {
var loopCount = 0
while (isLimitReached()) {
val waitLength = getWaitLength()
try {
Thread.sleep(waitLength)
} catch ( ex: InterruptedException ) {
return true
}
if (++loopCount >= maxWaitLoop) return true
}
return false
}
/** http responses should be passed to this so the actual rate limit window can be evaluated and used.
* The very first response and all 403 ones are good candidates if they can be expected to contain GitHub's rate limit headers.
*
* see: https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
*/
fun notifyHttpResponse(response: HttpURLConnection) {
if (response.responseMessage != "rate limit exceeded" && response.responseCode != 200) return
fun getHeaderLong(name: String, default: Long = 0L) =
response.headerFields[name]?.get(0)?.toLongOrNull() ?: default
val limit = getHeaderLong("X-RateLimit-Limit", maxRequestsPerInterval.toLong()).toInt()
val remaining = getHeaderLong("X-RateLimit-Remaining").toInt()
val reset = getHeaderLong("X-RateLimit-Reset")
if (limit != maxRequestsPerInterval)
debug("GitHub API Limit reported via http (%s) not equal assumed value (%s)", limit, maxRequestsPerInterval)
account = maxRequestsPerInterval - remaining
if (reset == 0L) return
firstRequest = (reset + 1L) * 1000L - intervalInMilliSeconds
}
}
/**
* Query GitHub for repositories marked "unciv-mod"
* @param amountPerPage Number of search results to return for this request.
* @param page The "page" number, starting at 1.
* @return Parsed [RepoSearch] json on success, `null` on failure.
* @see <a href="https://docs.github.com/en/rest/reference/search#search-repositories">Github API doc</a>
*/
fun tryGetGithubReposWithTopic(amountPerPage:Int, page:Int, searchRequest: String = ""): RepoSearch? {
// Add + here to separate the query text from its parameters
val searchText = if (searchRequest != "") "$searchRequest+" else ""
val link = "https://api.github.com/search/repositories?q=${searchText}%20topic:unciv-mod%20fork:true&sort:stars&per_page=$amountPerPage&page=$page"
var retries = 2
while (retries > 0) {
retries--
// obey rate limit
if (RateLimit.waitForLimit()) return null
// try download
val inputStream = download(link) {
if (it.responseCode == 403 || it.responseCode == 200 && page == 1 && retries == 1) {
// Pass the response headers to the rate limit handler so it can process the rate limit headers
RateLimit.notifyHttpResponse(it)
retries++ // An extra retry so the 403 is ignored in the retry count
}
} ?: continue
return json().fromJson(RepoSearch::class.java, inputStream.bufferedReader().readText())
}
return null
}
/** Get a Pixmap from a "preview" png or jpg file at the root of the repo, falling back to the
* repo owner's avatar [avatarUrl]. The file content url is constructed from [modUrl] and [defaultBranch]
* by replacing the host with `raw.githubusercontent.com`.
*/
fun tryGetPreviewImage(modUrl: String, defaultBranch: String, avatarUrl: String?): Pixmap? {
// Side note: github repos also have a "Social Preview" optionally assignable on the repo's
// settings page, but that info is inaccessible using the v3 API anonymously. The easiest way
// to get it would be to query the the repo's frontend page (modUrl), and parse out
// `head/meta[property=og:image]/@content`, which is one extra spurious roundtrip and a
// non-trivial waste of bandwidth.
// Thus we ask for a "preview" file as part of the repo contents instead.
val fileLocation = "$modUrl/$defaultBranch/preview"
.replace("github.com", "raw.githubusercontent.com")
try {
val file = download("$fileLocation.jpg")
?: download("$fileLocation.png")
// Note: avatar urls look like: https://avatars.githubusercontent.com/u/<number>?v=4
// So the image format is only recognizable from the response "Content-Type" header
// or by looking for magic markers in the bits - which the Pixmap constructor below does.
?: avatarUrl?.let { download(it) }
?: return null
val byteArray = file.readBytes()
val buffer = ByteBuffer.allocateDirect(byteArray.size).put(byteArray).position(0)
return Pixmap(buffer)
} catch (_: Throwable) {
return null
}
}
/** Class to receive a github API "Get a tree" response parsed as json */
// Parts of the response we ignore are commented out
private class Tree {
//val sha = ""
//val url = ""
class TreeFile {
//val path = ""
//val mode = 0
//val type = "" // blob / tree
//val sha = ""
//val url = ""
var size: Long = 0L
}
@Suppress("MemberNameEqualsClassName")
var tree = ArrayList<TreeFile>()
var truncated = false
}
/** Queries github for a tree and calculates the sum of the blob sizes.
* @return -1 on failure, else size rounded to kB
*/
fun getRepoSize(repo: Repo): Int {
// See https://docs.github.com/en/rest/git/trees#get-a-tree
val link = "https://api.github.com/repos/${repo.full_name}/git/trees/${repo.default_branch}?recursive=true"
var retries = 2
while (retries > 0) {
retries--
// obey rate limit
if (RateLimit.waitForLimit()) return -1
// try download
val inputStream = download(link) {
if (it.responseCode == 403 || it.responseCode == 200 && retries == 1) {
// Pass the response headers to the rate limit handler so it can process the rate limit headers
RateLimit.notifyHttpResponse(it)
retries++ // An extra retry so the 403 is ignored in the retry count
}
} ?: continue
val tree = json().fromJson(Tree::class.java, inputStream.bufferedReader().readText())
if (tree.truncated) return -1 // unlikely: >100k blobs or blob > 7MB
var totalSizeBytes = 0L
for (file in tree.tree)
totalSizeBytes += file.size
// overflow unlikely: >2TB
return ((totalSizeBytes + 512) / 1024).toInt()
}
return -1
}
/**
* Parsed GitHub repo search response
* @property total_count Total number of hits for the search (ignoring paging window)
* @property incomplete_results A flag set by github to indicate search was incomplete (never seen it on)
* @property items Array of [repositories][Repo]
* @see <a href="https://docs.github.com/en/rest/reference/search#search-repositories--code-samples">Github API doc</a>
*/
@Suppress("PropertyName")
class RepoSearch {
@Suppress("MemberVisibilityCanBePrivate")
var total_count = 0
var incomplete_results = false
var items = ArrayList<Repo>()
}
/** Part of [RepoSearch] in Github API response - one repository entry in [items][RepoSearch.items] */
@Suppress("PropertyName")
class Repo {
/** Unlike the rest of this class, this is not part of the API but added by us locally
* to track whether [getRepoSize] has been run successfully for this repo */
var hasUpdatedSize = false
var name = ""
var full_name = ""
var description: String? = null
var owner = RepoOwner()
var stargazers_count = 0
var default_branch = ""
var html_url = ""
var pushed_at = "" // don't use updated_at - see https://github.com/yairm210/Unciv/issues/6106
var size = 0
var topics = mutableListOf<String>()
//var stargazers_url = ""
//var homepage: String? = null // might use instead of go to repo?
//var has_wiki = false // a wiki could mean proper documentation for the mod?
/**
* Initialize `this` with an url, extracting all possible fields from it
* (html_url, author, repoName, branchName).
*
* Allow url formats:
* * Basic repo url:
* https://github.com/author/repoName
* * or complete 'zip' url from github's code->download zip menu:
* https://github.com/author/repoName/archive/refs/heads/branchName.zip
* * or the branch url same as one navigates to on github through the "branches" menu:
* https://github.com/author/repoName/tree/branchName
*
* In the case of the basic repo url, an API query is sent to determine the default branch.
* Other url forms will not go online.
*
* @return `this` to allow chaining, `null` for invalid links or any other failures
*/
fun parseUrl(url: String): Repo? {
fun processMatch(matchResult: MatchResult): Repo {
html_url = matchResult.groups[1]!!.value
owner.login = matchResult.groups[2]!!.value
name = matchResult.groups[3]!!.value
default_branch = matchResult.groups[4]!!.value
return this
}
html_url = url
default_branch = "master"
val matchZip = Regex("""^(.*/(.*)/(.*))/archive/(?:.*/)?([^.]+).zip$""").matchEntire(url)
if (matchZip != null && matchZip.groups.size > 4)
return processMatch(matchZip)
val matchBranch = Regex("""^(.*/(.*)/(.*))/tree/([^/]+)$""").matchEntire(url)
if (matchBranch != null && matchBranch.groups.size > 4)
return processMatch(matchBranch)
val matchRepo = Regex("""^.*//.*/(.+)/(.+)/?$""").matchEntire(url)
if (matchRepo != null && matchRepo.groups.size > 2) {
// Query API if we got the first URL format to get the correct default branch
val response = download("https://api.github.com/repos/${matchRepo.groups[1]!!.value}/${matchRepo.groups[2]!!.value}")
?: return null
val repoString = response.bufferedReader().readText()
return json().fromJson(Repo::class.java, repoString)
}
return null
}
}
/** Part of [Repo] in Github API response */
@Suppress("PropertyName")
class RepoOwner {
var login = ""
var avatar_url: String? = null
}
/**
* Query GitHub for topics named "unciv-mod*"
* @return Parsed [TopicSearchResponse] json on success, `null` on failure.
*/
fun tryGetGithubTopics(): TopicSearchResponse? {
// `+repositories:>1` means ignore unused or practically unused topics
val link = "https://api.github.com/search/topics?q=unciv-mod+repositories:%3E1&sort=name&order=asc"
var retries = 2
while (retries > 0) {
retries--
// obey rate limit
if (RateLimit.waitForLimit()) return null
// try download
val inputStream = download(link) {
if (it.responseCode == 403 || it.responseCode == 200 && retries == 1) {
// Pass the response headers to the rate limit handler so it can process the rate limit headers
RateLimit.notifyHttpResponse(it)
retries++ // An extra retry so the 403 is ignored in the retry count
}
} ?: continue
return json().fromJson(TopicSearchResponse::class.java, inputStream.bufferedReader().readText())
}
return null
}
/** Topic search response */
@Suppress("PropertyName")
class TopicSearchResponse {
// Commented out: Github returns them, but we're not interested
// var total_count = 0
// var incomplete_results = false
var items = ArrayList<Topic>()
class Topic {
var name = ""
var display_name: String? = null // Would need to be curated, which is alottawork
// var featured = false
// var curated = false
var created_at = "" // iso datetime with "Z" timezone
var updated_at = "" // iso datetime with "Z" timezone
}
}
/** Rewrite modOptions file for a mod we just installed to include metadata we got from the GitHub api
*
* (called on background thread)
*/
fun rewriteModOptions(repo: Repo, modFolder: FileHandle) {
val modOptionsFile = modFolder.child("jsons/ModOptions.json")
val modOptions = if (modOptionsFile.exists()) json().fromJsonFile(ModOptions::class.java, modOptionsFile) else ModOptions()
modOptions.modUrl = repo.html_url
modOptions.defaultBranch = repo.default_branch
modOptions.lastUpdated = repo.pushed_at
modOptions.author = repo.owner.login
modOptions.modSize = repo.size
modOptions.topics = repo.topics
modOptions.updateDeprecations()
json().toJson(modOptions, modOptionsFile)
}
private const val outerBlankReplacement = '='
// Github disallows **any** special chars and replaces them with '-' - so use something ascii the
// OS accepts but still is recognizable as non-original, to avoid confusion
/** Convert a [Repo] name to a local name for both display and folder name
*
* Replaces '-' with blanks but ensures no leading or trailing blanks.
* As mad modders know no limits, trailing "-" did indeed happen, causing things to break due to trailing blanks on a folder name.
* As "test-" and "test" are different allowed repository names, trimmed blanks are replaced with one equals sign per side.
* @param onlyOuterBlanks If `true` ignores inner dashes - only start and end are treated. Useful when modders have manually created local folder names using dashes.
*/
fun String.repoNameToFolderName(onlyOuterBlanks: Boolean = false): String {
var result = if (onlyOuterBlanks) this else replace('-', ' ')
if (result.endsWith(' ')) result = result.trimEnd() + outerBlankReplacement
if (result.startsWith(' ')) result = outerBlankReplacement + result.trimStart()
return result
}
/** Inverse of [repoNameToFolderName] */
// As of this writing, only used for loadMissingMods
fun String.folderNameToRepoName(): String {
var result = replace(' ', '-')
if (result.endsWith(outerBlankReplacement)) result = result.trimEnd(outerBlankReplacement) + '-'
if (result.startsWith(outerBlankReplacement)) result = '-' + result.trimStart(outerBlankReplacement)
return result
}
}
/** Utility - extract Zip archives
* @see [Zip.extractFolder]
*/
object Zip {
private const val bufferSize = 2048
/**
* Extract one Zip file recursively (nested Zip files are extracted in turn).
*
* The source Zip is not deleted, but successfully extracted nested ones are.
*
* **Warning**: Extracting into a non-empty destination folder will merge contents. Existing
* files also included in the archive will be partially overwritten, when the new data is shorter
* than the old you will get _mixed contents!_
*
* @param zipFile The Zip file to extract
* @param unzipDestination The folder to extract into, preferably empty (not enforced).
*/
fun extractFolder(zipFile: FileHandle, unzipDestination: FileHandle) {
// I went through a lot of similar answers that didn't work until I got to this gem by NeilMonday
// (with mild changes to fit the FileHandles)
// https://stackoverflow.com/questions/981578/how-to-unzip-files-recursively-in-java
debug("Extracting %s to %s", zipFile, unzipDestination)
// establish buffer for writing file
val data = ByteArray(bufferSize)
fun streamCopy(fromStream: InputStream, toHandle: FileHandle) {
val inputStream = BufferedInputStream(fromStream)
var currentByte: Int
// write the current file to disk
val fos = FileOutputStream(toHandle.file())
val dest = BufferedOutputStream(fos, bufferSize)
// read and write until last byte is encountered
while (inputStream.read(data, 0, bufferSize).also { currentByte = it } != -1) {
dest.write(data, 0, currentByte)
}
dest.flush()
dest.close()
inputStream.close()
}
val file = zipFile.file()
val zip = ZipFile(file)
//unzipDestination.mkdirs()
val zipFileEntries = zip.entries()
// Process each entry
while (zipFileEntries.hasMoreElements()) {
// grab a zip file entry
val entry = zipFileEntries.nextElement() as ZipEntry
val currentEntry = entry.name
val destFile = unzipDestination.child(currentEntry)
val destinationParent = destFile.parent()
// create the parent directory structure if needed
destinationParent.mkdirs()
if (!entry.isDirectory) {
streamCopy ( zip.getInputStream(entry), destFile)
}
// The new file has a current last modification time
// and not the one stored in the archive - we could:
// 'destFile.file().setLastModified(entry.time)'
// but later handling will throw these away anyway,
// and GitHub sets all timestamps to the download time.
if (currentEntry.endsWith(".zip")) {
// found a zip file, try to open
extractFolder(destFile, destinationParent)
destFile.delete()
}
}
zip.close() // Needed so we can delete the zip file later
}
}

4
core/src/com/unciv/ui/screens/savescreens/LoadGameScreen.kt
View File

@ -27,8 +27,8 @@ import com.unciv.ui.components.input.onClick
import com.unciv.ui.popups.LoadingPopup
import com.unciv.ui.popups.Popup
import com.unciv.ui.popups.ToastPopup
import com.unciv.ui.screens.pickerscreens.Github
import com.unciv.ui.screens.pickerscreens.Github.folderNameToRepoName
import com.unciv.logic.github.Github
import com.unciv.logic.github.Github.folderNameToRepoName
import com.unciv.utils.Concurrency
import com.unciv.utils.Log
import com.unciv.utils.launchOnGLThread