From c709ab939c5d30d044fc3d570b3fe3c1b9248905 Mon Sep 17 00:00:00 2001 From: Moritz Zwerger Date: Sat, 25 Nov 2023 16:21:05 +0100 Subject: [PATCH] download minosoft meta and pixlyzer from multiple urls (failover) My gitlab really blew up this night du to soooo many requests (not bad). But people were unable to play, because they could not download resources. Now github and gitlab.com are also tried as fallback. Closes GH-20, GH-22 --- .../resources/ResourceProfileMigrationTest.kt | 64 +++++++++++++++++++ .../minosoft/assets/meta/MinosoftMeta.kt | 20 ++++-- .../resources/ResourceProfileMigration.kt | 41 ++++++++++++ .../resources/ResourcesProfileManager.kt | 9 ++- .../profiles/resources/source/SourceC.kt | 13 +++- .../profile/storage/StorageProfileManager.kt | 3 +- .../registries/registries/PixLyzerUtil.kt | 21 ++++-- .../minosoft/util/http/DownloadUtil.kt | 34 ++++++++++ 8 files changed, 189 insertions(+), 16 deletions(-) create mode 100644 src/integration-test/kotlin/de/bixilon/minosoft/config/profile/profiles/resources/ResourceProfileMigrationTest.kt create mode 100644 src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/ResourceProfileMigration.kt create mode 100644 src/main/java/de/bixilon/minosoft/util/http/DownloadUtil.kt diff --git a/src/integration-test/kotlin/de/bixilon/minosoft/config/profile/profiles/resources/ResourceProfileMigrationTest.kt b/src/integration-test/kotlin/de/bixilon/minosoft/config/profile/profiles/resources/ResourceProfileMigrationTest.kt new file mode 100644 index 000000000..0102ec7ed --- /dev/null +++ b/src/integration-test/kotlin/de/bixilon/minosoft/config/profile/profiles/resources/ResourceProfileMigrationTest.kt @@ -0,0 +1,64 @@ +/* + * Minosoft + * Copyright (C) 2020-2023 Moritz Zwerger + * + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with this program. If not, see . + * + * This software is not affiliated with Mojang AB, the original developer of Minecraft. + */ + +package de.bixilon.minosoft.config.profile.profiles.resources + +import com.fasterxml.jackson.databind.node.ObjectNode +import de.bixilon.kutil.json.JsonObject +import de.bixilon.minosoft.util.json.Jackson +import org.testng.Assert.assertEquals +import org.testng.annotations.Test + +@Test(groups = ["profiles"]) +class ResourceProfileMigrationTest { + + + fun `m1 default urls`() { + val data = mapOf( + "source" to mapOf( + "pixlyzer" to "https://gitlab.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false", + "minosoft-meta" to "https://gitlab.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads", + ) + ) + val tree = Jackson.MAPPER.valueToTree(data) + ResourceProfileMigration.migrate1(tree) + val map = Jackson.MAPPER.convertValue(tree, Jackson.JSON_MAP_TYPE) + + assertEquals(map, mapOf( + "source" to emptyMap() // removing them will put the default value up + )) + } + + fun `m1 custom urls`() { + val data = mapOf( + "source" to mapOf( + "pixlyzer" to "https://custom.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false", + "minosoft-meta" to "https://custom.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads", + ) + ) + val tree = Jackson.MAPPER.valueToTree(data) + ResourceProfileMigration.migrate1(tree) + val map = Jackson.MAPPER.convertValue(tree, Jackson.JSON_MAP_TYPE) + + assertEquals(map, mapOf( + "source" to mapOf( + "pixlyzer" to listOf( + "https://custom.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false", + ), + "minosoft-meta" to listOf( + "https://custom.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads", + ), + ) + )) + } +} diff --git a/src/main/java/de/bixilon/minosoft/assets/meta/MinosoftMeta.kt b/src/main/java/de/bixilon/minosoft/assets/meta/MinosoftMeta.kt index 0d293141b..6146a1720 100644 --- a/src/main/java/de/bixilon/minosoft/assets/meta/MinosoftMeta.kt +++ b/src/main/java/de/bixilon/minosoft/assets/meta/MinosoftMeta.kt @@ -28,6 +28,10 @@ import de.bixilon.minosoft.config.profile.profiles.resources.ResourcesProfile import de.bixilon.minosoft.data.registries.identified.Namespaces.minosoft import de.bixilon.minosoft.protocol.versions.Version import de.bixilon.minosoft.protocol.versions.Versions +import de.bixilon.minosoft.util.http.DownloadUtil +import de.bixilon.minosoft.util.logging.Log +import de.bixilon.minosoft.util.logging.LogLevels +import de.bixilon.minosoft.util.logging.LogMessageType import java.io.ByteArrayInputStream object MinosoftMeta { @@ -44,13 +48,13 @@ object MinosoftMeta { return MBFBinaryReader(ByteArrayInputStream(this)).readMBF().data.unsafeCast() } - private fun MetaVersionEntry.load(profile: ResourcesProfile): JsonObject { - FileAssetsUtil.readOrNull(this.hash, FileAssetsTypes.META, compress = false)?.let { return it.load() } - - val data = FileAssetsUtil.read(profile.source.minosoftMeta.formatPlaceholder( + private fun verify(url: String, hash: String): JsonObject { + val url = url.formatPlaceholder( "hashPrefix" to hash.substring(0, 2), "fullHash" to hash, - ).toURL().openStream(), type = FileAssetsTypes.META, compress = false, hash = HashTypes.SHA256) + ).toURL() + Log.log(LogMessageType.ASSETS, LogLevels.VERBOSE) { "Downloading minosoft meta $url" } + val data = FileAssetsUtil.read(url.openStream(), type = FileAssetsTypes.META, compress = false, hash = HashTypes.SHA256) if (data.hash != hash) { throw IllegalStateException("Minosoft meta data mismatch (expected=$hash, hash=${data.hash}!") @@ -59,6 +63,12 @@ object MinosoftMeta { return data.data.load() } + private fun MetaVersionEntry.load(profile: ResourcesProfile): JsonObject { + FileAssetsUtil.readOrNull(this.hash, FileAssetsTypes.META, compress = false)?.let { return it.load() } + + return DownloadUtil.retry(profile.source.minosoftMeta) { verify(it, hash) } + } + fun MetaTypeEntry.load(profile: ResourcesProfile, version: Version): JsonObject? { var previous: MetaVersionEntry? = null var previousVersion: Version? = null diff --git a/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/ResourceProfileMigration.kt b/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/ResourceProfileMigration.kt new file mode 100644 index 000000000..aef171562 --- /dev/null +++ b/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/ResourceProfileMigration.kt @@ -0,0 +1,41 @@ +/* + * Minosoft + * Copyright (C) 2020-2023 Moritz Zwerger + * + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with this program. If not, see . + * + * This software is not affiliated with Mojang AB, the original developer of Minecraft. + */ + +package de.bixilon.minosoft.config.profile.profiles.resources + +import com.fasterxml.jackson.databind.node.ObjectNode +import de.bixilon.kutil.cast.CastUtil.nullCast +import de.bixilon.minosoft.util.json.Jackson + +object ResourceProfileMigration { + + /** + * Pixlyzer and minosoft meta urls are now a list + */ + fun migrate1(data: ObjectNode) { + val source = data.get("source").nullCast() ?: return + + source.remove("pixlyzer")?.asText()?.let { + if (it == "https://gitlab.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false") return@let + val array = Jackson.MAPPER.createArrayNode() + array.add(it) + source.replace("pixlyzer", array) + } + source.remove("minosoft-meta")?.asText()?.let { + if (it == "https://gitlab.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads") return@let + val array = Jackson.MAPPER.createArrayNode() + array.add(it) + source.replace("minosoft-meta", array) + } + } +} diff --git a/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/ResourcesProfileManager.kt b/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/ResourcesProfileManager.kt index a375f2148..7cc35f93c 100644 --- a/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/ResourcesProfileManager.kt +++ b/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/ResourcesProfileManager.kt @@ -13,9 +13,16 @@ package de.bixilon.minosoft.config.profile.profiles.resources +import com.fasterxml.jackson.databind.node.ObjectNode import de.bixilon.minosoft.config.profile.storage.StorageProfileManager object ResourcesProfileManager : StorageProfileManager() { override val type get() = ResourcesProfile - override val latestVersion get() = 1 + override val latestVersion get() = 2 + + + override fun migrate(version: Int, data: ObjectNode) = when (version) { + 1 -> ResourceProfileMigration.migrate1(data) + else -> Unit + } } diff --git a/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/source/SourceC.kt b/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/source/SourceC.kt index 5e2017e2e..e01c92173 100644 --- a/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/source/SourceC.kt +++ b/src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/source/SourceC.kt @@ -14,11 +14,20 @@ package de.bixilon.minosoft.config.profile.profiles.resources.source import de.bixilon.minosoft.config.profile.delegate.types.StringDelegate +import de.bixilon.minosoft.config.profile.delegate.types.list.ListDelegate import de.bixilon.minosoft.config.profile.profiles.resources.ResourcesProfile class SourceC(profile: ResourcesProfile) { - var pixlyzer by StringDelegate(profile, "https://gitlab.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false") - var minosoftMeta by StringDelegate(profile, "https://gitlab.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads") + var pixlyzer by ListDelegate(profile, mutableListOf( + "https://gitlab.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false", + "https://github.com/Bixilon/pixlyzer-data/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf", + "https://gitlab.com/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false", + )) + var minosoftMeta by ListDelegate(profile, mutableListOf( + "https://gitlab.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads", + "https://github.com/Bixilon/minosoft-meta-bin/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf", + "https://gitlab.com/Bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads", + )) var minecraftResources by StringDelegate(profile, "https://resources.download.minecraft.net/\${hashPrefix}/\${fullHash}") var mojangPackages by StringDelegate(profile, "https://launchermeta.mojang.com/v1/packages/\${fullHash}/\${filename}") var pistonObjects by StringDelegate(profile, "https://piston-data.mojang.com/v1/objects/\${fullHash}/\${filename}") diff --git a/src/main/java/de/bixilon/minosoft/config/profile/storage/StorageProfileManager.kt b/src/main/java/de/bixilon/minosoft/config/profile/storage/StorageProfileManager.kt index 774b2c8b0..c91ae5667 100644 --- a/src/main/java/de/bixilon/minosoft/config/profile/storage/StorageProfileManager.kt +++ b/src/main/java/de/bixilon/minosoft/config/profile/storage/StorageProfileManager.kt @@ -32,7 +32,6 @@ import de.bixilon.minosoft.config.profile.ProfileType import de.bixilon.minosoft.config.profile.ProfileUtil.isValidName import de.bixilon.minosoft.config.profile.profiles.Profile import de.bixilon.minosoft.data.registries.identified.Identified -import de.bixilon.minosoft.gui.eros.crash.ErosCrashReport.Companion.crash import de.bixilon.minosoft.protocol.ProtocolUtil.encodeNetwork import de.bixilon.minosoft.terminal.RunConfiguration import de.bixilon.minosoft.util.json.Jackson @@ -137,7 +136,7 @@ abstract class StorageProfileManager

: Iterable

, Identified { profiles[name] = profile } catch (error: Throwable) { Log.log(LogMessageType.PROFILES, LogLevels.FATAL) { error } - error.crash() + throw error } finally { lock.unlock() } diff --git a/src/main/java/de/bixilon/minosoft/data/registries/registries/PixLyzerUtil.kt b/src/main/java/de/bixilon/minosoft/data/registries/registries/PixLyzerUtil.kt index a834d4947..35b3e0837 100644 --- a/src/main/java/de/bixilon/minosoft/data/registries/registries/PixLyzerUtil.kt +++ b/src/main/java/de/bixilon/minosoft/data/registries/registries/PixLyzerUtil.kt @@ -25,6 +25,10 @@ import de.bixilon.minosoft.assets.util.HashTypes import de.bixilon.minosoft.assets.util.InputStreamUtil.readMBFMap import de.bixilon.minosoft.config.profile.profiles.resources.ResourcesProfile import de.bixilon.minosoft.protocol.versions.Version +import de.bixilon.minosoft.util.http.DownloadUtil +import de.bixilon.minosoft.util.logging.Log +import de.bixilon.minosoft.util.logging.LogLevels +import de.bixilon.minosoft.util.logging.LogMessageType import java.io.ByteArrayInputStream object PixLyzerUtil { @@ -33,14 +37,13 @@ object PixLyzerUtil { return ByteArrayInputStream(this).readMBFMap().toJsonObject() ?: throw IllegalStateException("Could not read pixlyzer data!") } - private fun verify(url: String, hash: String): Map { - FileAssetsUtil.readOrNull(hash, type = FileAssetsTypes.PIXLYZER, compress = false)?.let { return it.read() } - - - val data = FileAssetsUtil.read(url.formatPlaceholder( + private fun verify(url: String, hash: String): JsonObject { + val url = url.formatPlaceholder( "hashPrefix" to hash.substring(0, 2), "fullHash" to hash, - ).toURL().openStream(), type = FileAssetsTypes.PIXLYZER, compress = false, hash = HashTypes.SHA1) + ).toURL() + Log.log(LogMessageType.ASSETS, LogLevels.VERBOSE) { "Downloading pixlyzer data $url" } + val data = FileAssetsUtil.read(url.openStream(), type = FileAssetsTypes.PIXLYZER, compress = false, hash = HashTypes.SHA1) if (data.hash != hash) { throw IllegalStateException("Pixlyzer data mismatch (expected=$hash, hash=${data.hash}!") @@ -49,6 +52,12 @@ object PixLyzerUtil { return data.data.read() } + private fun verify(urls: List, hash: String): JsonObject { + FileAssetsUtil.readOrNull(hash, type = FileAssetsTypes.PIXLYZER, compress = false)?.let { return it.read() } + + return DownloadUtil.retry(urls) { verify(it, hash) } + } + fun loadPixlyzerData(profile: ResourcesProfile, version: Version): JsonObject { val pixlyzerHash = AssetsVersionProperties[version]?.pixlyzerHash ?: throw IllegalStateException("$version has no pixlyzer data available!") diff --git a/src/main/java/de/bixilon/minosoft/util/http/DownloadUtil.kt b/src/main/java/de/bixilon/minosoft/util/http/DownloadUtil.kt new file mode 100644 index 000000000..15e566ae4 --- /dev/null +++ b/src/main/java/de/bixilon/minosoft/util/http/DownloadUtil.kt @@ -0,0 +1,34 @@ +/* + * Minosoft + * Copyright (C) 2020-2023 Moritz Zwerger + * + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with this program. If not, see . + * + * This software is not affiliated with Mojang AB, the original developer of Minecraft. + */ + +package de.bixilon.minosoft.util.http + +import de.bixilon.kutil.json.JsonObject + +object DownloadUtil { + + fun retry(urls: List, loader: (String) -> JsonObject): Map { + if (urls.isEmpty()) throw IllegalArgumentException("No urls provided!") + + var first: Throwable? = null + for (url in urls) { + try { + return loader.invoke(url) + } catch (error: Throwable) { + first = error + error.printStackTrace() + } + } + throw first!! + } +}