AstralTransRocketries/minosoft
1
0
Fork 0
mirror of https://gitlab.bixilon.de/bixilon/minosoft.git synced 2025-09-13 01:16:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

download minosoft meta and pixlyzer from multiple urls (failover)

Browse Source 
My gitlab really blew up this night du to soooo many requests (not bad). But people were unable to play, because they could not download resources. Now github and gitlab.com are also tried as fallback.

Closes GH-20, GH-22
This commit is contained in:
Moritz Zwerger 2023-11-25 16:21:05 +01:00
parent 749936ec7b
commit c709ab939c
No known key found for this signature in database
GPG Key ID: 5CAD791931B09AC4
8 changed files with 189 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
src/integration-test/kotlin/de/bixilon/minosoft/config/profile/profiles/resources/ResourceProfileMigrationTest.kt Normal file
View File

@ -0,0 +1,64 @@
/*
* Minosoft
* Copyright (C) 2020-2023 Moritz Zwerger
*
* This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* This software is not affiliated with Mojang AB, the original developer of Minecraft.
*/
package de.bixilon.minosoft.config.profile.profiles.resources
import com.fasterxml.jackson.databind.node.ObjectNode
import de.bixilon.kutil.json.JsonObject
import de.bixilon.minosoft.util.json.Jackson
import org.testng.Assert.assertEquals
import org.testng.annotations.Test
@Test(groups = ["profiles"])
class ResourceProfileMigrationTest {
fun `m1 default urls`() {
val data = mapOf(
"source" to mapOf(
"pixlyzer" to "https://gitlab.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false",
"minosoft-meta" to "https://gitlab.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads",
)
)
val tree = Jackson.MAPPER.valueToTree<ObjectNode>(data)
ResourceProfileMigration.migrate1(tree)
val map = Jackson.MAPPER.convertValue<JsonObject>(tree, Jackson.JSON_MAP_TYPE)
assertEquals(map, mapOf(
"source" to emptyMap<String, Any>() // removing them will put the default value up
))
}
fun `m1 custom urls`() {
val data = mapOf(
"source" to mapOf(
"pixlyzer" to "https://custom.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false",
"minosoft-meta" to "https://custom.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads",
)
)
val tree = Jackson.MAPPER.valueToTree<ObjectNode>(data)
ResourceProfileMigration.migrate1(tree)
val map = Jackson.MAPPER.convertValue<JsonObject>(tree, Jackson.JSON_MAP_TYPE)
assertEquals(map, mapOf(
"source" to mapOf(
"pixlyzer" to listOf(
"https://custom.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false",
),
"minosoft-meta" to listOf(
"https://custom.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads",
),
)
))
}
}

20
src/main/java/de/bixilon/minosoft/assets/meta/MinosoftMeta.kt
View File

@ -28,6 +28,10 @@ import de.bixilon.minosoft.config.profile.profiles.resources.ResourcesProfile
import de.bixilon.minosoft.data.registries.identified.Namespaces.minosoft
import de.bixilon.minosoft.protocol.versions.Version
import de.bixilon.minosoft.protocol.versions.Versions
import de.bixilon.minosoft.util.http.DownloadUtil
import de.bixilon.minosoft.util.logging.Log
import de.bixilon.minosoft.util.logging.LogLevels
import de.bixilon.minosoft.util.logging.LogMessageType
import java.io.ByteArrayInputStream
object MinosoftMeta {
@ -44,13 +48,13 @@ object MinosoftMeta {
return MBFBinaryReader(ByteArrayInputStream(this)).readMBF().data.unsafeCast()
}
private fun MetaVersionEntry.load(profile: ResourcesProfile): JsonObject {
FileAssetsUtil.readOrNull(this.hash, FileAssetsTypes.META, compress = false)?.let { return it.load() }
val data = FileAssetsUtil.read(profile.source.minosoftMeta.formatPlaceholder(
private fun verify(url: String, hash: String): JsonObject {
val url = url.formatPlaceholder(
"hashPrefix" to hash.substring(0, 2),
"fullHash" to hash,
).toURL().openStream(), type = FileAssetsTypes.META, compress = false, hash = HashTypes.SHA256)
).toURL()
Log.log(LogMessageType.ASSETS, LogLevels.VERBOSE) { "Downloading minosoft meta $url" }
val data = FileAssetsUtil.read(url.openStream(), type = FileAssetsTypes.META, compress = false, hash = HashTypes.SHA256)
if (data.hash != hash) {
throw IllegalStateException("Minosoft meta data mismatch (expected=$hash, hash=${data.hash}!")
@ -59,6 +63,12 @@ object MinosoftMeta {
return data.data.load()
}
private fun MetaVersionEntry.load(profile: ResourcesProfile): JsonObject {
FileAssetsUtil.readOrNull(this.hash, FileAssetsTypes.META, compress = false)?.let { return it.load() }
return DownloadUtil.retry(profile.source.minosoftMeta) { verify(it, hash) }
}
fun MetaTypeEntry.load(profile: ResourcesProfile, version: Version): JsonObject? {
var previous: MetaVersionEntry? = null
var previousVersion: Version? = null

41
src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/ResourceProfileMigration.kt Normal file
View File

@ -0,0 +1,41 @@
/*
* Minosoft
* Copyright (C) 2020-2023 Moritz Zwerger
*
* This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* This software is not affiliated with Mojang AB, the original developer of Minecraft.
*/
package de.bixilon.minosoft.config.profile.profiles.resources
import com.fasterxml.jackson.databind.node.ObjectNode
import de.bixilon.kutil.cast.CastUtil.nullCast
import de.bixilon.minosoft.util.json.Jackson
object ResourceProfileMigration {
/**
* Pixlyzer and minosoft meta urls are now a list
*/
fun migrate1(data: ObjectNode) {
val source = data.get("source").nullCast<ObjectNode>() ?: return
source.remove("pixlyzer")?.asText()?.let {
if (it == "https://gitlab.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false") return@let
val array = Jackson.MAPPER.createArrayNode()
array.add(it)
source.replace("pixlyzer", array)
}
source.remove("minosoft-meta")?.asText()?.let {
if (it == "https://gitlab.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads") return@let
val array = Jackson.MAPPER.createArrayNode()
array.add(it)
source.replace("minosoft-meta", array)
}
}
}

9
src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/ResourcesProfileManager.kt
View File

@ -13,9 +13,16 @@
package de.bixilon.minosoft.config.profile.profiles.resources
import com.fasterxml.jackson.databind.node.ObjectNode
import de.bixilon.minosoft.config.profile.storage.StorageProfileManager
object ResourcesProfileManager : StorageProfileManager<ResourcesProfile>() {
override val type get() = ResourcesProfile
override val latestVersion get() = 1
override val latestVersion get() = 2
override fun migrate(version: Int, data: ObjectNode) = when (version) {
1 -> ResourceProfileMigration.migrate1(data)
else -> Unit
}
}

13
src/main/java/de/bixilon/minosoft/config/profile/profiles/resources/source/SourceC.kt
View File

@ -14,11 +14,20 @@
package de.bixilon.minosoft.config.profile.profiles.resources.source
import de.bixilon.minosoft.config.profile.delegate.types.StringDelegate
import de.bixilon.minosoft.config.profile.delegate.types.list.ListDelegate
import de.bixilon.minosoft.config.profile.profiles.resources.ResourcesProfile
class SourceC(profile: ResourcesProfile) {
var pixlyzer by StringDelegate(profile, "https://gitlab.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false")
var minosoftMeta by StringDelegate(profile, "https://gitlab.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads")
var pixlyzer by ListDelegate(profile, mutableListOf(
"https://gitlab.bixilon.de/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false",
"https://github.com/Bixilon/pixlyzer-data/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf",
"https://gitlab.com/bixilon/pixlyzer-data/-/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf?inline=false",
))
var minosoftMeta by ListDelegate(profile, mutableListOf(
"https://gitlab.bixilon.de/bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads",
"https://github.com/Bixilon/minosoft-meta-bin/raw/master/hash/\${hashPrefix}/\${fullHash}.mbf",
"https://gitlab.com/Bixilon/minosoft-meta-bin/-/raw/master/\${hashPrefix}/\${fullHash}?ref_type=heads",
))
var minecraftResources by StringDelegate(profile, "https://resources.download.minecraft.net/\${hashPrefix}/\${fullHash}")
var mojangPackages by StringDelegate(profile, "https://launchermeta.mojang.com/v1/packages/\${fullHash}/\${filename}")
var pistonObjects by StringDelegate(profile, "https://piston-data.mojang.com/v1/objects/\${fullHash}/\${filename}")

3
src/main/java/de/bixilon/minosoft/config/profile/storage/StorageProfileManager.kt
View File

@ -32,7 +32,6 @@ import de.bixilon.minosoft.config.profile.ProfileType
import de.bixilon.minosoft.config.profile.ProfileUtil.isValidName
import de.bixilon.minosoft.config.profile.profiles.Profile
import de.bixilon.minosoft.data.registries.identified.Identified
import de.bixilon.minosoft.gui.eros.crash.ErosCrashReport.Companion.crash
import de.bixilon.minosoft.protocol.ProtocolUtil.encodeNetwork
import de.bixilon.minosoft.terminal.RunConfiguration
import de.bixilon.minosoft.util.json.Jackson
@ -137,7 +136,7 @@ abstract class StorageProfileManager<P : Profile> : Iterable<P>, Identified {
profiles[name] = profile
} catch (error: Throwable) {
Log.log(LogMessageType.PROFILES, LogLevels.FATAL) { error }
error.crash()
throw error
} finally {
lock.unlock()
}

21
src/main/java/de/bixilon/minosoft/data/registries/registries/PixLyzerUtil.kt
View File

@ -25,6 +25,10 @@ import de.bixilon.minosoft.assets.util.HashTypes
import de.bixilon.minosoft.assets.util.InputStreamUtil.readMBFMap
import de.bixilon.minosoft.config.profile.profiles.resources.ResourcesProfile
import de.bixilon.minosoft.protocol.versions.Version
import de.bixilon.minosoft.util.http.DownloadUtil
import de.bixilon.minosoft.util.logging.Log
import de.bixilon.minosoft.util.logging.LogLevels
import de.bixilon.minosoft.util.logging.LogMessageType
import java.io.ByteArrayInputStream
object PixLyzerUtil {
@ -33,14 +37,13 @@ object PixLyzerUtil {
return ByteArrayInputStream(this).readMBFMap().toJsonObject() ?: throw IllegalStateException("Could not read pixlyzer data!")
}
private fun verify(url: String, hash: String): Map<String, Any> {
FileAssetsUtil.readOrNull(hash, type = FileAssetsTypes.PIXLYZER, compress = false)?.let { return it.read() }
val data = FileAssetsUtil.read(url.formatPlaceholder(
private fun verify(url: String, hash: String): JsonObject {
val url = url.formatPlaceholder(
"hashPrefix" to hash.substring(0, 2),
"fullHash" to hash,
).toURL().openStream(), type = FileAssetsTypes.PIXLYZER, compress = false, hash = HashTypes.SHA1)
).toURL()
Log.log(LogMessageType.ASSETS, LogLevels.VERBOSE) { "Downloading pixlyzer data $url" }
val data = FileAssetsUtil.read(url.openStream(), type = FileAssetsTypes.PIXLYZER, compress = false, hash = HashTypes.SHA1)
if (data.hash != hash) {
throw IllegalStateException("Pixlyzer data mismatch (expected=$hash, hash=${data.hash}!")
@ -49,6 +52,12 @@ object PixLyzerUtil {
return data.data.read()
}
private fun verify(urls: List<String>, hash: String): JsonObject {
FileAssetsUtil.readOrNull(hash, type = FileAssetsTypes.PIXLYZER, compress = false)?.let { return it.read() }
return DownloadUtil.retry(urls) { verify(it, hash) }
}
fun loadPixlyzerData(profile: ResourcesProfile, version: Version): JsonObject {
val pixlyzerHash = AssetsVersionProperties[version]?.pixlyzerHash ?: throw IllegalStateException("$version has no pixlyzer data available!")

34
src/main/java/de/bixilon/minosoft/util/http/DownloadUtil.kt Normal file
View File

@ -0,0 +1,34 @@
/*
* Minosoft
* Copyright (C) 2020-2023 Moritz Zwerger
*
* This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* This software is not affiliated with Mojang AB, the original developer of Minecraft.
*/
package de.bixilon.minosoft.util.http
import de.bixilon.kutil.json.JsonObject
object DownloadUtil {
fun retry(urls: List<String>, loader: (String) -> JsonObject): Map<String, Any> {
if (urls.isEmpty()) throw IllegalArgumentException("No urls provided!")
var first: Throwable? = null
for (url in urls) {
try {
return loader.invoke(url)
} catch (error: Throwable) {
first = error
error.printStackTrace()
}
}
throw first!!
}
}