Switchable gzipping of saved games (#6735)

* Switchable gzipping of saved games

* Switchable gzipping of saved games - consensus says default off

android/assets/jsons/translations/German.properties

@@ -581,6 +581,8 @@ Days = Tage
 Current saves = Gespeicherte Spiele
 Show autosaves = Zeige automatisch gespeicherte Spiele an
 Saved game name = Name des gespeicherten Spiels
+# This is the save game name the dialog will suggest
+[player] - [turns] turns = [player] ([turns] Runden)
 Copy to clipboard = In die Zwischenablage kopieren
 Copy saved game to clipboard = Gespeichertes Spiel in die Zwischenablage kopieren
 Could not load game = Spiel konnte nicht geladen werden

android/assets/jsons/translations/template.properties

@@ -584,6 +584,8 @@ Days =
 Current saves = 
 Show autosaves = 
 Saved game name = 
+# This is the save game name the dialog will suggest
+[player] - [turns] turns = 
 Copy to clipboard = 
 Copy saved game to clipboard = 
 Could not load game = 

android/src/com/unciv/app/CustomSaveLocationHelperAndroid.kt

@@ -6,7 +6,6 @@ import android.net.Uri
 import android.os.Build
 import androidx.annotation.GuardedBy
 import androidx.annotation.RequiresApi
-import com.unciv.json.json
 import com.unciv.logic.CustomSaveLocationHelper
 import com.unciv.logic.GameInfo
 import com.unciv.logic.GameSaver
@@ -74,7 +73,7 @@ class CustomSaveLocationHelperAndroid(private val activity: Activity) : CustomSa
         activity.contentResolver.openOutputStream(uri, "rwt")
                 ?.writer()
                 ?.use {
-                    it.write(json().toJson(gameInfo))
+                    it.write(GameSaver.gameInfoToString(gameInfo))
                 }
     }
 

core/src/com/unciv/MainMenuScreen.kt

@@ -27,7 +27,6 @@ import com.unciv.ui.utils.*
 import com.unciv.ui.utils.UncivTooltip.Companion.addTooltip
 
 class MainMenuScreen: BaseScreen() {
-    private val autosave = "Autosave"
     private val backgroundTable = Table().apply { background= ImageGetter.getBackground(Color.WHITE) }
     private val singleColumn = isCrampedPortrait()
 
@@ -90,7 +89,7 @@ class MainMenuScreen: BaseScreen() {
         val column1 = Table().apply { defaults().pad(10f).fillX() }
         val column2 = if(singleColumn) column1 else Table().apply { defaults().pad(10f).fillX() }
 
-        val autosaveGame = GameSaver.getSave(autosave, false)
+        val autosaveGame = GameSaver.getSave(GameSaver.autoSaveFileName, false)
         if (autosaveGame.exists()) {
             val resumeTable = getMenuButton("Resume","OtherIcons/Resume", 'r')
                 { autoLoadGame() }
@@ -163,7 +162,7 @@ class MainMenuScreen: BaseScreen() {
 
             var savedGame: GameInfo
             try {
-                savedGame = GameSaver.loadGameByName(autosave)
+                savedGame = GameSaver.loadGameByName(GameSaver.autoSaveFileName)
             } catch (oom: OutOfMemoryError) {
                 outOfMemory()
                 return@crashHandlingThread
@@ -171,7 +170,7 @@ class MainMenuScreen: BaseScreen() {
                 // This can help for situations when the autosave is corrupted
                 try {
                     val autosaves = GameSaver.getSaves()
-                        .filter { it.name() != autosave && it.name().startsWith(autosave) }
+                        .filter { it.name() != GameSaver.autoSaveFileName && it.name().startsWith(GameSaver.autoSaveFileName) }
                     savedGame =
                         GameSaver.loadGameFromFile(autosaves.maxByOrNull { it.lastModified() }!!)
                 } catch (oom: OutOfMemoryError) { // The autosave could have oom problems as well... smh

core/src/com/unciv/UncivGame.kt

@@ -216,7 +216,7 @@ class UncivGame(parameters: UncivGameParameters) : Game() {
         Thread.enumerate(threadList)
 
         if (isGameInfoInitialized()) {
-            val autoSaveThread = threadList.firstOrNull { it.name == "Autosave" }
+            val autoSaveThread = threadList.firstOrNull { it.name == GameSaver.autoSaveFileName }
             if (autoSaveThread != null && autoSaveThread.isAlive) {
                 // auto save is already in progress (e.g. started by onPause() event)
                 // let's allow it to finish and do not try to autosave second time

core/src/com/unciv/logic/GameSaver.kt

@@ -2,18 +2,24 @@ package com.unciv.logic
 
 import com.badlogic.gdx.Gdx
 import com.badlogic.gdx.files.FileHandle
-import com.badlogic.gdx.utils.Json
 import com.unciv.UncivGame
 import com.unciv.json.json
+import com.unciv.logic.multiplayer.OnlineMultiplayer
 import com.unciv.models.metadata.GameSettings
 import com.unciv.ui.crashhandling.crashHandlingThread
 import com.unciv.ui.crashhandling.postCrashHandlingRunnable
+import com.unciv.ui.saves.Gzip
 import java.io.File
 
+
 object GameSaver {
+    //region Data
+
     private const val saveFilesFolder = "SaveFiles"
     private const val multiplayerFilesFolder = "MultiplayerGames"
+    const val autoSaveFileName = "Autosave"
     const val settingsFileName = "GameSettings.json"
+    var saveZipped = false
 
     @Volatile
     var customSaveLocationHelper: CustomSaveLocationHelper? = null
@@ -22,13 +28,16 @@ object GameSaver {
      * See https://developer.android.com/training/data-storage/app-specific#external-access-files */
     var externalFilesDirForAndroid = ""
 
-    fun getSubfolder(multiplayer: Boolean = false) = if (multiplayer) multiplayerFilesFolder else saveFilesFolder
+    //endregion
+    //region Helpers
+
+    private fun getSubfolder(multiplayer: Boolean = false) = if (multiplayer) multiplayerFilesFolder else saveFilesFolder
 
     fun getSave(GameName: String, multiplayer: Boolean = false): FileHandle {
-        val localfile = Gdx.files.local("${getSubfolder(multiplayer)}/$GameName")
+        val localFile = Gdx.files.local("${getSubfolder(multiplayer)}/$GameName")
-        if (externalFilesDirForAndroid == "" || !Gdx.files.isExternalStorageAvailable) return localfile
+        if (externalFilesDirForAndroid == "" || !Gdx.files.isExternalStorageAvailable) return localFile
         val externalFile = Gdx.files.absolute(externalFilesDirForAndroid + "/${getSubfolder(multiplayer)}/$GameName")
-        if (localfile.exists() && !externalFile.exists()) return localfile
+        if (localFile.exists() && !externalFile.exists()) return localFile
         return externalFile
     }
 
@@ -38,15 +47,35 @@ object GameSaver {
         return localSaves + Gdx.files.absolute(externalFilesDirForAndroid + "/${getSubfolder(multiplayer)}").list().asSequence()
     }
 
+    fun canLoadFromCustomSaveLocation() = customSaveLocationHelper != null
+
+    fun deleteSave(GameName: String, multiplayer: Boolean = false) {
+        getSave(GameName, multiplayer).delete()
+    }
+
+    //endregion
+    //region Saving
+
     fun saveGame(game: GameInfo, GameName: String, saveCompletionCallback: ((Exception?) -> Unit)? = null) {
         try {
-            json().toJson(game, getSave(GameName))
+            getSave(GameName).writeString(gameInfoToString(game), false)
             saveCompletionCallback?.invoke(null)
         } catch (ex: Exception) {
             saveCompletionCallback?.invoke(ex)
         }
     }
 
+    /** Returns gzipped serialization of [game], optionally gzipped ([forceZip] overrides [saveZipped]) */
+    fun gameInfoToString(game: GameInfo, forceZip: Boolean? = null): String {
+        val plainJson = json().toJson(game)
+        return if (forceZip ?: saveZipped) Gzip.zip(plainJson) else plainJson
+    }
+
+    /** Returns gzipped serialization of preview [game] - only called from [OnlineMultiplayer] */
+    fun gameInfoToString(game: GameInfoPreview): String {
+        return Gzip.zip(json().toJson(game))
+    }
+
     /**
      * Overload of function saveGame to save a GameInfoPreview in the MultiplayerGames folder
      */
@@ -63,13 +92,14 @@ object GameSaver {
         customSaveLocationHelper!!.saveGame(game, GameName, forcePrompt = true, saveCompleteCallback = saveCompletionCallback)
     }
 
+    //endregion
+    //region Loading
+
     fun loadGameByName(GameName: String) =
             loadGameFromFile(getSave(GameName))
 
     fun loadGameFromFile(gameFile: FileHandle): GameInfo {
-        val game = json().fromJson(GameInfo::class.java, gameFile)
+        return gameInfoFromString(gameFile.readString())
-        game.setTransients()
-        return game
     }
 
     fun loadGamePreviewByName(GameName: String) =
@@ -85,16 +115,15 @@ object GameSaver {
         }
     }
 
-    fun canLoadFromCustomSaveLocation() = customSaveLocationHelper != null
-
     fun gameInfoFromString(gameData: String): GameInfo {
-        val game = json().fromJson(GameInfo::class.java, gameData)
+        return gameInfoFromStringWithoutTransients(gameData).apply {
-        game.setTransients()
+            setTransients()
-        return game
+        }
     }
 
+    /** Parses [gameData] as gzipped serialization of a [GameInfoPreview] - only called from [OnlineMultiplayer] */
     fun gameInfoPreviewFromString(gameData: String): GameInfoPreview {
-        return json().fromJson(GameInfoPreview::class.java, gameData)
+        return json().fromJson(GameInfoPreview::class.java, Gzip.unzip(gameData))
     }
 
     /**
@@ -102,15 +131,19 @@ object GameSaver {
      * The returned GameInfo can not be used for most circumstances because its not initialized!
      * It is therefore stateless and save to call for Multiplayer Turn Notifier, unlike gameInfoFromString().
      */
-    fun gameInfoFromStringWithoutTransients(gameData: String): GameInfo {
+    private fun gameInfoFromStringWithoutTransients(gameData: String): GameInfo {
-        return json().fromJson(GameInfo::class.java, gameData)
+        val unzippedJson = try {
+            Gzip.unzip(gameData)
+        } catch (ex: Exception) {
+            gameData
+        }
+        return json().fromJson(GameInfo::class.java, unzippedJson)
     }
 
-    fun deleteSave(GameName: String, multiplayer: Boolean = false) {
+    //endregion
-        getSave(GameName, multiplayer).delete()
+    //region Settings
-    }
 
-    fun getGeneralSettingsFile(): FileHandle {
+    private fun getGeneralSettingsFile(): FileHandle {
         return if (UncivGame.Current.consoleMode) FileHandle(settingsFileName)
         else Gdx.files.local(settingsFileName)
     }
@@ -139,13 +172,20 @@ object GameSaver {
         getGeneralSettingsFile().writeString(json().toJson(gameSettings), false)
     }
 
+    //endregion
+    //region Autosave
+
     fun autoSave(gameInfo: GameInfo, postRunnable: () -> Unit = {}) {
         // The save takes a long time (up to a few seconds on large games!) and we can do it while the player continues his game.
         // On the other hand if we alter the game data while it's being serialized we could get a concurrent modification exception.
         // So what we do is we clone all the game data and serialize the clone.
-        val gameInfoClone = gameInfo.clone()
+        autoSaveUnCloned(gameInfo.clone(), postRunnable)
-        crashHandlingThread(name = "Autosave") {
+    }
-            autoSaveSingleThreaded(gameInfoClone)
+
+    fun autoSaveUnCloned(gameInfo: GameInfo, postRunnable: () -> Unit = {}) {
+        // This is used when returning from WorldScreen to MainMenuScreen - no clone since UI access to it should be gone
+        crashHandlingThread(name = autoSaveFileName) {
+            autoSaveSingleThreaded(gameInfo)
             // do this on main thread
             postCrashHandlingRunnable ( postRunnable )
         }
@@ -153,21 +193,21 @@ object GameSaver {
 
     fun autoSaveSingleThreaded(gameInfo: GameInfo) {
         try {
-            saveGame(gameInfo, "Autosave")
+            saveGame(gameInfo, autoSaveFileName)
         } catch (oom: OutOfMemoryError) {
             return  // not much we can do here
         }
 
         // keep auto-saves for the last 10 turns for debugging purposes
         val newAutosaveFilename =
-            saveFilesFolder + File.separator + "Autosave-${gameInfo.currentPlayer}-${gameInfo.turns}"
+            saveFilesFolder + File.separator + autoSaveFileName + "-${gameInfo.currentPlayer}-${gameInfo.turns}"
-        getSave("Autosave").copyTo(Gdx.files.local(newAutosaveFilename))
+        getSave(autoSaveFileName).copyTo(Gdx.files.local(newAutosaveFilename))
 
         fun getAutosaves(): Sequence<FileHandle> {
-            return getSaves().filter { it.name().startsWith("Autosave") }
+            return getSaves().filter { it.name().startsWith(autoSaveFileName) }
         }
         while (getAutosaves().count() > 10) {
-            val saveToDelete = getAutosaves().minByOrNull { it: FileHandle -> it.lastModified() }!!
+            val saveToDelete = getAutosaves().minByOrNull { it.lastModified() }!!
             deleteSave(saveToDelete.name())
         }
     }

core/src/com/unciv/logic/multiplayer/Multiplayer.kt

@@ -3,11 +3,9 @@ package com.unciv.logic.multiplayer
 import com.badlogic.gdx.Net
 import com.unciv.Constants
 import com.unciv.UncivGame
-import com.unciv.json.json
 import com.unciv.logic.GameInfo
 import com.unciv.logic.GameInfoPreview
 import com.unciv.logic.GameSaver
-import com.unciv.ui.saves.Gzip
 import java.util.*
 
 interface IFileStorage {
@@ -87,7 +85,7 @@ class OnlineMultiplayer(var fileStorageIdentifier: String? = null) {
             tryUploadGamePreview(gameInfo.asPreview())
         }
 
-        val zippedGameInfo = Gzip.zip(json().toJson(gameInfo))
+        val zippedGameInfo = GameSaver.gameInfoToString(gameInfo, forceZip = true)
         fileStorage.saveFileData(gameInfo.gameId, zippedGameInfo)
     }
 
@@ -98,17 +96,17 @@ class OnlineMultiplayer(var fileStorageIdentifier: String? = null) {
      * @see GameInfo.asPreview
      */
     fun tryUploadGamePreview(gameInfo: GameInfoPreview) {
-        val zippedGameInfo = Gzip.zip(json().toJson(gameInfo))
+        val zippedGameInfo = GameSaver.gameInfoToString(gameInfo)
         fileStorage.saveFileData("${gameInfo.gameId}_Preview", zippedGameInfo)
     }
 
     fun tryDownloadGame(gameId: String): GameInfo {
         val zippedGameInfo = fileStorage.loadFileData(gameId)
-        return GameSaver.gameInfoFromString(Gzip.unzip(zippedGameInfo))
+        return GameSaver.gameInfoFromString(zippedGameInfo)
     }
 
     fun tryDownloadGamePreview(gameId: String): GameInfoPreview {
         val zippedGameInfo = fileStorage.loadFileData("${gameId}_Preview")
-        return GameSaver.gameInfoPreviewFromString(Gzip.unzip(zippedGameInfo))
+        return GameSaver.gameInfoPreviewFromString(zippedGameInfo)
     }
 }

core/src/com/unciv/ui/saves/LoadGameScreen.kt

@@ -89,10 +89,7 @@ class LoadGameScreen(previousScreen:BaseScreen) : PickerScreen(disableScroll = t
         loadFromClipboardButton.onClick {
             try {
                 val clipboardContentsString = Gdx.app.clipboard.contents.trim()
-                val decoded =
+                val loadedGame = GameSaver.gameInfoFromString(clipboardContentsString)
-                    if (clipboardContentsString.startsWith("{")) clipboardContentsString
-                    else Gzip.unzip(clipboardContentsString)
-                val loadedGame = GameSaver.gameInfoFromString(decoded)
                 UncivGame.Current.loadGame(loadedGame)
             } catch (ex: Exception) {
                 handleLoadGameException("Could not load game from clipboard!", ex)
@@ -216,7 +213,7 @@ class LoadGameScreen(previousScreen:BaseScreen) : PickerScreen(disableScroll = t
             postCrashHandlingRunnable {
                 saveTable.clear()
                 for (save in saves) {
-                    if (save.name().startsWith("Autosave") && !showAutosaves) continue
+                    if (save.name().startsWith(GameSaver.autoSaveFileName) && !showAutosaves) continue
                     val textButton = TextButton(save.name(), skin)
                     textButton.onClick { onSaveSelected(save) }
                     saveTable.add(textButton).pad(5f).row()

core/src/com/unciv/ui/saves/SaveGameScreen.kt

@@ -5,9 +5,7 @@ import com.badlogic.gdx.graphics.Color
 import com.badlogic.gdx.scenes.scene2d.ui.CheckBox
 import com.badlogic.gdx.scenes.scene2d.ui.Table
 import com.badlogic.gdx.scenes.scene2d.ui.TextField
-import com.badlogic.gdx.utils.Json
 import com.unciv.UncivGame
-import com.unciv.json.json
 import com.unciv.logic.GameInfo
 import com.unciv.logic.GameSaver
 import com.unciv.models.translations.tr
@@ -36,7 +34,7 @@ class SaveGameScreen(val gameInfo: GameInfo) : PickerScreen(disableScroll = true
 
         val newSave = Table()
         newSave.defaults().pad(5f, 10f)
-        val defaultSaveName = gameInfo.currentPlayer + " -  " + gameInfo.turns + " turns"
+        val defaultSaveName = "[${gameInfo.currentPlayer}] - [${gameInfo.turns}] turns".tr()
         gameNameTextField.text = defaultSaveName
 
         newSave.add("Saved game name".toLabel()).row()
@@ -46,15 +44,14 @@ class SaveGameScreen(val gameInfo: GameInfo) : PickerScreen(disableScroll = true
         copyJsonButton.onClick {
             thread(name="Copy to clipboard") { // the Gzip rarely leads to ANRs
                 try {
-                    val json = json().toJson(gameInfo)
+                    Gdx.app.clipboard.contents = GameSaver.gameInfoToString(gameInfo, forceZip = true)
-                    val base64Gzip = Gzip.zip(json)
-                    Gdx.app.clipboard.contents = base64Gzip
                 } catch (OOM: OutOfMemoryError) {
                     // you don't get a special toast, this isn't nearly common enough, this is a total edge-case
                 }
             }
         }
         newSave.add(copyJsonButton).row()
+
         if (GameSaver.canLoadFromCustomSaveLocation()) {
             val saveToCustomLocation = "Save to custom location".toTextButton()
             val errorLabel = "".toLabel(Color.RED)
@@ -79,7 +76,6 @@ class SaveGameScreen(val gameInfo: GameInfo) : PickerScreen(disableScroll = true
             newSave.add(errorLabel).row()
         }
 
-
         val showAutosavesCheckbox = CheckBox("Show autosaves".tr(), skin)
         showAutosavesCheckbox.isChecked = false
         showAutosavesCheckbox.onChange {
@@ -116,7 +112,7 @@ class SaveGameScreen(val gameInfo: GameInfo) : PickerScreen(disableScroll = true
         val saves = GameSaver.getSaves()
                 .sortedByDescending { it.lastModified() }
         for (saveGameFile in saves) {
-            if (saveGameFile.name().startsWith("Autosave") && !showAutosaves) continue
+            if (saveGameFile.name().startsWith(GameSaver.autoSaveFileName) && !showAutosaves) continue
             val textButton = saveGameFile.name().toTextButton()
             textButton.onClick {
                 gameNameTextField.text = saveGameFile.name()

core/src/com/unciv/ui/worldscreen/mainmenu/OptionsPopup.kt

@@ -10,6 +10,7 @@ import com.badlogic.gdx.utils.Align
 import com.unciv.Constants
 import com.unciv.MainMenuScreen
 import com.unciv.UncivGame
+import com.unciv.logic.GameSaver
 import com.unciv.logic.MapSaver
 import com.unciv.logic.civilization.PlayerType
 import com.unciv.logic.multiplayer.SimpleHttp
@@ -553,8 +554,8 @@ class OptionsPopup(
     private fun getDebugTab() = Table(BaseScreen.skin).apply {
         pad(10f)
         defaults().pad(5f)
-
         val game = UncivGame.Current
+
         val simulateButton = "Simulate until turn:".toTextButton()
         val simulateTextField = TextField(game.simulateUntilTurnForDebug.toString(), BaseScreen.skin)
         val invalidInputLabel = "This is not a valid integer!".toLabel().also { it.isVisible = false }
@@ -571,6 +572,7 @@ class OptionsPopup(
         add(simulateButton)
         add(simulateTextField).row()
         add(invalidInputLabel).colspan(2).row()
+
         add("Supercharged".toCheckBox(game.superchargedForDebug) {
             game.superchargedForDebug = it
         }).colspan(2).row()
@@ -582,9 +584,13 @@ class OptionsPopup(
                 game.gameInfo.gameParameters.godMode = it
             }).colspan(2).row()
         }
+        add("Save games compressed".toCheckBox(GameSaver.saveZipped) {
+            GameSaver.saveZipped = it
+        }).colspan(2).row()
         add("Save maps compressed".toCheckBox(MapSaver.saveZipped) {
             MapSaver.saveZipped = it
         }).colspan(2).row()
+
         add("Gdx Scene2D debug".toCheckBox(BaseScreen.enableSceneDebug) {
             BaseScreen.enableSceneDebug = it
         }).colspan(2).row()

core/src/com/unciv/ui/worldscreen/mainmenu/WorldScreenMenuPopup.kt

@@ -2,6 +2,7 @@ package com.unciv.ui.worldscreen.mainmenu
 
 import com.badlogic.gdx.Gdx
 import com.unciv.MainMenuScreen
+import com.unciv.logic.GameSaver
 import com.unciv.ui.civilopedia.CivilopediaScreen
 import com.unciv.models.metadata.GameSetupInfo
 import com.unciv.ui.newgamescreen.NewGameScreen
@@ -16,6 +17,7 @@ class WorldScreenMenuPopup(val worldScreen: WorldScreen) : Popup(worldScreen) {
         defaults().fillX()
 
         addButton("Main menu") {
+            GameSaver.autoSaveUnCloned(worldScreen.gameInfo)
             worldScreen.game.setScreen(MainMenuScreen())
         }
         addButton("Civilopedia") {

desktop/src/com/unciv/app/desktop/CustomSaveLocationHelperDesktop.kt

@@ -19,7 +19,7 @@ class CustomSaveLocationHelperDesktop : CustomSaveLocationHelper {
                 File(customSaveLocation).outputStream()
                         .writer()
                         .use { writer ->
-                            writer.write(json().toJson(gameInfo))
+                            writer.write(GameSaver.gameInfoToString(gameInfo))
                         }
                 saveCompleteCallback?.invoke(null)
             } catch (e: Exception) {