Fixed: Not able to upload 512MB+ custom apps anymore.

* Introduced the splitting zim file with 500MB for custom apps. Since the bundle has a limit for a file to upload it uses the play asset delivery mode, and for new custom apps, playStore does not allow to upload the apk (with apk we can upload more than 1GB file). So to address this issue we have introduced a feature in libzim where we can load the zim files via the fd list.

* We have modified our code to upload the files with 500MB chunks in the asset folder, and later we are accessing these files from the asset folder and creating the archive object with the help of the new libzim feature.

app/src/main/java/org/kiwix/kiwixmobile/nav/destination/reader/KiwixReaderFragment.kt

@@ -199,7 +199,7 @@ class KiwixReaderFragment : CoreReaderFragment() {
   override fun onResume() {
     super.onResume()
     if (zimReaderContainer?.zimFile == null &&
-      zimReaderContainer?.zimFileReader?.assetFileDescriptor == null
+      zimReaderContainer?.zimFileReader?.assetFileDescriptorList?.isEmpty() == true
     ) {
       exitBook()
     }

buildSrc/src/main/kotlin/custom/CustomApps.kt

@@ -48,8 +48,6 @@ fun ProductFlavors.create(customApps: List<CustomApp>) {
       buildConfigField("String", "ENFORCED_LANG", "\"${customApp.enforcedLanguage}\"")
       buildConfigField("String", "ABOUT_APP_URL", "\"${customApp.aboutAppUrl}\"")
       buildConfigField("String", "SUPPORT_URL", "\"${customApp.supportUrl}\"")
-      // Add asset file name in buildConfig file, we will use later to receive the zim file.
-      buildConfigField("String", "PLAY_ASSET_FILE", "\"${customApp.name}.zim\"")
       buildConfigField("Boolean", "DISABLE_SIDEBAR", "${customApp.disableSideBar}")
       buildConfigField("Boolean", "DISABLE_TABS", "${customApp.disableTabs}")
       buildConfigField("Boolean", "DISABLE_READ_ALOUD", "${customApp.disableReadAloud}")

core/src/main/java/org/kiwix/kiwixmobile/core/main/CoreReaderFragment.kt

@@ -1554,7 +1554,7 @@ abstract class CoreReaderFragment :
   protected fun openZimFile(
     file: File?,
     isCustomApp: Boolean = false,
-    assetFileDescriptor: AssetFileDescriptor? = null,
+    assetFileDescriptorList: List<AssetFileDescriptor> = emptyList(),
     filePath: String? = null
   ) {
     if (hasPermission(Manifest.permission.READ_EXTERNAL_STORAGE) || isCustomApp) {
@@ -1564,10 +1564,10 @@ abstract class CoreReaderFragment :
         reopenBook()
         openAndSetInContainer(file = file)
         updateTitle()
-      } else if (assetFileDescriptor != null) {
+      } else if (assetFileDescriptorList.isNotEmpty()) {
         reopenBook()
         openAndSetInContainer(
-          assetFileDescriptor = assetFileDescriptor,
+          assetFileDescriptorList = assetFileDescriptorList,
           filePath = filePath
         )
         updateTitle()
@@ -1602,7 +1602,7 @@ abstract class CoreReaderFragment :
 
   private fun openAndSetInContainer(
     file: File? = null,
-    assetFileDescriptor: AssetFileDescriptor? = null,
+    assetFileDescriptorList: List<AssetFileDescriptor> = emptyList(),
     filePath: String? = null
   ) {
     try {
@@ -1613,9 +1613,9 @@ abstract class CoreReaderFragment :
       e.printStackTrace()
     }
     zimReaderContainer?.let { zimReaderContainer ->
-      if (assetFileDescriptor != null) {
+      if (assetFileDescriptorList.isNotEmpty()) {
         zimReaderContainer.setZimFileDescriptor(
-          assetFileDescriptor,
+          assetFileDescriptorList,
           filePath = filePath
         )
       } else {

core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt

@@ -38,6 +38,7 @@ import org.kiwix.kiwixmobile.core.utils.files.Log
 import org.kiwix.libkiwix.JNIKiwixException
 import org.kiwix.libzim.Archive
 import org.kiwix.libzim.DirectAccessInfo
+import org.kiwix.libzim.FdInput
 import org.kiwix.libzim.Item
 import org.kiwix.libzim.SuggestionSearch
 import org.kiwix.libzim.SuggestionSearcher
@@ -56,7 +57,7 @@ private const val TAG = "ZimFileReader"
 @Suppress("LongParameterList")
 class ZimFileReader constructor(
   val zimFile: File?,
-  val assetFileDescriptor: AssetFileDescriptor? = null,
+  val assetFileDescriptorList: List<AssetFileDescriptor> = emptyList(),
   val assetDescriptorFilePath: String? = null,
   val jniKiwixReader: Archive,
   private val nightModeConfig: NightModeConfig,
@@ -65,7 +66,7 @@ class ZimFileReader constructor(
   interface Factory {
     suspend fun create(file: File): ZimFileReader?
     suspend fun create(
-      assetFileDescriptor: AssetFileDescriptor,
+      assetFileDescriptorList: List<AssetFileDescriptor>,
       filePath: String? = null
     ): ZimFileReader?
 
@@ -91,18 +92,19 @@ class ZimFileReader constructor(
         }
 
       override suspend fun create(
-        assetFileDescriptor: AssetFileDescriptor,
+        assetFileDescriptorList: List<AssetFileDescriptor>,
         filePath: String?
       ): ZimFileReader? = withContext(Dispatchers.IO) { // Bug Fix #3805
         try {
-          val archive = Archive(
+          val fdInputArray = getFdInputArrayFromAssetFileDescriptorList(assetFileDescriptorList)
-            assetFileDescriptor.parcelFileDescriptor.dup().fileDescriptor,
+          val archive = if (fdInputArray.size == 1) {
-            assetFileDescriptor.startOffset,
+            Archive(fdInputArray[0])
-            assetFileDescriptor.length
+          } else {
-          )
+            Archive(fdInputArray)
+          }
           ZimFileReader(
             null,
-            assetFileDescriptor,
+            assetFileDescriptorList,
             assetDescriptorFilePath = filePath,
             nightModeConfig = nightModeConfig,
             jniKiwixReader = archive,
@@ -116,6 +118,17 @@ class ZimFileReader constructor(
           null
         }
       }
+
+      private fun getFdInputArrayFromAssetFileDescriptorList(
+        assetFileDescriptorList: List<AssetFileDescriptor>
+      ): Array<FdInput> =
+        assetFileDescriptorList.map {
+          FdInput(
+            it.parcelFileDescriptor.dup().fileDescriptor,
+            it.startOffset,
+            it.length
+          )
+        }.toTypedArray()
     }
   }
 

core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt

@@ -46,12 +46,14 @@ class ZimReaderContainer @Inject constructor(private val zimFileReaderFactory: F
   }
 
   fun setZimFileDescriptor(
-    assetFileDescriptor: AssetFileDescriptor,
+    assetFileDescriptorList: List<AssetFileDescriptor>,
     filePath: String? = null
   ) {
     zimFileReader = runBlocking {
-      if (assetFileDescriptor.parcelFileDescriptor.dup().fileDescriptor.valid())
+      if (assetFileDescriptorList.isNotEmpty() &&
-        zimFileReaderFactory.create(assetFileDescriptor, filePath)
+        assetFileDescriptorList[0].parcelFileDescriptor.dup().fileDescriptor.valid()
+      )
+        zimFileReaderFactory.create(assetFileDescriptorList, filePath)
       else null
     }
   }

core/src/main/java/org/kiwix/kiwixmobile/core/webserver/KiwixServer.kt

@@ -51,7 +51,8 @@ class KiwixServer @Inject constructor(
             // Determine whether to create an Archive from an asset or a file path
             val archive = if (path == getDemoFilePathForCustomApp(context)) {
               // For custom apps using a demo file, create an Archive with FileDescriptor
-              val assetFileDescriptor = zimReaderContainer.zimFileReader?.assetFileDescriptor
+              val assetFileDescriptor =
+                zimReaderContainer.zimFileReader?.assetFileDescriptorList?.get(0)
               val startOffset = assetFileDescriptor?.startOffset ?: 0L
               val size = assetFileDescriptor?.length ?: 0L
               Archive(

custom/build.gradle.kts

@@ -103,17 +103,39 @@ fun ProductFlavor.fetchRequest(): Request {
   }
 }
 
-fun writeZimFileData(responseBody: ResponseBody, file: File) {
+fun writeZimFileData(
-  FileOutputStream(file).use { outputStream ->
+  responseBody: ResponseBody,
-    responseBody.byteStream().use { inputStream ->
+  file: File,
+  chunkSize: Long = 500 * 1024 * 1024 // create a chunk of 500MB
+) {
+  var outputStream: FileOutputStream? = null
   val buffer = ByteArray(4096)
   var bytesRead: Int
+  var totalBytesWritten = 0L
+  var chunkNumber = 0
+
+  responseBody.byteStream().use { inputStream ->
     while (inputStream.read(buffer).also { bytesRead = it } != -1) {
-        outputStream.write(buffer, 0, bytesRead)
+      if (outputStream == null || totalBytesWritten >= chunkSize) {
+        // Close the current chunk and open a new one
+        outputStream?.flush()
+        outputStream?.close()
+        chunkNumber++
+        val nextChunkFile = File(file.parent, "chunk$chunkNumber.zim")
+        nextChunkFile.createNewFile()
+        outputStream = FileOutputStream(nextChunkFile)
+        totalBytesWritten = 0 // Reset totalBytesWritten for the new chunk
       }
-      outputStream.flush()
+
+      // Write data to the output stream
+      outputStream?.write(buffer, 0, bytesRead)
+      totalBytesWritten += bytesRead
     }
   }
+
+  // Close the last chunk (if any)
+  outputStream?.flush()
+  outputStream?.close()
 }
 
 fun ProductFlavor.createDownloadTaskForPlayAssetDelivery(

custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomFileValidator.kt

@@ -21,9 +21,9 @@ package org.kiwix.kiwixmobile.custom.main
 import android.content.Context
 import android.content.pm.PackageManager
 import android.content.res.AssetFileDescriptor
+import android.content.res.AssetManager
 import androidx.core.content.ContextCompat
 import org.kiwix.kiwixmobile.core.utils.files.Log
-import org.kiwix.kiwixmobile.custom.BuildConfig
 import org.kiwix.kiwixmobile.custom.main.ValidationState.HasBothFiles
 import org.kiwix.kiwixmobile.custom.main.ValidationState.HasFile
 import org.kiwix.kiwixmobile.custom.main.ValidationState.HasNothing
@@ -37,16 +37,18 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
     when (val installationState = detectInstallationState()) {
       is HasBothFiles,
       is HasFile -> onFilesFound(installationState)
+
       HasNothing -> onNoFilesFound()
     }
 
   private fun detectInstallationState(
     obbFiles: List<File> = obbFiles(),
     zimFiles: List<File> = zimFiles(),
-    assetFileDescriptor: AssetFileDescriptor? = getAssetFileDescriptorFromPlayAssetDelivery()
+    assetFileDescriptorList: List<AssetFileDescriptor> =
+      getAssetFileDescriptorListFromPlayAssetDelivery()
   ): ValidationState {
     return when {
-      assetFileDescriptor != null -> HasFile(null, assetFileDescriptor)
+      assetFileDescriptorList.isNotEmpty() -> HasFile(null, assetFileDescriptorList)
       obbFiles.isNotEmpty() && zimFiles().isNotEmpty() -> HasBothFiles(obbFiles[0], zimFiles[0])
       obbFiles.isNotEmpty() -> HasFile(obbFiles[0])
       zimFiles.isNotEmpty() -> HasFile(zimFiles[0])
@@ -55,11 +57,15 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
   }
 
   @Suppress("MagicNumber")
-  private fun getAssetFileDescriptorFromPlayAssetDelivery(): AssetFileDescriptor? {
+  private fun getAssetFileDescriptorListFromPlayAssetDelivery(): List<AssetFileDescriptor> {
     try {
-      val context = context.createPackageContext(context.packageName, 0)
+      val assetManager = context.createPackageContext(context.packageName, 0).assets
-      val assetManager = context.assets
+      val assetFileDescriptorList: ArrayList<AssetFileDescriptor> = arrayListOf()
-      return assetManager.openFd(BuildConfig.PLAY_ASSET_FILE)
+      getChunksList(assetManager).forEach {
+        assetFileDescriptorList.add(assetManager.openFd(it))
+      }
+
+      return assetFileDescriptorList
     } catch (packageNameNotFoundException: PackageManager.NameNotFoundException) {
       Log.w(
         "ASSET_PACKAGE_DELIVERY",
@@ -68,7 +74,24 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
     } catch (ioException: IOException) {
       Log.w("ASSET_PACKAGE_DELIVERY", "Unable to copy the content of asset $ioException")
     }
-    return null
+    return emptyList()
+  }
+
+  private fun getChunksList(assetManager: AssetManager): List<String> {
+    val chunkFiles = mutableListOf<String>()
+
+    try {
+      // List of all files in the asset directory
+      val assets = assetManager.list("") ?: emptyArray()
+
+      // Filter and count chunk files.
+      assets.filterTo(chunkFiles) { it.startsWith("chunk") && it.endsWith(".zim") }
+      chunkFiles.sortBy { it.substringAfter("chunk").substringBefore(".zim").toInt() }
+    } catch (ioException: IOException) {
+      ioException.printStackTrace()
+    }
+
+    return chunkFiles
   }
 
   private fun obbFiles() = scanDirs(ContextCompat.getObbDirs(context), "obb")
@@ -104,7 +127,10 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
 
 sealed class ValidationState {
   data class HasBothFiles(val obbFile: File, val zimFile: File) : ValidationState()
-  data class HasFile(val file: File?, val assetFileDescriptor: AssetFileDescriptor? = null) :
+  data class HasFile(
+    val file: File?,
+    val assetFileDescriptorList: List<AssetFileDescriptor> = emptyList()
+  ) :
     ValidationState()
 
   object HasNothing : ValidationState()

custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomReaderFragment.kt

@@ -181,8 +181,8 @@ class CustomReaderFragment : CoreReaderFragment() {
       onFilesFound = {
         when (it) {
           is ValidationState.HasFile -> {
-            if (it.assetFileDescriptor != null) {
+            if (it.assetFileDescriptorList.isNotEmpty()) {
-              openZimFile(null, true, it.assetFileDescriptor)
+              openZimFile(null, true, it.assetFileDescriptorList)
             } else {
               openZimFile(it.file, true)
             }