From 69a7099bc60276015500b06f6fd3d2e2d0b7d379 Mon Sep 17 00:00:00 2001 From: MohitMaliFtechiz Date: Mon, 22 Apr 2024 18:48:37 +0530 Subject: [PATCH] Fixed: Not able to upload 512MB+ custom apps anymore. * Introduced the splitting zim file with 500MB for custom apps. Since the bundle has a limit for a file to upload it uses the play asset delivery mode, and for new custom apps, playStore does not allow to upload the apk (with apk we can upload more than 1GB file). So to address this issue we have introduced a feature in libzim where we can load the zim files via the fd list. * We have modified our code to upload the files with 500MB chunks in the asset folder, and later we are accessing these files from the asset folder and creating the archive object with the help of the new libzim feature. --- .../destination/reader/KiwixReaderFragment.kt | 2 +- buildSrc/src/main/kotlin/custom/CustomApps.kt | 2 - .../core/main/CoreReaderFragment.kt | 12 ++--- .../kiwixmobile/core/reader/ZimFileReader.kt | 31 +++++++++---- .../core/reader/ZimReaderContainer.kt | 8 ++-- .../kiwixmobile/core/webserver/KiwixServer.kt | 3 +- custom/build.gradle.kts | 38 ++++++++++++---- .../custom/main/CustomFileValidator.kt | 44 +++++++++++++++---- .../custom/main/CustomReaderFragment.kt | 4 +- 9 files changed, 103 insertions(+), 41 deletions(-) diff --git a/app/src/main/java/org/kiwix/kiwixmobile/nav/destination/reader/KiwixReaderFragment.kt b/app/src/main/java/org/kiwix/kiwixmobile/nav/destination/reader/KiwixReaderFragment.kt index 59d8190af..9d64a98e3 100644 --- a/app/src/main/java/org/kiwix/kiwixmobile/nav/destination/reader/KiwixReaderFragment.kt +++ b/app/src/main/java/org/kiwix/kiwixmobile/nav/destination/reader/KiwixReaderFragment.kt @@ -199,7 +199,7 @@ class KiwixReaderFragment : CoreReaderFragment() { override fun onResume() { super.onResume() if (zimReaderContainer?.zimFile == null && - zimReaderContainer?.zimFileReader?.assetFileDescriptor == null + zimReaderContainer?.zimFileReader?.assetFileDescriptorList?.isEmpty() == true ) { exitBook() } diff --git a/buildSrc/src/main/kotlin/custom/CustomApps.kt b/buildSrc/src/main/kotlin/custom/CustomApps.kt index d13c84827..7f4faa4d1 100644 --- a/buildSrc/src/main/kotlin/custom/CustomApps.kt +++ b/buildSrc/src/main/kotlin/custom/CustomApps.kt @@ -48,8 +48,6 @@ fun ProductFlavors.create(customApps: List) { buildConfigField("String", "ENFORCED_LANG", "\"${customApp.enforcedLanguage}\"") buildConfigField("String", "ABOUT_APP_URL", "\"${customApp.aboutAppUrl}\"") buildConfigField("String", "SUPPORT_URL", "\"${customApp.supportUrl}\"") - // Add asset file name in buildConfig file, we will use later to receive the zim file. - buildConfigField("String", "PLAY_ASSET_FILE", "\"${customApp.name}.zim\"") buildConfigField("Boolean", "DISABLE_SIDEBAR", "${customApp.disableSideBar}") buildConfigField("Boolean", "DISABLE_TABS", "${customApp.disableTabs}") buildConfigField("Boolean", "DISABLE_READ_ALOUD", "${customApp.disableReadAloud}") diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/main/CoreReaderFragment.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/main/CoreReaderFragment.kt index 11249f14a..fad6e3430 100644 --- a/core/src/main/java/org/kiwix/kiwixmobile/core/main/CoreReaderFragment.kt +++ b/core/src/main/java/org/kiwix/kiwixmobile/core/main/CoreReaderFragment.kt @@ -1554,7 +1554,7 @@ abstract class CoreReaderFragment : protected fun openZimFile( file: File?, isCustomApp: Boolean = false, - assetFileDescriptor: AssetFileDescriptor? = null, + assetFileDescriptorList: List = emptyList(), filePath: String? = null ) { if (hasPermission(Manifest.permission.READ_EXTERNAL_STORAGE) || isCustomApp) { @@ -1564,10 +1564,10 @@ abstract class CoreReaderFragment : reopenBook() openAndSetInContainer(file = file) updateTitle() - } else if (assetFileDescriptor != null) { + } else if (assetFileDescriptorList.isNotEmpty()) { reopenBook() openAndSetInContainer( - assetFileDescriptor = assetFileDescriptor, + assetFileDescriptorList = assetFileDescriptorList, filePath = filePath ) updateTitle() @@ -1602,7 +1602,7 @@ abstract class CoreReaderFragment : private fun openAndSetInContainer( file: File? = null, - assetFileDescriptor: AssetFileDescriptor? = null, + assetFileDescriptorList: List = emptyList(), filePath: String? = null ) { try { @@ -1613,9 +1613,9 @@ abstract class CoreReaderFragment : e.printStackTrace() } zimReaderContainer?.let { zimReaderContainer -> - if (assetFileDescriptor != null) { + if (assetFileDescriptorList.isNotEmpty()) { zimReaderContainer.setZimFileDescriptor( - assetFileDescriptor, + assetFileDescriptorList, filePath = filePath ) } else { diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt index 1412ebeb7..35dcf54aa 100644 --- a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt +++ b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt @@ -38,6 +38,7 @@ import org.kiwix.kiwixmobile.core.utils.files.Log import org.kiwix.libkiwix.JNIKiwixException import org.kiwix.libzim.Archive import org.kiwix.libzim.DirectAccessInfo +import org.kiwix.libzim.FdInput import org.kiwix.libzim.Item import org.kiwix.libzim.SuggestionSearch import org.kiwix.libzim.SuggestionSearcher @@ -56,7 +57,7 @@ private const val TAG = "ZimFileReader" @Suppress("LongParameterList") class ZimFileReader constructor( val zimFile: File?, - val assetFileDescriptor: AssetFileDescriptor? = null, + val assetFileDescriptorList: List = emptyList(), val assetDescriptorFilePath: String? = null, val jniKiwixReader: Archive, private val nightModeConfig: NightModeConfig, @@ -65,7 +66,7 @@ class ZimFileReader constructor( interface Factory { suspend fun create(file: File): ZimFileReader? suspend fun create( - assetFileDescriptor: AssetFileDescriptor, + assetFileDescriptorList: List, filePath: String? = null ): ZimFileReader? @@ -91,18 +92,19 @@ class ZimFileReader constructor( } override suspend fun create( - assetFileDescriptor: AssetFileDescriptor, + assetFileDescriptorList: List, filePath: String? ): ZimFileReader? = withContext(Dispatchers.IO) { // Bug Fix #3805 try { - val archive = Archive( - assetFileDescriptor.parcelFileDescriptor.dup().fileDescriptor, - assetFileDescriptor.startOffset, - assetFileDescriptor.length - ) + val fdInputArray = getFdInputArrayFromAssetFileDescriptorList(assetFileDescriptorList) + val archive = if (fdInputArray.size == 1) { + Archive(fdInputArray[0]) + } else { + Archive(fdInputArray) + } ZimFileReader( null, - assetFileDescriptor, + assetFileDescriptorList, assetDescriptorFilePath = filePath, nightModeConfig = nightModeConfig, jniKiwixReader = archive, @@ -116,6 +118,17 @@ class ZimFileReader constructor( null } } + + private fun getFdInputArrayFromAssetFileDescriptorList( + assetFileDescriptorList: List + ): Array = + assetFileDescriptorList.map { + FdInput( + it.parcelFileDescriptor.dup().fileDescriptor, + it.startOffset, + it.length + ) + }.toTypedArray() } } diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt index 074ede743..e21341430 100644 --- a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt +++ b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt @@ -46,12 +46,14 @@ class ZimReaderContainer @Inject constructor(private val zimFileReaderFactory: F } fun setZimFileDescriptor( - assetFileDescriptor: AssetFileDescriptor, + assetFileDescriptorList: List, filePath: String? = null ) { zimFileReader = runBlocking { - if (assetFileDescriptor.parcelFileDescriptor.dup().fileDescriptor.valid()) - zimFileReaderFactory.create(assetFileDescriptor, filePath) + if (assetFileDescriptorList.isNotEmpty() && + assetFileDescriptorList[0].parcelFileDescriptor.dup().fileDescriptor.valid() + ) + zimFileReaderFactory.create(assetFileDescriptorList, filePath) else null } } diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/webserver/KiwixServer.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/webserver/KiwixServer.kt index d54235ddd..bcac156e3 100644 --- a/core/src/main/java/org/kiwix/kiwixmobile/core/webserver/KiwixServer.kt +++ b/core/src/main/java/org/kiwix/kiwixmobile/core/webserver/KiwixServer.kt @@ -51,7 +51,8 @@ class KiwixServer @Inject constructor( // Determine whether to create an Archive from an asset or a file path val archive = if (path == getDemoFilePathForCustomApp(context)) { // For custom apps using a demo file, create an Archive with FileDescriptor - val assetFileDescriptor = zimReaderContainer.zimFileReader?.assetFileDescriptor + val assetFileDescriptor = + zimReaderContainer.zimFileReader?.assetFileDescriptorList?.get(0) val startOffset = assetFileDescriptor?.startOffset ?: 0L val size = assetFileDescriptor?.length ?: 0L Archive( diff --git a/custom/build.gradle.kts b/custom/build.gradle.kts index 0d1650402..b95a64bcc 100644 --- a/custom/build.gradle.kts +++ b/custom/build.gradle.kts @@ -103,17 +103,39 @@ fun ProductFlavor.fetchRequest(): Request { } } -fun writeZimFileData(responseBody: ResponseBody, file: File) { - FileOutputStream(file).use { outputStream -> - responseBody.byteStream().use { inputStream -> - val buffer = ByteArray(4096) - var bytesRead: Int - while (inputStream.read(buffer).also { bytesRead = it } != -1) { - outputStream.write(buffer, 0, bytesRead) +fun writeZimFileData( + responseBody: ResponseBody, + file: File, + chunkSize: Long = 500 * 1024 * 1024 // create a chunk of 500MB +) { + var outputStream: FileOutputStream? = null + val buffer = ByteArray(4096) + var bytesRead: Int + var totalBytesWritten = 0L + var chunkNumber = 0 + + responseBody.byteStream().use { inputStream -> + while (inputStream.read(buffer).also { bytesRead = it } != -1) { + if (outputStream == null || totalBytesWritten >= chunkSize) { + // Close the current chunk and open a new one + outputStream?.flush() + outputStream?.close() + chunkNumber++ + val nextChunkFile = File(file.parent, "chunk$chunkNumber.zim") + nextChunkFile.createNewFile() + outputStream = FileOutputStream(nextChunkFile) + totalBytesWritten = 0 // Reset totalBytesWritten for the new chunk } - outputStream.flush() + + // Write data to the output stream + outputStream?.write(buffer, 0, bytesRead) + totalBytesWritten += bytesRead } } + + // Close the last chunk (if any) + outputStream?.flush() + outputStream?.close() } fun ProductFlavor.createDownloadTaskForPlayAssetDelivery( diff --git a/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomFileValidator.kt b/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomFileValidator.kt index 5a105d5ab..288b61643 100644 --- a/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomFileValidator.kt +++ b/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomFileValidator.kt @@ -21,9 +21,9 @@ package org.kiwix.kiwixmobile.custom.main import android.content.Context import android.content.pm.PackageManager import android.content.res.AssetFileDescriptor +import android.content.res.AssetManager import androidx.core.content.ContextCompat import org.kiwix.kiwixmobile.core.utils.files.Log -import org.kiwix.kiwixmobile.custom.BuildConfig import org.kiwix.kiwixmobile.custom.main.ValidationState.HasBothFiles import org.kiwix.kiwixmobile.custom.main.ValidationState.HasFile import org.kiwix.kiwixmobile.custom.main.ValidationState.HasNothing @@ -37,16 +37,18 @@ class CustomFileValidator @Inject constructor(private val context: Context) { when (val installationState = detectInstallationState()) { is HasBothFiles, is HasFile -> onFilesFound(installationState) + HasNothing -> onNoFilesFound() } private fun detectInstallationState( obbFiles: List = obbFiles(), zimFiles: List = zimFiles(), - assetFileDescriptor: AssetFileDescriptor? = getAssetFileDescriptorFromPlayAssetDelivery() + assetFileDescriptorList: List = + getAssetFileDescriptorListFromPlayAssetDelivery() ): ValidationState { return when { - assetFileDescriptor != null -> HasFile(null, assetFileDescriptor) + assetFileDescriptorList.isNotEmpty() -> HasFile(null, assetFileDescriptorList) obbFiles.isNotEmpty() && zimFiles().isNotEmpty() -> HasBothFiles(obbFiles[0], zimFiles[0]) obbFiles.isNotEmpty() -> HasFile(obbFiles[0]) zimFiles.isNotEmpty() -> HasFile(zimFiles[0]) @@ -55,11 +57,15 @@ class CustomFileValidator @Inject constructor(private val context: Context) { } @Suppress("MagicNumber") - private fun getAssetFileDescriptorFromPlayAssetDelivery(): AssetFileDescriptor? { + private fun getAssetFileDescriptorListFromPlayAssetDelivery(): List { try { - val context = context.createPackageContext(context.packageName, 0) - val assetManager = context.assets - return assetManager.openFd(BuildConfig.PLAY_ASSET_FILE) + val assetManager = context.createPackageContext(context.packageName, 0).assets + val assetFileDescriptorList: ArrayList = arrayListOf() + getChunksList(assetManager).forEach { + assetFileDescriptorList.add(assetManager.openFd(it)) + } + + return assetFileDescriptorList } catch (packageNameNotFoundException: PackageManager.NameNotFoundException) { Log.w( "ASSET_PACKAGE_DELIVERY", @@ -68,7 +74,24 @@ class CustomFileValidator @Inject constructor(private val context: Context) { } catch (ioException: IOException) { Log.w("ASSET_PACKAGE_DELIVERY", "Unable to copy the content of asset $ioException") } - return null + return emptyList() + } + + private fun getChunksList(assetManager: AssetManager): List { + val chunkFiles = mutableListOf() + + try { + // List of all files in the asset directory + val assets = assetManager.list("") ?: emptyArray() + + // Filter and count chunk files. + assets.filterTo(chunkFiles) { it.startsWith("chunk") && it.endsWith(".zim") } + chunkFiles.sortBy { it.substringAfter("chunk").substringBefore(".zim").toInt() } + } catch (ioException: IOException) { + ioException.printStackTrace() + } + + return chunkFiles } private fun obbFiles() = scanDirs(ContextCompat.getObbDirs(context), "obb") @@ -104,7 +127,10 @@ class CustomFileValidator @Inject constructor(private val context: Context) { sealed class ValidationState { data class HasBothFiles(val obbFile: File, val zimFile: File) : ValidationState() - data class HasFile(val file: File?, val assetFileDescriptor: AssetFileDescriptor? = null) : + data class HasFile( + val file: File?, + val assetFileDescriptorList: List = emptyList() + ) : ValidationState() object HasNothing : ValidationState() diff --git a/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomReaderFragment.kt b/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomReaderFragment.kt index 84f5c7118..429d2ba2c 100644 --- a/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomReaderFragment.kt +++ b/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomReaderFragment.kt @@ -181,8 +181,8 @@ class CustomReaderFragment : CoreReaderFragment() { onFilesFound = { when (it) { is ValidationState.HasFile -> { - if (it.assetFileDescriptor != null) { - openZimFile(null, true, it.assetFileDescriptor) + if (it.assetFileDescriptorList.isNotEmpty()) { + openZimFile(null, true, it.assetFileDescriptorList) } else { openZimFile(it.file, true) }