kiwix/kiwix-android
1
0
Fork 0
mirror of https://github.com/kiwix/kiwix-android.git synced 2025-09-22 03:54:18 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fixed: Not able to upload 512MB+ custom apps anymore.

Browse Source 
* Introduced the splitting zim file with 500MB for custom apps. Since the bundle has a limit for a file to upload it uses the play asset delivery mode, and for new custom apps, playStore does not allow to upload the apk (with apk we can upload more than 1GB file). So to address this issue we have introduced a feature in libzim where we can load the zim files via the fd list.

* We have modified our code to upload the files with 500MB chunks in the asset folder, and later we are accessing these files from the asset folder and creating the archive object with the help of the new libzim feature.
This commit is contained in:
MohitMaliFtechiz 2024-04-22 18:48:37 +05:30
parent d09987612e
commit 69a7099bc6
9 changed files with 103 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/src/main/java/org/kiwix/kiwixmobile/nav/destination/reader/KiwixReaderFragment.kt
View File

@ -199,7 +199,7 @@ class KiwixReaderFragment : CoreReaderFragment() {
override fun onResume() {
super.onResume()
if (zimReaderContainer?.zimFile == null &&
zimReaderContainer?.zimFileReader?.assetFileDescriptor == null
zimReaderContainer?.zimFileReader?.assetFileDescriptorList?.isEmpty() == true
) {
exitBook()
}

2
buildSrc/src/main/kotlin/custom/CustomApps.kt
View File

@ -48,8 +48,6 @@ fun ProductFlavors.create(customApps: List<CustomApp>) {
buildConfigField("String", "ENFORCED_LANG", "\"${customApp.enforcedLanguage}\"")
buildConfigField("String", "ABOUT_APP_URL", "\"${customApp.aboutAppUrl}\"")
buildConfigField("String", "SUPPORT_URL", "\"${customApp.supportUrl}\"")
// Add asset file name in buildConfig file, we will use later to receive the zim file.
buildConfigField("String", "PLAY_ASSET_FILE", "\"${customApp.name}.zim\"")
buildConfigField("Boolean", "DISABLE_SIDEBAR", "${customApp.disableSideBar}")
buildConfigField("Boolean", "DISABLE_TABS", "${customApp.disableTabs}")
buildConfigField("Boolean", "DISABLE_READ_ALOUD", "${customApp.disableReadAloud}")

12
core/src/main/java/org/kiwix/kiwixmobile/core/main/CoreReaderFragment.kt
View File

@ -1554,7 +1554,7 @@ abstract class CoreReaderFragment :
protected fun openZimFile(
file: File?,
isCustomApp: Boolean = false,
assetFileDescriptor: AssetFileDescriptor? = null,
assetFileDescriptorList: List<AssetFileDescriptor> = emptyList(),
filePath: String? = null
) {
if (hasPermission(Manifest.permission.READ_EXTERNAL_STORAGE) || isCustomApp) {
@ -1564,10 +1564,10 @@ abstract class CoreReaderFragment :
reopenBook()
openAndSetInContainer(file = file)
updateTitle()
} else if (assetFileDescriptor != null) {
} else if (assetFileDescriptorList.isNotEmpty()) {
reopenBook()
openAndSetInContainer(
assetFileDescriptor = assetFileDescriptor,
assetFileDescriptorList = assetFileDescriptorList,
filePath = filePath
)
updateTitle()
@ -1602,7 +1602,7 @@ abstract class CoreReaderFragment :
private fun openAndSetInContainer(
file: File? = null,
assetFileDescriptor: AssetFileDescriptor? = null,
assetFileDescriptorList: List<AssetFileDescriptor> = emptyList(),
filePath: String? = null
) {
try {
@ -1613,9 +1613,9 @@ abstract class CoreReaderFragment :
e.printStackTrace()
}
zimReaderContainer?.let { zimReaderContainer ->
if (assetFileDescriptor != null) {
if (assetFileDescriptorList.isNotEmpty()) {
zimReaderContainer.setZimFileDescriptor(
assetFileDescriptor,
assetFileDescriptorList,
filePath = filePath
)
} else {

31
core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt
View File

@ -38,6 +38,7 @@ import org.kiwix.kiwixmobile.core.utils.files.Log
import org.kiwix.libkiwix.JNIKiwixException
import org.kiwix.libzim.Archive
import org.kiwix.libzim.DirectAccessInfo
import org.kiwix.libzim.FdInput
import org.kiwix.libzim.Item
import org.kiwix.libzim.SuggestionSearch
import org.kiwix.libzim.SuggestionSearcher
@ -56,7 +57,7 @@ private const val TAG = "ZimFileReader"
@Suppress("LongParameterList")
class ZimFileReader constructor(
val zimFile: File?,
val assetFileDescriptor: AssetFileDescriptor? = null,
val assetFileDescriptorList: List<AssetFileDescriptor> = emptyList(),
val assetDescriptorFilePath: String? = null,
val jniKiwixReader: Archive,
private val nightModeConfig: NightModeConfig,
@ -65,7 +66,7 @@ class ZimFileReader constructor(
interface Factory {
suspend fun create(file: File): ZimFileReader?
suspend fun create(
assetFileDescriptor: AssetFileDescriptor,
assetFileDescriptorList: List<AssetFileDescriptor>,
filePath: String? = null
): ZimFileReader?
@ -91,18 +92,19 @@ class ZimFileReader constructor(
}
override suspend fun create(
assetFileDescriptor: AssetFileDescriptor,
assetFileDescriptorList: List<AssetFileDescriptor>,
filePath: String?
): ZimFileReader? = withContext(Dispatchers.IO) { // Bug Fix #3805
try {
val archive = Archive(
assetFileDescriptor.parcelFileDescriptor.dup().fileDescriptor,
assetFileDescriptor.startOffset,
assetFileDescriptor.length
)
val fdInputArray = getFdInputArrayFromAssetFileDescriptorList(assetFileDescriptorList)
val archive = if (fdInputArray.size == 1) {
Archive(fdInputArray[0])
} else {
Archive(fdInputArray)
}
ZimFileReader(
null,
assetFileDescriptor,
assetFileDescriptorList,
assetDescriptorFilePath = filePath,
nightModeConfig = nightModeConfig,
jniKiwixReader = archive,
@ -116,6 +118,17 @@ class ZimFileReader constructor(
null
}
}
private fun getFdInputArrayFromAssetFileDescriptorList(
assetFileDescriptorList: List<AssetFileDescriptor>
): Array<FdInput> =
assetFileDescriptorList.map {
FdInput(
it.parcelFileDescriptor.dup().fileDescriptor,
it.startOffset,
it.length
)
}.toTypedArray()
}
}

8
core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt
View File

@ -46,12 +46,14 @@ class ZimReaderContainer @Inject constructor(private val zimFileReaderFactory: F
}
fun setZimFileDescriptor(
assetFileDescriptor: AssetFileDescriptor,
assetFileDescriptorList: List<AssetFileDescriptor>,
filePath: String? = null
) {
zimFileReader = runBlocking {
if (assetFileDescriptor.parcelFileDescriptor.dup().fileDescriptor.valid())
zimFileReaderFactory.create(assetFileDescriptor, filePath)
if (assetFileDescriptorList.isNotEmpty() &&
assetFileDescriptorList[0].parcelFileDescriptor.dup().fileDescriptor.valid()
)
zimFileReaderFactory.create(assetFileDescriptorList, filePath)
else null
}
}

3
core/src/main/java/org/kiwix/kiwixmobile/core/webserver/KiwixServer.kt
View File

@ -51,7 +51,8 @@ class KiwixServer @Inject constructor(
// Determine whether to create an Archive from an asset or a file path
val archive = if (path == getDemoFilePathForCustomApp(context)) {
// For custom apps using a demo file, create an Archive with FileDescriptor
val assetFileDescriptor = zimReaderContainer.zimFileReader?.assetFileDescriptor
val assetFileDescriptor =
zimReaderContainer.zimFileReader?.assetFileDescriptorList?.get(0)
val startOffset = assetFileDescriptor?.startOffset ?: 0L
val size = assetFileDescriptor?.length ?: 0L
Archive(

38
custom/build.gradle.kts
View File

@ -103,17 +103,39 @@ fun ProductFlavor.fetchRequest(): Request {
}
}
fun writeZimFileData(responseBody: ResponseBody, file: File) {
FileOutputStream(file).use { outputStream ->
responseBody.byteStream().use { inputStream ->
val buffer = ByteArray(4096)
var bytesRead: Int
while (inputStream.read(buffer).also { bytesRead = it } != -1) {
outputStream.write(buffer, 0, bytesRead)
fun writeZimFileData(
responseBody: ResponseBody,
file: File,
chunkSize: Long = 500 * 1024 * 1024 // create a chunk of 500MB
) {
var outputStream: FileOutputStream? = null
val buffer = ByteArray(4096)
var bytesRead: Int
var totalBytesWritten = 0L
var chunkNumber = 0
responseBody.byteStream().use { inputStream ->
while (inputStream.read(buffer).also { bytesRead = it } != -1) {
if (outputStream == null || totalBytesWritten >= chunkSize) {
// Close the current chunk and open a new one
outputStream?.flush()
outputStream?.close()
chunkNumber++
val nextChunkFile = File(file.parent, "chunk$chunkNumber.zim")
nextChunkFile.createNewFile()
outputStream = FileOutputStream(nextChunkFile)
totalBytesWritten = 0 // Reset totalBytesWritten for the new chunk
}
outputStream.flush()
// Write data to the output stream
outputStream?.write(buffer, 0, bytesRead)
totalBytesWritten += bytesRead
}
}
// Close the last chunk (if any)
outputStream?.flush()
outputStream?.close()
}
fun ProductFlavor.createDownloadTaskForPlayAssetDelivery(

44
custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomFileValidator.kt
View File

@ -21,9 +21,9 @@ package org.kiwix.kiwixmobile.custom.main
import android.content.Context
import android.content.pm.PackageManager
import android.content.res.AssetFileDescriptor
import android.content.res.AssetManager
import androidx.core.content.ContextCompat
import org.kiwix.kiwixmobile.core.utils.files.Log
import org.kiwix.kiwixmobile.custom.BuildConfig
import org.kiwix.kiwixmobile.custom.main.ValidationState.HasBothFiles
import org.kiwix.kiwixmobile.custom.main.ValidationState.HasFile
import org.kiwix.kiwixmobile.custom.main.ValidationState.HasNothing
@ -37,16 +37,18 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
when (val installationState = detectInstallationState()) {
is HasBothFiles,
is HasFile -> onFilesFound(installationState)
HasNothing -> onNoFilesFound()
}
private fun detectInstallationState(
obbFiles: List<File> = obbFiles(),
zimFiles: List<File> = zimFiles(),
assetFileDescriptor: AssetFileDescriptor? = getAssetFileDescriptorFromPlayAssetDelivery()
assetFileDescriptorList: List<AssetFileDescriptor> =
getAssetFileDescriptorListFromPlayAssetDelivery()
): ValidationState {
return when {
assetFileDescriptor != null -> HasFile(null, assetFileDescriptor)
assetFileDescriptorList.isNotEmpty() -> HasFile(null, assetFileDescriptorList)
obbFiles.isNotEmpty() && zimFiles().isNotEmpty() -> HasBothFiles(obbFiles[0], zimFiles[0])
obbFiles.isNotEmpty() -> HasFile(obbFiles[0])
zimFiles.isNotEmpty() -> HasFile(zimFiles[0])
@ -55,11 +57,15 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
}
@Suppress("MagicNumber")
private fun getAssetFileDescriptorFromPlayAssetDelivery(): AssetFileDescriptor? {
private fun getAssetFileDescriptorListFromPlayAssetDelivery(): List<AssetFileDescriptor> {
try {
val context = context.createPackageContext(context.packageName, 0)
val assetManager = context.assets
return assetManager.openFd(BuildConfig.PLAY_ASSET_FILE)
val assetManager = context.createPackageContext(context.packageName, 0).assets
val assetFileDescriptorList: ArrayList<AssetFileDescriptor> = arrayListOf()
getChunksList(assetManager).forEach {
assetFileDescriptorList.add(assetManager.openFd(it))
}
return assetFileDescriptorList
} catch (packageNameNotFoundException: PackageManager.NameNotFoundException) {
Log.w(
"ASSET_PACKAGE_DELIVERY",
@ -68,7 +74,24 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
} catch (ioException: IOException) {
Log.w("ASSET_PACKAGE_DELIVERY", "Unable to copy the content of asset $ioException")
}
return null
return emptyList()
}
private fun getChunksList(assetManager: AssetManager): List<String> {
val chunkFiles = mutableListOf<String>()
try {
// List of all files in the asset directory
val assets = assetManager.list("") ?: emptyArray()
// Filter and count chunk files.
assets.filterTo(chunkFiles) { it.startsWith("chunk") && it.endsWith(".zim") }
chunkFiles.sortBy { it.substringAfter("chunk").substringBefore(".zim").toInt() }
} catch (ioException: IOException) {
ioException.printStackTrace()
}
return chunkFiles
}
private fun obbFiles() = scanDirs(ContextCompat.getObbDirs(context), "obb")
@ -104,7 +127,10 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
sealed class ValidationState {
data class HasBothFiles(val obbFile: File, val zimFile: File) : ValidationState()
data class HasFile(val file: File?, val assetFileDescriptor: AssetFileDescriptor? = null) :
data class HasFile(
val file: File?,
val assetFileDescriptorList: List<AssetFileDescriptor> = emptyList()
) :
ValidationState()
object HasNothing : ValidationState()

4
custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomReaderFragment.kt
View File

@ -181,8 +181,8 @@ class CustomReaderFragment : CoreReaderFragment() {
onFilesFound = {
when (it) {
is ValidationState.HasFile -> {
if (it.assetFileDescriptor != null) {
openZimFile(null, true, it.assetFileDescriptor)
if (it.assetFileDescriptorList.isNotEmpty()) {
openZimFile(null, true, it.assetFileDescriptorList)
} else {
openZimFile(it.file, true)
}