Writing zim file in chunks with play asset delivery mode.

* We are reading the chunks and loading the data in ZimFileReader.
This commit is contained in:
MohitMali 2023-11-07 18:35:30 +05:30
parent 1b1d874c89
commit 54f8354104
2 changed files with 78 additions and 10 deletions

View File

@ -103,17 +103,40 @@ fun ProductFlavor.fetchRequest(): Request {
} }
} }
fun writeZimFileData(responseBody: ResponseBody, file: File) { fun writeZimFileData(responseBody: ResponseBody, file: File, chunkSize: Long = 100 * 1024 * 1024) {
FileOutputStream(file).use { outputStream -> var outputStream: FileOutputStream? = null
responseBody.byteStream().use { inputStream -> val buffer = ByteArray(4096)
val buffer = ByteArray(4096) var bytesRead: Int
var bytesRead: Int var totalBytesWritten = 0L
while (inputStream.read(buffer).also { bytesRead = it } != -1) { var chunkNumber = 0
outputStream.write(buffer, 0, bytesRead)
responseBody.byteStream().use { inputStream ->
while (inputStream.read(buffer).also { bytesRead = it } != -1) {
if (outputStream == null) {
// Create a new chunk file
val nextChunkFile = File(file.parent, "chunk$chunkNumber.zim")
nextChunkFile.createNewFile()
outputStream = FileOutputStream(nextChunkFile)
}
// Write the buffer to the output stream
outputStream?.write(buffer, 0, bytesRead)
totalBytesWritten += bytesRead
// Check if we've reached the chunk size, and if so, close the current chunk
if (totalBytesWritten >= chunkSize) {
outputStream?.flush()
outputStream?.close()
outputStream = null
chunkNumber++
totalBytesWritten = 0 // Reset the totalBytesWritten for the next chunk
} }
outputStream.flush()
} }
} }
// Close the last chunk (if any)
outputStream?.flush()
outputStream?.close()
} }
fun ProductFlavor.createDownloadTaskForPlayAssetDelivery( fun ProductFlavor.createDownloadTaskForPlayAssetDelivery(

View File

@ -21,13 +21,15 @@ package org.kiwix.kiwixmobile.custom.main
import android.content.Context import android.content.Context
import android.content.pm.PackageManager import android.content.pm.PackageManager
import android.content.res.AssetFileDescriptor import android.content.res.AssetFileDescriptor
import android.content.res.AssetManager
import android.util.Log import android.util.Log
import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat
import org.kiwix.kiwixmobile.custom.BuildConfig import org.kiwix.kiwixmobile.core.utils.files.FileUtils
import org.kiwix.kiwixmobile.custom.main.ValidationState.HasBothFiles import org.kiwix.kiwixmobile.custom.main.ValidationState.HasBothFiles
import org.kiwix.kiwixmobile.custom.main.ValidationState.HasFile import org.kiwix.kiwixmobile.custom.main.ValidationState.HasFile
import org.kiwix.kiwixmobile.custom.main.ValidationState.HasNothing import org.kiwix.kiwixmobile.custom.main.ValidationState.HasNothing
import java.io.File import java.io.File
import java.io.FileOutputStream
import java.io.IOException import java.io.IOException
import javax.inject.Inject import javax.inject.Inject
@ -59,7 +61,34 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
try { try {
val context = context.createPackageContext(context.packageName, 0) val context = context.createPackageContext(context.packageName, 0)
val assetManager = context.assets val assetManager = context.assets
return assetManager.openFd(BuildConfig.PLAY_ASSET_FILE) val assetFileDescriptorList: ArrayList<AssetFileDescriptor> = arrayListOf()
getChunksList(assetManager).forEach {
assetFileDescriptorList.add(assetManager.openFd(it))
}
val combinedFilePath = FileUtils.getDemoFilePathForCustomApp(context)
val combinedFileOutputStream = FileOutputStream(combinedFilePath)
val chunkSize = 100 * 1024 * 1024
for (chunkNumber in 0 until assetFileDescriptorList.size) {
val chunkFileName = "chunk$chunkNumber.zim"
val chunkFileInputStream =
context.assets.open(chunkFileName)
val buffer = ByteArray(4096)
var bytesRead: Int
while (chunkFileInputStream.read(buffer).also { bytesRead = it } != -1) {
combinedFileOutputStream.write(
buffer,
0,
bytesRead
)
}
chunkFileInputStream.close()
}
return assetFileDescriptorList[0]
} catch (packageNameNotFoundException: PackageManager.NameNotFoundException) { } catch (packageNameNotFoundException: PackageManager.NameNotFoundException) {
Log.w( Log.w(
"ASSET_PACKAGE_DELIVERY", "ASSET_PACKAGE_DELIVERY",
@ -71,6 +100,22 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
return null return null
} }
private fun getChunksList(assetManager: AssetManager): MutableList<String> {
val chunkFiles = mutableListOf<String>()
try {
// List all files in the asset directory
val assets = assetManager.list("") ?: emptyArray()
// Filter and count chunk files based on your naming convention
assets.filterTo(chunkFiles) { it.startsWith("chunk") && it.endsWith(".zim") }
} catch (ioException: IOException) {
ioException.printStackTrace()
}
return chunkFiles
}
private fun obbFiles() = scanDirs(ContextCompat.getObbDirs(context), "obb") private fun obbFiles() = scanDirs(ContextCompat.getObbDirs(context), "obb")
private fun zimFiles(): List<File> { private fun zimFiles(): List<File> {