Writing zim file in chunks with play asset delivery mode.

* We are reading the chunks and loading the data in ZimFileReader.
This commit is contained in:
MohitMali 2023-11-07 18:35:30 +05:30
parent 1b1d874c89
commit 54f8354104
2 changed files with 78 additions and 10 deletions

View File

@ -103,17 +103,40 @@ fun ProductFlavor.fetchRequest(): Request {
}
}
fun writeZimFileData(responseBody: ResponseBody, file: File) {
FileOutputStream(file).use { outputStream ->
responseBody.byteStream().use { inputStream ->
val buffer = ByteArray(4096)
var bytesRead: Int
while (inputStream.read(buffer).also { bytesRead = it } != -1) {
outputStream.write(buffer, 0, bytesRead)
fun writeZimFileData(responseBody: ResponseBody, file: File, chunkSize: Long = 100 * 1024 * 1024) {
var outputStream: FileOutputStream? = null
val buffer = ByteArray(4096)
var bytesRead: Int
var totalBytesWritten = 0L
var chunkNumber = 0
responseBody.byteStream().use { inputStream ->
while (inputStream.read(buffer).also { bytesRead = it } != -1) {
if (outputStream == null) {
// Create a new chunk file
val nextChunkFile = File(file.parent, "chunk$chunkNumber.zim")
nextChunkFile.createNewFile()
outputStream = FileOutputStream(nextChunkFile)
}
// Write the buffer to the output stream
outputStream?.write(buffer, 0, bytesRead)
totalBytesWritten += bytesRead
// Check if we've reached the chunk size, and if so, close the current chunk
if (totalBytesWritten >= chunkSize) {
outputStream?.flush()
outputStream?.close()
outputStream = null
chunkNumber++
totalBytesWritten = 0 // Reset the totalBytesWritten for the next chunk
}
outputStream.flush()
}
}
// Close the last chunk (if any)
outputStream?.flush()
outputStream?.close()
}
fun ProductFlavor.createDownloadTaskForPlayAssetDelivery(

View File

@ -21,13 +21,15 @@ package org.kiwix.kiwixmobile.custom.main
import android.content.Context
import android.content.pm.PackageManager
import android.content.res.AssetFileDescriptor
import android.content.res.AssetManager
import android.util.Log
import androidx.core.content.ContextCompat
import org.kiwix.kiwixmobile.custom.BuildConfig
import org.kiwix.kiwixmobile.core.utils.files.FileUtils
import org.kiwix.kiwixmobile.custom.main.ValidationState.HasBothFiles
import org.kiwix.kiwixmobile.custom.main.ValidationState.HasFile
import org.kiwix.kiwixmobile.custom.main.ValidationState.HasNothing
import java.io.File
import java.io.FileOutputStream
import java.io.IOException
import javax.inject.Inject
@ -59,7 +61,34 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
try {
val context = context.createPackageContext(context.packageName, 0)
val assetManager = context.assets
return assetManager.openFd(BuildConfig.PLAY_ASSET_FILE)
val assetFileDescriptorList: ArrayList<AssetFileDescriptor> = arrayListOf()
getChunksList(assetManager).forEach {
assetFileDescriptorList.add(assetManager.openFd(it))
}
val combinedFilePath = FileUtils.getDemoFilePathForCustomApp(context)
val combinedFileOutputStream = FileOutputStream(combinedFilePath)
val chunkSize = 100 * 1024 * 1024
for (chunkNumber in 0 until assetFileDescriptorList.size) {
val chunkFileName = "chunk$chunkNumber.zim"
val chunkFileInputStream =
context.assets.open(chunkFileName)
val buffer = ByteArray(4096)
var bytesRead: Int
while (chunkFileInputStream.read(buffer).also { bytesRead = it } != -1) {
combinedFileOutputStream.write(
buffer,
0,
bytesRead
)
}
chunkFileInputStream.close()
}
return assetFileDescriptorList[0]
} catch (packageNameNotFoundException: PackageManager.NameNotFoundException) {
Log.w(
"ASSET_PACKAGE_DELIVERY",
@ -71,6 +100,22 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
return null
}
private fun getChunksList(assetManager: AssetManager): MutableList<String> {
val chunkFiles = mutableListOf<String>()
try {
// List all files in the asset directory
val assets = assetManager.list("") ?: emptyArray()
// Filter and count chunk files based on your naming convention
assets.filterTo(chunkFiles) { it.startsWith("chunk") && it.endsWith(".zim") }
} catch (ioException: IOException) {
ioException.printStackTrace()
}
return chunkFiles
}
private fun obbFiles() = scanDirs(ContextCompat.getObbDirs(context), "obb")
private fun zimFiles(): List<File> {