From c478b8646a1bd3590c478b8f167c54b75ff042b7 Mon Sep 17 00:00:00 2001 From: MohitMaliFtechiz Date: Mon, 8 Jul 2024 21:14:55 +0530 Subject: [PATCH 1/3] Fixed: Playing video is too slow to start. * This video's size was already coming in the data, and we were again loading the url for getting the size of content while returning the webViewResponse to webView for rendering so we have refactored our code to use the same existing data, and if existing data does not have the size then we are getting the video size from libzim. * Rendering the HTML data on the IO thread. * We were creating the file objects twice to get the inputStream of a video file which takes more time to render. Now we are using the same file object for getting the inputStream. * These changes reduced the video loading time --- .../kiwixmobile/core/reader/ZimFileReader.kt | 23 +++++++++---------- .../core/reader/ZimReaderContainer.kt | 14 ++++++++++- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt index 7b596ad8e..9b25492ea 100644 --- a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt +++ b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt @@ -23,9 +23,9 @@ import android.net.Uri import android.os.ParcelFileDescriptor import android.util.Base64 import androidx.core.net.toUri -import io.reactivex.Completable -import io.reactivex.schedulers.Schedulers +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.kiwix.kiwixmobile.core.CoreApp import org.kiwix.kiwixmobile.core.NightModeConfig @@ -276,14 +276,14 @@ class ZimFileReader constructor( } private fun loadAsset(uri: String): InputStream? { - val article = try { + val item = try { jniKiwixReader.getEntryByPath(uri.filePath).getItem(true) } catch (exception: Exception) { Log.e(TAG, "Could not get Item for uri = $uri \n original exception = $exception") null } val infoPair = try { - article?.directAccessInformation + item?.directAccessInformation } catch (ignore: Exception) { Log.e( TAG, @@ -292,12 +292,13 @@ class ZimFileReader constructor( ) null } - if (infoPair == null || !File(infoPair.filename).exists()) { + val file = infoPair?.filename?.let(::File) + if (infoPair == null || file == null || !file.exists()) { return loadAssetFromCache(uri) } - return article?.size?.let { + return item?.size?.let { AssetFileDescriptor( - infoPair.parcelFileDescriptor, + infoPair.parcelFileDescriptor(file), infoPair.offset, it ).createInputStream() @@ -317,7 +318,7 @@ class ZimFileReader constructor( @SuppressLint("CheckResult") private fun streamZimContentToPipe(uri: String, outputStream: OutputStream) { - Completable.fromAction { + CoroutineScope(Dispatchers.IO).launch { try { outputStream.use { if (uri.endsWith(UNINITIALISER_ADDRESS)) { @@ -335,8 +336,6 @@ class ZimFileReader constructor( Log.e(TAG, "error writing pipe for $uri", ioException) } } - .subscribeOn(Schedulers.io()) - .subscribe({ }, Throwable::printStackTrace) } fun getItem(url: String): Item? = @@ -447,8 +446,8 @@ val String.truncateMimeType: String val String.replaceWithEncodedString: String get() = replace("?", "%3F") -private val DirectAccessInfo.parcelFileDescriptor: ParcelFileDescriptor? - get() = ParcelFileDescriptor.open(File(filename), ParcelFileDescriptor.MODE_READ_ONLY) +private fun DirectAccessInfo.parcelFileDescriptor(file: File): ParcelFileDescriptor? = + ParcelFileDescriptor.open(file, ParcelFileDescriptor.MODE_READ_ONLY) // Default illustration size for ZIM file favicons const val ILLUSTRATION_SIZE = 48 diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt index 514e5c346..1b26a6eab 100644 --- a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt +++ b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt @@ -74,7 +74,19 @@ class ZimReaderContainer @Inject constructor(private val zimFileReaderFactory: F val headers = mutableMapOf("Accept-Ranges" to "bytes") if ("Range" in requestHeaders.keys) { setStatusCodeAndReasonPhrase(HttpURLConnection.HTTP_PARTIAL, "Partial Content") - val fullSize = zimFileReader?.getItem(url)?.size ?: 0L + val fullSize = when { + // check if the previously loaded data has the size then return it. + // It will prevent the again data loading for those videos that are already loaded. + // See #3909 + data?.available() != null && data.available().toLong() != 0L -> + data.available().toLong() + + else -> { + // if the loaded data does not have the size, especially for YT videos. + // Then get the content size from libzim and set it to the headers. + zimFileReader?.getItem(url)?.size ?: 0L + } + } val lastByte = fullSize - 1 val byteRanges = requestHeaders.getValue("Range").substringAfter("=").split("-") headers["Content-Range"] = "bytes ${byteRanges[0]}-$lastByte/$fullSize" From 2e39330ba055592f27abd9a6f82b158fa618aede Mon Sep 17 00:00:00 2001 From: MohitMaliFtechiz Date: Wed, 10 Jul 2024 19:22:09 +0530 Subject: [PATCH 2/3] Returning inputStream in better way to webView for rendering. * Loading the video on the background thread so that it will not block the UI thread, and the user can easily use the UI(For better video loading in case of a large video file) while ensuring that it returns the inputStream. --- .../kiwixmobile/core/reader/ZimFileReader.kt | 12 +++++------ .../core/reader/ZimReaderContainer.kt | 21 ++++--------------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt index 9b25492ea..c12ef9540 100644 --- a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt +++ b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt @@ -219,16 +219,16 @@ class ZimFileReader constructor( } @Suppress("UnreachableCode") - fun load(uri: String): InputStream? { + suspend fun load(uri: String): InputStream? = withContext(Dispatchers.IO) { val extension = uri.substringAfterLast(".") if (assetExtensions.any { it == extension }) { try { - return loadAsset(uri) + return@withContext loadAsset(uri) } catch (ioException: IOException) { Log.e(TAG, "failed to write video for $uri", ioException) } } - return loadContent(uri) + return@withContext loadContent(uri) } fun getMimeTypeFromUrl(uri: String): String? = getItem(uri)?.mimetype @@ -275,7 +275,7 @@ class ZimFileReader constructor( throw IOException("Could not open pipe for $uri", ioException) } - private fun loadAsset(uri: String): InputStream? { + private suspend fun loadAsset(uri: String): InputStream? = withContext(Dispatchers.IO) { val item = try { jniKiwixReader.getEntryByPath(uri.filePath).getItem(true) } catch (exception: Exception) { @@ -294,9 +294,9 @@ class ZimFileReader constructor( } val file = infoPair?.filename?.let(::File) if (infoPair == null || file == null || !file.exists()) { - return loadAssetFromCache(uri) + return@withContext loadAssetFromCache(uri) } - return item?.size?.let { + return@withContext item?.size?.let { AssetFileDescriptor( infoPair.parcelFileDescriptor(file), infoPair.offset, diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt index 1b26a6eab..46027b840 100644 --- a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt +++ b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt @@ -63,30 +63,17 @@ class ZimReaderContainer @Inject constructor(private val zimFileReaderFactory: F fun getRandomArticleUrl() = zimFileReader?.getRandomArticleUrl() fun isRedirect(url: String): Boolean = zimFileReader?.isRedirect(url) == true fun getRedirect(url: String): String = zimFileReader?.getRedirect(url) ?: "" - fun load(url: String, requestHeaders: Map): WebResourceResponse { - val data = zimFileReader?.load(url) - return WebResourceResponse( + fun load(url: String, requestHeaders: Map): WebResourceResponse = runBlocking { + return@runBlocking WebResourceResponse( zimFileReader?.getMimeTypeFromUrl(url), Charsets.UTF_8.name(), - data + zimFileReader?.load(url) ) .apply { val headers = mutableMapOf("Accept-Ranges" to "bytes") if ("Range" in requestHeaders.keys) { setStatusCodeAndReasonPhrase(HttpURLConnection.HTTP_PARTIAL, "Partial Content") - val fullSize = when { - // check if the previously loaded data has the size then return it. - // It will prevent the again data loading for those videos that are already loaded. - // See #3909 - data?.available() != null && data.available().toLong() != 0L -> - data.available().toLong() - - else -> { - // if the loaded data does not have the size, especially for YT videos. - // Then get the content size from libzim and set it to the headers. - zimFileReader?.getItem(url)?.size ?: 0L - } - } + val fullSize = zimFileReader?.getItem(url)?.size ?: 0L val lastByte = fullSize - 1 val byteRanges = requestHeaders.getValue("Range").substringAfter("=").split("-") headers["Content-Range"] = "bytes ${byteRanges[0]}-$lastByte/$fullSize" From 8ce2dfee3c34e81531b8d533803310545a6ba13a Mon Sep 17 00:00:00 2001 From: MohitMaliFtechiz Date: Mon, 15 Jul 2024 18:30:33 +0530 Subject: [PATCH 3/3] Returning the inputStream for above 1MB item to webView. --- .../kiwixmobile/core/reader/ZimFileReader.kt | 61 +++++++++++++++---- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt index c12ef9540..43eb2b802 100644 --- a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt +++ b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt @@ -23,6 +23,7 @@ import android.net.Uri import android.os.ParcelFileDescriptor import android.util.Base64 import androidx.core.net.toUri +import eu.mhutti1.utils.storage.Kb import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch @@ -42,6 +43,7 @@ import org.kiwix.libzim.FdInput import org.kiwix.libzim.Item import org.kiwix.libzim.SuggestionSearch import org.kiwix.libzim.SuggestionSearcher +import java.io.ByteArrayInputStream import java.io.File import java.io.FileInputStream import java.io.IOException @@ -218,7 +220,6 @@ class ZimFileReader constructor( null } - @Suppress("UnreachableCode") suspend fun load(uri: String): InputStream? = withContext(Dispatchers.IO) { val extension = uri.substringAfterLast(".") if (assetExtensions.any { it == extension }) { @@ -228,7 +229,29 @@ class ZimFileReader constructor( Log.e(TAG, "failed to write video for $uri", ioException) } } - return@withContext loadContent(uri) + return@withContext loadContent(uri, extension) + } + + @Suppress("UnreachableCode", "NestedBlockDepth", "ReturnCount") + private fun loadContent(uri: String, extension: String): InputStream? { + val item = getItem(uri) + if (compressedExtensions.any { it != extension }) { + item?.size?.let { + // Check if the item size exceeds 1 MB + if (it / Kb > 1024) { + // Retrieve direct access information for the item + val infoPair = getDirectAccessInfoOfItem(item, uri) + val file = infoPair?.filename?.let(::File) + // If no file found or file does not exist, return input stream from item data + if (infoPair == null || file == null || !file.exists()) { + return@loadContent ByteArrayInputStream(item.data?.data) + } + // Return the input stream from the direct access information + return@loadContent getInputStreamFromDirectAccessInfo(item, file, infoPair) + } + } + } + return loadContent(item, uri) } fun getMimeTypeFromUrl(uri: String): String? = getItem(uri)?.mimetype @@ -267,10 +290,10 @@ class ZimFileReader constructor( private fun extractQueryParam(url: String): String = "?" + url.substringAfterLast("?", "") - private fun loadContent(uri: String) = + private fun loadContent(item: Item?, uri: String) = try { val outputStream = PipedOutputStream() - PipedInputStream(outputStream).also { streamZimContentToPipe(uri, outputStream) } + PipedInputStream(outputStream).also { streamZimContentToPipe(item, uri, outputStream) } } catch (ioException: IOException) { throw IOException("Could not open pipe for $uri", ioException) } @@ -282,7 +305,16 @@ class ZimFileReader constructor( Log.e(TAG, "Could not get Item for uri = $uri \n original exception = $exception") null } - val infoPair = try { + val infoPair = getDirectAccessInfoOfItem(item, uri) + val file = infoPair?.filename?.let(::File) + if (infoPair == null || file == null || !file.exists()) { + return@withContext loadAssetFromCache(uri) + } + return@withContext getInputStreamFromDirectAccessInfo(item, file, infoPair) + } + + private fun getDirectAccessInfoOfItem(item: Item?, uri: String): DirectAccessInfo? = + try { item?.directAccessInformation } catch (ignore: Exception) { Log.e( @@ -292,18 +324,19 @@ class ZimFileReader constructor( ) null } - val file = infoPair?.filename?.let(::File) - if (infoPair == null || file == null || !file.exists()) { - return@withContext loadAssetFromCache(uri) - } - return@withContext item?.size?.let { + + private fun getInputStreamFromDirectAccessInfo( + item: Item?, + file: File, + infoPair: DirectAccessInfo + ): InputStream? = + item?.size?.let { AssetFileDescriptor( infoPair.parcelFileDescriptor(file), infoPair.offset, it ).createInputStream() } - } @Throws(IOException::class) private fun loadAssetFromCache(uri: String): FileInputStream { @@ -317,14 +350,14 @@ class ZimFileReader constructor( private fun getContent(url: String) = getItem(url)?.data?.data @SuppressLint("CheckResult") - private fun streamZimContentToPipe(uri: String, outputStream: OutputStream) { + private fun streamZimContentToPipe(item: Item?, uri: String, outputStream: OutputStream) { CoroutineScope(Dispatchers.IO).launch { try { outputStream.use { if (uri.endsWith(UNINITIALISER_ADDRESS)) { it.write(UNINITIALISE_HTML.toByteArray()) } else { - getItem(uri)?.let { item -> + item?.let { item -> if ("text/css" == item.mimetype && nightModeConfig.isNightModeActive()) { it.write(INVERT_IMAGES_VIDEO.toByteArray()) } @@ -414,6 +447,8 @@ class ZimFileReader constructor( """.trimIndent() private val assetExtensions = listOf("3gp", "mp4", "m4a", "webm", "mkv", "ogg", "ogv", "svg", "warc") + private val compressedExtensions = + listOf("zip", "7z", "gz", "rar", "sitx") } }