#1659 Autoload next video - replace content provider with WebResource input streams

build.gradle.kts

@@ -6,9 +6,6 @@ buildscript {
   dependencies {
     classpath(Libs.com_android_tools_build_gradle)
     classpath(Libs.kotlin_gradle_plugin)
-    classpath(Libs.ktlint_gradle)
-    classpath(Libs.jacoco_android)
-    classpath(Libs.detekt_gradle_plugin)
     // NOTE: Do not place your application dependencies here; they belong
     // in the individual module build.gradle files
   }

core/detekt_baseline.xml

@@ -4,6 +4,7 @@
   <Whitelist>
     <ID>EmptyFunctionBlock:BooksOnDiskViewHolder.kt$BookOnDiskViewHolder.BookViewHolder${ }</ID>
     <ID>EmptyFunctionBlock:FetchDownloadMonitor.kt$FetchDownloadMonitor.&lt;no name provided&gt;${}</ID>
+    <ID>ForbiddenComment:JNIInitialiser.kt$JNIInitialiser$// TODO: Consider surfacing to user</ID>
     <ID>LongParameterList:MainMenu.kt$MainMenu.Factory$( menu: Menu, webViews: MutableList&lt;KiwixWebView&gt;, urlIsValid: Boolean, menuClickListener: MenuClickListener, disableReadAloud: Boolean, disableTabs: Boolean )</ID>
     <ID>MagicNumber:ArticleCount.kt$ArticleCount$1000.0</ID>
     <ID>MagicNumber:ArticleCount.kt$ArticleCount$3</ID>
@@ -12,6 +13,7 @@
     <ID>MagicNumber:DownloaderModule.kt$DownloaderModule$5</ID>
     <ID>MagicNumber:FetchDownloadRequester.kt$10</ID>
     <ID>MagicNumber:FileUtils.kt$FileUtils$3</ID>
+    <ID>MagicNumber:JNIInitialiser.kt$JNIInitialiser$1024</ID>
     <ID>MagicNumber:KiloByte.kt$KiloByte$1024.0</ID>
     <ID>MagicNumber:MainMenu.kt$MainMenu$99</ID>
     <ID>MagicNumber:SearchResultGenerator.kt$ZimSearchResultGenerator$200</ID>
@@ -20,6 +22,7 @@
     <ID>MagicNumber:Seconds.kt$Seconds$60.0</ID>
     <ID>NestedBlockDepth:FileUtils.kt$FileUtils$deleteZimFile</ID>
     <ID>NestedBlockDepth:ImageUtils.kt$ImageUtils$getBitmapFromView</ID>
+    <ID>NestedBlockDepth:JNIInitialiser.kt$JNIInitialiser$loadICUData</ID>
     <ID>NestedBlockDepth:StorageDeviceUtils.kt$StorageDeviceUtils$canWrite</ID>
     <ID>PackageNaming:ArticleCount.kt$package org.kiwix.kiwixmobile.core.zim_manager.fileselect_view</ID>
     <ID>PackageNaming:BookOnDiskDelegate.kt$package org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.adapter</ID>
@@ -36,6 +39,7 @@
     <ID>ReturnCount:FileUtils.kt$FileUtils$@JvmStatic fun hasPart(file: File): Boolean</ID>
     <ID>ReturnCount:FileUtils.kt$FileUtils$@Synchronized private fun deleteZimFileParts(path: String): Boolean</ID>
     <ID>ReturnCount:ImageUtils.kt$ImageUtils$private fun getBitmapFromView(width: Int, height: Int, viewToDrawFrom: View): Bitmap?</ID>
+    <ID>TooGenericExceptionCaught:JNIInitialiser.kt$JNIInitialiser$e: Exception</ID>
     <ID>TooGenericExceptionThrown:AbstractContentProvider.kt$AbstractContentProvider$throw RuntimeException("Operation not supported")</ID>
     <ID>TooGenericExceptionThrown:AdapterDelegateManager.kt$AdapterDelegateManager$throw RuntimeException("No delegate registered for $item")</ID>
     <ID>TooGenericExceptionThrown:Bytes.kt$Bytes$throw RuntimeException("impossible value $size")</ID>
@@ -51,7 +55,6 @@
     <ID>TooManyFunctions:NewBookDao.kt$NewBookDao$NewBookDao</ID>
     <ID>TooManyFunctions:Repository.kt$Repository$Repository</ID>
     <ID>TooManyFunctions:ZimFileReader.kt$ZimFileReader$ZimFileReader</ID>
-    <ID>TooManyFunctions:ZimReaderContainer.kt$ZimReaderContainer$ZimReaderContainer</ID>
     <ID>TopLevelPropertyNaming:Bytes.kt$const val Eb = Pb * 1024</ID>
     <ID>TopLevelPropertyNaming:Bytes.kt$const val Gb = Mb * 1024</ID>
     <ID>TopLevelPropertyNaming:Bytes.kt$const val Kb = 1 * 1024L</ID>

core/src/main/AndroidManifest.xml

@@ -29,11 +29,6 @@
 
     <activity android:name=".bookmark.BookmarksActivity" />
 
-    <provider
-      android:name=".reader.ZimContentProvider"
-      android:authorities="${applicationId}.zim.base"
-      android:exported="true" />
-
     <activity
       android:name=".error.ErrorActivity"
       android:process=":error_activity" />

core/src/main/java/org/kiwix/kiwixmobile/core/CoreApp.java

@@ -49,6 +49,8 @@ public abstract class CoreApp extends Application {
   NightModeConfig nightModeConfig;
   @Inject
   KiwixDatabase kiwixDatabase;
+  @Inject
+  JNIInitialiser jniInitialiser;
 
   public static CoreApp getInstance() {
     return app;
@@ -80,6 +82,7 @@ public abstract class CoreApp extends Application {
     AndroidThreeTen.init(this);
     writeLogFile();
     coreComponent.inject(this);
+    jniInitialiser.init();
     kiwixDatabase.forceMigration();
     downloadMonitor.init();
     nightModeConfig.init();

core/src/main/java/org/kiwix/kiwixmobile/core/JNIInitialiser.kt

@@ -0,0 +1,61 @@
+/*
+ * Kiwix Android
+ * Copyright (c) 2020 Kiwix <android.kiwix.org>
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+package org.kiwix.kiwixmobile.core
+
+import android.content.Context
+import android.util.Log
+import org.kiwix.kiwixlib.JNIKiwix
+import org.kiwix.kiwixmobile.core.utils.TAG_KIWIX
+import java.io.File
+import java.io.FileOutputStream
+import javax.inject.Inject
+
+internal class JNIInitialiser @Inject constructor(context: Context, jniKiwix: JNIKiwix) {
+  init {
+    loadICUData(context)?.let(jniKiwix::setDataDirectory)
+  }
+
+  private fun loadICUData(context: Context): String? {
+    return try {
+      val icuDir = File(context.filesDir, "icu")
+      if (!icuDir.exists()) {
+        icuDir.mkdirs()
+      }
+      val icuFileNames = context.assets.list("icu") ?: emptyArray()
+      for (icuFileName in icuFileNames) {
+        val icuDataFile = File(icuDir, icuFileName)
+        if (!icuDataFile.exists()) {
+          FileOutputStream(icuDataFile).use { outputStream ->
+            context.assets.open("icu/$icuFileName").use { inputStream ->
+              inputStream.copyTo(outputStream, 1024)
+            }
+          }
+        }
+      }
+      icuDir.absolutePath
+    } catch (e: Exception) {
+      Log.w(TAG_KIWIX, "Error copying icu data file", e)
+      // TODO: Consider surfacing to user
+      null
+    }
+  }
+
+  fun init() {
+    //  empty method so class is not reported as unused
+  }
+}

core/src/main/java/org/kiwix/kiwixmobile/core/di/components/CoreComponent.kt

@@ -48,7 +48,6 @@ import org.kiwix.kiwixmobile.core.help.HelpActivity
 import org.kiwix.kiwixmobile.core.history.HistoryModule
 import org.kiwix.kiwixmobile.core.main.AddNoteDialog
 import org.kiwix.kiwixmobile.core.main.KiwixWebView
-import org.kiwix.kiwixmobile.core.reader.ZimContentProvider
 import org.kiwix.kiwixmobile.core.reader.ZimFileReader
 import org.kiwix.kiwixmobile.core.reader.ZimReaderContainer
 import org.kiwix.kiwixmobile.core.search.SearchActivity
@@ -102,7 +101,6 @@ interface CoreComponent {
   fun notificationManager(): NotificationManager
 
   fun inject(application: CoreApp)
-  fun inject(zimContentProvider: ZimContentProvider)
   fun inject(kiwixWebView: KiwixWebView)
   fun inject(storageSelectDialog: StorageSelectDialog)
   fun inject(addNoteDialog: AddNoteDialog)

core/src/main/java/org/kiwix/kiwixmobile/core/main/CoreWebViewClient.java

@@ -24,8 +24,10 @@ import android.util.Log;
 import android.view.LayoutInflater;
 import android.view.View;
 import android.webkit.MimeTypeMap;
+import android.webkit.WebResourceResponse;
 import android.webkit.WebView;
 import android.webkit.WebViewClient;
+import androidx.annotation.Nullable;
 import java.util.HashMap;
 import org.kiwix.kiwixmobile.core.CoreApp;
 import org.kiwix.kiwixmobile.core.R;
@@ -137,4 +139,14 @@ public abstract class CoreWebViewClient extends WebViewClient {
     view.removeAllViews();
     view.addView(home);
   }
+
+  @Nullable
+  @Override
+  public WebResourceResponse shouldInterceptRequest(WebView view, String url) {
+    if (url.startsWith("content")) {
+      return zimReaderContainer.load(url);
+    } else {
+      return super.shouldInterceptRequest(view, url);
+    }
+  }
 }

core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimContentProvider.java

@@ -1,120 +0,0 @@
-/*
- * Kiwix Android
- * Copyright (c) 2019 Kiwix <android.kiwix.org>
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-package org.kiwix.kiwixmobile.core.reader;
-
-import android.content.Context;
-import android.net.Uri;
-import android.os.ParcelFileDescriptor;
-import android.util.Log;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import javax.inject.Inject;
-import org.kiwix.kiwixlib.JNIKiwix;
-import org.kiwix.kiwixmobile.core.CoreApp;
-import org.kiwix.kiwixmobile.core.data.AbstractContentProvider;
-
-import static org.kiwix.kiwixmobile.core.utils.ConstantsKt.TAG_KIWIX;
-
-public class ZimContentProvider extends AbstractContentProvider {
-
-  @Inject
-  public JNIKiwix jniKiwix;
-  @Inject
-  ZimReaderContainer zimReaderContainer;
-
-  @Override
-  public boolean onCreate() {
-    CoreApp.getCoreComponent().inject(this);
-    setIcuDataDirectory();
-    return true;
-  }
-
-  @Override
-  public String getType(Uri uri) {
-    return zimReaderContainer.readMimeType(uri);
-  }
-
-  @Override
-  public ParcelFileDescriptor openFile(Uri uri, String mode) {
-    return zimReaderContainer.load(uri);
-  }
-
-  private void setIcuDataDirectory() {
-    String icuDirPath = loadICUData(getContext());
-    if (icuDirPath != null) {
-      Log.d(TAG_KIWIX, "Setting the ICU directory path to " + icuDirPath);
-      jniKiwix.setDataDirectory(icuDirPath);
-    }
-  }
-
-  private String loadICUData(Context context) {
-    try {
-      File icuDir = new File(context.getFilesDir(), "icu");
-      if (!icuDir.exists()) {
-        icuDir.mkdirs();
-      }
-      String[] icuFileNames = context.getAssets().list("icu");
-      for (int i = 0; i < icuFileNames.length; i++) {
-        String icuFileName = icuFileNames[i];
-        File icuDataFile = new File(icuDir, icuFileName);
-        if (!icuDataFile.exists()) {
-          InputStream in = context.getAssets().open("icu/" + icuFileName);
-          OutputStream out = new FileOutputStream(icuDataFile);
-          byte[] buf = new byte[1024];
-          int len;
-          while ((len = in.read(buf)) > 0) {
-            out.write(buf, 0, len);
-          }
-          in.close();
-          out.flush();
-          out.close();
-        }
-      }
-      return icuDir.getAbsolutePath();
-    } catch (Exception e) {
-      Log.w(TAG_KIWIX, "Error copying icu data file", e);
-      //TODO: Consider surfacing to user
-      return null;
-    }
-  }
-
-  private static String getFulltextIndexPath(String file) {
-    String[] names = { file, file };
-
-    /* File might be a ZIM chunk like foobar.zimaa */
-    if (!names[0].substring(names[0].length() - 3).equals("zim")) {
-      names[0] = names[0].substring(0, names[0].length() - 2);
-    }
-
-    /* Try to find a *.idx fulltext file/directory beside the ZIM
-     * file. Returns <zimfile>.zim.idx or <zimfile>.zimaa.idx. */
-    for (String name : names) {
-      File f = new File(name + ".idx");
-      if (f.exists() && f.isDirectory()) {
-        return f.getPath();
-      }
-    }
-
-    /* If no separate fulltext index file found then returns the ZIM
-     * file path itself (embedded fulltext index) */
-    return file;
-  }
-}

core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt

@@ -17,14 +17,13 @@
  */
 package org.kiwix.kiwixmobile.core.reader
 
+import android.content.res.AssetFileDescriptor
 import android.net.Uri
 import android.os.ParcelFileDescriptor
-import android.os.ParcelFileDescriptor.AutoCloseOutputStream
-import android.os.ParcelFileDescriptor.dup
 import android.util.Log
 import android.webkit.MimeTypeMap
 import androidx.core.net.toUri
-import io.reactivex.Single
+import io.reactivex.Completable
 import io.reactivex.schedulers.Schedulers
 import org.kiwix.kiwixlib.JNIKiwixException
 import org.kiwix.kiwixlib.JNIKiwixInt
@@ -38,10 +37,12 @@ import org.kiwix.kiwixmobile.core.reader.ZimFileReader.Companion.CONTENT_URI
 import org.kiwix.kiwixmobile.core.search.SearchSuggestion
 import org.kiwix.kiwixmobile.core.utils.files.FileUtils
 import java.io.File
-import java.io.FileDescriptor
+import java.io.FileInputStream
-import java.io.FileOutputStream
 import java.io.IOException
-import java.io.RandomAccessFile
+import java.io.InputStream
+import java.io.OutputStream
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
 import javax.inject.Inject
 
 private const val TAG = "ZimFileReader"
@@ -80,7 +81,7 @@ class ZimFileReader constructor(
   val description: String get() = jniKiwixReader.description
   val favicon: String? get() = jniKiwixReader.favicon
   val language: String get() = jniKiwixReader.language
-  val tags: String get() = "${getContent(Uri.parse("M/Tags"))}"
+  val tags: String get() = "${getContentAndMimeType("M/Tags")}"
   private val mediaCount: Int?
     get() = try {
       jniKiwixReader.mediaCount
@@ -112,8 +113,8 @@ class ZimFileReader constructor(
   fun getRandomArticleUrl(): String? =
     valueOfJniStringAfter(jniKiwixReader::getRandomPage)
 
-  fun load(uri: Uri): ParcelFileDescriptor {
+  fun load(uri: String): InputStream? {
-    if ("$uri".matches(VIDEO_REGEX)) {
+    if (uri.matches(VIDEO_REGEX)) {
       try {
         return loadVideo(uri)
       } catch (ioException: IOException) {
@@ -123,7 +124,7 @@ class ZimFileReader constructor(
     return loadContent(uri)
   }
 
-  fun readMimeType(uri: Uri) = "$uri".removeArguments().let {
+  fun readMimeType(uri: String) = uri.removeArguments().let {
     it.mimeType?.takeIf(String::isNotEmpty) ?: mimeTypeFromReader(it)
   }.also { Log.d(TAG, "getting mimetype for $uri = $it") }
 
@@ -139,73 +140,72 @@ class ZimFileReader constructor(
   private fun toRedirect(url: String) =
     "$CONTENT_URI${jniKiwixReader.checkUrl(url.toUri().filePath)}".toUri()
 
-  private fun loadContent(uri: Uri) =
+  private fun loadContent(uri: String) =
     try {
-      ParcelFileDescriptor.createPipe().also {
+      PipedInputStream(PipedOutputStream().also { streamZimContentToPipe(uri, it) })
-        streamZimContentToPipe(uri, AutoCloseOutputStream(it[1]))
-      }[0]
     } catch (ioException: IOException) {
       throw IOException("Could not open pipe for $uri", ioException)
     }
 
-  private fun loadVideo(uri: Uri): ParcelFileDescriptor {
+  private fun loadVideo(uri: String): InputStream? {
     val infoPair = jniKiwixReader.getDirectAccessInformation(uri.filePath)
     if (infoPair == null || !File(infoPair.filename).exists()) {
       return loadVideoFromCache(uri)
     }
-    return dup(infoPair.fileDescriptor)
+    return AssetFileDescriptor(
+      infoPair.parcelFileDescriptor,
+      infoPair.offset,
+      articleSize(uri)
+    ).createInputStream()
+  }
+
+  private fun articleSize(uri: String) = with(JNIKiwixInt()) {
+    jniKiwixReader.getContentPart(uri.filePath, 0, 0, this)
+    value.toLong()
   }
 
   @Throws(IOException::class)
-  private fun loadVideoFromCache(uri: Uri): ParcelFileDescriptor {
+  private fun loadVideoFromCache(uri: String): FileInputStream {
-    val outputFile = File(
+    return File(
       FileUtils.getFileCacheDir(CoreApp.getInstance()),
-      "$uri".substringAfterLast("/")
+      uri.substringAfterLast("/")
-    )
+    ).apply { writeBytes(getContent(uri)) }
-    FileOutputStream(outputFile).use { it.write(getContent(uri)) }
+      .inputStream()
-    return ParcelFileDescriptor.open(outputFile, ParcelFileDescriptor.MODE_READ_ONLY)
   }
 
-  private fun streamZimContentToPipe(
+  private fun getContent(url: String) = getContentAndMimeType(url).let { (content, _) -> content }
-    uri: Uri,
+
-    outputStream: AutoCloseOutputStream
+  private fun streamZimContentToPipe(uri: String, outputStream: OutputStream) {
-  ) {
+    Completable.fromCallable {
-    Single.just(Unit)
-      .subscribeOn(Schedulers.io())
-      .observeOn(Schedulers.io())
-      .subscribe(
-        {
       try {
         outputStream.use {
-              val mime = JNIKiwixString()
+          getContentAndMimeType(uri).let { (content: ByteArray, mimeType: String) ->
-              val size = JNIKiwixInt()
+            if ("text/css" == mimeType && nightModeConfig.isNightModeActive()) {
-              val url = JNIKiwixString(uri.filePath.removeArguments())
-              val content = getContent(url = url, mime = mime, size = size)
-              if ("text/css" == mime.value && nightModeConfig.isNightModeActive()) {
               it.write(INVERT_IMAGES_VIDEO.toByteArray(Charsets.UTF_8))
             }
             it.write(content)
-              Log.d(
+          }
-                TAG,
-                "reading  ${url.value}(mime: ${mime.value}, size: ${size.value}) finished."
-              )
         }
       } catch (ioException: IOException) {
         Log.e(TAG, "error writing pipe for $uri", ioException)
       }
-        },
+    }
-        Throwable::printStackTrace
+      .subscribeOn(Schedulers.io())
-      )
+      .subscribe({ }, Throwable::printStackTrace)
   }
 
-  private fun getContent(uri: Uri) = getContent(JNIKiwixString(uri.filePath.removeArguments()))
+  private fun getContentAndMimeType(uri: String) = with(JNIKiwixString()) {
+    getContent(url = JNIKiwixString(uri.filePath.removeArguments()), mime = this) to value
+  }
 
   private fun getContent(
     url: JNIKiwixString = JNIKiwixString(),
     jniKiwixString: JNIKiwixString = JNIKiwixString(),
     mime: JNIKiwixString = JNIKiwixString(),
     size: JNIKiwixInt = JNIKiwixInt()
-  ) = jniKiwixReader.getContent(url, jniKiwixString, mime, size)
+  ) = jniKiwixReader.getContent(url, jniKiwixString, mime, size).also {
+    Log.d(TAG, "reading  ${url.value}(mime: ${mime.value}, size: ${size.value}) finished.")
+  }
 
   private fun valueOfJniStringAfter(jniStringFunction: (JNIKiwixString) -> Boolean) =
     JNIKiwixString().takeIf { jniStringFunction(it) }?.value
@@ -233,6 +233,7 @@ class ZimFileReader constructor(
     */
     @JvmField
     val UI_URI: Uri? = Uri.parse("content://org.kiwix.ui/")
+
     @JvmField
     val CONTENT_URI: Uri? =
       Uri.parse("content://${CoreApp.getInstance().packageName}.zim.base/")
@@ -256,8 +257,6 @@ class ZimFileReader constructor(
 }
 
 private fun String.removeArguments() = substringBefore("?")
-private val Pair.fileDescriptor: FileDescriptor?
-  get() = RandomAccessFile(filename, "r").apply { seek(offset.toLong()) }.fd
 private val Uri.filePath: String
   get() = toString().filePath
 private val String.filePath: String
@@ -266,3 +265,5 @@ private val String.mimeType: String?
   get() = MimeTypeMap.getSingleton().getMimeTypeFromExtension(
     MimeTypeMap.getFileExtensionFromUrl(this)
   )
+private val Pair.parcelFileDescriptor: ParcelFileDescriptor?
+  get() = ParcelFileDescriptor.open(File(filename), ParcelFileDescriptor.MODE_READ_ONLY)

core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt

@@ -17,7 +17,7 @@
  */
 package org.kiwix.kiwixmobile.core.reader
 
-import android.net.Uri
+import android.webkit.WebResourceResponse
 import org.kiwix.kiwixlib.JNIKiwixSearcher
 import org.kiwix.kiwixmobile.core.reader.ZimFileReader.Factory
 import java.io.File
@@ -48,10 +48,6 @@ class ZimReaderContainer @Inject constructor(
       else null
   }
 
-  fun readMimeType(uri: Uri) = zimFileReader?.readMimeType(uri)
-
-  fun load(uri: Uri) = zimFileReader?.load(uri)
-
   fun searchSuggestions(prefix: String, count: Int) =
     zimFileReader?.searchSuggestions(prefix, count) ?: false
 
@@ -67,8 +63,15 @@ class ZimReaderContainer @Inject constructor(
   fun getNextResult() = jniKiwixSearcher?.nextResult?.let { SearchResult(it.title) }
   fun isRedirect(url: String): Boolean = zimFileReader?.isRedirect(url) == true
   fun getRedirect(url: String): String = zimFileReader?.getRedirect(url) ?: ""
+  fun load(url: String) =
+    WebResourceResponse(
+      zimFileReader?.readMimeType(url),
+      Charsets.UTF_8.name(),
+      zimFileReader?.load(url)
+    )
 
   val zimFile get() = zimFileReader?.zimFile
+
   val zimCanonicalPath get() = zimFileReader?.zimFile?.canonicalPath
   val zimFileTitle get() = zimFileReader?.title
   val mainPage get() = zimFileReader?.mainPage