Add fat32 checker that checks on storage change and on deletion/file removal. Fix library stream breaking on error

app/src/main/java/org/kiwix/kiwixmobile/database/NetworkLanguageDao.java

@@ -24,7 +24,6 @@ import com.yahoo.squidb.sql.Query;
 import io.reactivex.Flowable;
 import io.reactivex.processors.BehaviorProcessor;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import javax.inject.Inject;
 import org.jetbrains.annotations.NotNull;
@@ -42,7 +41,7 @@ public class NetworkLanguageDao extends BaseDao {
 
   @Override
   protected void onUpdateToTable() {
-      allLanguageProcessor.onNext(fetchAllLanguages());
+    allLanguageProcessor.onNext(fetchAllLanguages());
   }
 
   public Flowable<List<Language>> allLanguages() {
@@ -75,21 +74,20 @@ public class NetworkLanguageDao extends BaseDao {
   }
 
   public void saveFilteredLanguages(List<Language> languages) {
-    kiwixDatabase.beginTransaction();
-    kiwixDatabase.deleteAll(NetworkLanguageDatabaseEntity.class);
-    Collections.sort(languages,
-        (language, t1) -> language.getLanguage().compareTo(t1.getLanguage()));
-
-    for (int i = 0; i < languages.size(); i++) {
-      Language language = languages.get(i);
-      NetworkLanguageDatabaseEntity networkLanguageDatabaseEntity =
-          new NetworkLanguageDatabaseEntity();
-      networkLanguageDatabaseEntity.setLanguageISO3(language.getLanguageCode());
-      networkLanguageDatabaseEntity.setIsEnabled(language.getActive());
-      networkLanguageDatabaseEntity.setNumberOfOccurences(language.getOccurencesOfLanguage());
-      kiwixDatabase.persist(networkLanguageDatabaseEntity);
+    if (languages.size() > 0) {
+      kiwixDatabase.beginTransaction();
+      kiwixDatabase.deleteAll(NetworkLanguageDatabaseEntity.class);
+      for (int i = 0; i < languages.size(); i++) {
+        Language language = languages.get(i);
+        NetworkLanguageDatabaseEntity networkLanguageDatabaseEntity =
+            new NetworkLanguageDatabaseEntity();
+        networkLanguageDatabaseEntity.setLanguageISO3(language.getLanguageCode());
+        networkLanguageDatabaseEntity.setIsEnabled(language.getActive());
+        networkLanguageDatabaseEntity.setNumberOfOccurences(language.getOccurencesOfLanguage());
+        kiwixDatabase.persist(networkLanguageDatabaseEntity);
+      }
+      kiwixDatabase.setTransactionSuccessful();
+      kiwixDatabase.endTransaction();
     }
-    kiwixDatabase.setTransactionSuccessful();
-    kiwixDatabase.endTransaction();
   }
 }

app/src/main/java/org/kiwix/kiwixmobile/network/KiwixService.java

@@ -19,6 +19,7 @@ package org.kiwix.kiwixmobile.network;
 
 import io.reactivex.Flowable;
 import io.reactivex.Observable;
+import io.reactivex.Single;
 import io.reactivex.schedulers.Schedulers;
 import okhttp3.OkHttpClient;
 import org.kiwix.kiwixmobile.library.entity.LibraryNetworkEntity;
@@ -30,7 +31,7 @@ import retrofit2.http.GET;
 import retrofit2.http.Url;
 
 public interface KiwixService {
-  @GET("/library/library_zim.xml") Flowable<LibraryNetworkEntity> getLibrary();
+  @GET("/library/library_zim.xml") Single<LibraryNetworkEntity> getLibrary();
 
   @GET Observable<MetaLinkNetworkEntity> getMetaLinks(@Url String url);
 

app/src/main/java/org/kiwix/kiwixmobile/utils/SharedPreferenceUtil.java

@@ -4,7 +4,7 @@ import android.content.Context;
 import android.content.SharedPreferences;
 import android.os.Environment;
 import android.preference.PreferenceManager;
-
+import io.reactivex.processors.BehaviorProcessor;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -13,7 +13,6 @@ import static org.kiwix.kiwixmobile.utils.Constants.PREF_BACK_TO_TOP;
 import static org.kiwix.kiwixmobile.utils.Constants.PREF_BOTTOM_TOOLBAR;
 import static org.kiwix.kiwixmobile.utils.Constants.PREF_EXTERNAL_LINK_POPUP;
 import static org.kiwix.kiwixmobile.utils.Constants.PREF_FULLSCREEN;
-import static org.kiwix.kiwixmobile.utils.Constants.PREF_FULL_TEXT_SEARCH;
 import static org.kiwix.kiwixmobile.utils.Constants.PREF_HIDE_TOOLBAR;
 import static org.kiwix.kiwixmobile.utils.Constants.PREF_IS_FIRST_RUN;
 import static org.kiwix.kiwixmobile.utils.Constants.PREF_LANG;
@@ -33,11 +32,13 @@ import static org.kiwix.kiwixmobile.utils.Constants.PREF_ZOOM_ENABLED;
 public class SharedPreferenceUtil {
   private SharedPreferences sharedPreferences;
   private SharedPreferences.Editor editor;
+  public final BehaviorProcessor<String> prefStorages;
 
   @Inject
   public SharedPreferenceUtil(Context context) {
     sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context);
     editor = sharedPreferences.edit();
+    prefStorages = BehaviorProcessor.createDefault(getPrefStorage());
   }
 
   public void remove(String key) {
@@ -131,6 +132,7 @@ public class SharedPreferenceUtil {
 
   public void putPrefStorage(String storage) {
     editor.putString(PREF_STORAGE, storage).apply();
+    prefStorages.onNext(storage);
   }
 
   public void putPrefFullScreen(boolean fullScreen) {

app/src/main/java/org/kiwix/kiwixmobile/zim_manager/Fat32Checker.kt

@@ -0,0 +1,101 @@
+/*
+ * Kiwix Android
+ * Copyright (C) 2018  Kiwix <android.kiwix.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.kiwix.kiwixmobile.zim_manager
+
+import android.os.FileObserver
+import android.util.Log
+import io.reactivex.Flowable
+import io.reactivex.functions.BiFunction
+import io.reactivex.processors.BehaviorProcessor
+import org.kiwix.kiwixmobile.utils.SharedPreferenceUtil
+import org.kiwix.kiwixmobile.zim_manager.Fat32Checker.FileSystemState.CanWrite4GbFile
+import org.kiwix.kiwixmobile.zim_manager.Fat32Checker.FileSystemState.CannotWrite4GbFile
+import org.kiwix.kiwixmobile.zim_manager.Fat32Checker.FileSystemState.NotEnoughSpaceFor4GbFile
+import java.io.File
+import java.io.RandomAccessFile
+import javax.inject.Inject
+
+class Fat32Checker @Inject constructor(sharedPreferenceUtil: SharedPreferenceUtil) {
+  private val _fileSystemStates: BehaviorProcessor<FileSystemState> = BehaviorProcessor.create()
+  val fileSystemStates = _fileSystemStates.distinctUntilChanged()
+  var fileObserver: FileObserver? = null
+  private val requestCheckSystemFileType = BehaviorProcessor.createDefault(Unit)
+
+  init {
+    Flowable.combineLatest(
+        sharedPreferenceUtil.prefStorages.distinctUntilChanged(),
+        requestCheckSystemFileType,
+        BiFunction { storage: String, _: Unit -> storage }
+    )
+        .subscribe(
+            {
+              val systemState = toFileSystemState(it)
+              _fileSystemStates.onNext(systemState)
+              fileObserver = if (systemState == NotEnoughSpaceFor4GbFile) fileObserver(it) else null
+
+            },
+            Throwable::printStackTrace
+        )
+  }
+
+  private fun fileObserver(it: String?): FileObserver {
+    return object : FileObserver(it, MOVED_FROM or DELETE) {
+      override fun onEvent(
+        event: Int,
+        path: String?
+      ) {
+        requestCheckSystemFileType.onNext(Unit)
+      }
+    }.apply { startWatching() }
+  }
+
+  private fun toFileSystemState(it: String) =
+    when {
+      File(it).freeSpace > FOUR_GIGABYTES ->
+        if (canCreate4GbFile(it)) CanWrite4GbFile
+        else CannotWrite4GbFile
+      else -> NotEnoughSpaceFor4GbFile
+    }
+
+  private fun canCreate4GbFile(storage: String): Boolean {
+    val path = "$storage/large_file_test.txt"
+    File(path).delete()
+    try {
+      RandomAccessFile(path, "rw").use {
+        it.setLength(FOUR_GIGABYTES)
+        return true
+      }
+    } catch (e: Exception) {
+      Log.d("Fat32Checker", e.message)
+      return false
+    } finally {
+      File(path).delete()
+    }
+  }
+
+  companion object {
+    const val FOUR_GIGABYTES = 4L * 1024L * 1024L * 1024L
+    const val FOUR_GIGABYTES_IN_KILOBYTES = 4L * 1024L * 1024L
+  }
+
+  sealed class FileSystemState() {
+    object NotEnoughSpaceFor4GbFile : FileSystemState()
+    object CanWrite4GbFile : FileSystemState()
+    object CannotWrite4GbFile : FileSystemState()
+  }
+}

app/src/main/java/org/kiwix/kiwixmobile/zim_manager/ZimManageActivity.kt

@@ -99,7 +99,9 @@ class ZimManageActivity : BaseActivity() {
       LanguageSelectDialog.Builder(this, dialogStyle())
           .apply {
             onOkClicked = {
-              Flowable.fromCallable { languagesDao.saveFilteredLanguages(it) }
+              Flowable.fromCallable {
+                languagesDao.saveFilteredLanguages(it.sortedBy(Language::language))
+              }
                   .subscribeOn(Schedulers.io())
                   .subscribe()
             }

app/src/main/java/org/kiwix/kiwixmobile/zim_manager/ZimManageViewModel.kt

@@ -25,7 +25,7 @@ import io.reactivex.Flowable
 import io.reactivex.disposables.CompositeDisposable
 import io.reactivex.disposables.Disposable
 import io.reactivex.functions.BiFunction
-import io.reactivex.functions.Function5
+import io.reactivex.functions.Function6
 import io.reactivex.processors.BehaviorProcessor
 import io.reactivex.processors.PublishProcessor
 import io.reactivex.schedulers.Schedulers
@@ -44,6 +44,11 @@ import org.kiwix.kiwixmobile.library.entity.LibraryNetworkEntity
 import org.kiwix.kiwixmobile.library.entity.LibraryNetworkEntity.Book
 import org.kiwix.kiwixmobile.network.KiwixService
 import org.kiwix.kiwixmobile.utils.BookUtils
+import org.kiwix.kiwixmobile.zim_manager.Fat32Checker.FileSystemState
+import org.kiwix.kiwixmobile.zim_manager.Fat32Checker.FileSystemState.CanWrite4GbFile
+import org.kiwix.kiwixmobile.zim_manager.Fat32Checker.FileSystemState.CannotWrite4GbFile
+import org.kiwix.kiwixmobile.zim_manager.Fat32Checker.FileSystemState.NotEnoughSpaceFor4GbFile
+import org.kiwix.kiwixmobile.zim_manager.NetworkState.CONNECTED
 import org.kiwix.kiwixmobile.zim_manager.fileselect_view.StorageObserver
 import org.kiwix.kiwixmobile.zim_manager.library_view.adapter.Language
 import org.kiwix.kiwixmobile.zim_manager.library_view.adapter.LibraryListItem
@@ -64,7 +69,8 @@ class ZimManageViewModel @Inject constructor(
   private val kiwixService: KiwixService,
   private val context: Application,
   private val connectivityBroadcastReceiver: ConnectivityBroadcastReceiver,
-  private val bookUtils: BookUtils
+  private val bookUtils: BookUtils,
+  private val fat32Checker: Fat32Checker
 ) : ViewModel() {
 
   val libraryItems: MutableLiveData<List<LibraryListItem>> = MutableLiveData()
@@ -97,20 +103,44 @@ class ZimManageViewModel @Inject constructor(
     val downloads: Flowable<MutableList<DownloadModel>> = downloadDao.downloads()
     val downloadStatuses = downloadStatuses(downloads)
     val booksFromDao: Flowable<List<Book>> = books()
-    val library = libraryFromNetwork()
+    val networkLibrary = PublishProcessor.create<LibraryNetworkEntity>()
     return arrayOf(
         updateDownloadItems(downloadStatuses),
         removeCompletedDownloadsFromDb(downloadStatuses),
         removeNonExistingDownloadsFromDb(downloadStatuses, downloads),
         updateBookItems(booksFromDao),
         checkFileSystemForBooksOnRequest(booksFromDao),
-        updateLibraryItems(booksFromDao, downloads, library),
-        updateLanguagesInDao(library),
+        updateLibraryItems(booksFromDao, downloads, networkLibrary),
+        updateLanguagesInDao(networkLibrary),
         updateNetworkStates(),
-        updateLanguageItemsForDialog()
+        updateLanguageItemsForDialog(),
+        requestsAndConnectivtyChangesToLibraryRequests(networkLibrary)
     )
   }
 
+  private fun requestsAndConnectivtyChangesToLibraryRequests(library: PublishProcessor<LibraryNetworkEntity>) =
+    Flowable.combineLatest(
+        requestDownloadLibrary,
+        connectivityBroadcastReceiver.networkStates.distinctUntilChanged().filter(
+            CONNECTED::equals
+        ),
+        BiFunction<Unit, NetworkState, Unit> { _, _ -> Unit }
+    )
+        .subscribeOn(Schedulers.io())
+        .observeOn(Schedulers.io())
+        .subscribe(
+            {
+              kiwixService.library.subscribe(
+                  { library.onNext(it) },
+                  {
+                    it.printStackTrace()
+                    library.onNext(LibraryNetworkEntity().apply { book = LinkedList() })
+                  }
+              )
+            },
+            Throwable::printStackTrace
+        )
+
   private fun removeNonExistingDownloadsFromDb(
     downloadStatuses: Flowable<List<DownloadStatus>>,
     downloads: Flowable<MutableList<DownloadModel>>
@@ -164,20 +194,6 @@ class ZimManageViewModel @Inject constructor(
         networkStates::postValue, Throwable::printStackTrace
     )
 
-  private fun libraryFromNetwork() =
-    Flowable.combineLatest(
-        requestDownloadLibrary,
-        connectivityBroadcastReceiver.networkStates.distinctUntilChanged().filter(
-            NetworkState.CONNECTED::equals
-        ),
-        BiFunction<Unit, NetworkState, Unit> { _, _ -> Unit }
-    )
-        .subscribeOn(Schedulers.io())
-        .doOnNext { libraryListIsRefreshing.postValue(true) }
-        .switchMap { kiwixService.library }
-        .doOnError(Throwable::printStackTrace)
-        .onErrorResumeNext(Flowable.just(LibraryNetworkEntity().apply { book = LinkedList() }))
-
   private fun updateLibraryItems(
     booksFromDao: Flowable<List<Book>>,
     downloads: Flowable<MutableList<DownloadModel>>,
@@ -191,7 +207,8 @@ class ZimManageViewModel @Inject constructor(
           .doOnNext { libraryListIsRefreshing.postValue(true) }
           .debounce(500, MILLISECONDS)
           .observeOn(Schedulers.io()),
-      Function5(this::combineLibrarySources)
+      fat32Checker.fileSystemStates,
+      Function6(this::combineLibrarySources)
   )
       .doOnNext { libraryListIsRefreshing.postValue(false) }
       .subscribeOn(Schedulers.io())
@@ -209,6 +226,7 @@ class ZimManageViewModel @Inject constructor(
           languageDao.allLanguages(),
           BiFunction(this::combineToLanguageList)
       )
+      .map { it.sortedBy(Language::language) }
       .subscribe(
           languageDao::saveFilteredLanguages,
           Throwable::printStackTrace
@@ -274,7 +292,8 @@ class ZimManageViewModel @Inject constructor(
     activeDownloads: List<DownloadModel>,
     allLanguages: List<Language>,
     libraryNetworkEntity: LibraryNetworkEntity,
-    filter: String
+    filter: String,
+    fileSystemState: FileSystemState
   ): List<LibraryListItem> {
     val downloadedBooksIds = booksOnFileSystem.map { it.id }
     val downloadingBookIds = activeDownloads.map { it.book.id }
@@ -283,6 +302,13 @@ class ZimManageViewModel @Inject constructor(
     val booksUnfilteredByLanguage =
       applyUserFilter(
           libraryNetworkEntity.books
+              .filter {
+                when (fileSystemState) {
+                  CannotWrite4GbFile -> it.size.toLongOrNull() ?: 0L < Fat32Checker.FOUR_GIGABYTES_IN_KILOBYTES
+                  NotEnoughSpaceFor4GbFile,
+                  CanWrite4GbFile -> true
+                }
+              }
               .filterNot { downloadedBooksIds.contains(it.id) }
               .filterNot { downloadingBookIds.contains(it.id) }
               .filterNot { it.url.contains("/stack_exchange/") },// Temp filter see #694, filter)