Move check on adding a download pre db insertion. Handle downloads cancelled outside application. Use transaction to mediate refresh rate of observables

app/src/main/java/org/kiwix/kiwixmobile/database/BaseDao.java

@@ -39,8 +39,6 @@ abstract class BaseDao {
           }
         });
     updates
-        .debounce(10, TimeUnit.MILLISECONDS)
-        .onBackpressureDrop()
         .subscribeOn(Schedulers.io())
         .observeOn(Schedulers.io())
         .subscribe(unit -> {

app/src/main/java/org/kiwix/kiwixmobile/database/BookDao.java

@@ -19,7 +19,6 @@ package org.kiwix.kiwixmobile.database;
 
 import com.yahoo.squidb.data.SquidCursor;
 import com.yahoo.squidb.sql.Query;
-import com.yahoo.squidb.sql.Table;
 import com.yahoo.squidb.sql.TableStatement;
 import io.reactivex.Flowable;
 import io.reactivex.processors.BehaviorProcessor;
@@ -27,7 +26,6 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import javax.inject.Inject;
-import org.jetbrains.annotations.NotNull;
 import org.kiwix.kiwixmobile.database.entity.BookDatabaseEntity;
 import org.kiwix.kiwixmobile.library.entity.LibraryNetworkEntity.Book;
 
@@ -91,32 +89,39 @@ public class BookDao extends BaseDao {
   }
 
   public List<Book> getBooks() {
-    SquidCursor<BookDatabaseEntity> bookCursor = kiwixDatabase.query(
+    kiwixDatabase.beginTransaction();
-        BookDatabaseEntity.class,
-        Query.select());
     ArrayList<Book> books = new ArrayList<>();
-    while (bookCursor.moveToNext()) {
+    try(SquidCursor<BookDatabaseEntity> bookCursor = kiwixDatabase.query(
-      Book book = new Book();
+        BookDatabaseEntity.class,
-      setBookDetails(book, bookCursor);
+        Query.select())) {
-      if (!hasParts(book.file)) {
+      while (bookCursor.moveToNext()) {
-        if (book.file.exists()) {
+        Book book = new Book();
-          books.add(book);
+        setBookDetails(book, bookCursor);
-        } else {
+        if (!hasParts(book.file)) {
-          kiwixDatabase.deleteWhere(BookDatabaseEntity.class, BookDatabaseEntity.URL.eq(book.file));
+          if (book.file.exists()) {
+            books.add(book);
+          } else {
+            kiwixDatabase.deleteWhere(BookDatabaseEntity.class,
+                BookDatabaseEntity.URL.eq(book.file));
+          }
         }
       }
     }
-    bookCursor.close();
+    kiwixDatabase.setTransactionSuccessful();
+    kiwixDatabase.endTransaction();
     return books;
   }
 
   public void saveBooks(List<Book> books) {
+    kiwixDatabase.beginTransaction();
     for (Book book : books) {
       if (book != null) {
         BookDatabaseEntity bookDatabaseEntity = new BookDatabaseEntity();
         setBookDatabaseEntity(book, bookDatabaseEntity);
       }
     }
+    kiwixDatabase.setTransactionSuccessful();
+    kiwixDatabase.endTransaction();
   }
 
   public void deleteBook(String id) {

app/src/main/java/org/kiwix/kiwixmobile/database/DownloadDao.java

@@ -49,16 +49,14 @@ public class DownloadDao extends BaseDao{
   }
 
   public void insert(final DownloadModel downloadModel) {
-    if (doesNotAlreadyExist(downloadModel)) {
       kiwixDatabase.persistWithOnConflict(databaseEntity(downloadModel),
           TableStatement.ConflictAlgorithm.REPLACE);
-    }
   }
 
-  private boolean doesNotAlreadyExist(DownloadModel downloadModel) {
+  public boolean doesNotAlreadyExist(LibraryNetworkEntity.Book book) {
     return kiwixDatabase.count(
         DownloadDatabaseEntity.class,
-        DownloadDatabaseEntity.BOOK_ID.eq(downloadModel.getBook().getId())
+        DownloadDatabaseEntity.BOOK_ID.eq(book.getId())
     ) == 0;
   }
 

app/src/main/java/org/kiwix/kiwixmobile/database/NetworkLanguageDao.java

@@ -37,12 +37,12 @@ public class NetworkLanguageDao extends BaseDao {
 
   @Inject
   public NetworkLanguageDao(KiwixDatabase kiwikDatabase) {
-    super(kiwikDatabase,NetworkLanguageDatabaseEntity.TABLE);
+    super(kiwikDatabase, NetworkLanguageDatabaseEntity.TABLE);
   }
 
   @Override
   protected void onUpdateToTable() {
-    allLanguageProcessor.onNext(fetchAllLanguages());
+      allLanguageProcessor.onNext(fetchAllLanguages());
   }
 
   public Flowable<List<Language>> allLanguages() {
@@ -75,10 +75,13 @@ public class NetworkLanguageDao extends BaseDao {
   }
 
   public void saveFilteredLanguages(List<Language> languages) {
+    kiwixDatabase.beginTransaction();
     kiwixDatabase.deleteAll(NetworkLanguageDatabaseEntity.class);
     Collections.sort(languages,
         (language, t1) -> language.getLanguage().compareTo(t1.getLanguage()));
-    for (Language language : languages) {
+
+    for (int i = 0; i < languages.size(); i++) {
+      Language language = languages.get(i);
       NetworkLanguageDatabaseEntity networkLanguageDatabaseEntity =
           new NetworkLanguageDatabaseEntity();
       networkLanguageDatabaseEntity.setLanguageISO3(language.getLanguageCode());
@@ -86,5 +89,7 @@ public class NetworkLanguageDao extends BaseDao {
       networkLanguageDatabaseEntity.setNumberOfOccurences(language.getOccurencesOfLanguage());
       kiwixDatabase.persist(networkLanguageDatabaseEntity);
     }
+    kiwixDatabase.setTransactionSuccessful();
+    kiwixDatabase.endTransaction();
   }
 }

app/src/main/java/org/kiwix/kiwixmobile/downloader/DownloaderImpl.kt

@@ -36,15 +36,18 @@ class DownloaderImpl @Inject constructor(
   override fun download(book: LibraryNetworkEntity.Book) {
     kiwixService.getMetaLinks(book.url)
         .subscribeOn(Schedulers.io())
+        .observeOn(Schedulers.io())
         .take(1)
         .subscribe(
             {
-              val downloadId = downloadRequester.enqueue(
+              if(downloadDao.doesNotAlreadyExist(book)){
-                  DownloadRequest(it, book)
+                val downloadId = downloadRequester.enqueue(
-              )
+                    DownloadRequest(it, book)
-              downloadDao.insert(
+                )
-                  DownloadModel(downloadId, book)
+                downloadDao.insert(
-              )
+                    DownloadModel(downloadId, book)
+                )
+              }
             },
             Throwable::printStackTrace
         )

app/src/main/java/org/kiwix/kiwixmobile/zim_manager/ZimManageViewModel.kt

@@ -101,6 +101,7 @@ class ZimManageViewModel @Inject constructor(
     return arrayOf(
         updateDownloadItems(downloadStatuses),
         removeCompletedDownloadsFromDb(downloadStatuses),
+        removeNonExistingDownloadsFromDb(downloadStatuses, downloads),
         updateBookItems(booksFromDao),
         checkFileSystemForBooksOnRequest(booksFromDao),
         updateLibraryItems(booksFromDao, downloads, library),
@@ -110,6 +111,46 @@ class ZimManageViewModel @Inject constructor(
     )
   }
 
+  private fun removeNonExistingDownloadsFromDb(
+    downloadStatuses: Flowable<List<DownloadStatus>>,
+    downloads: Flowable<MutableList<DownloadModel>>
+  ) = downloadStatuses
+      .withLatestFrom(
+          downloads,
+          BiFunction(this::combineToDownloadsWithoutStatuses)
+      )
+      .buffer(3, SECONDS)
+      .map(this::downloadIdsWithNoStatusesOverBufferPeriod)
+      .subscribe(
+          {
+            downloadDao.delete(*it.toTypedArray())
+          },
+          Throwable::printStackTrace
+      )
+
+  private fun downloadIdsWithNoStatusesOverBufferPeriod(it: List<MutableList<Long>>) =
+    it.flatten()
+        .fold(mutableMapOf<Long, Int>(),
+            { acc, id -> acc.increment(id) })
+        .filter { (_, value) -> value == it.size }
+        .map { (key, _) -> key }
+
+  private fun combineToDownloadsWithoutStatuses(
+    statuses: List<DownloadStatus>,
+    downloads: List<DownloadModel>
+  ): MutableList<Long> {
+    val downloadIdsWithStatuses = statuses.map { it.downloadId }
+    return downloads.fold(
+        mutableListOf(),
+        { acc, downloadModel ->
+          if (!downloadIdsWithStatuses.contains(downloadModel.downloadId)) {
+            acc.add(downloadModel.downloadId)
+          }
+          acc
+        }
+    )
+  }
+
   private fun updateLanguageItemsForDialog() = requestLanguagesDialog
       .withLatestFrom(languageDao.allLanguages(),
           BiFunction<Unit, List<Language>, List<Language>> { _, languages -> languages })
@@ -144,13 +185,11 @@ class ZimManageViewModel @Inject constructor(
   ) = Flowable.combineLatest(
       booksFromDao,
       downloads,
-      languageDao.allLanguages()
+      languageDao.allLanguages().filter { it.isNotEmpty() },
-          .debounce(100, MILLISECONDS)
-          .filter { it.isNotEmpty() },
       library,
       requestFiltering
           .doOnNext { libraryListIsRefreshing.postValue(true) }
-          .debounce(1, SECONDS)
+          .debounce(500, MILLISECONDS)
           .observeOn(Schedulers.io()),
       Function5(this::combineLibrarySources)
   )
@@ -178,22 +217,26 @@ class ZimManageViewModel @Inject constructor(
   private fun combineToLanguageList(
     booksFromNetwork: List<Book>,
     allLanguages: List<Language>
-  ): List<Language> {
+  ) = when {
-    val networkLanguageCounts = booksFromNetwork.mapNotNull { it.language }
+    booksFromNetwork.isEmpty() && allLanguages.isEmpty() -> defaultLanguage()
+    booksFromNetwork.isEmpty() && allLanguages.isNotEmpty() -> emptyList()
+    booksFromNetwork.isNotEmpty() && allLanguages.isEmpty() ->
+      fromLocalesWithNetworkMatchesSetActiveBy(
+          networkLanguageCounts(booksFromNetwork), defaultLanguage()
+      )
+    booksFromNetwork.isNotEmpty() && allLanguages.isNotEmpty() ->
+      fromLocalesWithNetworkMatchesSetActiveBy(
+          networkLanguageCounts(booksFromNetwork), allLanguages
+      )
+    else -> throw RuntimeException("Impossible state")
+  }
+
+  private fun networkLanguageCounts(booksFromNetwork: List<Book>) =
+    booksFromNetwork.mapNotNull { it.language }
         .fold(
             mutableMapOf<String, Int>(),
             { acc, language -> acc.increment(language) }
         )
-    return when {
-      booksFromNetwork.isEmpty() && allLanguages.isEmpty() -> defaultLanguage()
-      booksFromNetwork.isEmpty() && allLanguages.isNotEmpty() -> allLanguages
-      booksFromNetwork.isNotEmpty() && allLanguages.isEmpty() ->
-        fromLocalesWithNetworkMatchesSetActiveBy(networkLanguageCounts, defaultLanguage())
-      booksFromNetwork.isNotEmpty() && allLanguages.isNotEmpty() ->
-        fromLocalesWithNetworkMatchesSetActiveBy(networkLanguageCounts, allLanguages)
-      else -> throw RuntimeException("Impossible state")
-    }
-  }
 
   private fun <K> MutableMap<K, Int>.increment(key: K) =
     apply { set(key, getOrElse(key, { 0 }) + 1) }
@@ -201,18 +244,16 @@ class ZimManageViewModel @Inject constructor(
   private fun fromLocalesWithNetworkMatchesSetActiveBy(
     networkLanguageCounts: MutableMap<String, Int>,
     listToActivateBy: List<Language>
-  ): List<Language> {
+  ) = Locale.getISOLanguages()
-    return Locale.getISOLanguages()
+      .map { Locale(it) }
-        .map { Locale(it) }
+      .filter { networkLanguageCounts.containsKey(it.isO3Language) }
-        .filter { networkLanguageCounts.containsKey(it.isO3Language) }
+      .map { locale ->
-        .map { locale ->
+        Language(
-          Language(
+            locale.isO3Language,
-              locale.isO3Language,
+            languageIsActive(listToActivateBy, locale),
-              languageIsActive(listToActivateBy, locale),
+            networkLanguageCounts.getOrElse(locale.isO3Language, { 0 })
-              networkLanguageCounts.getOrElse(locale.isO3Language, { 0 })
+        )
-          )
+      }
-        }
-  }
 
   private fun defaultLanguage() =
     listOf(

app/src/main/java/org/kiwix/kiwixmobile/zim_manager/fileselect_view/BooksViewHolder.kt

@@ -2,7 +2,6 @@ package org.kiwix.kiwixmobile.zim_manager.fileselect_view
 
 import android.support.v7.widget.RecyclerView.ViewHolder
 import android.view.View
-import butterknife.internal.DebouncingOnClickListener
 import kotlinx.android.extensions.LayoutContainer
 import kotlinx.android.synthetic.main.library_item.creator
 import kotlinx.android.synthetic.main.library_item.date
@@ -20,7 +19,7 @@ import org.kiwix.kiwixmobile.extensions.setTextAndVisibility
 import org.kiwix.kiwixmobile.library.entity.LibraryNetworkEntity.Book
 import org.kiwix.kiwixmobile.utils.BookUtils
 import org.kiwix.kiwixmobile.utils.NetworkUtils
-import org.kiwix.kiwixmobile.zim_manager.library_view.adapter.MegaByte
+import org.kiwix.kiwixmobile.zim_manager.library_view.adapter.KiloByte
 
 class BooksViewHolder(
   override val containerView: View,
@@ -37,7 +36,7 @@ class BooksViewHolder(
     creator.setTextAndVisibility(book.creator)
     publisher.setTextAndVisibility(book.publisher)
     date.setTextAndVisibility(book.date)
-    size.setTextAndVisibility(MegaByte(book.size).humanReadable)
+    size.setTextAndVisibility(KiloByte(book.size).humanReadable)
     language.text = bookUtils.getLanguage(book.getLanguage())
     fileName.text = NetworkUtils.parseURL(
         KiwixApplication.getInstance(), book.file.path

app/src/main/java/org/kiwix/kiwixmobile/zim_manager/library_view/LibraryFragment.kt

@@ -213,11 +213,11 @@ class LibraryFragment : BaseFragment() {
       arguments = Bundle().apply {
         putString(
             StorageSelectDialog.STORAGE_DIALOG_INTERNAL,
-            getString(string.internal_storage)
+            this@LibraryFragment.getString(string.internal_storage)
         )
         putString(
             StorageSelectDialog.STORAGE_DIALOG_EXTERNAL,
-            getString(string.external_storage)
+            this@LibraryFragment.getString(string.external_storage)
         )
         putInt(StorageSelectDialog.STORAGE_DIALOG_THEME, StyleUtils.dialogStyle())
       }

app/src/main/java/org/kiwix/kiwixmobile/zim_manager/library_view/adapter/KiloByte.kt

@@ -2,9 +2,9 @@ package org.kiwix.kiwixmobile.zim_manager.library_view.adapter
 
 import java.text.DecimalFormat
 
-inline class MegaByte(val megabyteString: String?) {
+inline class KiloByte(val kilobyteString: String?) {
   val humanReadable
-    get() = megabyteString?.toLongOrNull()?.let {
+    get() = kilobyteString?.toLongOrNull()?.let {
       val units = arrayOf("KB", "MB", "GB", "TB")
       val conversion = (Math.log10(it.toDouble()) / Math.log10(1024.0)).toInt()
       (DecimalFormat("#,##0.#")

app/src/main/java/org/kiwix/kiwixmobile/zim_manager/library_view/adapter/LibraryViewHolder.kt

@@ -2,7 +2,6 @@ package org.kiwix.kiwixmobile.zim_manager.library_view.adapter
 
 import android.support.v7.widget.RecyclerView
 import android.view.View
-import butterknife.internal.DebouncingOnClickListener
 import kotlinx.android.extensions.LayoutContainer
 import kotlinx.android.synthetic.main.library_divider.divider_text
 import kotlinx.android.synthetic.main.library_item.creator
@@ -40,7 +39,7 @@ sealed class LibraryViewHolder<T : LibraryListItem>(override val containerView:
       creator.setTextAndVisibility(item.book.creator)
       publisher.setTextAndVisibility(item.book.publisher)
       date.setTextAndVisibility(item.book.date)
-      size.setTextAndVisibility(MegaByte(item.book.size).humanReadable)
+      size.setTextAndVisibility(KiloByte(item.book.size).humanReadable)
       language.text = bookUtils.getLanguage(item.book.getLanguage())
       fileName.text = NetworkUtils.parseURL(
           KiwixApplication.getInstance(), item.book.file?.path ?: ""