Migrated on-disk BookStores/ZimStores from ObjectBox to libkiwix.

This commit is contained in:
MohitMaliFtechiz 2025-06-10 00:28:45 +05:30
parent 5b64e21127
commit 8370f72d35
12 changed files with 351 additions and 26 deletions

View File

@ -16,13 +16,13 @@
*
*/
package org.kiwix.kiwixmobile.core.zim_manager
package org.kiwix.kiwixmobile.zimManager
import org.kiwix.kiwixmobile.core.entity.LibkiwixBook
import org.kiwix.libkiwix.Library
import org.kiwix.libkiwix.Manager
class OnlineLibraryManager(val library: Library, val manager: Manager) {
class OnlineLibraryManager(private val library: Library, private val manager: Manager) {
suspend fun parseOPDSStream(content: String?, urlHost: String): Boolean =
runCatching {
manager.readOpds(content, urlHost)

View File

@ -96,7 +96,6 @@ import org.kiwix.kiwixmobile.core.zim_manager.ConnectivityBroadcastReceiver
import org.kiwix.kiwixmobile.core.zim_manager.Language
import org.kiwix.kiwixmobile.core.zim_manager.NetworkState
import org.kiwix.kiwixmobile.core.zim_manager.NetworkState.CONNECTED
import org.kiwix.kiwixmobile.core.zim_manager.OnlineLibraryManager
import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.BooksOnDiskListItem
import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.BooksOnDiskListItem.BookOnDisk
import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.SelectionMode.MULTI

View File

@ -50,7 +50,6 @@
<ID>PackageNaming:TagsView.kt$package org.kiwix.kiwixmobile.core.zim_manager</ID>
<ID>PackageNaming:ConnectivityBroadcastReceiver.kt$package org.kiwix.kiwixmobile.core.zim_manager</ID>
<ID>PackageNaming:NetworkState.kt$package org.kiwix.kiwixmobile.core.zim_manager</ID>
<ID>PackageNaming:OnlineLibraryManager.kt$package org.kiwix.kiwixmobile.core.zim_manager</ID>
<ID>ReturnCount:FileUtils.kt$FileUtils$@JvmStatic fun getAllZimParts(book: Book): List&lt;File></ID>
<ID>ReturnCount:FileUtils.kt$FileUtils$@JvmStatic suspend fun getLocalFilePathByUri( context: Context, uri: Uri ): String?</ID>
<ID>ReturnCount:FileUtils.kt$FileUtils$@JvmStatic suspend fun hasPart(file: File): Boolean</ID>

View File

@ -0,0 +1,239 @@
/*
* Kiwix Android
* Copyright (c) 2025 Kiwix <android.kiwix.org>
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
package org.kiwix.kiwixmobile.core.dao
import android.os.Build
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.flowOn
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.mapLatest
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import org.kiwix.kiwixmobile.core.dao.LibkiwixBookmarks.Companion.TAG
import org.kiwix.kiwixmobile.core.di.modules.LOCAL_BOOKS_LIBRARY
import org.kiwix.kiwixmobile.core.di.modules.LOCAL_BOOKS_MANAGER
import org.kiwix.kiwixmobile.core.entity.LibkiwixBook
import org.kiwix.kiwixmobile.core.extensions.isFileExist
import org.kiwix.kiwixmobile.core.utils.SharedPreferenceUtil
import org.kiwix.kiwixmobile.core.utils.files.Log
import org.kiwix.kiwixmobile.core.zim_manager.fileselect_view.BooksOnDiskListItem.BookOnDisk
import org.kiwix.libkiwix.Book
import org.kiwix.libkiwix.Library
import org.kiwix.libkiwix.Manager
import java.io.File
import javax.inject.Inject
import javax.inject.Named
class LibkiwixBookOnDisk @Inject constructor(
@Named(LOCAL_BOOKS_LIBRARY) private val library: Library,
@Named(LOCAL_BOOKS_MANAGER) private val manager: Manager,
private val sharedPreferenceUtil: SharedPreferenceUtil
) {
private var libraryBooksList: List<String> = arrayListOf()
private var localBooksList: List<LibkiwixBook> = arrayListOf()
/**
* Request new data from Libkiwix when changes occur inside it; otherwise,
* return the previous data to avoid unnecessary data load on Libkiwix.
*/
private var booksChanged: Boolean = false
private val localBookFolderPath: String by lazy {
if (Build.DEVICE.contains("generic")) {
// Workaround for emulators: Emulators have limited memory and
// restrictions on creating folders, so we will use the default
// path for saving the bookmark file.
sharedPreferenceUtil.context.filesDir.path
} else {
"${sharedPreferenceUtil.defaultStorage()}/ZIMFiles/"
}
}
private val libraryFile: File by lazy {
File("$localBookFolderPath/library.xml")
}
init {
// Check if ZIM files folder exist if not then create the folder first.
if (runBlocking { !File(localBookFolderPath).isFileExist() }) File(localBookFolderPath).mkdir()
// Check if library file exist if not then create the file to save the library with book information.
if (runBlocking { !libraryFile.isFileExist() }) libraryFile.createNewFile()
// set up manager to read the library from this file
manager.readFile(libraryFile.canonicalPath)
}
@Suppress("InjectDispatcher")
private val localBooksFlow: MutableStateFlow<List<LibkiwixBook>> by lazy {
MutableStateFlow<List<LibkiwixBook>>(emptyList()).also { flow ->
CoroutineScope(Dispatchers.IO).launch {
runCatching {
flow.emit(getBooksList())
}.onFailure { it.printStackTrace() }
}
}
}
private suspend fun getBooksList(dispatcher: CoroutineDispatcher = Dispatchers.IO): List<LibkiwixBook> =
withContext(dispatcher) {
if (!booksChanged && localBooksList.isNotEmpty()) {
// No changes, return the cached data
return@withContext localBooksList.distinctBy(LibkiwixBook::path)
}
// Retrieve the list of books from the library.
val booksIds = library.booksIds.toList()
// Create a list to store LibkiwixBook objects.
localBooksList =
booksIds.mapNotNull { bookId ->
val book = library.getBookById(bookId)
return@mapNotNull LibkiwixBook(book).also {
// set the books change to false to avoid reloading the data from libkiwix
booksChanged = false
}
}
return@withContext localBooksList.distinctBy(LibkiwixBook::path)
}
@OptIn(ExperimentalCoroutinesApi::class)
fun books(dispatcher: CoroutineDispatcher = Dispatchers.IO) =
localBooksFlow
.mapLatest { booksList ->
removeBooksThatAreInTrashFolder(booksList)
removeBooksThatDoNotExist(booksList.toMutableList())
booksList.mapNotNull { book ->
try {
if (book.zimReaderSource.exists() &&
!isInTrashFolder(book.zimReaderSource.toDatabase())
) {
book
} else {
null
}
} catch (_: Exception) {
null
}
}
}
.map { it.map(::BookOnDisk) }
.flowOn(dispatcher)
suspend fun getBooks() = getBooksList().map(::BookOnDisk)
suspend fun insert(libkiwixBooks: List<Book>) {
withContext(Dispatchers.IO) {
val existingBookIds = library.booksIds.toSet()
val existingBookPaths = existingBookIds
.mapNotNull { id -> library.getBookById(id)?.path }
.toSet()
val newBooks = libkiwixBooks.filterNot { book ->
book.id in existingBookIds || book.path in existingBookPaths
}
newBooks.forEach { book ->
runCatching {
library.addBook(book)
Log.d(TAG, "Added book to library: ${book.title}, ID=${book.id}")
}.onFailure {
Log.e(TAG, "Failed to add book: ${book.title} - ${it.message}")
}
}
if (newBooks.isNotEmpty()) {
writeBookMarksAndSaveLibraryToFile()
updateLocalBooksFlow()
}
}
}
private fun addBookToLibraryIfNotExist(libKiwixBook: Book?) {
libKiwixBook?.let { book ->
if (!isBookAlreadyExistInLibrary(book.id)) {
library.addBook(libKiwixBook).also {
// now library has changed so update our library list.
libraryBooksList = library.booksIds.toList()
Log.e(
TAG,
"Added Book to Library:\n" +
"ZIM File Path: ${book.path}\n" +
"Book ID: ${book.id}\n" +
"Book Title: ${book.title}"
)
}
}
}
}
private fun isBookAlreadyExistInLibrary(bookId: String): Boolean {
if (libraryBooksList.isEmpty()) {
// store booksIds in a list to avoid multiple data call on libkiwix
libraryBooksList = library.booksIds.toList()
}
return libraryBooksList.any { it == bookId }
}
private suspend fun removeBooksThatDoNotExist(books: MutableList<LibkiwixBook>) {
delete(books.filterNot { it.zimReaderSource.exists() })
}
// Remove the existing books from database which are showing on the library screen.
private suspend fun removeBooksThatAreInTrashFolder(books: List<LibkiwixBook>) {
delete(books.filter { isInTrashFolder(it.zimReaderSource.toDatabase()) })
}
// Check if any existing ZIM file showing on the library screen which is inside the trash folder.
private suspend fun isInTrashFolder(filePath: String) =
Regex("/\\.Trash/").containsMatchIn(filePath)
private suspend fun delete(books: List<LibkiwixBook>) {
runCatching {
books.forEach {
library.removeBookById(it.id)
}
}.onFailure { it.printStackTrace() }
writeBookMarksAndSaveLibraryToFile()
// TODO test when getting books it will not goes to circular dependencies mode.
updateLocalBooksFlow()
}
fun delete(bookId: String) {
runCatching {
library.removeBookById(bookId)
}.onFailure { it.printStackTrace() }
}
/**
* Asynchronously writes the library data to their respective file in a background thread
* to prevent potential data loss and ensures that the library holds the updated ZIM file data.
*/
private suspend fun writeBookMarksAndSaveLibraryToFile() {
// Save the library, which contains ZIM file data.
library.writeToFile(libraryFile.canonicalPath)
// set the bookmark change to true so that libkiwix will return the new data.
booksChanged = true
}
private suspend fun updateLocalBooksFlow() {
localBooksFlow.emit(getBooksList())
}
}

View File

@ -33,6 +33,8 @@ import kotlinx.coroutines.withContext
import org.kiwix.kiwixmobile.core.CoreApp
import org.kiwix.kiwixmobile.core.DarkModeConfig
import org.kiwix.kiwixmobile.core.R
import org.kiwix.kiwixmobile.core.di.modules.BOOKMARK_LIBRARY
import org.kiwix.kiwixmobile.core.di.modules.BOOKMARK_MANAGER
import org.kiwix.kiwixmobile.core.extensions.ActivityExtensions.isCustomApp
import org.kiwix.kiwixmobile.core.extensions.deleteFile
import org.kiwix.kiwixmobile.core.extensions.getFavicon
@ -53,11 +55,12 @@ import org.kiwix.libzim.Archive
import org.kiwix.libzim.SuggestionSearcher
import java.io.File
import javax.inject.Inject
import javax.inject.Named
class LibkiwixBookmarks @Inject constructor(
val library: Library,
manager: Manager,
val sharedPreferenceUtil: SharedPreferenceUtil,
@Named(BOOKMARK_LIBRARY) private val library: Library,
@Named(BOOKMARK_MANAGER) private val manager: Manager,
private val sharedPreferenceUtil: SharedPreferenceUtil,
private val bookDao: NewBookDao,
private val zimReaderContainer: ZimReaderContainer?
) : PageDao {
@ -69,16 +72,14 @@ class LibkiwixBookmarks @Inject constructor(
private var bookmarkList: List<LibkiwixBookmarkItem> = arrayListOf()
private var libraryBooksList: List<String> = arrayListOf()
@Suppress("InjectDispatcher", "TooGenericExceptionCaught")
@Suppress("InjectDispatcher")
private val bookmarkListFlow: MutableStateFlow<List<LibkiwixBookmarkItem>> by lazy {
MutableStateFlow<List<LibkiwixBookmarkItem>>(emptyList()).also { flow ->
CoroutineScope(Dispatchers.IO).launch {
try {
runCatching {
val bookmarks = getBookmarksList()
flow.emit(bookmarks)
} catch (e: Exception) {
e.printStackTrace()
}
}.onFailure { it.printStackTrace() }
}
}
}

View File

@ -24,7 +24,9 @@ import io.objectbox.kotlin.boxFor
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.kiwix.kiwixmobile.core.CoreApp
import org.kiwix.kiwixmobile.core.dao.LibkiwixBookOnDisk
import org.kiwix.kiwixmobile.core.dao.LibkiwixBookmarks
import org.kiwix.kiwixmobile.core.dao.entities.BookOnDiskEntity
import org.kiwix.kiwixmobile.core.dao.entities.BookmarkEntity
import org.kiwix.kiwixmobile.core.page.bookmark.adapter.LibkiwixBookmarkItem
import org.kiwix.kiwixmobile.core.utils.SharedPreferenceUtil
@ -43,14 +45,44 @@ class ObjectBoxToLibkiwixMigrator {
@Inject
lateinit var libkiwixBookmarks: LibkiwixBookmarks
@Inject
lateinit var libkiwixBookOnDisk: LibkiwixBookOnDisk
private val migrationMutex = Mutex()
suspend fun migrateBookmarksToLibkiwix() {
suspend fun migrateObjectBoxDataToLibkiwix() {
CoreApp.coreComponent.inject(this)
migrateBookMarks(boxStore.boxFor())
if (!sharedPreferenceUtil.prefIsBookmarksMigrated) {
migrateBookMarks(boxStore.boxFor())
}
if (!sharedPreferenceUtil.prefIsBookOnDiskMigrated) {
migrateLocalBooks(boxStore.boxFor())
}
// TODO we will migrate here for other entities
}
suspend fun migrateLocalBooks(box: Box<BookOnDiskEntity>) {
val bookOnDiskList = box.all
migrationMutex.withLock {
runCatching {
val libkiwixBooks = bookOnDiskList.map {
val archive = Archive(it.file.path)
Book().apply {
update(archive)
}
}
libkiwixBookOnDisk.insert(libkiwixBooks)
}.onFailure {
Log.e(
"MIGRATING_BOOK_ON_DISK",
"there is an error while migrating the bookOnDisk \n" +
"Original exception is = $it"
)
}
}
sharedPreferenceUtil.putPrefBookOnDiskMigrated(true)
}
suspend fun migrateBookMarks(box: Box<BookmarkEntity>) {
val bookMarksList = box.all
// run migration with mutex to do the migration one by one.

View File

@ -30,6 +30,7 @@ import org.kiwix.kiwixmobile.core.dao.DownloadRoomDao
import org.kiwix.kiwixmobile.core.dao.HistoryDao
import org.kiwix.kiwixmobile.core.dao.HistoryRoomDao
import org.kiwix.kiwixmobile.core.dao.LibkiwixBookmarks
import org.kiwix.kiwixmobile.core.dao.LibkiwixBookOnDisk
import org.kiwix.kiwixmobile.core.dao.NewBookDao
import org.kiwix.kiwixmobile.core.dao.NewBookmarksDao
import org.kiwix.kiwixmobile.core.dao.NewLanguagesDao
@ -99,6 +100,7 @@ interface CoreComponent {
fun connectivityManager(): ConnectivityManager
fun objectBoxToLibkiwixMigrator(): ObjectBoxToLibkiwixMigrator
fun libkiwixBookmarks(): LibkiwixBookmarks
fun libkiwixBooks(): LibkiwixBookOnDisk
fun recentSearchRoomDao(): RecentSearchRoomDao
fun historyRoomDao(): HistoryRoomDao
fun webViewHistoryRoomDao(): WebViewHistoryRoomDao

View File

@ -21,12 +21,14 @@ import android.content.Context
import dagger.Module
import dagger.Provides
import org.kiwix.kiwixmobile.core.dao.LibkiwixBookmarks
import org.kiwix.kiwixmobile.core.dao.LibkiwixBookOnDisk
import org.kiwix.kiwixmobile.core.dao.NewBookDao
import org.kiwix.kiwixmobile.core.reader.ZimReaderContainer
import org.kiwix.kiwixmobile.core.utils.SharedPreferenceUtil
import org.kiwix.libkiwix.JNIKiwix
import org.kiwix.libkiwix.Library
import org.kiwix.libkiwix.Manager
import javax.inject.Named
import javax.inject.Singleton
@Module
@ -36,20 +38,47 @@ class JNIModule {
@Provides
@Singleton
fun provideLibrary(): Library = Library()
@Named(BOOKMARK_LIBRARY)
fun provideBookmarkLibrary(): Library = Library()
@Provides
@Singleton
fun providesManager(library: Library): Manager = Manager(library)
@Named(BOOKMARK_MANAGER)
fun providesBookmarkManager(@Named(BOOKMARK_LIBRARY) library: Library): Manager =
Manager(library)
@Provides
@Singleton
fun providesLibkiwixBookmarks(
library: Library,
manager: Manager,
@Named(BOOKMARK_LIBRARY) library: Library,
@Named(BOOKMARK_MANAGER) manager: Manager,
sharedPreferenceUtil: SharedPreferenceUtil,
bookDao: NewBookDao,
zimReaderContainer: ZimReaderContainer
): LibkiwixBookmarks =
LibkiwixBookmarks(library, manager, sharedPreferenceUtil, bookDao, zimReaderContainer)
@Provides
@Singleton
@Named(LOCAL_BOOKS_LIBRARY)
fun provideLocalBooksLibrary(): Library = Library()
@Provides
@Singleton
@Named(LOCAL_BOOKS_MANAGER)
fun providesLocalBooksManager(@Named(LOCAL_BOOKS_LIBRARY) library: Library): Manager =
Manager(library)
@Provides
@Singleton
fun providesLibkiwixBooks(
@Named(LOCAL_BOOKS_LIBRARY) library: Library,
@Named(LOCAL_BOOKS_MANAGER) manager: Manager,
sharedPreferenceUtil: SharedPreferenceUtil,
): LibkiwixBookOnDisk = LibkiwixBookOnDisk(library, manager, sharedPreferenceUtil)
}
const val BOOKMARK_LIBRARY = "bookmarkLibrary"
const val BOOKMARK_MANAGER = "bookmarkManager"
const val LOCAL_BOOKS_LIBRARY = "localBooksLibrary"
const val LOCAL_BOOKS_MANAGER = "localBooksManager"

View File

@ -19,6 +19,7 @@
package org.kiwix.kiwixmobile.core.entity
import org.kiwix.kiwixmobile.core.extensions.getFavicon
import org.kiwix.kiwixmobile.core.reader.ZimReaderSource
import org.kiwix.libkiwix.Book
import java.io.File
@ -43,6 +44,7 @@ data class LibkiwixBook(
private var _bookName: String? = null,
private var _favicon: String = "",
private var _tags: String? = null,
private var _path: String? = "",
var searchMatches: Int = 0,
var file: File? = null
) {
@ -130,6 +132,15 @@ data class LibkiwixBook(
_tags = tags
}
var path: String?
get() = _path ?: nativeBook?.path
set(path) {
_path = path
}
val zimReaderSource: ZimReaderSource
get() = ZimReaderSource(File(path.orEmpty()))
// Two books are equal if their ids match
override fun equals(other: Any?): Boolean {
if (other is LibkiwixBook) {

View File

@ -121,9 +121,8 @@ abstract class CoreMainActivity : BaseActivity(), WebViewProvider {
super.onCreate(savedInstanceState)
if (!BuildConfig.DEBUG) {
val appContext = applicationContext
Thread.setDefaultUncaughtExceptionHandler {
paramThread: Thread?,
paramThrowable: Throwable? ->
Thread.setDefaultUncaughtExceptionHandler { paramThread: Thread?,
paramThrowable: Throwable? ->
val intent = Intent(appContext, ErrorActivity::class.java)
val extras = Bundle()
extras.putSerializable(ErrorActivity.EXCEPTION_KEY, paramThrowable)
@ -137,11 +136,9 @@ abstract class CoreMainActivity : BaseActivity(), WebViewProvider {
}
setMainActivityToCoreApp()
if (!sharedPreferenceUtil.prefIsBookmarksMigrated) {
// run the migration on background thread to avoid any UI related issues.
CoroutineScope(Dispatchers.IO).launch {
objectBoxToLibkiwixMigrator.migrateBookmarksToLibkiwix()
}
// run the migration on background thread to avoid any UI related issues.
CoroutineScope(Dispatchers.IO).launch {
objectBoxToLibkiwixMigrator.migrateObjectBoxDataToLibkiwix()
}
// run the migration on background thread to avoid any UI related issues.
CoroutineScope(Dispatchers.IO).launch {

View File

@ -114,6 +114,9 @@ class SharedPreferenceUtil @Inject constructor(val context: Context) {
val prefIsAppDirectoryMigrated: Boolean
get() = sharedPreferences.getBoolean(PREF_APP_DIRECTORY_TO_PUBLIC_MIGRATED, false)
val prefIsBookOnDiskMigrated: Boolean
get() = sharedPreferences.getBoolean(PREF_BOOK_ON_DISK_MIGRATED, false)
val prefStorage: String
get() {
val storage = sharedPreferences.getString(PREF_STORAGE, null)
@ -163,6 +166,9 @@ class SharedPreferenceUtil @Inject constructor(val context: Context) {
fun putPrefAppDirectoryMigrated(isMigrated: Boolean) =
sharedPreferences.edit { putBoolean(PREF_APP_DIRECTORY_TO_PUBLIC_MIGRATED, isMigrated) }
fun putPrefBookOnDiskMigrated(isMigrated: Boolean) =
sharedPreferences.edit { putBoolean(PREF_BOOK_ON_DISK_MIGRATED, isMigrated) }
fun putPrefLanguage(language: String) =
sharedPreferences.edit { putString(PREF_LANG, language) }
@ -345,6 +351,7 @@ class SharedPreferenceUtil @Inject constructor(val context: Context) {
const val PREF_HISTORY_MIGRATED = "pref_history_migrated"
const val PREF_NOTES_MIGRATED = "pref_notes_migrated"
const val PREF_APP_DIRECTORY_TO_PUBLIC_MIGRATED = "pref_app_directory_to_public_migrated"
const val PREF_BOOK_ON_DISK_MIGRATED = "pref_book_on_disk_migrated"
const val PREF_SHOW_COPY_MOVE_STORAGE_SELECTION_DIALOG = "pref_show_copy_move_storage_dialog"
private const val PREF_LATER_CLICKED_MILLIS = "pref_later_clicked_millis"
const val PREF_LAST_DONATION_POPUP_SHOWN_IN_MILLISECONDS =

View File

@ -54,6 +54,10 @@ sealed class BooksOnDiskListItem {
book.language.convertToLocal()
}
@Deprecated(
"Now we are using the libkiwix to store and retrieve the local " +
"books so this constructor is no longer used."
)
constructor(bookOnDiskEntity: BookOnDiskEntity) : this(
databaseId = bookOnDiskEntity.id,
file = bookOnDiskEntity.file,
@ -70,5 +74,10 @@ sealed class BooksOnDiskListItem {
book = zimFileReader.toBook(),
zimReaderSource = zimFileReader.zimReaderSource
)
constructor(libkiwixBook: LibkiwixBook) : this(
book = libkiwixBook,
zimReaderSource = libkiwixBook.zimReaderSource
)
}
}