From d9a10b30a624fba43e3cce6c3f0124adf2213dc8 Mon Sep 17 00:00:00 2001 From: MohitMaliFtechiz Date: Thu, 3 Jul 2025 00:40:57 +0530 Subject: [PATCH] Refactored the code to fetch the online library from OPDS same like Kiwix server, it first fetches the 50 items from the OPDS catelog, and load more when needed. --- .../zimManager/OnlineLibraryManager.kt | 107 +++++++++++++++--- .../zimManager/ZimManageViewModel.kt | 19 ++-- .../core/data/remote/KiwixService.kt | 11 +- 3 files changed, 112 insertions(+), 25 deletions(-) diff --git a/app/src/main/java/org/kiwix/kiwixmobile/zimManager/OnlineLibraryManager.kt b/app/src/main/java/org/kiwix/kiwixmobile/zimManager/OnlineLibraryManager.kt index 247491af5..b9b412c93 100644 --- a/app/src/main/java/org/kiwix/kiwixmobile/zimManager/OnlineLibraryManager.kt +++ b/app/src/main/java/org/kiwix/kiwixmobile/zimManager/OnlineLibraryManager.kt @@ -18,11 +18,18 @@ package org.kiwix.kiwixmobile.zimManager +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.kiwix.kiwixmobile.core.data.remote.KiwixService.Companion.OPDS_LIBRARY_ENDPOINT +import org.kiwix.kiwixmobile.core.downloader.downloadManager.ZERO import org.kiwix.kiwixmobile.core.entity.LibkiwixBook import org.kiwix.kiwixmobile.di.modules.ONLINE_BOOKS_LIBRARY import org.kiwix.kiwixmobile.di.modules.ONLINE_BOOKS_MANAGER import org.kiwix.libkiwix.Library import org.kiwix.libkiwix.Manager +import org.xmlpull.v1.XmlPullParser +import org.xmlpull.v1.XmlPullParserFactory +import java.io.StringReader import javax.inject.Inject import javax.inject.Named @@ -30,23 +37,26 @@ class OnlineLibraryManager @Inject constructor( @Named(ONLINE_BOOKS_LIBRARY) private val library: Library, @Named(ONLINE_BOOKS_MANAGER) private val manager: Manager, ) { - suspend fun parseOPDSStream(content: String?, urlHost: String): Boolean = - runCatching { - manager.readOpds(content, urlHost) - }.onFailure { - it.printStackTrace() - }.isSuccess - suspend fun getOnlineBooks(): List { - val onlineBooksList = arrayListOf() + var totalResult = 0 + suspend fun parseOPDSStreamAndGetBooks( + content: String?, + urlHost: String + ): ArrayList? = runCatching { - library.booksIds.forEach { bookId -> - val book = library.getBookById(bookId) + content?.let { totalResult = extractTotalResults(it) } + val onlineBooksList = arrayListOf() + val tempLibrary = Library() + val tempManager = Manager(tempLibrary) + tempManager.readOpds(content, urlHost) + tempLibrary.booksIds.forEach { bookId -> + val book = tempLibrary.getBookById(bookId) onlineBooksList.add(LibkiwixBook(book)) } - }.onFailure { it.printStackTrace() } - return onlineBooksList - } + onlineBooksList + }.onFailure { + it.printStackTrace() + }.getOrNull() suspend fun getOnlineBooksLanguage(): List { return runCatching { @@ -55,4 +65,75 @@ class OnlineLibraryManager @Inject constructor( it.printStackTrace() }.getOrDefault(emptyList()) } + + /** + * Builds the URL for fetching the OPDS library entries with pagination and optional filters. + * + * @param baseUrl The base URL of the Kiwix library server (e.g., "https://opds.library.kiwix.org/"). + * @param start The index from which to start fetching entries (default is 0). + * @param count The number of entries to fetch per page (default is 50). + * @param query Optional search query for filtering results by text. + * @param lang Optional language code filter (e.g., "en", "ita"). + * @param category Optional category filter (e.g., "wikipedia", "books"). + * @return A full URL string with query parameters applied. + * + * Example: + * buildLibraryUrl("https://library.kiwix.org", start = 100, count = 50, lang = "en") + * returns: "https://library.kiwix.org/v2/entries?start=100&count=50&lang=en" + */ + fun buildLibraryUrl( + baseUrl: String, + start: Int = 0, + count: Int = 50, + query: String? = null, + lang: String? = null, + category: String? = null + ): String { + val params = mutableListOf("start=$start", "count=$count") + query?.takeIf { it.isNotBlank() }?.let { params += "q=$it" } + lang?.takeIf { it.isNotBlank() }?.let { params += "lang=$it" } + category?.takeIf { it.isNotBlank() }?.let { params += "category=$it" } + + return "$baseUrl/$OPDS_LIBRARY_ENDPOINT?${params.joinToString("&")}" + } + + /** + * Calculates the total number of pages needed for pagination. + * + * @param totalResults Total number of items available (e.g., 3408). + * @param pageSize Number of items per page (e.g., 50). + * @return The total number of pages required to show all items. + * + * Example: + * calculateTotalPages(3408, 50) returns 69 + */ + fun calculateTotalPages(totalResults: Int, pageSize: Int): Int = + (totalResults + pageSize - 1) / pageSize + + /** + * Calculates the starting index (offset) for a given page number. + * + * @param pageIndex The page number starting from 0 (e.g., pageIndex = 2 means page 3). + * @param pageSize Number of items per page (e.g., 50). + * @return The offset index to be used in a paginated request (e.g., start=100). + * + * Example: + * getStartOffset(2, 50) returns 100 + */ + fun getStartOffset(pageIndex: Int, pageSize: Int): Int = pageIndex * pageSize + + private suspend fun extractTotalResults(xml: String): Int = withContext(Dispatchers.IO) { + val factory = XmlPullParserFactory.newInstance() + val parser = factory.newPullParser() + parser.setInput(StringReader(xml)) + + var eventType = parser.eventType + while (eventType != XmlPullParser.END_DOCUMENT) { + if (eventType == XmlPullParser.START_TAG && parser.name == "totalResults") { + return@withContext parser.nextText().toIntOrNull() ?: ZERO + } + eventType = parser.next() + } + return@withContext ZERO + } } diff --git a/app/src/main/java/org/kiwix/kiwixmobile/zimManager/ZimManageViewModel.kt b/app/src/main/java/org/kiwix/kiwixmobile/zimManager/ZimManageViewModel.kt index 2b6c20635..b4a471bec 100644 --- a/app/src/main/java/org/kiwix/kiwixmobile/zimManager/ZimManageViewModel.kt +++ b/app/src/main/java/org/kiwix/kiwixmobile/zimManager/ZimManageViewModel.kt @@ -70,7 +70,8 @@ import org.kiwix.kiwixmobile.core.dao.LanguageRoomDao import org.kiwix.kiwixmobile.core.dao.LibkiwixBookOnDisk import org.kiwix.kiwixmobile.core.data.DataSource import org.kiwix.kiwixmobile.core.data.remote.KiwixService -import org.kiwix.kiwixmobile.core.data.remote.KiwixService.Companion.OPDS_LIBRARY_NETWORK_PATH +import org.kiwix.kiwixmobile.core.data.remote.KiwixService.Companion.ITEMS_PER_PAGE +import org.kiwix.kiwixmobile.core.data.remote.KiwixService.Companion.OPDS_LIBRARY_ENDPOINT import org.kiwix.kiwixmobile.core.data.remote.ProgressResponseBody import org.kiwix.kiwixmobile.core.data.remote.UserAgentInterceptor import org.kiwix.kiwixmobile.core.di.modules.CALL_TIMEOUT @@ -254,7 +255,7 @@ class ZimManageViewModel @Inject constructor( private fun getContentLengthOfLibraryXmlFile(): Long { val headRequest = Request.Builder() - .url("$KIWIX_OPDS_LIBRARY_URL$OPDS_LIBRARY_NETWORK_PATH") + .url("$KIWIX_OPDS_LIBRARY_URL$OPDS_LIBRARY_ENDPOINT?count=$ITEMS_PER_PAGE") .head() .header("Accept-Encoding", "identity") .build() @@ -447,18 +448,22 @@ class ZimManageViewModel @Inject constructor( kiwixService: KiwixService ): Flow> = flow { downloadProgress.postValue(context.getString(R.string.starting_downloading_remote_library)) - val response = kiwixService.getLibrary() + // TODO get the filter from online library and pass it here to get the online content based on filters. + val buildUrl = onlineLibraryManager.buildLibraryUrl( + KIWIX_OPDS_LIBRARY_URL, + ) + val response = kiwixService.getLibraryPage(buildUrl) val resolvedUrl = response.raw().networkResponse?.request?.url ?: response.raw().request.url val baseHostUrl = "${resolvedUrl.scheme}://${resolvedUrl.host}" downloadProgress.postValue(context.getString(R.string.parsing_remote_library)) val libraryXml = response.body() - val isLibraryParsed = onlineLibraryManager.parseOPDSStream(libraryXml, baseHostUrl) + val onlineBooks = onlineLibraryManager.parseOPDSStreamAndGetBooks(libraryXml, baseHostUrl) emit( - if (isLibraryParsed) { - onlineLibraryManager.getOnlineBooks() - } else { + if (onlineBooks.isNullOrEmpty()) { emptyList() + } else { + onlineBooks } ) } diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/data/remote/KiwixService.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/data/remote/KiwixService.kt index 1ef98b7e6..808549b5f 100644 --- a/core/src/main/java/org/kiwix/kiwixmobile/core/data/remote/KiwixService.kt +++ b/core/src/main/java/org/kiwix/kiwixmobile/core/data/remote/KiwixService.kt @@ -29,8 +29,10 @@ import retrofit2.http.GET import retrofit2.http.Url interface KiwixService { - @GET(OPDS_LIBRARY_NETWORK_PATH) - suspend fun getLibrary(): Response + @GET + suspend fun getLibraryPage( + @Url url: String + ): Response @GET suspend fun getMetaLinks( @@ -52,8 +54,7 @@ interface KiwixService { } companion object { - // To fetch the full OPDS catalog. - // TODO we will change this to pagination later once we migrate to OPDS properly. - const val OPDS_LIBRARY_NETWORK_PATH = "v2/entries?count=-1" + const val OPDS_LIBRARY_ENDPOINT = "v2/entries" + const val ITEMS_PER_PAGE = 50 } }