diff --git a/src/manager/kiwix-manage.cpp b/src/manager/kiwix-manage.cpp index d6bdf7a..b47a4e1 100644 --- a/src/manager/kiwix-manage.cpp +++ b/src/manager/kiwix-manage.cpp @@ -22,33 +22,39 @@ #endif #include #include +#include #include +#include #include #include +#include +#include + using namespace std; -enum supportedAction { NONE, ADD, SHOW, REMOVE }; +enum supportedAction { NONE, ADD, SHOW, REMOVE, DOWNLOAD }; -void show(kiwix::Library library) +void show(kiwix::Library* library) { - std::vector::iterator itr; + auto booksIds = library->getBooksIds(); unsigned int inc = 1; - for (itr = library.books.begin(); itr != library.books.end(); ++itr) { + for(auto& id: booksIds) { + auto& book = library->getBookById(id); std::cout << "#" << inc++ << std::endl - << "id:\t\t" << itr->id << std::endl - << "path:\t\t" << itr->path << std::endl - << "indexpath:\t" << itr->indexPath << std::endl - << "url:\t\t" << itr->url << std::endl - << "title:\t\t" << itr->title << std::endl - << "name:\t\t" << itr->name << std::endl - << "tags:\t\t" << itr->tags << std::endl - << "description:\t" << itr->description << std::endl - << "creator:\t" << itr->creator << std::endl - << "date:\t\t" << itr->date << std::endl - << "articleCount:\t" << itr->articleCount << std::endl - << "mediaCount:\t" << itr->mediaCount << std::endl - << "size:\t\t" << itr->size << " KB" << std::endl + << "id:\t\t" << book.getId() << std::endl + << "path:\t\t" << book.getPath() << std::endl + << "indexpath:\t" << book.getIndexPath() << std::endl + << "url:\t\t" << book.getUrl() << std::endl + << "title:\t\t" << book.getTitle() << std::endl + << "name:\t\t" << book.getName() << std::endl + << "tags:\t\t" << book.getTags() << std::endl + << "description:\t" << book.getDescription() << std::endl + << "creator:\t" << book.getCreator() << std::endl + << "date:\t\t" << book.getDate() << std::endl + << "articleCount:\t" << book.getArticleCount() << std::endl + << "mediaCount:\t" << book.getMediaCount() << std::endl + << "size:\t\t" << book.getSize() << " KB" << std::endl << std::endl; } } @@ -68,23 +74,21 @@ void usage() } -bool handle_show(kiwix::Manager* libraryManager, const std::string& libraryPath, +bool handle_show(kiwix::Library* library, const std::string& libraryPath, int argc, char* argv[]) { - show(libraryManager->cloneLibrary()); + show(library); return(0); } -bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath, +bool handle_add(kiwix::Library* library, const std::string& libraryPath, int argc, char* argv[]) { string zimPath; string zimPathToSave = "."; string indexPath; - kiwix::supportedIndexType indexBackend = kiwix::XAPIAN; string url; string origID = ""; - bool setCurrent = false; int option_index = 0; int c = 0; bool resultCode = 0; @@ -100,12 +104,10 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath, = {{"url", required_argument, 0, 'u'}, {"origId", required_argument, 0, 'o'}, {"indexPath", required_argument, 0, 'i'}, - {"indexBackend", required_argument, 0, 'b'}, {"zimPathToSave", required_argument, 0, 'z'}, - {"current", no_argument, 0, 'c'}, {0, 0, 0, 0}}; - c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index); + c = getopt_long(argc, argv, "cz:u:i:", long_options, &option_index); if (c != -1) { switch (c) { @@ -117,22 +119,10 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath, origID = optarg; break; - case 'c': - setCurrent = true; - break; - case 'i': indexPath = optarg; break; - case 'b': - if (!strcmp(optarg, "xapian")) { - indexBackend = kiwix::XAPIAN; - } else { - usage(); - } - break; - case 'z': zimPathToSave = optarg; break; @@ -143,15 +133,18 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath, } if (!zimPath.empty()) { + kiwix::Manager manager(library); zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave; - string bookId = libraryManager->addBookFromPathAndGetId( + string bookId = manager.addBookFromPathAndGetId( zimPath, zimPathToSave, url, false); if (!bookId.empty()) { - if (setCurrent) - libraryManager->setCurrentBookId(bookId); /* Save the index infos if necessary */ - if (!indexPath.empty()) - libraryManager->setBookIndex(bookId, indexPath, indexBackend); + if (!indexPath.empty()) { + if (isRelativePath(indexPath)) { + indexPath = computeAbsolutePath(indexPath, getCurrentDirectory()); + } + library->getBookById(bookId).setIndexPath(indexPath); + } } else { cerr << "Unable to build or save library file '" << libraryPath << "'" << endl; @@ -165,28 +158,25 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath, return(resultCode); } -bool handle_remove(kiwix::Manager* libraryManager, const std::string& libraryPath, +bool handle_remove(kiwix::Library* library, const std::string& libraryPath, int argc, char* argv[]) { - unsigned int bookIndex = 0; - const unsigned int totalBookCount = libraryManager->getBookCount(true, true); + std::string bookId; + const unsigned int totalBookCount = library->getBookCount(true, true); bool exitCode = 0; if (argc > 3) { - bookIndex = atoi(argv[3]); + bookId = argv[3]; } - if (bookIndex > 0 && bookIndex <= totalBookCount) { - libraryManager->removeBookByIndex(bookIndex - 1); - } else { + if (!library->removeBookById(bookId)) { if (totalBookCount > 0) { std::cerr - << "Invalid book index number. Please give a number between 1 and " - << totalBookCount << std::endl; + << "Invalid book id." << std::endl; exitCode = 1; } else { std::cerr - << "Invalid book index number. Library is empty, no book to delete." + << "Invalid book id. Library is empty, no book to delete." << std::endl; exitCode = 1; } @@ -195,11 +185,67 @@ bool handle_remove(kiwix::Manager* libraryManager, const std::string& libraryPat return(exitCode); } +bool handle_download(kiwix::Library* library, const std::string& libraryPath, + int argc, char* argv[]) +{ + std::string bookId; + bool exitCode = false; + + if (argc > 3) { + bookId = argv[3]; + } + + auto& book = library->getBookById(bookId); + auto did = book.getDownloadId(); + kiwix::Downloader downloader; + kiwix::Download* download = nullptr; + if (!did.empty()) { + std::cout << "try resume " << did << std::endl; + try { + download = downloader.getDownload(did); + } catch(...) {} + } + if (nullptr == download || download->getStatus() == kiwix::Download::K_UNKNOWN) { + download = downloader.startDownload(book.getUrl()); + book.setDownloadId(download->getDid()); + } + int step = 60*5; + while (step--) { + download->updateStatus(); + if (download->getStatus() == kiwix::Download::K_COMPLETE) { + auto followingId = download->getFollowedBy(); + if (followingId.empty()) { + book.setPath(download->getPath()); + book.setDownloadId(""); + std::cout << "File downloaded to " << book.getPath() << std::endl; + break; + } else { + download = downloader.getDownload(followingId); + } + } else if (download->getStatus() == kiwix::Download::K_ACTIVE) { + std::cout << download->getDid() << " : " + << kiwix::beautifyFileSize(download->getCompletedLength()) << "/" + << kiwix::beautifyFileSize(download->getTotalLength()) + << " (" << kiwix::beautifyFileSize(download->getDownloadSpeed()) << "/s) " + << " [" << kiwix::beautifyFileSize(download->getVerifiedLength()) << "]" + << "[" << step << "] \n" << std::flush; + } else if (download->getStatus() == kiwix::Download::K_ERROR) { + std::cout << "File Error" << std::endl; + exitCode = true; + break; + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + downloader.close(); + return exitCode; +} + int main(int argc, char** argv) { string libraryPath = ""; supportedAction action = NONE; - kiwix::Manager libraryManager; + kiwix::Library library; /* Argument parsing */ if (argc > 2) { @@ -212,6 +258,8 @@ int main(int argc, char** argv) action = SHOW; else if (actionString == "remove" || actionString == "delete") action = REMOVE; + else if (actionString == "download") + action = DOWNLOAD; } /* Print usage)) if necessary */ @@ -224,21 +272,31 @@ int main(int argc, char** argv) libraryPath = isRelativePath(libraryPath) ? computeAbsolutePath(getCurrentDirectory(), libraryPath) : libraryPath; - libraryManager.readFile(libraryPath, false); + kiwix::Manager manager(&library); + manager.readFile(libraryPath, false); /* SHOW */ bool exitCode = 0; - if (action == SHOW) { - exitCode = handle_show(&libraryManager, libraryPath, argc, argv); - } else if (action == ADD) { - exitCode = handle_add(&libraryManager, libraryPath, argc, argv); - } else if (action == REMOVE) { - exitCode = handle_remove(&libraryManager, libraryPath, argc, argv); + switch (action) { + case SHOW: + exitCode = handle_show(&library, libraryPath, argc, argv); + break; + case ADD: + exitCode = handle_add(&library, libraryPath, argc, argv); + break; + case REMOVE: + exitCode = handle_remove(&library, libraryPath, argc, argv); + break; + case DOWNLOAD: + exitCode = handle_download(&library, libraryPath, argc, argv); + break; + case NONE: + break; } /* Rewrite the library file */ - if (action == REMOVE || action == ADD) { - libraryManager.writeFile(libraryPath); + if (action == REMOVE || action == ADD || action == DOWNLOAD) { + library.writeToFile(libraryPath); } exit(exitCode); diff --git a/src/server/kiwix-serve.cpp b/src/server/kiwix-serve.cpp index 2d5cd3a..1345c30 100644 --- a/src/server/kiwix-serve.cpp +++ b/src/server/kiwix-serve.cpp @@ -101,20 +101,11 @@ static std::map extMimeTypes; static std::map readers; static std::map searchers; static kiwix::Searcher* globalSearcher = nullptr; -static kiwix::Manager libraryManager; +static kiwix::Library library; static pthread_mutex_t searchLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t compressorLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER; -template -inline std::string _tostring(const T& value) -{ - std::ostringstream stream; - stream << value; - return stream.str(); -} - - std::pair get_from_humanReadableBookId(const std::string& humanReadableBookId) { kiwix::Searcher* searcher @@ -385,7 +376,7 @@ static struct MHD_Response* build_callback_response_from_entry( MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str()); MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_LENGTH, - _tostring(range_len).c_str()); + kiwix::to_string(range_len).c_str()); /* Specify the mime type */ MHD_add_response_header( @@ -694,33 +685,51 @@ static struct MHD_Response* handle_catalog(RequestContext* request) kiwix::OPDSDumper opdsDumper; opdsDumper.setRootLocation(rootLocation); opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml"); + opdsDumper.setId(kiwix::to_string(uuid)); + opdsDumper.setLibrary(&library); mimeType = "application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8"; - kiwix::Library library_to_dump; + std::vector bookIdsToDump; if (url == "root.xml") { opdsDumper.setTitle("All zims"); uuid = zim::Uuid::generate(host); - library_to_dump = libraryManager.cloneLibrary(); + + bookIdsToDump = library.listBooksIds( + kiwix::VALID|kiwix::LOCAL|kiwix::REMOTE); } else if (url == "search") { std::string query; + std::string language; + size_t count(10); + size_t startIndex(0); try { query = request->get_argument("q"); - } catch (const std::out_of_range&) { - return build_404(request, ""); - } + } catch (const std::out_of_range&) {} + try { + language = request->get_argument("lang"); + } catch (const std::out_of_range&) {} + try { + count = stoul(request->get_argument("count")); + } catch (...) {} + try { + startIndex = stoul(request->get_argument("start")); + } catch (...) {} opdsDumper.setTitle("Search result for " + query); uuid = zim::Uuid::generate(); - library_to_dump = libraryManager.filter(query); + bookIdsToDump = library.listBooksIds( + kiwix::VALID|kiwix::LOCAL|kiwix::REMOTE, + kiwix::UNSORTED, + query, + language); + auto totalResults = bookIdsToDump.size(); + bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex); + if (count>0 && bookIdsToDump.size() > count) { + bookIdsToDump.resize(count); + } + opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size()); } else { return build_404(request, ""); } - { - std::stringstream ss; - ss << uuid; - opdsDumper.setId(ss.str()); - } - opdsDumper.setLibrary(library_to_dump); - content = opdsDumper.dumpOPDSFeed(); + content = opdsDumper.dumpOPDSFeed(bookIdsToDump); } bool deflated = request->can_compress() && compress_content(content, mimeType); @@ -1021,6 +1030,7 @@ int main(int argc, char** argv) } /* Setup the library manager and get the list of books */ + kiwix::Manager manager(&library); if (libraryFlag) { vector libraryPaths = kiwix::split(libraryPath, ";"); vector::iterator itr; @@ -1034,7 +1044,7 @@ int main(int argc, char** argv) = isRelativePath(*itr) ? computeAbsolutePath(getCurrentDirectory(), *itr) : *itr; - retVal = libraryManager.readFile(libraryPath, true); + retVal = manager.readFile(libraryPath, true); } catch (...) { retVal = false; } @@ -1048,74 +1058,69 @@ int main(int argc, char** argv) } /* Check if the library is not empty (or only remote books)*/ - if (libraryManager.getBookCount(true, false) == 0) { + if (library.getBookCount(true, false) == 0) { cerr << "The XML library file '" << libraryPath << "' is empty (or has only remote books)." << endl; } } else { std::vector::iterator it; for (it = zimPathes.begin(); it != zimPathes.end(); it++) { - if (!libraryManager.addBookFromPath(*it, *it, "", false)) { + if (!manager.addBookFromPath(*it, *it, "", false)) { cerr << "Unable to add the ZIM file '" << *it << "' to the internal library." << endl; exit(1); } } if (!indexPath.empty()) { - libraryManager.setBookIndex(libraryManager.getBooksIds()[0], indexPath); + if (isRelativePath(indexPath)) { + indexPath = computeAbsolutePath(indexPath, getCurrentDirectory()); + } + library.getBookById(library.getBooksIds()[0]).setIndexPath(indexPath); } } /* Instance the readers and searcher and build the corresponding maps */ - vector booksIds = libraryManager.getBooksIds(); - vector::iterator itr; - kiwix::Book currentBook; + vector booksIds = library.listBooksIds(kiwix::LOCAL); globalSearcher = new kiwix::Searcher(); globalSearcher->setProtocolPrefix(rootLocation + "/"); globalSearcher->setSearchProtocolPrefix(rootLocation + "/" + "search?"); - for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) { - bool zimFileOk = false; - libraryManager.getBookById(*itr, currentBook); - std::string zimPath = currentBook.pathAbsolute; + for (auto& bookId: booksIds) { + auto& currentBook = library.getBookById(bookId); + auto zimPath = currentBook.getPath(); + auto indexPath = currentBook.getIndexPath(); - if (!zimPath.empty()) { - indexPath = currentBook.indexPathAbsolute; + /* Instanciate the ZIM file handler */ + kiwix::Reader* reader = NULL; + try { + reader = new kiwix::Reader(zimPath); + } catch (...) { + cerr << "Unable to open the ZIM file '" << zimPath << "'." << endl; + continue; + } - /* Instanciate the ZIM file handler */ - kiwix::Reader* reader = NULL; + auto humanReadableId = currentBook.getHumanReadableIdFromPath(); + readers[humanReadableId] = reader; + + if (reader->hasFulltextIndex()) { + kiwix::Searcher* searcher = new kiwix::Searcher(humanReadableId); + searcher->setProtocolPrefix(rootLocation + "/"); + searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?"); + searcher->add_reader(reader, humanReadableId); + globalSearcher->add_reader(reader, humanReadableId); + searchers[humanReadableId] = searcher; + } else if ( !indexPath.empty() ) { try { - reader = new kiwix::Reader(zimPath); - zimFileOk = true; + kiwix::Searcher* searcher = new kiwix::Searcher(indexPath, reader, humanReadableId); + searcher->setProtocolPrefix(rootLocation + "/"); + searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?"); + searchers[humanReadableId] = searcher; } catch (...) { - cerr << "Unable to open the ZIM file '" << zimPath << "'." << endl; - } - - if (zimFileOk) { - string humanReadableId = currentBook.getHumanReadableIdFromPath(); - readers[humanReadableId] = reader; - - if ( reader->hasFulltextIndex()) { - kiwix::Searcher* searcher = new kiwix::Searcher(humanReadableId); - searcher->setProtocolPrefix(rootLocation + "/"); - searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?"); - searcher->add_reader(reader, humanReadableId); - globalSearcher->add_reader(reader, humanReadableId); - searchers[humanReadableId] = searcher; - } else if ( !indexPath.empty() ) { - try { - kiwix::Searcher* searcher = new kiwix::Searcher(indexPath, reader, humanReadableId); - searcher->setProtocolPrefix(rootLocation + "/"); - searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?"); - searchers[humanReadableId] = searcher; - } catch (...) { - cerr << "Unable to open the search index '" << indexPath << "'." - << endl; - searchers[humanReadableId] = nullptr; - } - } else { - searchers[humanReadableId] = nullptr; - } + cerr << "Unable to open the search index '" << indexPath << "'." + << endl; + searchers[humanReadableId] = nullptr; } + } else { + searchers[humanReadableId] = nullptr; } } @@ -1123,24 +1128,25 @@ int main(int argc, char** argv) string welcomeBooksHtml = "" "
"; - for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) { - libraryManager.getBookById(*itr, currentBook); + for (auto& bookId: booksIds) { + auto& currentBook = library.getBookById(bookId); - if (!currentBook.path.empty() + if (!currentBook.getPath().empty() && readers.find(currentBook.getHumanReadableIdFromPath()) != readers.end()) { welcomeBooksHtml += "" "" "
" - "
" - "
" + currentBook.title + "
" - "
" + currentBook.description + "
" + "
" + "
" + currentBook.getTitle() + "
" + "
" + currentBook.getDescription() + "
" "
" - "" + kiwix::beautifyInteger(atoi(currentBook.articleCount.c_str())) + " articles, " + kiwix::beautifyInteger(atoi(currentBook.mediaCount.c_str())) + " medias" + "" + kiwix::beautifyInteger(currentBook.getArticleCount()) + " articles, " + kiwix::beautifyInteger(currentBook.getMediaCount()) + " medias" "
" "
" "
" -""; +"\n"; } } welcomeBooksHtml += "" diff --git a/static/server/opensearchdescription.xml b/static/server/opensearchdescription.xml index 27c67e9..1f29bc6 100644 --- a/static/server/opensearchdescription.xml +++ b/static/server/opensearchdescription.xml @@ -4,5 +4,6 @@ Search zim files in the catalog. + indexOffset="0" + template="/__ROOT_LOCATION__/catalog/search?q={searchTerms}&lang={language}&count={count}&start={startIndex}"/>