Merge pull request #229 from kiwix/new_kiwix-lib_api

WIP New kiwix lib api

src/manager/kiwix-manage.cpp

@@ -22,33 +22,39 @@
 #endif
 #include <getopt.h>
 #include <kiwix/common/pathTools.h>
+#include <kiwix/common/stringTools.h>
 #include <kiwix/manager.h>
+#include <kiwix/downloader.h>
 #include <cstdlib>
 #include <iostream>
 
+#include <thread>
+#include <chrono>
+
 using namespace std;
 
-enum supportedAction { NONE, ADD, SHOW, REMOVE };
+enum supportedAction { NONE, ADD, SHOW, REMOVE, DOWNLOAD };
 
-void show(kiwix::Library library)
+void show(kiwix::Library* library)
 {
-  std::vector<kiwix::Book>::iterator itr;
+  auto booksIds = library->getBooksIds();
   unsigned int inc = 1;
-  for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
+  for(auto& id: booksIds) {
+    auto& book = library->getBookById(id);
     std::cout << "#" << inc++ << std::endl
-              << "id:\t\t" << itr->id << std::endl
+              << "id:\t\t" << book.getId() << std::endl
-              << "path:\t\t" << itr->path << std::endl
+              << "path:\t\t" << book.getPath() << std::endl
-              << "indexpath:\t" << itr->indexPath << std::endl
+              << "indexpath:\t" << book.getIndexPath() << std::endl
-              << "url:\t\t" << itr->url << std::endl
+              << "url:\t\t" << book.getUrl() << std::endl
-              << "title:\t\t" << itr->title << std::endl
+              << "title:\t\t" << book.getTitle() << std::endl
-              << "name:\t\t" << itr->name << std::endl
+              << "name:\t\t" << book.getName() << std::endl
-              << "tags:\t\t" << itr->tags << std::endl
+              << "tags:\t\t" << book.getTags() << std::endl
-              << "description:\t" << itr->description << std::endl
+              << "description:\t" << book.getDescription() << std::endl
-              << "creator:\t" << itr->creator << std::endl
+              << "creator:\t" << book.getCreator() << std::endl
-              << "date:\t\t" << itr->date << std::endl
+              << "date:\t\t" << book.getDate() << std::endl
-              << "articleCount:\t" << itr->articleCount << std::endl
+              << "articleCount:\t" << book.getArticleCount() << std::endl
-              << "mediaCount:\t" << itr->mediaCount << std::endl
+              << "mediaCount:\t" << book.getMediaCount() << std::endl
-              << "size:\t\t" << itr->size << " KB" << std::endl
+              << "size:\t\t" << book.getSize() << " KB" << std::endl
               << std::endl;
   }
 }
@@ -68,23 +74,21 @@ void usage()
 }
 
 
-bool handle_show(kiwix::Manager* libraryManager, const std::string& libraryPath,
+bool handle_show(kiwix::Library* library, const std::string& libraryPath,
                  int argc, char* argv[])
 {
-  show(libraryManager->cloneLibrary());
+  show(library);
   return(0);
 }
 
-bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
+bool handle_add(kiwix::Library* library, const std::string& libraryPath,
                 int argc, char* argv[])
 {
   string zimPath;
   string zimPathToSave = ".";
   string indexPath;
-  kiwix::supportedIndexType indexBackend = kiwix::XAPIAN;
   string url;
   string origID = "";
-  bool setCurrent = false;
   int option_index = 0;
   int c = 0;
   bool resultCode = 0;
@@ -100,12 +104,10 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
         = {{"url", required_argument, 0, 'u'},
            {"origId", required_argument, 0, 'o'},
            {"indexPath", required_argument, 0, 'i'},
-           {"indexBackend", required_argument, 0, 'b'},
            {"zimPathToSave", required_argument, 0, 'z'},
-           {"current", no_argument, 0, 'c'},
            {0, 0, 0, 0}};
 
-    c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index);
+    c = getopt_long(argc, argv, "cz:u:i:", long_options, &option_index);
 
     if (c != -1) {
       switch (c) {
@@ -117,22 +119,10 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
           origID = optarg;
           break;
 
-        case 'c':
-          setCurrent = true;
-          break;
-
         case 'i':
           indexPath = optarg;
           break;
 
-        case 'b':
-          if (!strcmp(optarg, "xapian")) {
-            indexBackend = kiwix::XAPIAN;
-          } else {
-            usage();
-          }
-          break;
-
         case 'z':
           zimPathToSave = optarg;
           break;
@@ -143,15 +133,18 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
   }
 
   if (!zimPath.empty()) {
+    kiwix::Manager manager(library);
     zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
-    string bookId = libraryManager->addBookFromPathAndGetId(
+    string bookId = manager.addBookFromPathAndGetId(
         zimPath, zimPathToSave, url, false);
      if (!bookId.empty()) {
-      if (setCurrent)
-        libraryManager->setCurrentBookId(bookId);
       /* Save the index infos if necessary */
-      if (!indexPath.empty())
+      if (!indexPath.empty()) {
-        libraryManager->setBookIndex(bookId, indexPath, indexBackend);
+        if (isRelativePath(indexPath)) {
+          indexPath = computeAbsolutePath(indexPath, getCurrentDirectory());
+        }
+        library->getBookById(bookId).setIndexPath(indexPath);
+      }
      } else {
       cerr << "Unable to build or save library file '" << libraryPath << "'"
            << endl;
@@ -165,28 +158,25 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
   return(resultCode);
 }
 
-bool handle_remove(kiwix::Manager* libraryManager, const std::string& libraryPath,
+bool handle_remove(kiwix::Library* library, const std::string& libraryPath,
                    int argc, char* argv[])
 {
-  unsigned int bookIndex = 0;
+  std::string bookId;
-  const unsigned int totalBookCount = libraryManager->getBookCount(true, true);
+  const unsigned int totalBookCount = library->getBookCount(true, true);
   bool exitCode = 0;
 
   if (argc > 3) {
-    bookIndex = atoi(argv[3]);
+    bookId = argv[3];
   }
 
-  if (bookIndex > 0 && bookIndex <= totalBookCount) {
+  if (!library->removeBookById(bookId)) {
-    libraryManager->removeBookByIndex(bookIndex - 1);
-  } else {
     if (totalBookCount > 0) {
       std::cerr
-          << "Invalid book index number. Please give a number between 1 and "
+          << "Invalid book id." << std::endl;
-          << totalBookCount << std::endl;
       exitCode = 1;
     } else {
       std::cerr
-          << "Invalid book index number. Library is empty, no book to delete."
+          << "Invalid book id. Library is empty, no book to delete."
           << std::endl;
       exitCode = 1;
     }
@@ -195,11 +185,67 @@ bool handle_remove(kiwix::Manager* libraryManager, const std::string& libraryPat
   return(exitCode);
 }
 
+bool handle_download(kiwix::Library* library, const std::string& libraryPath,
+                     int argc, char* argv[])
+{
+  std::string bookId;
+  bool exitCode = false;
+
+  if (argc > 3) {
+     bookId = argv[3];
+  }
+
+  auto& book = library->getBookById(bookId);
+  auto did = book.getDownloadId();
+  kiwix::Downloader downloader;
+  kiwix::Download* download = nullptr;
+  if (!did.empty()) {
+    std::cout << "try resume " << did << std::endl;
+    try {
+      download = downloader.getDownload(did);
+    } catch(...) {}
+  }
+  if (nullptr == download || download->getStatus() == kiwix::Download::K_UNKNOWN) {
+    download = downloader.startDownload(book.getUrl());
+    book.setDownloadId(download->getDid());
+  }
+  int step = 60*5;
+  while (step--) {
+    download->updateStatus();
+    if (download->getStatus() == kiwix::Download::K_COMPLETE) {
+      auto followingId = download->getFollowedBy();
+      if (followingId.empty()) {
+        book.setPath(download->getPath());
+        book.setDownloadId("");
+        std::cout << "File downloaded to " << book.getPath() << std::endl;
+        break;
+      } else {
+        download = downloader.getDownload(followingId);
+      }
+    } else if (download->getStatus() == kiwix::Download::K_ACTIVE) {
+      std::cout << download->getDid() << " : "
+                << kiwix::beautifyFileSize(download->getCompletedLength()) << "/"
+                << kiwix::beautifyFileSize(download->getTotalLength())
+                << " (" << kiwix::beautifyFileSize(download->getDownloadSpeed()) << "/s) "
+                << " [" << kiwix::beautifyFileSize(download->getVerifiedLength()) << "]"
+                << "[" << step << "]    \n" << std::flush;
+    } else if (download->getStatus() == kiwix::Download::K_ERROR) {
+      std::cout << "File Error" << std::endl;
+      exitCode = true;
+      break;
+    }
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+  }
+
+  downloader.close();
+  return exitCode;
+}
+
 int main(int argc, char** argv)
 {
   string libraryPath = "";
   supportedAction action = NONE;
-  kiwix::Manager libraryManager;
+  kiwix::Library library;
 
   /* Argument parsing */
   if (argc > 2) {
@@ -212,6 +258,8 @@ int main(int argc, char** argv)
       action = SHOW;
     else if (actionString == "remove" || actionString == "delete")
       action = REMOVE;
+    else if (actionString == "download")
+      action = DOWNLOAD;
   }
 
   /* Print usage)) if necessary */
@@ -224,21 +272,31 @@ int main(int argc, char** argv)
   libraryPath = isRelativePath(libraryPath)
                     ? computeAbsolutePath(getCurrentDirectory(), libraryPath)
                     : libraryPath;
-  libraryManager.readFile(libraryPath, false);
+  kiwix::Manager manager(&library);
+  manager.readFile(libraryPath, false);
 
   /* SHOW */
   bool exitCode = 0;
-  if (action == SHOW) {
+  switch (action) {
-    exitCode = handle_show(&libraryManager, libraryPath, argc, argv);
+    case SHOW:
-  } else if (action == ADD) {
+      exitCode = handle_show(&library, libraryPath, argc, argv);
-    exitCode = handle_add(&libraryManager, libraryPath, argc, argv);
+      break;
-  } else if (action == REMOVE) {
+    case ADD:
-    exitCode = handle_remove(&libraryManager, libraryPath, argc, argv);
+      exitCode = handle_add(&library, libraryPath, argc, argv);
+      break;
+    case REMOVE:
+      exitCode = handle_remove(&library, libraryPath, argc, argv);
+      break;
+    case DOWNLOAD:
+      exitCode = handle_download(&library, libraryPath, argc, argv);
+      break;
+    case NONE:
+      break;
   }
 
   /* Rewrite the library file */
-  if (action == REMOVE || action == ADD) {
+  if (action == REMOVE || action == ADD || action == DOWNLOAD) {
-    libraryManager.writeFile(libraryPath);
+    library.writeToFile(libraryPath);
   }
 
   exit(exitCode);

src/server/kiwix-serve.cpp

@@ -101,20 +101,11 @@ static std::map<std::string, std::string> extMimeTypes;
 static std::map<std::string, kiwix::Reader*> readers;
 static std::map<std::string, kiwix::Searcher*> searchers;
 static kiwix::Searcher* globalSearcher = nullptr;
-static kiwix::Manager libraryManager;
+static kiwix::Library library;
 static pthread_mutex_t searchLock = PTHREAD_MUTEX_INITIALIZER;
 static pthread_mutex_t compressorLock = PTHREAD_MUTEX_INITIALIZER;
 static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER;
 
-template<typename T>
-inline std::string _tostring(const T& value)
-{
-    std::ostringstream stream;
-    stream << value;
-    return stream.str();
-}
-
-
 std::pair<kiwix::Reader*, kiwix::Searcher*>
 get_from_humanReadableBookId(const std::string& humanReadableBookId) {
   kiwix::Searcher* searcher
@@ -385,7 +376,7 @@ static struct MHD_Response* build_callback_response_from_entry(
   MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
 
   MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_LENGTH,
-    _tostring(range_len).c_str());
+    kiwix::to_string(range_len).c_str());
 
   /* Specify the mime type */
   MHD_add_response_header(
@@ -694,33 +685,51 @@ static struct MHD_Response* handle_catalog(RequestContext* request)
     kiwix::OPDSDumper opdsDumper;
     opdsDumper.setRootLocation(rootLocation);
     opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
+    opdsDumper.setId(kiwix::to_string(uuid));
+    opdsDumper.setLibrary(&library);
     mimeType = "application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8";
-    kiwix::Library library_to_dump;
+    std::vector<std::string> bookIdsToDump;
     if (url == "root.xml") {
       opdsDumper.setTitle("All zims");
       uuid = zim::Uuid::generate(host);
-      library_to_dump = libraryManager.cloneLibrary();
+
+      bookIdsToDump = library.listBooksIds(
+        kiwix::VALID|kiwix::LOCAL|kiwix::REMOTE);
     } else if (url == "search") {
       std::string query;
+      std::string language;
+      size_t count(10);
+      size_t startIndex(0);
       try {
         query = request->get_argument("q");
-      } catch (const std::out_of_range&) {
+      } catch (const std::out_of_range&) {}
-        return build_404(request, "");
+      try {
-      }
+        language = request->get_argument("lang");
+      } catch (const std::out_of_range&) {}
+      try {
+        count = stoul(request->get_argument("count"));
+      } catch (...) {}
+      try {
+        startIndex = stoul(request->get_argument("start"));
+      } catch (...) {}
       opdsDumper.setTitle("Search result for " + query);
       uuid = zim::Uuid::generate();
-      library_to_dump = libraryManager.filter(query);
+      bookIdsToDump = library.listBooksIds(
+        kiwix::VALID|kiwix::LOCAL|kiwix::REMOTE,
+        kiwix::UNSORTED,
+        query,
+        language);
+      auto totalResults = bookIdsToDump.size();
+      bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
+      if (count>0 && bookIdsToDump.size() > count) {
+        bookIdsToDump.resize(count);
+      }
+      opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
     } else {
       return build_404(request, "");
     }
 
-    {
+    content = opdsDumper.dumpOPDSFeed(bookIdsToDump);
-      std::stringstream ss;
-      ss << uuid;
-      opdsDumper.setId(ss.str());
-    }
-    opdsDumper.setLibrary(library_to_dump);
-    content = opdsDumper.dumpOPDSFeed();
   }
 
   bool deflated = request->can_compress() && compress_content(content, mimeType);
@@ -1021,6 +1030,7 @@ int main(int argc, char** argv)
   }
 
   /* Setup the library manager and get the list of books */
+  kiwix::Manager manager(&library);
   if (libraryFlag) {
     vector<string> libraryPaths = kiwix::split(libraryPath, ";");
     vector<string>::iterator itr;
@@ -1034,7 +1044,7 @@ int main(int argc, char** argv)
               = isRelativePath(*itr)
                     ? computeAbsolutePath(getCurrentDirectory(), *itr)
                     : *itr;
-          retVal = libraryManager.readFile(libraryPath, true);
+          retVal = manager.readFile(libraryPath, true);
         } catch (...) {
           retVal = false;
         }
@@ -1048,74 +1058,69 @@ int main(int argc, char** argv)
     }
 
     /* Check if the library is not empty (or only remote books)*/
-    if (libraryManager.getBookCount(true, false) == 0) {
+    if (library.getBookCount(true, false) == 0) {
       cerr << "The XML library file '" << libraryPath
            << "' is empty (or has only remote books)." << endl;
     }
   } else {
     std::vector<std::string>::iterator it;
     for (it = zimPathes.begin(); it != zimPathes.end(); it++) {
-      if (!libraryManager.addBookFromPath(*it, *it, "", false)) {
+      if (!manager.addBookFromPath(*it, *it, "", false)) {
         cerr << "Unable to add the ZIM file '" << *it
              << "' to the internal library." << endl;
         exit(1);
       }
     }
     if (!indexPath.empty()) {
-      libraryManager.setBookIndex(libraryManager.getBooksIds()[0], indexPath);
+      if (isRelativePath(indexPath)) {
+        indexPath = computeAbsolutePath(indexPath, getCurrentDirectory());
+      }
+      library.getBookById(library.getBooksIds()[0]).setIndexPath(indexPath);
     }
   }
 
   /* Instance the readers and searcher and build the corresponding maps */
-  vector<string> booksIds = libraryManager.getBooksIds();
+  vector<string> booksIds = library.listBooksIds(kiwix::LOCAL);
-  vector<string>::iterator itr;
-  kiwix::Book currentBook;
   globalSearcher = new kiwix::Searcher();
   globalSearcher->setProtocolPrefix(rootLocation + "/");
   globalSearcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
-  for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) {
+  for (auto& bookId: booksIds) {
-    bool zimFileOk = false;
+    auto& currentBook = library.getBookById(bookId);
-    libraryManager.getBookById(*itr, currentBook);
+    auto zimPath = currentBook.getPath();
-    std::string zimPath = currentBook.pathAbsolute;
+    auto indexPath = currentBook.getIndexPath();
 
-    if (!zimPath.empty()) {
+    /* Instanciate the ZIM file handler */
-      indexPath = currentBook.indexPathAbsolute;
+    kiwix::Reader* reader = NULL;
+    try {
+      reader = new kiwix::Reader(zimPath);
+    } catch (...) {
+      cerr << "Unable to open the ZIM file '" << zimPath << "'." << endl;
+      continue;
+    }
 
-      /* Instanciate the ZIM file handler */
+    auto humanReadableId = currentBook.getHumanReadableIdFromPath();
-      kiwix::Reader* reader = NULL;
+    readers[humanReadableId] = reader;
+
+    if (reader->hasFulltextIndex()) {
+      kiwix::Searcher* searcher = new kiwix::Searcher(humanReadableId);
+      searcher->setProtocolPrefix(rootLocation + "/");
+      searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
+      searcher->add_reader(reader, humanReadableId);
+      globalSearcher->add_reader(reader, humanReadableId);
+      searchers[humanReadableId] = searcher;
+    } else if ( !indexPath.empty() ) {
       try {
-        reader = new kiwix::Reader(zimPath);
+        kiwix::Searcher* searcher = new kiwix::Searcher(indexPath, reader, humanReadableId);
-        zimFileOk = true;
+        searcher->setProtocolPrefix(rootLocation + "/");
+        searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
+        searchers[humanReadableId] = searcher;
       } catch (...) {
-        cerr << "Unable to open the ZIM file '" << zimPath << "'." << endl;
+        cerr << "Unable to open the search index '" << indexPath << "'."
-      }
+             << endl;
-
+        searchers[humanReadableId] = nullptr;
-      if (zimFileOk) {
-        string humanReadableId = currentBook.getHumanReadableIdFromPath();
-        readers[humanReadableId] = reader;
-
-        if ( reader->hasFulltextIndex()) {
-          kiwix::Searcher* searcher = new kiwix::Searcher(humanReadableId);
-          searcher->setProtocolPrefix(rootLocation + "/");
-          searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
-          searcher->add_reader(reader, humanReadableId);
-          globalSearcher->add_reader(reader, humanReadableId);
-          searchers[humanReadableId] = searcher;
-        } else if ( !indexPath.empty() ) {
-          try {
-            kiwix::Searcher* searcher = new kiwix::Searcher(indexPath, reader, humanReadableId);
-            searcher->setProtocolPrefix(rootLocation + "/");
-            searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
-            searchers[humanReadableId] = searcher;
-          } catch (...) {
-            cerr << "Unable to open the search index '" << indexPath << "'."
-                 << endl;
-            searchers[humanReadableId] = nullptr;
-          }
-        } else {
-            searchers[humanReadableId] = nullptr;
-        }
       }
+    } else {
+        searchers[humanReadableId] = nullptr;
     }
   }
 
@@ -1123,24 +1128,25 @@ int main(int argc, char** argv)
   string welcomeBooksHtml
       = ""
         "<div class='book__list'>";
-  for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) {
+  for (auto& bookId: booksIds) {
-    libraryManager.getBookById(*itr, currentBook);
+    auto& currentBook = library.getBookById(bookId);
 
-    if (!currentBook.path.empty()
+    if (!currentBook.getPath().empty()
         && readers.find(currentBook.getHumanReadableIdFromPath())
                != readers.end()) {
       welcomeBooksHtml += ""
 "<a href='" + rootLocation + "/" + currentBook.getHumanReadableIdFromPath() + "/'>"
     "<div class='book'>"
-        "<div class='book__background' style='background-image: url(data:" + currentBook.faviconMimeType+ ";base64," + currentBook.favicon + ");'>"
+        "<div class='book__background' style=\"background-image: url('/meta?content="
-            "<div class='book__title' title='" + currentBook.title + "'>" + currentBook.title + "</div>"
+             + currentBook.getHumanReadableIdFromPath() + "&name=favicon');\">"
-            "<div class='book__description' title='" + currentBook.description + "'>" + currentBook.description + "</div>"
+            "<div class='book__title' title='" + currentBook.getTitle() + "'>" + currentBook.getTitle() + "</div>"
+            "<div class='book__description' title='" + currentBook.getDescription() + "'>" + currentBook.getDescription() + "</div>"
             "<div class='book__info'>"
-                "" + kiwix::beautifyInteger(atoi(currentBook.articleCount.c_str())) + " articles, " + kiwix::beautifyInteger(atoi(currentBook.mediaCount.c_str())) + " medias"
+                "" + kiwix::beautifyInteger(currentBook.getArticleCount()) + " articles, " + kiwix::beautifyInteger(currentBook.getMediaCount()) + " medias"
             "</div>"
         "</div>"
     "</div>"
-"</a>";
+"</a>\n";
     }
   }
   welcomeBooksHtml += ""

static/server/opensearchdescription.xml

@@ -4,5 +4,6 @@
   <Description>Search zim files in the catalog.</Description>
   <Url type="application/atom+xml;profile=opds-catalog" 
        xmlns:atom="http://www.w3.org/2005/Atom"
-       template="/__ROOT_LOCATION__/catalog/search?q={searchTerms}"/>
+       indexOffset="0"
+       template="/__ROOT_LOCATION__/catalog/search?q={searchTerms}&lang={language}&count={count}&start={startIndex}"/>
 </OpenSearchDescription>