Merge pull request #178 from kiwix/opds

Opds

meson.build

@@ -1,7 +1,7 @@
 project('kiwix-tools', 'cpp',
   version : '0.4.0',
   license : 'GPL',
-  default_options: ['c_std=c11', 'cpp_std=c++11'])
+  default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 

src/manager/kiwix-manage.cpp

@@ -67,14 +67,128 @@ void usage()
   cerr << "\tkiwix-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]" << endl;
 }
 
+
+void handle_show(kiwix::Manager* libraryManager, const std::string& libraryPath,
+                 int argc, char* argv[])
+{
+  show(libraryManager->cloneLibrary());
+}
+
+void handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
+                int argc, char* argv[])
+{
+  string zimPath;
+  string zimPathToSave = ".";
+  string indexPath;
+  kiwix::supportedIndexType indexBackend = kiwix::XAPIAN;
+  string url;
+  string origID = "";
+  bool setCurrent = false;
+  int option_index = 0;
+  int c = 0;
+
+  if (argc > 3) {
+    zimPath = argv[3];
+  }
+
+  /* Options parsing */
+  optind = 2;
+  while (42) {
+    static struct option long_options[]
+        = {{"url", required_argument, 0, 'u'},
+           {"origId", required_argument, 0, 'o'},
+           {"indexPath", required_argument, 0, 'i'},
+           {"indexBackend", required_argument, 0, 'b'},
+           {"zimPathToSave", required_argument, 0, 'z'},
+           {"current", no_argument, 0, 'c'},
+           {0, 0, 0, 0}};
+
+    c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index);
+
+    if (c != -1) {
+      switch (c) {
+        case 'u':
+          url = optarg;
+          break;
+
+        case 'o':
+          origID = optarg;
+          break;
+
+        case 'c':
+          setCurrent = true;
+          break;
+
+        case 'i':
+          indexPath = optarg;
+          break;
+
+        case 'b':
+          if (!strcmp(optarg, "xapian")) {
+            indexBackend = kiwix::XAPIAN;
+          } else {
+            usage();
+          }
+          break;
+
+        case 'z':
+          zimPathToSave = optarg;
+          break;
+      }
+    } else {
+      break;
+    }
+  }
+
+  if (!zimPath.empty()) {
+    zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
+    string bookId = libraryManager->addBookFromPathAndGetId(
+        zimPath, zimPathToSave, url, false);
+     if (!bookId.empty()) {
+      if (setCurrent)
+        libraryManager->setCurrentBookId(bookId);
+      /* Save the index infos if necessary */
+      if (!indexPath.empty())
+        libraryManager->setBookIndex(bookId, indexPath, indexBackend);
+     } else {
+      cerr << "Unable to build or save library file '" << libraryPath << "'"
+           << endl;
+    }
+  } else {
+    std::cerr << "Invalid zim file path" << std::endl;
+  }
+}
+
+void handle_remove(kiwix::Manager* libraryManager, const std::string& libraryPath,
+                   int argc, char* argv[])
+{
+  unsigned int bookIndex = 0;
+  const unsigned int totalBookCount = libraryManager->getBookCount(true, true);
+
+  if (argc > 3) {
+    bookIndex = atoi(argv[3]);
+  }
+
+  if (bookIndex > 0 && bookIndex <= totalBookCount) {
+    libraryManager->removeBookByIndex(bookIndex - 1);
+  } else {
+    if (totalBookCount > 0) {
+      std::cerr
+          << "Invalid book index number. Please give a number between 1 and "
+          << totalBookCount << std::endl;
+    } else {
+      std::cerr
+          << "Invalid book index number. Library is empty, no book to delete."
+          << std::endl;
+    }
+  }
+}
+
 int main(int argc, char** argv)
 {
   string libraryPath = "";
   supportedAction action = NONE;
-  string zimPath = "";
   kiwix::Manager libraryManager;
-  int option_index = 0;
-  int c = 0;
 
   /* Argument parsing */
   if (argc > 2) {
@@ -103,111 +217,11 @@ int main(int argc, char** argv)
 
   /* SHOW */
   if (action == SHOW) {
-    show(libraryManager.cloneLibrary());
+    handle_show(&libraryManager, libraryPath, argc, argv);
   } else if (action == ADD) {
-    string zimPath;
+    handle_add(&libraryManager, libraryPath, argc, argv);
-    string zimPathToSave = ".";
-    string indexPath;
-    kiwix::supportedIndexType indexBackend = kiwix::XAPIAN;
-    string url;
-    string origID = "";
-    bool setCurrent = false;
-
-    if (argc > 3) {
-      zimPath = argv[3];
-    }
-
-    /* Options parsing */
-    optind = 2;
-    while (42) {
-      static struct option long_options[]
-          = {{"url", required_argument, 0, 'u'},
-             {"origId", required_argument, 0, 'o'},
-             {"indexPath", required_argument, 0, 'i'},
-             {"indexBackend", required_argument, 0, 'b'},
-             {"zimPathToSave", required_argument, 0, 'z'},
-             {"current", no_argument, 0, 'c'},
-             {0, 0, 0, 0}};
-
-      c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index);
-
-      if (c != -1) {
-        switch (c) {
-          case 'u':
-            url = optarg;
-            break;
-
-          case 'o':
-            origID = optarg;
-            break;
-
-          case 'c':
-            setCurrent = true;
-            break;
-
-          case 'i':
-            indexPath = optarg;
-            break;
-
-          case 'b':
-            if (!strcmp(optarg, "xapian")) {
-              indexBackend = kiwix::XAPIAN;
-            } else {
-              usage();
-            }
-            break;
-
-          case 'z':
-            zimPathToSave = optarg;
-            break;
-        }
-      } else {
-        break;
-      }
-    }
-
-    if (!zimPath.empty()) {
-      zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
-      string bookId = libraryManager.addBookFromPathAndGetId(
-          zimPath, zimPathToSave, url, false);
-
-      if (!bookId.empty()) {
-        if (setCurrent)
-          libraryManager.setCurrentBookId(bookId);
-
-        /* Save the index infos if necessary */
-        if (!indexPath.empty())
-          libraryManager.setBookIndex(bookId, indexPath, indexBackend);
-
-      } else {
-        cerr << "Unable to build or save library file '" << libraryPath << "'"
-             << endl;
-      }
-    } else {
-      std::cerr << "Invalid zim file path" << std::endl;
-    }
-
   } else if (action == REMOVE) {
-    unsigned int bookIndex = 0;
+    handle_remove(&libraryManager, libraryPath, argc, argv);
-    const unsigned int totalBookCount = libraryManager.getBookCount(true, true);
-
-    if (argc > 3) {
-      bookIndex = atoi(argv[3]);
-    }
-
-    if (bookIndex > 0 && bookIndex <= totalBookCount) {
-      libraryManager.removeBookByIndex(bookIndex - 1);
-    } else {
-      if (totalBookCount > 0) {
-        std::cerr
-            << "Invalid book index number. Please give a number between 1 and "
-            << totalBookCount << std::endl;
-      } else {
-        std::cerr
-            << "Invalid book index number. Library is empty, no book to delete."
-            << std::endl;
-      }
-    }
   }
 
   /* Rewrite the library file */

src/reader/kiwix-read.cpp

@@ -79,10 +79,8 @@ int main(int argc, char** argv)
 
   /* Start to read an article */
   if (reader != NULL) {
-    string mainPageUrl = reader->getMainPageUrl();
     string content;
     string contentType;
-    unsigned int contentLength = 0;
     string suggestion;
 
     if (pattern != NULL) {
@@ -94,13 +92,6 @@ int main(int argc, char** argv)
       }
     }
 
-    /*
-    if (reader->getContentByUrl(mainPageUrl, content, contentLength,
-    contentType)) {
-      cout << content << endl;
-    }
-    */
-
     delete reader;
   } else {
     cerr << "Unable instanciate the Kiwix reader." << endl;

src/server/kiwix-serve.cpp

@@ -51,6 +51,7 @@ extern "C" {
 #include <kiwix/manager.h>
 #include <kiwix/reader.h>
 #include <kiwix/searcher.h>
+#include <kiwix/opds_dumper.h>
 #include <pthread.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -92,12 +93,14 @@ using namespace std;
 static bool noLibraryButtonFlag = false;
 static bool noSearchBarFlag = false;
 static string welcomeHTML;
+static string catalogOpenSearchDescription;
 static std::atomic_bool isVerbose(false);
 static std::string rootLocation = "";
 static std::map<std::string, std::string> extMimeTypes;
 static std::map<std::string, kiwix::Reader*> readers;
 static std::map<std::string, kiwix::Searcher*> searchers;
 static kiwix::Searcher* globalSearcher = nullptr;
+static kiwix::Manager libraryManager;
 static pthread_mutex_t searchLock = PTHREAD_MUTEX_INITIALIZER;
 static pthread_mutex_t compressorLock = PTHREAD_MUTEX_INITIALIZER;
 static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER;
@@ -218,8 +221,8 @@ static struct MHD_Response* build_response(const void* data,
                                            bool cacheEnabled)
 {
   /* Create the response */
-  struct MHD_Response* response = MHD_create_response_from_data(
+  struct MHD_Response* response = MHD_create_response_from_buffer(
-      length, const_cast<void*>(data), MHD_NO, MHD_YES);
+      length, const_cast<void*>(data), MHD_RESPMEM_MUST_COPY);
 
   /* Make a redirection if necessary otherwise send the content */
   if (!httpRedirection.empty()) {
@@ -382,6 +385,50 @@ get_from_humanReadableBookId(const std::string& humanReadableBookId) {
   return std::pair<kiwix::Reader*, kiwix::Searcher*>(reader, searcher);
 }
 
+static struct MHD_Response* handle_meta(RequestContext* request)
+{
+  std::string humanReadableBookId;
+  std::string meta_name;
+  try {
+    humanReadableBookId = request->get_argument("content");
+    meta_name = request->get_argument("name");
+  } catch (const std::out_of_range& e) {
+    return build_404(request, humanReadableBookId);
+  }
+
+  auto reader = get_from_humanReadableBookId(humanReadableBookId).first;
+  if (reader == nullptr) {
+    return build_404(request, humanReadableBookId);
+  }
+
+  std::string content;
+  std::string mimeType = "text";
+
+  if (meta_name == "title") {
+    content = reader->getTitle();
+  } else if (meta_name == "description") {
+    content = reader->getDescription();
+  } else if (meta_name == "language") {
+    content = reader->getLanguage();
+  } else if (meta_name == "name") {
+    content = reader->getName();
+  } else if (meta_name == "tags") {
+    content = reader->getTags();
+  } else if (meta_name == "date") {
+    content = reader->getDate();
+  } else if (meta_name == "creator") {
+    content = reader->getCreator();
+  } else if (meta_name == "publisher") {
+    content = reader->getPublisher();
+  } else if (meta_name == "favicon") {
+    reader->getFavicon(content, mimeType);
+  } else {
+    return build_404(request, humanReadableBookId);
+  }
+
+  return build_response(content.data(), content.size(), "", mimeType, false, true);
+}
+
 static struct MHD_Response* handle_suggest(RequestContext* request)
 {
   if (isVerbose.load()) {
@@ -593,6 +640,66 @@ static struct MHD_Response* handle_random(RequestContext* request)
   return build_response("", 0, httpRedirection, "", false, false);
 }
 
+static struct MHD_Response* handle_catalog(RequestContext* request)
+{
+  if (isVerbose.load()) {
+    printf("** running handle_catalog");
+  }
+
+  std::string host;
+  std::string url;
+  try {
+    host = request->get_header("Host");
+    url  = request->get_url_part(1);
+  } catch (const std::out_of_range&) {
+    return build_404(request, "");
+  }
+
+  std::string content;
+  std::string mimeType;
+
+  if (url == "searchdescription.xml") {
+    content = catalogOpenSearchDescription;
+    mimeType = "application/opensearchdescription+xml";
+  } else {
+    zim::Uuid uuid;
+    kiwix::OPDSDumper opdsDumper;
+    opdsDumper.setRootLocation(rootLocation);
+    opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
+    mimeType = "application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8";
+    kiwix::Library library_to_dump;
+    if (url == "root.xml") {
+      opdsDumper.setTitle("All zims");
+      uuid = zim::Uuid::generate(host);
+      library_to_dump = libraryManager.cloneLibrary();
+    } else if (url == "search") {
+      std::string query;
+      try {
+        query = request->get_argument("q");
+      } catch (const std::out_of_range&) {
+        return build_404(request, "");
+      }
+      opdsDumper.setTitle("Search result for " + query);
+      uuid = zim::Uuid::generate();
+      library_to_dump = libraryManager.filter(query);
+    } else {
+      return build_404(request, "");
+    }
+
+    {
+      std::stringstream ss;
+      ss << uuid;
+      opdsDumper.setId(ss.str());
+    }
+    opdsDumper.setLibrary(library_to_dump);
+    content = opdsDumper.dumpOPDSFeed();
+  }
+
+  bool deflated = request->can_compress() && compress_content(content, mimeType);
+  return build_response(
+      content.data(), content.size(), "", mimeType, deflated, false);
+}
+
 static struct MHD_Response* handle_content(RequestContext* request)
 {
   if (isVerbose.load()) {
@@ -750,6 +857,10 @@ static int accessHandlerCallback(void* cls,
   } else {
     if (startswith(request.get_url(), "/skin/")) {
       response = handle_skin(&request);
+    } else if (startswith(request.get_url(), "/catalog")) {
+      response = handle_catalog(&request);
+    } else if (request.get_url() == "/meta") {
+      response = handle_meta(&request);
     } else if (request.get_url() == "/search") {
       response = handle_search(&request);
     } else if (request.get_url() == "/suggest") {
@@ -784,12 +895,11 @@ int main(int argc, char** argv)
   string rootPath;
   string interface;
   int serverPort = 80;
-  int daemonFlag = false;
+  int daemonFlag [[gnu::unused]] = false;
   int libraryFlag = false;
   string PPIDString;
   unsigned int PPID = 0;
   unsigned int nb_threads = std::thread::hardware_concurrency();
-  kiwix::Manager libraryManager;
 
   static struct option long_options[]
       = {{"daemon", no_argument, 0, 'd'},
@@ -1029,6 +1139,11 @@ int main(int argc, char** argv)
 
   introduceTaskbar(welcomeHTML, "");
 
+  /* Compute the OpenSearch description */
+  catalogOpenSearchDescription = RESOURCE::opensearchdescription_xml;
+  catalogOpenSearchDescription = replaceRegex(catalogOpenSearchDescription, rootLocation, "__ROOT_LOCATION__");
+
+
 #ifndef _WIN32
   /* Fork if necessary */
   if (daemonFlag) {

static/server/opensearchdescription.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
+  <ShortName>Zim catalog search</ShortName>
+  <Description>Search zim files in the catalog.</Description>
+  <Url type="application/atom+xml;profile=opds-catalog" 
+       xmlns:atom="http://www.w3.org/2005/Atom"
+       template="/__ROOT_LOCATION__/catalog/search?q={searchTerms}"/>
+</OpenSearchDescription>

static/server/resources_list.txt

@@ -22,3 +22,4 @@ include.html.part
 taskbar.css
 taskbar.html.part
 global_taskbar.html.part
+opensearchdescription.xml

travis/install_deps.sh

@@ -3,7 +3,7 @@
 set -e
 
 REPO_NAME=${TRAVIS_REPO_SLUG#*/}
-ARCHIVE_NAME=deps_${PLATFORM}_${REPO_NAME}.tar.gz
+ARCHIVE_NAME=deps_${TRAVIS_OS_NAME}_${PLATFORM}_${REPO_NAME}.tar.gz
 
 # Packages.
 case ${PLATFORM} in