Merge pull request #178 from kiwix/opds

Opds

meson.build

@@ -1,7 +1,7 @@
 project('kiwix-tools', 'cpp',
   version : '0.4.0',
   license : 'GPL',
-  default_options: ['c_std=c11', 'cpp_std=c++11'])
+  default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true'])
 
 compiler = meson.get_compiler('cpp')
 

src/manager/kiwix-manage.cpp

@@ -67,44 +67,16 @@ void usage()
   cerr << "\tkiwix-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]" << endl;
 }
 
-int main(int argc, char** argv)
+
+void handle_show(kiwix::Manager* libraryManager, const std::string& libraryPath,
+                 int argc, char* argv[])
 {
-  string libraryPath = "";
+  show(libraryManager->cloneLibrary());
-  supportedAction action = NONE;
+}
-  string zimPath = "";
-  kiwix::Manager libraryManager;
-  int option_index = 0;
-  int c = 0;
 
-  /* Argument parsing */
+void handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
-  if (argc > 2) {
+                int argc, char* argv[])
-    libraryPath = argv[1];
+{
-    string actionString = argv[2];
-
-    if (actionString == "add")
-      action = ADD;
-    else if (actionString == "show")
-      action = SHOW;
-    else if (actionString == "remove" || actionString == "delete")
-      action = REMOVE;
-  }
-
-  /* Print usage)) if necessary */
-  if (libraryPath == "" || action == NONE) {
-    usage();
-    exit(1);
-  }
-
-  /* Try to read the file */
-  libraryPath = isRelativePath(libraryPath)
-                    ? computeAbsolutePath(getCurrentDirectory(), libraryPath)
-                    : libraryPath;
-  libraryManager.readFile(libraryPath, false);
-
-  /* SHOW */
-  if (action == SHOW) {
-    show(libraryManager.cloneLibrary());
-  } else if (action == ADD) {
   string zimPath;
   string zimPathToSave = ".";
   string indexPath;
@@ -112,6 +84,8 @@ int main(int argc, char** argv)
   string url;
   string origID = "";
   bool setCurrent = false;
+  int option_index = 0;
+  int c = 0;
 
   if (argc > 3) {
     zimPath = argv[3];
@@ -168,17 +142,14 @@ int main(int argc, char** argv)
 
   if (!zimPath.empty()) {
     zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
-      string bookId = libraryManager.addBookFromPathAndGetId(
+    string bookId = libraryManager->addBookFromPathAndGetId(
         zimPath, zimPathToSave, url, false);
-
      if (!bookId.empty()) {
       if (setCurrent)
-          libraryManager.setCurrentBookId(bookId);
+        libraryManager->setCurrentBookId(bookId);
-
       /* Save the index infos if necessary */
       if (!indexPath.empty())
-          libraryManager.setBookIndex(bookId, indexPath, indexBackend);
+        libraryManager->setBookIndex(bookId, indexPath, indexBackend);
-
      } else {
       cerr << "Unable to build or save library file '" << libraryPath << "'"
            << endl;
@@ -186,17 +157,20 @@ int main(int argc, char** argv)
   } else {
     std::cerr << "Invalid zim file path" << std::endl;
   }
+}
 
-  } else if (action == REMOVE) {
+void handle_remove(kiwix::Manager* libraryManager, const std::string& libraryPath,
+                   int argc, char* argv[])
+{
   unsigned int bookIndex = 0;
-    const unsigned int totalBookCount = libraryManager.getBookCount(true, true);
+  const unsigned int totalBookCount = libraryManager->getBookCount(true, true);
 
   if (argc > 3) {
     bookIndex = atoi(argv[3]);
   }
 
   if (bookIndex > 0 && bookIndex <= totalBookCount) {
-      libraryManager.removeBookByIndex(bookIndex - 1);
+    libraryManager->removeBookByIndex(bookIndex - 1);
   } else {
     if (totalBookCount > 0) {
       std::cerr
@@ -208,6 +182,46 @@ int main(int argc, char** argv)
           << std::endl;
     }
   }
+}
+
+int main(int argc, char** argv)
+{
+  string libraryPath = "";
+  supportedAction action = NONE;
+  kiwix::Manager libraryManager;
+
+  /* Argument parsing */
+  if (argc > 2) {
+    libraryPath = argv[1];
+    string actionString = argv[2];
+
+    if (actionString == "add")
+      action = ADD;
+    else if (actionString == "show")
+      action = SHOW;
+    else if (actionString == "remove" || actionString == "delete")
+      action = REMOVE;
+  }
+
+  /* Print usage)) if necessary */
+  if (libraryPath == "" || action == NONE) {
+    usage();
+    exit(1);
+  }
+
+  /* Try to read the file */
+  libraryPath = isRelativePath(libraryPath)
+                    ? computeAbsolutePath(getCurrentDirectory(), libraryPath)
+                    : libraryPath;
+  libraryManager.readFile(libraryPath, false);
+
+  /* SHOW */
+  if (action == SHOW) {
+    handle_show(&libraryManager, libraryPath, argc, argv);
+  } else if (action == ADD) {
+    handle_add(&libraryManager, libraryPath, argc, argv);
+  } else if (action == REMOVE) {
+    handle_remove(&libraryManager, libraryPath, argc, argv);
   }
 
   /* Rewrite the library file */

src/reader/kiwix-read.cpp

@@ -79,10 +79,8 @@ int main(int argc, char** argv)
 
   /* Start to read an article */
   if (reader != NULL) {
-    string mainPageUrl = reader->getMainPageUrl();
     string content;
     string contentType;
-    unsigned int contentLength = 0;
     string suggestion;
 
     if (pattern != NULL) {
@@ -94,13 +92,6 @@ int main(int argc, char** argv)
       }
     }
 
-    /*
-    if (reader->getContentByUrl(mainPageUrl, content, contentLength,
-    contentType)) {
-      cout << content << endl;
-    }
-    */
-
     delete reader;
   } else {
     cerr << "Unable instanciate the Kiwix reader." << endl;

src/server/kiwix-serve.cpp

@@ -51,6 +51,7 @@ extern "C" {
 #include <kiwix/manager.h>
 #include <kiwix/reader.h>
 #include <kiwix/searcher.h>
+#include <kiwix/opds_dumper.h>
 #include <pthread.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -92,12 +93,14 @@ using namespace std;
 static bool noLibraryButtonFlag = false;
 static bool noSearchBarFlag = false;
 static string welcomeHTML;
+static string catalogOpenSearchDescription;
 static std::atomic_bool isVerbose(false);
 static std::string rootLocation = "";
 static std::map<std::string, std::string> extMimeTypes;
 static std::map<std::string, kiwix::Reader*> readers;
 static std::map<std::string, kiwix::Searcher*> searchers;
 static kiwix::Searcher* globalSearcher = nullptr;
+static kiwix::Manager libraryManager;
 static pthread_mutex_t searchLock = PTHREAD_MUTEX_INITIALIZER;
 static pthread_mutex_t compressorLock = PTHREAD_MUTEX_INITIALIZER;
 static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER;
@@ -218,8 +221,8 @@ static struct MHD_Response* build_response(const void* data,
                                            bool cacheEnabled)
 {
   /* Create the response */
-  struct MHD_Response* response = MHD_create_response_from_data(
+  struct MHD_Response* response = MHD_create_response_from_buffer(
-      length, const_cast<void*>(data), MHD_NO, MHD_YES);
+      length, const_cast<void*>(data), MHD_RESPMEM_MUST_COPY);
 
   /* Make a redirection if necessary otherwise send the content */
   if (!httpRedirection.empty()) {
@@ -382,6 +385,50 @@ get_from_humanReadableBookId(const std::string& humanReadableBookId) {
   return std::pair<kiwix::Reader*, kiwix::Searcher*>(reader, searcher);
 }
 
+static struct MHD_Response* handle_meta(RequestContext* request)
+{
+  std::string humanReadableBookId;
+  std::string meta_name;
+  try {
+    humanReadableBookId = request->get_argument("content");
+    meta_name = request->get_argument("name");
+  } catch (const std::out_of_range& e) {
+    return build_404(request, humanReadableBookId);
+  }
+
+  auto reader = get_from_humanReadableBookId(humanReadableBookId).first;
+  if (reader == nullptr) {
+    return build_404(request, humanReadableBookId);
+  }
+
+  std::string content;
+  std::string mimeType = "text";
+
+  if (meta_name == "title") {
+    content = reader->getTitle();
+  } else if (meta_name == "description") {
+    content = reader->getDescription();
+  } else if (meta_name == "language") {
+    content = reader->getLanguage();
+  } else if (meta_name == "name") {
+    content = reader->getName();
+  } else if (meta_name == "tags") {
+    content = reader->getTags();
+  } else if (meta_name == "date") {
+    content = reader->getDate();
+  } else if (meta_name == "creator") {
+    content = reader->getCreator();
+  } else if (meta_name == "publisher") {
+    content = reader->getPublisher();
+  } else if (meta_name == "favicon") {
+    reader->getFavicon(content, mimeType);
+  } else {
+    return build_404(request, humanReadableBookId);
+  }
+
+  return build_response(content.data(), content.size(), "", mimeType, false, true);
+}
+
 static struct MHD_Response* handle_suggest(RequestContext* request)
 {
   if (isVerbose.load()) {
@@ -593,6 +640,66 @@ static struct MHD_Response* handle_random(RequestContext* request)
   return build_response("", 0, httpRedirection, "", false, false);
 }
 
+static struct MHD_Response* handle_catalog(RequestContext* request)
+{
+  if (isVerbose.load()) {
+    printf("** running handle_catalog");
+  }
+
+  std::string host;
+  std::string url;
+  try {
+    host = request->get_header("Host");
+    url  = request->get_url_part(1);
+  } catch (const std::out_of_range&) {
+    return build_404(request, "");
+  }
+
+  std::string content;
+  std::string mimeType;
+
+  if (url == "searchdescription.xml") {
+    content = catalogOpenSearchDescription;
+    mimeType = "application/opensearchdescription+xml";
+  } else {
+    zim::Uuid uuid;
+    kiwix::OPDSDumper opdsDumper;
+    opdsDumper.setRootLocation(rootLocation);
+    opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
+    mimeType = "application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8";
+    kiwix::Library library_to_dump;
+    if (url == "root.xml") {
+      opdsDumper.setTitle("All zims");
+      uuid = zim::Uuid::generate(host);
+      library_to_dump = libraryManager.cloneLibrary();
+    } else if (url == "search") {
+      std::string query;
+      try {
+        query = request->get_argument("q");
+      } catch (const std::out_of_range&) {
+        return build_404(request, "");
+      }
+      opdsDumper.setTitle("Search result for " + query);
+      uuid = zim::Uuid::generate();
+      library_to_dump = libraryManager.filter(query);
+    } else {
+      return build_404(request, "");
+    }
+
+    {
+      std::stringstream ss;
+      ss << uuid;
+      opdsDumper.setId(ss.str());
+    }
+    opdsDumper.setLibrary(library_to_dump);
+    content = opdsDumper.dumpOPDSFeed();
+  }
+
+  bool deflated = request->can_compress() && compress_content(content, mimeType);
+  return build_response(
+      content.data(), content.size(), "", mimeType, deflated, false);
+}
+
 static struct MHD_Response* handle_content(RequestContext* request)
 {
   if (isVerbose.load()) {
@@ -750,6 +857,10 @@ static int accessHandlerCallback(void* cls,
   } else {
     if (startswith(request.get_url(), "/skin/")) {
       response = handle_skin(&request);
+    } else if (startswith(request.get_url(), "/catalog")) {
+      response = handle_catalog(&request);
+    } else if (request.get_url() == "/meta") {
+      response = handle_meta(&request);
     } else if (request.get_url() == "/search") {
       response = handle_search(&request);
     } else if (request.get_url() == "/suggest") {
@@ -784,12 +895,11 @@ int main(int argc, char** argv)
   string rootPath;
   string interface;
   int serverPort = 80;
-  int daemonFlag = false;
+  int daemonFlag [[gnu::unused]] = false;
   int libraryFlag = false;
   string PPIDString;
   unsigned int PPID = 0;
   unsigned int nb_threads = std::thread::hardware_concurrency();
-  kiwix::Manager libraryManager;
 
   static struct option long_options[]
       = {{"daemon", no_argument, 0, 'd'},
@@ -1029,6 +1139,11 @@ int main(int argc, char** argv)
 
   introduceTaskbar(welcomeHTML, "");
 
+  /* Compute the OpenSearch description */
+  catalogOpenSearchDescription = RESOURCE::opensearchdescription_xml;
+  catalogOpenSearchDescription = replaceRegex(catalogOpenSearchDescription, rootLocation, "__ROOT_LOCATION__");
+
+
 #ifndef _WIN32
   /* Fork if necessary */
   if (daemonFlag) {

static/server/opensearchdescription.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
+  <ShortName>Zim catalog search</ShortName>
+  <Description>Search zim files in the catalog.</Description>
+  <Url type="application/atom+xml;profile=opds-catalog" 
+       xmlns:atom="http://www.w3.org/2005/Atom"
+       template="/__ROOT_LOCATION__/catalog/search?q={searchTerms}"/>
+</OpenSearchDescription>

static/server/resources_list.txt

@@ -22,3 +22,4 @@ include.html.part
 taskbar.css
 taskbar.html.part
 global_taskbar.html.part
+opensearchdescription.xml

travis/install_deps.sh

@@ -3,7 +3,7 @@
 set -e
 
 REPO_NAME=${TRAVIS_REPO_SLUG#*/}
-ARCHIVE_NAME=deps_${PLATFORM}_${REPO_NAME}.tar.gz
+ARCHIVE_NAME=deps_${TRAVIS_OS_NAME}_${PLATFORM}_${REPO_NAME}.tar.gz
 
 # Packages.
 case ${PLATFORM} in