From 53b2dadfcebb35cbb1a9f72dbf2216a568f846b3 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 19 Apr 2018 15:59:42 +0200 Subject: [PATCH 1/6] Compile without warning. --- meson.build | 2 +- src/reader/kiwix-read.cpp | 9 --------- src/server/kiwix-serve.cpp | 6 +++--- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/meson.build b/meson.build index af6eb97..88b2fe9 100644 --- a/meson.build +++ b/meson.build @@ -1,7 +1,7 @@ project('kiwix-tools', 'cpp', version : '0.4.0', license : 'GPL', - default_options: ['c_std=c11', 'cpp_std=c++11']) + default_options: ['c_std=c11', 'cpp_std=c++11', 'werror=true']) compiler = meson.get_compiler('cpp') diff --git a/src/reader/kiwix-read.cpp b/src/reader/kiwix-read.cpp index 7b8761c..4025bbb 100644 --- a/src/reader/kiwix-read.cpp +++ b/src/reader/kiwix-read.cpp @@ -79,10 +79,8 @@ int main(int argc, char** argv) /* Start to read an article */ if (reader != NULL) { - string mainPageUrl = reader->getMainPageUrl(); string content; string contentType; - unsigned int contentLength = 0; string suggestion; if (pattern != NULL) { @@ -94,13 +92,6 @@ int main(int argc, char** argv) } } - /* - if (reader->getContentByUrl(mainPageUrl, content, contentLength, - contentType)) { - cout << content << endl; - } - */ - delete reader; } else { cerr << "Unable instanciate the Kiwix reader." << endl; diff --git a/src/server/kiwix-serve.cpp b/src/server/kiwix-serve.cpp index eeb56dd..b0706f3 100644 --- a/src/server/kiwix-serve.cpp +++ b/src/server/kiwix-serve.cpp @@ -218,8 +218,8 @@ static struct MHD_Response* build_response(const void* data, bool cacheEnabled) { /* Create the response */ - struct MHD_Response* response = MHD_create_response_from_data( - length, const_cast(data), MHD_NO, MHD_YES); + struct MHD_Response* response = MHD_create_response_from_buffer( + length, const_cast(data), MHD_RESPMEM_MUST_COPY); /* Make a redirection if necessary otherwise send the content */ if (!httpRedirection.empty()) { @@ -784,7 +784,7 @@ int main(int argc, char** argv) string rootPath; string interface; int serverPort = 80; - int daemonFlag = false; + int daemonFlag [[gnu::unused]] = false; int libraryFlag = false; string PPIDString; unsigned int PPID = 0; From 2acd2767532c342bb1dd84be4378e0fda0d91404 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 26 Mar 2018 18:27:45 +0200 Subject: [PATCH 2/6] [KIWIX-SERVE] Serve a zim metadata on the '/meta' url. This can be used to get some metadata about a zim with a simple http request. --- src/server/kiwix-serve.cpp | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/server/kiwix-serve.cpp b/src/server/kiwix-serve.cpp index b0706f3..8051e07 100644 --- a/src/server/kiwix-serve.cpp +++ b/src/server/kiwix-serve.cpp @@ -382,6 +382,50 @@ get_from_humanReadableBookId(const std::string& humanReadableBookId) { return std::pair(reader, searcher); } +static struct MHD_Response* handle_meta(RequestContext* request) +{ + std::string humanReadableBookId; + std::string meta_name; + try { + humanReadableBookId = request->get_argument("content"); + meta_name = request->get_argument("name"); + } catch (const std::out_of_range& e) { + return build_404(request, humanReadableBookId); + } + + auto reader = get_from_humanReadableBookId(humanReadableBookId).first; + if (reader == nullptr) { + return build_404(request, humanReadableBookId); + } + + std::string content; + std::string mimeType = "text"; + + if (meta_name == "title") { + content = reader->getTitle(); + } else if (meta_name == "description") { + content = reader->getDescription(); + } else if (meta_name == "language") { + content = reader->getLanguage(); + } else if (meta_name == "name") { + content = reader->getName(); + } else if (meta_name == "tags") { + content = reader->getTags(); + } else if (meta_name == "date") { + content = reader->getDate(); + } else if (meta_name == "creator") { + content = reader->getCreator(); + } else if (meta_name == "publisher") { + content = reader->getPublisher(); + } else if (meta_name == "favicon") { + reader->getFavicon(content, mimeType); + } else { + return build_404(request, humanReadableBookId); + } + + return build_response(content.data(), content.size(), "", mimeType, false, true); +} + static struct MHD_Response* handle_suggest(RequestContext* request) { if (isVerbose.load()) { @@ -750,6 +794,8 @@ static int accessHandlerCallback(void* cls, } else { if (startswith(request.get_url(), "/skin/")) { response = handle_skin(&request); + } else if (request.get_url() == "/meta") { + response = handle_meta(&request); } else if (request.get_url() == "/search") { response = handle_search(&request); } else if (request.get_url() == "/suggest") { From fb8c14a1e4f546442581bbdee7c6753fa7495d2e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 26 Mar 2018 18:59:39 +0200 Subject: [PATCH 3/6] Make kiwix-serve serve an opds stream of all zims. --- src/server/kiwix-serve.cpp | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/server/kiwix-serve.cpp b/src/server/kiwix-serve.cpp index 8051e07..1828c24 100644 --- a/src/server/kiwix-serve.cpp +++ b/src/server/kiwix-serve.cpp @@ -51,6 +51,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -98,6 +99,7 @@ static std::map extMimeTypes; static std::map readers; static std::map searchers; static kiwix::Searcher* globalSearcher = nullptr; +static kiwix::OPDSDumper* opdsDumper = nullptr; static pthread_mutex_t searchLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t compressorLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER; @@ -637,6 +639,34 @@ static struct MHD_Response* handle_random(RequestContext* request) return build_response("", 0, httpRedirection, "", false, false); } +static struct MHD_Response* handle_catalog(RequestContext* request) +{ + if (isVerbose.load()) { + printf("** running handle_catalog"); + } + + std::string host; + try { + host = request->get_header("Host"); + } catch (const std::out_of_range&) { + return build_404(request, ""); + } + + { + auto uuid = zim::Uuid::generate(host); + std::stringstream ss; + ss << uuid; + opdsDumper->setId(ss.str()); + } + std::string content = opdsDumper->dumpOPDSFeed(); + + std::string mimeType = "application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8"; + + bool deflated = request->can_compress() && compress_content(content, mimeType); + return build_response( + content.data(), content.size(), "", mimeType, deflated, false); +} + static struct MHD_Response* handle_content(RequestContext* request) { if (isVerbose.load()) { @@ -794,6 +824,8 @@ static int accessHandlerCallback(void* cls, } else { if (startswith(request.get_url(), "/skin/")) { response = handle_skin(&request); + } else if (startswith(request.get_url(), "/catalog")) { + response = handle_catalog(&request); } else if (request.get_url() == "/meta") { response = handle_meta(&request); } else if (request.get_url() == "/search") { @@ -1075,6 +1107,11 @@ int main(int argc, char** argv) introduceTaskbar(welcomeHTML, ""); + /* Compute the OPDS feed */ + opdsDumper = new kiwix::OPDSDumper(libraryManager.cloneLibrary()); + opdsDumper->setTitle("All zims"); + opdsDumper->setRootLocation(rootLocation); + #ifndef _WIN32 /* Fork if necessary */ if (daemonFlag) { @@ -1235,6 +1272,7 @@ int main(int argc, char** argv) } while (waiting); delete globalSearcher; + delete opdsDumper; /* Stop the daemon */ MHD_stop_daemon(daemon); From 4bd18ce4667bfb0fc7180c9cf7ab57153080ec04 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 26 Mar 2018 21:31:20 +0200 Subject: [PATCH 4/6] Add open search support to search in the catalog. --- src/server/kiwix-serve.cpp | 61 +++++++++++++++++++------ static/server/opensearchdescription.xml | 8 ++++ static/server/resources_list.txt | 1 + 3 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 static/server/opensearchdescription.xml diff --git a/src/server/kiwix-serve.cpp b/src/server/kiwix-serve.cpp index 1828c24..1750c2f 100644 --- a/src/server/kiwix-serve.cpp +++ b/src/server/kiwix-serve.cpp @@ -93,13 +93,14 @@ using namespace std; static bool noLibraryButtonFlag = false; static bool noSearchBarFlag = false; static string welcomeHTML; +static string catalogOpenSearchDescription; static std::atomic_bool isVerbose(false); static std::string rootLocation = ""; static std::map extMimeTypes; static std::map readers; static std::map searchers; static kiwix::Searcher* globalSearcher = nullptr; -static kiwix::OPDSDumper* opdsDumper = nullptr; +static kiwix::Manager libraryManager; static pthread_mutex_t searchLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t compressorLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER; @@ -646,21 +647,53 @@ static struct MHD_Response* handle_catalog(RequestContext* request) } std::string host; + std::string url; try { host = request->get_header("Host"); + url = request->get_url_part(1); } catch (const std::out_of_range&) { return build_404(request, ""); } - { - auto uuid = zim::Uuid::generate(host); - std::stringstream ss; - ss << uuid; - opdsDumper->setId(ss.str()); - } - std::string content = opdsDumper->dumpOPDSFeed(); + std::string content; + std::string mimeType; - std::string mimeType = "application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8"; + if (url == "searchdescription.xml") { + content = catalogOpenSearchDescription; + mimeType = "application/opensearchdescription+xml"; + } else { + zim::Uuid uuid; + kiwix::OPDSDumper opdsDumper; + opdsDumper.setRootLocation(rootLocation); + opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml"); + mimeType = "application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8"; + kiwix::Library library_to_dump; + if (url == "root.xml") { + opdsDumper.setTitle("All zims"); + uuid = zim::Uuid::generate(host); + library_to_dump = libraryManager.cloneLibrary(); + } else if (url == "search") { + std::string query; + try { + query = request->get_argument("q"); + } catch (const std::out_of_range&) { + return build_404(request, ""); + } + opdsDumper.setTitle("Search result for " + query); + uuid = zim::Uuid::generate(); + library_to_dump = libraryManager.filter(query); + } else { + return build_404(request, ""); + } + + { + std::stringstream ss; + ss << uuid; + opdsDumper.setId(ss.str()); + } + opdsDumper.setLibrary(library_to_dump); + content = opdsDumper.dumpOPDSFeed(); + } bool deflated = request->can_compress() && compress_content(content, mimeType); return build_response( @@ -867,7 +900,6 @@ int main(int argc, char** argv) string PPIDString; unsigned int PPID = 0; unsigned int nb_threads = std::thread::hardware_concurrency(); - kiwix::Manager libraryManager; static struct option long_options[] = {{"daemon", no_argument, 0, 'd'}, @@ -1107,10 +1139,10 @@ int main(int argc, char** argv) introduceTaskbar(welcomeHTML, ""); - /* Compute the OPDS feed */ - opdsDumper = new kiwix::OPDSDumper(libraryManager.cloneLibrary()); - opdsDumper->setTitle("All zims"); - opdsDumper->setRootLocation(rootLocation); + /* Compute the OpenSearch description */ + catalogOpenSearchDescription = RESOURCE::opensearchdescription_xml; + catalogOpenSearchDescription = replaceRegex(catalogOpenSearchDescription, rootLocation, "__ROOT_LOCATION__"); + #ifndef _WIN32 /* Fork if necessary */ @@ -1272,7 +1304,6 @@ int main(int argc, char** argv) } while (waiting); delete globalSearcher; - delete opdsDumper; /* Stop the daemon */ MHD_stop_daemon(daemon); diff --git a/static/server/opensearchdescription.xml b/static/server/opensearchdescription.xml new file mode 100644 index 0000000..27c67e9 --- /dev/null +++ b/static/server/opensearchdescription.xml @@ -0,0 +1,8 @@ + + + Zim catalog search + Search zim files in the catalog. + + diff --git a/static/server/resources_list.txt b/static/server/resources_list.txt index 97a741f..9df07e1 100644 --- a/static/server/resources_list.txt +++ b/static/server/resources_list.txt @@ -22,3 +22,4 @@ include.html.part taskbar.css taskbar.html.part global_taskbar.html.part +opensearchdescription.xml From a01906d2736082900efd7b9b4e0ce0ab121d31b3 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 27 Mar 2018 09:22:16 +0200 Subject: [PATCH 5/6] [manage] Move handling of action in separated function. --- src/manager/kiwix-manage.cpp | 226 +++++++++++++++++++---------------- 1 file changed, 120 insertions(+), 106 deletions(-) diff --git a/src/manager/kiwix-manage.cpp b/src/manager/kiwix-manage.cpp index 6fb2b14..d441b62 100644 --- a/src/manager/kiwix-manage.cpp +++ b/src/manager/kiwix-manage.cpp @@ -67,14 +67,128 @@ void usage() cerr << "\tkiwix-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]" << endl; } + +void handle_show(kiwix::Manager* libraryManager, const std::string& libraryPath, + int argc, char* argv[]) +{ + show(libraryManager->cloneLibrary()); +} + +void handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath, + int argc, char* argv[]) +{ + string zimPath; + string zimPathToSave = "."; + string indexPath; + kiwix::supportedIndexType indexBackend = kiwix::XAPIAN; + string url; + string origID = ""; + bool setCurrent = false; + int option_index = 0; + int c = 0; + + if (argc > 3) { + zimPath = argv[3]; + } + + /* Options parsing */ + optind = 2; + while (42) { + static struct option long_options[] + = {{"url", required_argument, 0, 'u'}, + {"origId", required_argument, 0, 'o'}, + {"indexPath", required_argument, 0, 'i'}, + {"indexBackend", required_argument, 0, 'b'}, + {"zimPathToSave", required_argument, 0, 'z'}, + {"current", no_argument, 0, 'c'}, + {0, 0, 0, 0}}; + + c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index); + + if (c != -1) { + switch (c) { + case 'u': + url = optarg; + break; + + case 'o': + origID = optarg; + break; + + case 'c': + setCurrent = true; + break; + + case 'i': + indexPath = optarg; + break; + + case 'b': + if (!strcmp(optarg, "xapian")) { + indexBackend = kiwix::XAPIAN; + } else { + usage(); + } + break; + + case 'z': + zimPathToSave = optarg; + break; + } + } else { + break; + } + } + + if (!zimPath.empty()) { + zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave; + string bookId = libraryManager->addBookFromPathAndGetId( + zimPath, zimPathToSave, url, false); + if (!bookId.empty()) { + if (setCurrent) + libraryManager->setCurrentBookId(bookId); + /* Save the index infos if necessary */ + if (!indexPath.empty()) + libraryManager->setBookIndex(bookId, indexPath, indexBackend); + } else { + cerr << "Unable to build or save library file '" << libraryPath << "'" + << endl; + } + } else { + std::cerr << "Invalid zim file path" << std::endl; + } +} + +void handle_remove(kiwix::Manager* libraryManager, const std::string& libraryPath, + int argc, char* argv[]) +{ + unsigned int bookIndex = 0; + const unsigned int totalBookCount = libraryManager->getBookCount(true, true); + + if (argc > 3) { + bookIndex = atoi(argv[3]); + } + + if (bookIndex > 0 && bookIndex <= totalBookCount) { + libraryManager->removeBookByIndex(bookIndex - 1); + } else { + if (totalBookCount > 0) { + std::cerr + << "Invalid book index number. Please give a number between 1 and " + << totalBookCount << std::endl; + } else { + std::cerr + << "Invalid book index number. Library is empty, no book to delete." + << std::endl; + } + } +} + int main(int argc, char** argv) { string libraryPath = ""; supportedAction action = NONE; - string zimPath = ""; kiwix::Manager libraryManager; - int option_index = 0; - int c = 0; /* Argument parsing */ if (argc > 2) { @@ -103,111 +217,11 @@ int main(int argc, char** argv) /* SHOW */ if (action == SHOW) { - show(libraryManager.cloneLibrary()); + handle_show(&libraryManager, libraryPath, argc, argv); } else if (action == ADD) { - string zimPath; - string zimPathToSave = "."; - string indexPath; - kiwix::supportedIndexType indexBackend = kiwix::XAPIAN; - string url; - string origID = ""; - bool setCurrent = false; - - if (argc > 3) { - zimPath = argv[3]; - } - - /* Options parsing */ - optind = 2; - while (42) { - static struct option long_options[] - = {{"url", required_argument, 0, 'u'}, - {"origId", required_argument, 0, 'o'}, - {"indexPath", required_argument, 0, 'i'}, - {"indexBackend", required_argument, 0, 'b'}, - {"zimPathToSave", required_argument, 0, 'z'}, - {"current", no_argument, 0, 'c'}, - {0, 0, 0, 0}}; - - c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index); - - if (c != -1) { - switch (c) { - case 'u': - url = optarg; - break; - - case 'o': - origID = optarg; - break; - - case 'c': - setCurrent = true; - break; - - case 'i': - indexPath = optarg; - break; - - case 'b': - if (!strcmp(optarg, "xapian")) { - indexBackend = kiwix::XAPIAN; - } else { - usage(); - } - break; - - case 'z': - zimPathToSave = optarg; - break; - } - } else { - break; - } - } - - if (!zimPath.empty()) { - zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave; - string bookId = libraryManager.addBookFromPathAndGetId( - zimPath, zimPathToSave, url, false); - - if (!bookId.empty()) { - if (setCurrent) - libraryManager.setCurrentBookId(bookId); - - /* Save the index infos if necessary */ - if (!indexPath.empty()) - libraryManager.setBookIndex(bookId, indexPath, indexBackend); - - } else { - cerr << "Unable to build or save library file '" << libraryPath << "'" - << endl; - } - } else { - std::cerr << "Invalid zim file path" << std::endl; - } - + handle_add(&libraryManager, libraryPath, argc, argv); } else if (action == REMOVE) { - unsigned int bookIndex = 0; - const unsigned int totalBookCount = libraryManager.getBookCount(true, true); - - if (argc > 3) { - bookIndex = atoi(argv[3]); - } - - if (bookIndex > 0 && bookIndex <= totalBookCount) { - libraryManager.removeBookByIndex(bookIndex - 1); - } else { - if (totalBookCount > 0) { - std::cerr - << "Invalid book index number. Please give a number between 1 and " - << totalBookCount << std::endl; - } else { - std::cerr - << "Invalid book index number. Library is empty, no book to delete." - << std::endl; - } - } + handle_remove(&libraryManager, libraryPath, argc, argv); } /* Rewrite the library file */ From bd8d0c3805a3b670428e8dc556462b8beb836d18 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 23 Apr 2018 10:28:42 +0200 Subject: [PATCH 6/6] Get the dependencies archive using the correct new name scheme. Now that we are compiling to ios, deendencies archive names have TRAVIS_OS_NAME included. --- travis/install_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/install_deps.sh b/travis/install_deps.sh index 1ed9d0c..c658762 100755 --- a/travis/install_deps.sh +++ b/travis/install_deps.sh @@ -3,7 +3,7 @@ set -e REPO_NAME=${TRAVIS_REPO_SLUG#*/} -ARCHIVE_NAME=deps_${PLATFORM}_${REPO_NAME}.tar.gz +ARCHIVE_NAME=deps_${TRAVIS_OS_NAME}_${PLATFORM}_${REPO_NAME}.tar.gz # Packages. case ${PLATFORM} in