Format all the code using clang-format.

Add a script `format_code.sh` to easily format the code.
This commit is contained in:
Matthieu Gautier 2017-07-05 15:52:32 +02:00
parent 856bfc675a
commit 4e3ff03059
7 changed files with 697 additions and 562 deletions

12
.clang-format Normal file
View File

@ -0,0 +1,12 @@
BasedOnStyle: Google
BinPackArguments: false
BinPackParameters: false
BreakBeforeBinaryOperators: All
BreakBeforeBraces: Linux
DerivePointerAlignment: false
SpacesInContainerLiterals: false
Standard: Cpp11
AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false

15
format_code.sh Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/bash
files=(
"src/installer/kiwix-install.cpp"
"src/searcher/kiwix-search.cpp"
"src/reader/kiwix-read.cpp"
"src/manager/kiwix-manage.cpp"
"src/server/kiwix-serve.cpp"
)
for i in "${files[@]}"
do
echo $i
clang-format -i -style=file $i
done

View File

@ -19,21 +19,23 @@
#include <getopt.h> #include <getopt.h>
#include <kiwix/common/pathTools.h> #include <kiwix/common/pathTools.h>
#include <kiwix/reader.h>
#include <kiwix/manager.h> #include <kiwix/manager.h>
#include <kiwix/reader.h>
enum supportedAction { NONE, ADDCONTENT }; enum supportedAction { NONE, ADDCONTENT };
void usage() { void usage()
cout << "Usage: kiwix-install [--verbose] addcontent ZIM_PATH KIWIX_PATH" << endl; {
cout << "Usage: kiwix-install [--verbose] addcontent ZIM_PATH KIWIX_PATH"
<< endl;
exit(1); exit(1);
} }
int main(int argc, char **argv) { int main(int argc, char** argv)
{
/* Init the variables */ /* Init the variables */
const char *contentPath = NULL; const char* contentPath = NULL;
const char *kiwixPath = NULL; const char* kiwixPath = NULL;
supportedAction action = NONE; supportedAction action = NONE;
bool verboseFlag = false; bool verboseFlag = false;
int option_index = 0; int option_index = 0;
@ -41,11 +43,8 @@ int main(int argc, char **argv) {
/* Argument parsing */ /* Argument parsing */
while (42) { while (42) {
static struct option long_options[]
static struct option long_options[] = { = {{"verbose", no_argument, 0, 'v'}, {0, 0, 0, 0}};
{"verbose", no_argument, 0, 'v'},
{0, 0, 0, 0}
};
if (c != -1) { if (c != -1) {
c = getopt_long(argc, argv, "vi", long_options, &option_index); c = getopt_long(argc, argv, "vi", long_options, &option_index);
@ -82,38 +81,48 @@ int main(int argc, char **argv) {
/* Make the action */ /* Make the action */
if (action == ADDCONTENT) { if (action == ADDCONTENT) {
/* Check if the content path exists and is readable */ /* Check if the content path exists and is readable */
if (verboseFlag) { std::cout << "Check if the ZIM file exists..." << std::endl; } if (verboseFlag) {
std::cout << "Check if the ZIM file exists..." << std::endl;
}
if (!fileExists(contentPath)) { if (!fileExists(contentPath)) {
cerr << "The content path '" << contentPath << "' does not exist or is not readable." << endl; cerr << "The content path '" << contentPath
<< "' does not exist or is not readable." << endl;
exit(1); exit(1);
} }
/* Check if this is a ZIM file */ /* Check if this is a ZIM file */
try { try {
if (verboseFlag) { std::cout << "Check if the ZIM file is valid..." << std::endl; } if (verboseFlag) {
kiwix::Reader *reader = new kiwix::Reader(contentPath); std::cout << "Check if the ZIM file is valid..." << std::endl;
}
kiwix::Reader* reader = new kiwix::Reader(contentPath);
delete reader; delete reader;
} catch (exception &e) { } catch (exception& e) {
cerr << "The content available at '" << contentPath << "' is not a ZIM file." << endl; cerr << "The content available at '" << contentPath
<< "' is not a ZIM file." << endl;
exit(1); exit(1);
} }
string contentFilename = getLastPathElement(contentPath); string contentFilename = getLastPathElement(contentPath);
/* Check if kiwixPath/kiwix/kiwix.exe exists */ /* Check if kiwixPath/kiwix/kiwix.exe exists */
if (verboseFlag) { std::cout << "Check if the Kiwix path is valid..." << std::endl; } if (verboseFlag) {
std::cout << "Check if the Kiwix path is valid..." << std::endl;
}
string kiwixBinaryPath = computeAbsolutePath(kiwixPath, "kiwix/kiwix.exe"); string kiwixBinaryPath = computeAbsolutePath(kiwixPath, "kiwix/kiwix.exe");
if (!fileExists(kiwixBinaryPath)) { if (!fileExists(kiwixBinaryPath)) {
kiwixBinaryPath = computeAbsolutePath(kiwixPath, "kiwix/kiwix"); kiwixBinaryPath = computeAbsolutePath(kiwixPath, "kiwix/kiwix");
if (!fileExists(kiwixBinaryPath)) { if (!fileExists(kiwixBinaryPath)) {
cerr << "Unable to find the Kiwix binary at '" << kiwixBinaryPath << "[.exe]'." << endl; cerr << "Unable to find the Kiwix binary at '" << kiwixBinaryPath
<< "[.exe]'." << endl;
exit(1); exit(1);
} }
} }
/* Check if the directory "data" structure exists */ /* Check if the directory "data" structure exists */
if (verboseFlag) { std::cout << "Check the target data directory structure..." << std::endl; } if (verboseFlag) {
std::cout << "Check the target data directory structure..." << std::endl;
}
string dataPath = computeAbsolutePath(kiwixPath, "data/"); string dataPath = computeAbsolutePath(kiwixPath, "data/");
if (!fileExists(dataPath)) { if (!fileExists(dataPath)) {
makeDirectory(dataPath); makeDirectory(dataPath);
@ -132,22 +141,31 @@ int main(int argc, char **argv) {
} }
/* Copy the file to the data/content directory */ /* Copy the file to the data/content directory */
if (verboseFlag) { std::cout << "Copy ZIM file to the target directory..." << std::endl; } if (verboseFlag) {
string newContentPath = computeAbsolutePath(dataContentPath, contentFilename); std::cout << "Copy ZIM file to the target directory..." << std::endl;
if (!fileExists(newContentPath) || getFileSize(contentPath) != getFileSize(newContentPath)) { }
string newContentPath
= computeAbsolutePath(dataContentPath, contentFilename);
if (!fileExists(newContentPath)
|| getFileSize(contentPath) != getFileSize(newContentPath)) {
copyFile(contentPath, newContentPath); copyFile(contentPath, newContentPath);
} }
/* Add the file to the library.xml */ /* Add the file to the library.xml */
if (verboseFlag) { std::cout << "Create the library XML file..." << std::endl; } if (verboseFlag) {
std::cout << "Create the library XML file..." << std::endl;
}
kiwix::Manager libraryManager; kiwix::Manager libraryManager;
string libraryPath = computeAbsolutePath(dataLibraryPath, contentFilename + ".xml"); string libraryPath
string bookId = libraryManager.addBookFromPathAndGetId(newContentPath, "../content/" + contentFilename, "", false); = computeAbsolutePath(dataLibraryPath, contentFilename + ".xml");
string bookId = libraryManager.addBookFromPathAndGetId(
newContentPath, "../content/" + contentFilename, "", false);
if (!bookId.empty()) { if (!bookId.empty()) {
libraryManager.setCurrentBookId(bookId); libraryManager.setCurrentBookId(bookId);
libraryManager.writeFile(libraryPath); libraryManager.writeFile(libraryPath);
} else { } else {
cerr << "Unable to build or save library file '" << libraryPath << "'" << endl; cerr << "Unable to build or save library file '" << libraryPath << "'"
<< endl;
} }
} }

View File

@ -21,47 +21,54 @@
#include <unistd.h> #include <unistd.h>
#endif #endif
#include <getopt.h> #include <getopt.h>
#include <iostream>
#include <cstdlib>
#include <kiwix/common/pathTools.h> #include <kiwix/common/pathTools.h>
#include <kiwix/manager.h> #include <kiwix/manager.h>
#include <cstdlib>
#include <iostream>
using namespace std; using namespace std;
enum supportedAction { NONE, ADD, SHOW, REMOVE }; enum supportedAction { NONE, ADD, SHOW, REMOVE };
void show(kiwix::Library library)
void show(kiwix::Library library) { {
std::vector<kiwix::Book>::iterator itr; std::vector<kiwix::Book>::iterator itr;
unsigned int inc = 1; unsigned int inc = 1;
for ( itr = library.books.begin(); itr != library.books.end(); ++itr ) { for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
std::cout << "#" << inc++ std::cout << "#" << inc++ << std::endl
<< std::endl << "id:\t\t" << itr->id << "id:\t\t" << itr->id << std::endl
<< std::endl << "path:\t\t" << itr->path << "path:\t\t" << itr->path << std::endl
<< std::endl << "indexpath:\t" << itr->indexPath << "indexpath:\t" << itr->indexPath << std::endl
<< std::endl << "url:\t\t" << itr->url << "url:\t\t" << itr->url << std::endl
<< std::endl << "title:\t\t" << itr->title << "title:\t\t" << itr->title << std::endl
<< std::endl << "name:\t\t" << itr->name << "name:\t\t" << itr->name << std::endl
<< std::endl << "tags:\t\t" << itr->tags << "tags:\t\t" << itr->tags << std::endl
<< std::endl << "description:\t" << itr->description << "description:\t" << itr->description << std::endl
<< std::endl << "creator:\t" << itr->creator << "creator:\t" << itr->creator << std::endl
<< std::endl << "date:\t\t" << itr->date << "date:\t\t" << itr->date << std::endl
<< std::endl << "articleCount:\t" << itr->articleCount << "articleCount:\t" << itr->articleCount << std::endl
<< std::endl << "mediaCount:\t" << itr->mediaCount << "mediaCount:\t" << itr->mediaCount << std::endl
<< std::endl << "size:\t\t" << itr->size << " KB" << "size:\t\t" << itr->size << " KB" << std::endl
<< std::endl << std::endl; << std::endl;
} }
} }
void usage() { void usage()
{
cerr << "Usage:" << endl; cerr << "Usage:" << endl;
cerr << "\tkiwix-manage LIBRARY_PATH add ZIM_PATH [--zimPathToSave=../content/foobar.zim] [--current] [--indexBackend=xapian] [--indexPath=FULLTEXT_IDX_PATH] [--url=http://...metalink]" << endl; cerr << "\tkiwix-manage LIBRARY_PATH add ZIM_PATH "
cerr << "\tkiwix-manage LIBRARY_PATH show [CONTENTID1] [CONTENTID2] ... (show everything if no param.)" << endl; "[--zimPathToSave=../content/foobar.zim] [--current] "
"[--indexBackend=xapian] [--indexPath=FULLTEXT_IDX_PATH] "
"[--url=http://...metalink]"
<< endl;
cerr << "\tkiwix-manage LIBRARY_PATH show [CONTENTID1] [CONTENTID2] ... "
"(show everything if no param.)"
<< endl;
cerr << "\tkiwix-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]" << endl; cerr << "\tkiwix-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]" << endl;
} }
int main(int argc, char **argv) { int main(int argc, char** argv)
{
string libraryPath = ""; string libraryPath = "";
supportedAction action = NONE; supportedAction action = NONE;
string zimPath = ""; string zimPath = "";
@ -89,9 +96,9 @@ int main(int argc, char **argv) {
} }
/* Try to read the file */ /* Try to read the file */
libraryPath = isRelativePath(libraryPath) ? libraryPath = isRelativePath(libraryPath)
computeAbsolutePath(getCurrentDirectory(), libraryPath) : ? computeAbsolutePath(getCurrentDirectory(), libraryPath)
libraryPath; : libraryPath;
libraryManager.readFile(libraryPath, false); libraryManager.readFile(libraryPath, false);
/* SHOW */ /* SHOW */
@ -103,31 +110,28 @@ int main(int argc, char **argv) {
string indexPath; string indexPath;
kiwix::supportedIndexType indexBackend = kiwix::XAPIAN; kiwix::supportedIndexType indexBackend = kiwix::XAPIAN;
string url; string url;
string origID=""; string origID = "";
bool setCurrent = false; bool setCurrent = false;
if (argc>3) { if (argc > 3) {
zimPath = argv[3]; zimPath = argv[3];
} }
/* Options parsing */ /* Options parsing */
optind = 2; optind = 2;
while (42) { while (42) {
static struct option long_options[]
static struct option long_options[] = { = {{"url", required_argument, 0, 'u'},
{"url", required_argument, 0, 'u'},
{"origId", required_argument, 0, 'o'}, {"origId", required_argument, 0, 'o'},
{"indexPath", required_argument, 0, 'i'}, {"indexPath", required_argument, 0, 'i'},
{"indexBackend", required_argument, 0, 'b'}, {"indexBackend", required_argument, 0, 'b'},
{"zimPathToSave", required_argument, 0, 'z'}, {"zimPathToSave", required_argument, 0, 'z'},
{"current", no_argument, 0, 'c'}, {"current", no_argument, 0, 'c'},
{0, 0, 0, 0} {0, 0, 0, 0}};
};
c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index); c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index);
if (c != -1) { if (c != -1) {
switch (c) { switch (c) {
case 'u': case 'u':
url = optarg; url = optarg;
@ -156,7 +160,6 @@ int main(int argc, char **argv) {
case 'z': case 'z':
zimPathToSave = optarg; zimPathToSave = optarg;
break; break;
} }
} else { } else {
break; break;
@ -165,10 +168,10 @@ int main(int argc, char **argv) {
if (!zimPath.empty()) { if (!zimPath.empty()) {
zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave; zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
string bookId = libraryManager.addBookFromPathAndGetId(zimPath, zimPathToSave, url, false); string bookId = libraryManager.addBookFromPathAndGetId(
zimPath, zimPathToSave, url, false);
if (!bookId.empty()) { if (!bookId.empty()) {
if (setCurrent) if (setCurrent)
libraryManager.setCurrentBookId(bookId); libraryManager.setCurrentBookId(bookId);
@ -177,7 +180,8 @@ int main(int argc, char **argv) {
libraryManager.setBookIndex(bookId, indexPath, indexBackend); libraryManager.setBookIndex(bookId, indexPath, indexBackend);
} else { } else {
cerr << "Unable to build or save library file '" << libraryPath << "'" << endl; cerr << "Unable to build or save library file '" << libraryPath << "'"
<< endl;
} }
} else { } else {
std::cerr << "Invalid zim file path" << std::endl; std::cerr << "Invalid zim file path" << std::endl;
@ -187,7 +191,7 @@ int main(int argc, char **argv) {
unsigned int bookIndex = 0; unsigned int bookIndex = 0;
const unsigned int totalBookCount = libraryManager.getBookCount(true, true); const unsigned int totalBookCount = libraryManager.getBookCount(true, true);
if (argc>3) { if (argc > 3) {
bookIndex = atoi(argv[3]); bookIndex = atoi(argv[3]);
} }
@ -195,9 +199,13 @@ int main(int argc, char **argv) {
libraryManager.removeBookByIndex(bookIndex - 1); libraryManager.removeBookByIndex(bookIndex - 1);
} else { } else {
if (totalBookCount > 0) { if (totalBookCount > 0) {
std::cerr << "Invalid book index number. Please give a number between 1 and " << totalBookCount << std::endl; std::cerr
<< "Invalid book index number. Please give a number between 1 and "
<< totalBookCount << std::endl;
} else { } else {
std::cerr << "Invalid book index number. Library is empty, no book to delete." << std::endl; std::cerr
<< "Invalid book index number. Library is empty, no book to delete."
<< std::endl;
} }
} }
} }

View File

@ -18,39 +18,40 @@
*/ */
#include <getopt.h> #include <getopt.h>
#include <unistd.h>
#include <string>
#include <map>
#include <kiwix/common/tree.h> #include <kiwix/common/tree.h>
#include <kiwix/reader.h> #include <kiwix/reader.h>
#include <unistd.h>
#include <map>
#include <string>
void usage() { void usage()
{
cout << "Usage: kiwix-read --suggest=<PATTERN> ZIM_FILE_PATH" << endl; cout << "Usage: kiwix-read --suggest=<PATTERN> ZIM_FILE_PATH" << endl;
exit(1); exit(1);
} }
void updateSuggestionTree(tree< pair<string, unsigned> > &suggestionTree, string suggestion) { void updateSuggestionTree(tree<pair<string, unsigned>>& suggestionTree,
string suggestion)
{
return; return;
} }
int main(int argc, char **argv) { int main(int argc, char** argv)
{
/* Init the variables */ /* Init the variables */
const char *filePath = NULL; const char* filePath = NULL;
const char *pattern = NULL; const char* pattern = NULL;
int option_index = 0; int option_index = 0;
int c = 0; int c = 0;
kiwix::Reader *reader = NULL; kiwix::Reader* reader = NULL;
/* Argument parsing */ /* Argument parsing */
while (42) { while (42) {
static struct option long_options[]
static struct option long_options[] = { = {{"verbose", no_argument, 0, 'v'},
{"verbose", no_argument, 0, 'v'},
{"suggest", required_argument, 0, 's'}, {"suggest", required_argument, 0, 's'},
{0, 0, 0, 0} {0, 0, 0, 0}};
};
if (c != -1) { if (c != -1) {
c = getopt_long(argc, argv, "vs:", long_options, &option_index); c = getopt_long(argc, argv, "vs:", long_options, &option_index);
@ -101,13 +102,14 @@ int main(int argc, char **argv) {
} }
/* /*
if (reader->getContentByUrl(mainPageUrl, content, contentLength, contentType)) { if (reader->getContentByUrl(mainPageUrl, content, contentLength,
contentType)) {
cout << content << endl; cout << content << endl;
} }
*/ */
// tree< pair<string, unsigned> > tree; // tree< pair<string, unsigned> > tree;
//updateSuggestionTree(tree, string(suggestion)); // updateSuggestionTree(tree, string(suggestion));
delete reader; delete reader;
} else { } else {

View File

@ -18,36 +18,35 @@
*/ */
#include <getopt.h> #include <getopt.h>
#include <unistd.h>
#include <kiwix/reader.h> #include <kiwix/reader.h>
#include <kiwix/searcher.h> #include <kiwix/searcher.h>
#include <unistd.h>
void usage() { void usage()
{
cout << "Usage: kiwix-search [--verbose|-v] ZIM_PATH SEARCH" << endl; cout << "Usage: kiwix-search [--verbose|-v] ZIM_PATH SEARCH" << endl;
exit(1); exit(1);
} }
int main(int argc, char **argv) { int main(int argc, char** argv)
{
/* Init the variables */ /* Init the variables */
//const char *indexPath = "/home/itamar/.www.kiwix.org/kiwix/43k0i1j4.default/6d2e587b-d586-dc6a-dc6a-e4ef035a1495d15c.index"; // const char *indexPath =
//const char *indexPath = "/home/itamar/testindex"; // "/home/itamar/.www.kiwix.org/kiwix/43k0i1j4.default/6d2e587b-d586-dc6a-dc6a-e4ef035a1495d15c.index";
const char *zimPath = NULL; // const char *indexPath = "/home/itamar/testindex";
const char *search = NULL; const char* zimPath = NULL;
const char* search = NULL;
bool verboseFlag = false; bool verboseFlag = false;
int option_index = 0; int option_index = 0;
int c = 0; int c = 0;
kiwix::Searcher *searcher = NULL; kiwix::Searcher* searcher = NULL;
kiwix::Reader *reader = NULL; kiwix::Reader* reader = NULL;
/* Argument parsing */ /* Argument parsing */
while (42) { while (42) {
static struct option long_options[]
static struct option long_options[] = { = {{"verbose", no_argument, 0, 'v'}, {0, 0, 0, 0}};
{"verbose", no_argument, 0, 'v'},
{0, 0, 0, 0}
};
if (c != -1) { if (c != -1) {
c = getopt_long(argc, argv, "vb:", long_options, &option_index); c = getopt_long(argc, argv, "vb:", long_options, &option_index);
@ -82,10 +81,11 @@ int main(int argc, char **argv) {
try { try {
reader = new kiwix::Reader(zimPath); reader = new kiwix::Reader(zimPath);
} catch (...) { } catch (...) {
/* Cannot open the zimPath, maybe it is a plain old xapian database directory */ /* Cannot open the zimPath, maybe it is a plain old xapian database
* directory */
} }
if ( reader ) { if (reader) {
searcher = new kiwix::Searcher("", reader); searcher = new kiwix::Searcher("", reader);
} else { } else {
try { try {
@ -101,7 +101,7 @@ int main(int argc, char **argv) {
string searchString(search); string searchString(search);
searcher->search(searchString, 0, 10); searcher->search(searchString, 0, 10);
kiwix::Result* p_result; kiwix::Result* p_result;
while ( (p_result = searcher->getNextResult()) ) { while ((p_result = searcher->getNextResult())) {
cout << p_result->get_title() << endl; cout << p_result->get_title() << endl;
delete p_result; delete p_result;
} }

View File

@ -20,8 +20,8 @@
#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100 #define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
#ifdef __APPLE__ #ifdef __APPLE__
#import <sys/types.h>
#import <sys/sysctl.h> #import <sys/sysctl.h>
#import <sys/types.h>
#define MIBSIZE 4 #define MIBSIZE 4
#endif #endif
@ -33,8 +33,8 @@
#include <winsock2.h> #include <winsock2.h>
#include <ws2tcpip.h> // otherwise socklen_t is not a recognized type #include <ws2tcpip.h> // otherwise socklen_t is not a recognized type
//#include <Windows.h> // otherwise int is not a recognized type //#include <Windows.h> // otherwise int is not a recognized type
//typedef int off_t; // typedef int off_t;
//typedef SSIZE_T ssize_t; // typedef SSIZE_T ssize_t;
typedef UINT64 uint64_t; typedef UINT64 uint64_t;
typedef UINT16 uint16_t; typedef UINT16 uint16_t;
extern "C" { extern "C" {
@ -43,40 +43,40 @@ extern "C" {
#endif #endif
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <getopt.h> #include <getopt.h>
#include <iostream> #include <kiwix/common/otherTools.h>
#include <string>
#include <vector>
#include <map>
#include <fstream>
#include <iostream>
#include <sstream>
#include <zim/zim.h>
#include <zim/file.h>
#include <zim/article.h>
#include <zim/fileiterator.h>
#include <pthread.h>
#include <zlib.h>
#include <kiwix/reader.h>
#include <kiwix/manager.h>
#include <kiwix/searcher.h>
#include <kiwix/common/pathTools.h> #include <kiwix/common/pathTools.h>
#include <kiwix/common/regexTools.h> #include <kiwix/common/regexTools.h>
#include <kiwix/common/stringTools.h> #include <kiwix/common/stringTools.h>
#include <kiwix/common/otherTools.h> #include <kiwix/manager.h>
#include <kiwix/reader.h>
#include <kiwix/searcher.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <zim/article.h>
#include <zim/file.h>
#include <zim/fileiterator.h>
#include <zim/zim.h>
#include <zlib.h>
#include <fstream>
#include <iostream>
#include <iostream>
#include <map>
#include <sstream>
#include <string>
#include <vector>
#include "server-resources.h" #include "server-resources.h"
#ifndef _WIN32 #ifndef _WIN32
#include <stdint.h>
#include <unistd.h>
#include <microhttpd.h>
#include <arpa/inet.h> #include <arpa/inet.h>
#include <sys/socket.h>
#include <netdb.h>
#include <ifaddrs.h> #include <ifaddrs.h>
#include <microhttpd.h>
#include <netdb.h>
#include <stdint.h>
#include <sys/socket.h>
#include <unistd.h>
#endif #endif
#ifdef interface #ifdef interface
@ -101,17 +101,19 @@ static pthread_mutex_t verboseFlagLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t mimeTypeLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t mimeTypeLock = PTHREAD_MUTEX_INITIALIZER;
/* Try to get the mimeType from the file extension */ /* Try to get the mimeType from the file extension */
static std::string getMimeTypeForFile(const std::string& filename) { static std::string getMimeTypeForFile(const std::string& filename)
{
std::string mimeType = "text/plain"; std::string mimeType = "text/plain";
unsigned int pos = filename.find_last_of("."); unsigned int pos = filename.find_last_of(".");
if (pos != std::string::npos) { if (pos != std::string::npos) {
std::string extension = filename.substr(pos+1); std::string extension = filename.substr(pos + 1);
pthread_mutex_lock(&mimeTypeLock); pthread_mutex_lock(&mimeTypeLock);
if (extMimeTypes.find(extension) != extMimeTypes.end()) { if (extMimeTypes.find(extension) != extMimeTypes.end()) {
mimeType = extMimeTypes[extension]; mimeType = extMimeTypes[extension];
} else if (extMimeTypes.find(kiwix::lcAll(extension)) != extMimeTypes.end()) { } else if (extMimeTypes.find(kiwix::lcAll(extension))
!= extMimeTypes.end()) {
mimeType = extMimeTypes[kiwix::lcAll(extension)]; mimeType = extMimeTypes[kiwix::lcAll(extension)];
} }
pthread_mutex_unlock(&mimeTypeLock); pthread_mutex_unlock(&mimeTypeLock);
@ -120,25 +122,33 @@ static std::string getMimeTypeForFile(const std::string& filename) {
return mimeType; return mimeType;
} }
void introduceTaskbar(string &content, const string &humanReadableBookId) { void introduceTaskbar(string& content, const string& humanReadableBookId)
{
if (!noSearchBarFlag) { if (!noSearchBarFlag) {
content = appendToFirstOccurence(content, "<head>", content = appendToFirstOccurence(
replaceRegex(RESOURCE::include_html_part, content,
humanReadableBookId, "__CONTENT__"));
content = appendToFirstOccurence(content,
"<head>", "<head>",
"<style>" + replaceRegex(
RESOURCE::taskbar_css + RESOURCE::include_html_part, humanReadableBookId, "__CONTENT__"));
(noLibraryButtonFlag ? " #kiwix_serve_taskbar_library_button { display: none }" : "") + content = appendToFirstOccurence(
"</style>"); content,
content = appendToFirstOccurence(content, "<body[^>]*>", "<head>",
replaceRegex(RESOURCE::taskbar_html_part, "<style>" + RESOURCE::taskbar_css
humanReadableBookId, "__CONTENT__")); + (noLibraryButtonFlag
? " #kiwix_serve_taskbar_library_button { display: none }"
: "")
+ "</style>");
content = appendToFirstOccurence(
content,
"<body[^>]*>",
replaceRegex(
RESOURCE::taskbar_html_part, humanReadableBookId, "__CONTENT__"));
} }
} }
/* Should display debug information? */ /* Should display debug information? */
bool isVerbose() { bool isVerbose()
{
bool value; bool value;
pthread_mutex_lock(&verboseFlagLock); pthread_mutex_lock(&verboseFlagLock);
value = verboseFlag; value = verboseFlag;
@ -146,25 +156,23 @@ bool isVerbose() {
return value; return value;
} }
static static bool compress_content(string& content, const string& mimeType)
bool compress_content(string &content,
const string &mimeType)
{ {
static std::vector<Bytef> compr_buffer; static std::vector<Bytef> compr_buffer;
/* Should be deflate */ /* Should be deflate */
bool deflated = mimeType.find("text/") != string::npos || bool deflated = mimeType.find("text/") != string::npos
mimeType.find("application/javascript") != string::npos || || mimeType.find("application/javascript") != string::npos
mimeType.find("application/json") != string::npos; || mimeType.find("application/json") != string::npos;
if ( ! deflated ) if (!deflated)
return false; return false;
/* Compute the lengh */ /* Compute the lengh */
unsigned int contentLength = content.size(); unsigned int contentLength = content.size();
/* If the content is too short, no need to compress it */ /* If the content is too short, no need to compress it */
if ( contentLength <= KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE) if (contentLength <= KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE)
return false; return false;
uLong bufferBound = compressBound(contentLength); uLong bufferBound = compressBound(contentLength);
@ -173,15 +181,18 @@ bool compress_content(string &content,
pthread_mutex_lock(&compressorLock); pthread_mutex_lock(&compressorLock);
compr_buffer.reserve(bufferBound); compr_buffer.reserve(bufferBound);
uLongf comprLen = compr_buffer.capacity(); uLongf comprLen = compr_buffer.capacity();
int err = compress(&compr_buffer[0], &comprLen, (const Bytef*)(content.data()), contentLength); int err = compress(&compr_buffer[0],
&comprLen,
(const Bytef*)(content.data()),
contentLength);
if (err == Z_OK && comprLen > 2 && comprLen < (contentLength+2)) { if (err == Z_OK && comprLen > 2 && comprLen < (contentLength + 2)) {
/* /!\ Internet Explorer has a bug with deflate compression. /* /!\ Internet Explorer has a bug with deflate compression.
It can not handle the first two bytes (compression headers) It can not handle the first two bytes (compression headers)
We need to chunk them off (move the content 2bytes) We need to chunk them off (move the content 2bytes)
It has no incidence on other browsers It has no incidence on other browsers
See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */ See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
content = string((char *)&compr_buffer[2], comprLen-2); content = string((char*)&compr_buffer[2], comprLen - 2);
} else { } else {
deflated = false; deflated = false;
} }
@ -190,9 +201,7 @@ bool compress_content(string &content,
return deflated; return deflated;
} }
static struct MHD_Response* build_response(const void* data,
static
struct MHD_Response* build_response(const void* data,
unsigned int length, unsigned int length,
const std::string& httpRedirection, const std::string& httpRedirection,
const std::string& mimeType, const std::string& mimeType,
@ -200,65 +209,70 @@ struct MHD_Response* build_response(const void* data,
bool cacheEnabled) bool cacheEnabled)
{ {
/* Create the response */ /* Create the response */
struct MHD_Response * response = MHD_create_response_from_data(length, struct MHD_Response* response = MHD_create_response_from_data(
const_cast<void*>(data), length, const_cast<void*>(data), MHD_NO, MHD_YES);
MHD_NO,
MHD_YES);
/* Make a redirection if necessary otherwise send the content */ /* Make a redirection if necessary otherwise send the content */
if (!httpRedirection.empty()) { if (!httpRedirection.empty()) {
MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, httpRedirection.c_str()); MHD_add_response_header(
response, MHD_HTTP_HEADER_LOCATION, httpRedirection.c_str());
} else { } else {
/* Add if necessary the content-encoding */ /* Add if necessary the content-encoding */
if (deflated) { if (deflated) {
MHD_add_response_header(response, MHD_HTTP_HEADER_VARY, "Accept-Encoding"); MHD_add_response_header(
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate"); response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
MHD_add_response_header(
response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
} }
/* Tell the client that byte ranges are accepted */ /* Tell the client that byte ranges are accepted */
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
/* Specify the mime type */ /* Specify the mime type */
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, mimeType.c_str()); MHD_add_response_header(
response, MHD_HTTP_HEADER_CONTENT_TYPE, mimeType.c_str());
} }
/* Force to close the connection - cf. 100% CPU usage with v. 4.4 (in Lucid) */ /* Force to close the connection - cf. 100% CPU usage with v. 4.4 (in Lucid)
//MHD_add_response_header(response, MHD_HTTP_HEADER_CONNECTION, "close"); */
// MHD_add_response_header(response, MHD_HTTP_HEADER_CONNECTION, "close");
/* Allow cross-domain requests */ /* Allow cross-domain requests */
//MHD_add_response_header(response, MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*"); // MHD_add_response_header(response,
// MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
MHD_add_response_header(response, "Access-Control-Allow-Origin", "*"); MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
if (cacheEnabled) { /* Force cache */ if (cacheEnabled) { /* Force cache */
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, "max-age=2723040, public"); MHD_add_response_header(
response, MHD_HTTP_HEADER_CACHE_CONTROL, "max-age=2723040, public");
} else { /* Prevent cache (for random page) */ } else { /* Prevent cache (for random page) */
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, "no-cache, no-store, must-revalidate"); MHD_add_response_header(response,
MHD_HTTP_HEADER_CACHE_CONTROL,
"no-cache, no-store, must-revalidate");
} }
return response; return response;
} }
ssize_t callback_reader_from_blob(void *cls, ssize_t callback_reader_from_blob(void* cls,
uint64_t pos, uint64_t pos,
char *buf, char* buf,
size_t max) size_t max)
{ {
zim::Blob* blob = static_cast<zim::Blob*>(cls); zim::Blob* blob = static_cast<zim::Blob*>(cls);
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&readerLock);
size_t max_size_to_set = min<size_t>(max, blob->size()-pos); size_t max_size_to_set = min<size_t>(max, blob->size() - pos);
if (max_size_to_set <= 0) if (max_size_to_set <= 0) {
{
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&readerLock);
return MHD_CONTENT_READER_END_WITH_ERROR; return MHD_CONTENT_READER_END_WITH_ERROR;
} }
memcpy(buf, blob->data()+pos, max_size_to_set); memcpy(buf, blob->data() + pos, max_size_to_set);
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&readerLock);
return max_size_to_set; return max_size_to_set;
} }
void callback_free_blob(void *cls) void callback_free_blob(void* cls)
{ {
zim::Blob* blob = static_cast<zim::Blob*>(cls); zim::Blob* blob = static_cast<zim::Blob*>(cls);
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&readerLock);
@ -266,13 +280,13 @@ void callback_free_blob(void *cls)
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&readerLock);
} }
static static struct MHD_Response* build_callback_response_from_blob(
struct MHD_Response* build_callback_response_from_blob(zim::Blob& blob, zim::Blob& blob, const std::string& mimeType)
const std::string& mimeType)
{ {
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&readerLock);
zim::Blob* p_blob = new zim::Blob(blob); zim::Blob* p_blob = new zim::Blob(blob);
struct MHD_Response * response = MHD_create_response_from_callback(blob.size(), struct MHD_Response* response
= MHD_create_response_from_callback(blob.size(),
16384, 16384,
callback_reader_from_blob, callback_reader_from_blob,
p_blob, p_blob,
@ -282,22 +296,25 @@ struct MHD_Response* build_callback_response_from_blob(zim::Blob& blob,
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
/* Specify the mime type */ /* Specify the mime type */
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, mimeType.c_str()); MHD_add_response_header(
response, MHD_HTTP_HEADER_CONTENT_TYPE, mimeType.c_str());
/* Allow cross-domain requests */ /* Allow cross-domain requests */
//MHD_add_response_header(response, MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*"); // MHD_add_response_header(response,
// MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
MHD_add_response_header(response, "Access-Control-Allow-Origin", "*"); MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, "max-age=2723040, public"); MHD_add_response_header(
response, MHD_HTTP_HEADER_CACHE_CONTROL, "max-age=2723040, public");
return response; return response;
} }
static static struct MHD_Response* handle_suggest(
struct MHD_Response* handle_suggest(struct MHD_Connection * connection, struct MHD_Connection* connection,
int& httpResponseCode, int& httpResponseCode,
kiwix::Reader *reader, kiwix::Reader* reader,
kiwix::Searcher *searcher, kiwix::Searcher* searcher,
const std::string& urlStr, const std::string& urlStr,
const std::string& humanReadableBookId, const std::string& humanReadableBookId,
bool acceptEncodingDeflate) bool acceptEncodingDeflate)
@ -309,7 +326,8 @@ struct MHD_Response* handle_suggest(struct MHD_Connection * connection,
std::string suggestion; std::string suggestion;
/* Get the suggestion pattern from the HTTP request */ /* Get the suggestion pattern from the HTTP request */
const char* cTerm = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "term"); const char* cTerm
= MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "term");
std::string term = cTerm == NULL ? "" : cTerm; std::string term = cTerm == NULL ? "" : cTerm;
if (isVerbose()) { if (isVerbose()) {
std::cout << "Searching suggestions for: \"" << term << "\"" << endl; std::cout << "Searching suggestions for: \"" << term << "\"" << endl;
@ -321,27 +339,30 @@ struct MHD_Response* handle_suggest(struct MHD_Connection * connection,
while (reader->getNextSuggestion(suggestion)) { while (reader->getNextSuggestion(suggestion)) {
kiwix::stringReplacement(suggestion, "\"", "\\\""); kiwix::stringReplacement(suggestion, "\"", "\\\"");
content += (content == "[" ? "" : ","); content += (content == "[" ? "" : ",");
content += "{\"value\":\"" + suggestion + "\",\"label\":\"" + suggestion + "\"}"; content += "{\"value\":\"" + suggestion + "\",\"label\":\"" + suggestion
+ "\"}";
suggestionCount++; suggestionCount++;
} }
/* Propose the fulltext search if possible */ /* Propose the fulltext search if possible */
if (searcher != NULL) { if (searcher != NULL) {
content += (suggestionCount == 0 ? "" : ","); content += (suggestionCount == 0 ? "" : ",");
content += "{\"value\":\"" + std::string(term) + " \", \"label\":\"containing '" + std::string(term) + "'...\"}"; content += "{\"value\":\"" + std::string(term)
+ " \", \"label\":\"containing '" + std::string(term)
+ "'...\"}";
} }
content += "]"; content += "]";
mimeType = "application/json; charset=utf-8"; mimeType = "application/json; charset=utf-8";
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType); bool deflated = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(content.data(), content.size(), "", mimeType, deflated, true); return build_response(
content.data(), content.size(), "", mimeType, deflated, true);
} }
static static struct MHD_Response* handle_skin(struct MHD_Connection* connection,
struct MHD_Response* handle_skin(struct MHD_Connection * connection,
int& httpResponseCode, int& httpResponseCode,
kiwix::Reader *reader, kiwix::Reader* reader,
kiwix::Searcher *searcher, kiwix::Searcher* searcher,
const std::string& urlStr, const std::string& urlStr,
const std::string& humanReadableBookId, const std::string& humanReadableBookId,
bool acceptEncodingDeflate) bool acceptEncodingDeflate)
@ -349,14 +370,15 @@ struct MHD_Response* handle_skin(struct MHD_Connection * connection,
std::string content = getResource(urlStr.substr(6)); std::string content = getResource(urlStr.substr(6));
std::string mimeType = getMimeTypeForFile(urlStr); std::string mimeType = getMimeTypeForFile(urlStr);
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType); bool deflated = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(content.data(), content.size(), "", mimeType, deflated, true); return build_response(
content.data(), content.size(), "", mimeType, deflated, true);
} }
static static struct MHD_Response* handle_search(
struct MHD_Response* handle_search(struct MHD_Connection * connection, struct MHD_Connection* connection,
int& httpResponseCode, int& httpResponseCode,
kiwix::Reader *reader, kiwix::Reader* reader,
kiwix::Searcher *searcher, kiwix::Searcher* searcher,
const std::string& urlStr, const std::string& urlStr,
const std::string& humanReadableBookId, const std::string& humanReadableBookId,
bool acceptEncodingDeflate) bool acceptEncodingDeflate)
@ -366,8 +388,10 @@ struct MHD_Response* handle_search(struct MHD_Connection * connection,
std::string httpRedirection; std::string httpRedirection;
/* Retrieve the pattern to search */ /* Retrieve the pattern to search */
const char* pattern = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "pattern"); const char* pattern = MHD_lookup_connection_value(
std::string patternString = kiwix::urlDecode(pattern == NULL ? "" : string(pattern)); connection, MHD_GET_ARGUMENT_KIND, "pattern");
std::string patternString
= kiwix::urlDecode(pattern == NULL ? "" : string(pattern));
std::string patternCorrespondingUrl; std::string patternCorrespondingUrl;
/* Try first to load directly the article */ /* Try first to load directly the article */
@ -384,7 +408,8 @@ struct MHD_Response* handle_search(struct MHD_Connection * connection,
/* If article found then redirect directly to it */ /* If article found then redirect directly to it */
if (!patternCorrespondingUrl.empty()) { if (!patternCorrespondingUrl.empty()) {
httpRedirection = "/" + humanReadableBookId + "/" + patternCorrespondingUrl; httpRedirection
= "/" + humanReadableBookId + "/" + patternCorrespondingUrl;
httpResponseCode = MHD_HTTP_FOUND; httpResponseCode = MHD_HTTP_FOUND;
return build_response("", 0, httpRedirection, "", false, true); return build_response("", 0, httpRedirection, "", false, true);
} }
@ -392,8 +417,10 @@ struct MHD_Response* handle_search(struct MHD_Connection * connection,
/* Make the search */ /* Make the search */
if (patternCorrespondingUrl.empty() && searcher != NULL) { if (patternCorrespondingUrl.empty() && searcher != NULL) {
const char* start = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "start"); const char* start = MHD_lookup_connection_value(
const char* end = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "end"); connection, MHD_GET_ARGUMENT_KIND, "start");
const char* end
= MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "end");
unsigned int startNumber = start != NULL ? atoi(start) : 0; unsigned int startNumber = start != NULL ? atoi(start) : 0;
unsigned int endNumber = end != NULL ? atoi(end) : 25; unsigned int endNumber = end != NULL ? atoi(end) : 25;
@ -418,14 +445,19 @@ struct MHD_Response* handle_search(struct MHD_Connection * connection,
introduceTaskbar(content, humanReadableBookId); introduceTaskbar(content, humanReadableBookId);
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType); bool deflated = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(content.data(), content.size(), httpRedirection, mimeType, deflated, true); return build_response(content.data(),
content.size(),
httpRedirection,
mimeType,
deflated,
true);
} }
static static struct MHD_Response* handle_random(
struct MHD_Response* handle_random(struct MHD_Connection * connection, struct MHD_Connection* connection,
int& httpResponseCode, int& httpResponseCode,
kiwix::Reader *reader, kiwix::Reader* reader,
kiwix::Searcher *searcher, kiwix::Searcher* searcher,
const std::string& urlStr, const std::string& urlStr,
const std::string& humanReadableBookId, const std::string& humanReadableBookId,
bool acceptEncodingDeflate) bool acceptEncodingDeflate)
@ -436,16 +468,17 @@ struct MHD_Response* handle_random(struct MHD_Connection * connection,
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&readerLock);
std::string randomUrl = reader->getRandomPageUrl(); std::string randomUrl = reader->getRandomPageUrl();
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&readerLock);
httpRedirection = "/" + humanReadableBookId + "/" + kiwix::urlEncode(randomUrl); httpRedirection
= "/" + humanReadableBookId + "/" + kiwix::urlEncode(randomUrl);
} }
return build_response("", 0, httpRedirection, "", false, false); return build_response("", 0, httpRedirection, "", false, false);
} }
static static struct MHD_Response* handle_content(
struct MHD_Response* handle_content(struct MHD_Connection * connection, struct MHD_Connection* connection,
int& httpResponseCode, int& httpResponseCode,
kiwix::Reader *reader, kiwix::Reader* reader,
kiwix::Searcher *searcher, kiwix::Searcher* searcher,
const std::string& urlStr, const std::string& urlStr,
const std::string& humanReadableBookId, const std::string& humanReadableBookId,
bool acceptEncodingDeflate) bool acceptEncodingDeflate)
@ -463,7 +496,7 @@ struct MHD_Response* handle_content(struct MHD_Connection * connection,
if (found) { if (found) {
/* If redirect */ /* If redirect */
unsigned int loopCounter = 0; unsigned int loopCounter = 0;
while (article.isRedirect() && loopCounter++<42) { while (article.isRedirect() && loopCounter++ < 42) {
article = article.getRedirectArticle(); article = article.getRedirectArticle();
} }
@ -481,19 +514,26 @@ struct MHD_Response* handle_content(struct MHD_Connection * connection,
if (isVerbose()) if (isVerbose())
cout << "Failed to find " << urlStr << endl; cout << "Failed to find " << urlStr << endl;
content = "<!DOCTYPE html>\n<html><head><meta content=\"text/html;charset=UTF-8\" http-equiv=\"content-type\" /><title>Content not found</title></head><body><h1>Not Found</h1><p>The requested URL \"" + urlStr + "\" was not found on this server.</p></body></html>"; content
= "<!DOCTYPE html>\n<html><head><meta "
"content=\"text/html;charset=UTF-8\" http-equiv=\"content-type\" "
"/><title>Content not found</title></head><body><h1>Not "
"Found</h1><p>The requested URL \""
+ urlStr + "\" was not found on this server.</p></body></html>";
mimeType = "text/html"; mimeType = "text/html";
httpResponseCode = MHD_HTTP_NOT_FOUND; httpResponseCode = MHD_HTTP_NOT_FOUND;
introduceTaskbar(content, humanReadableBookId); introduceTaskbar(content, humanReadableBookId);
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType); bool deflated
return build_response(content.data(), content.size(), "", mimeType, deflated, false); = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(
content.data(), content.size(), "", mimeType, deflated, false);
} }
try { try {
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&readerLock);
mimeType = article.getMimeType(); mimeType = article.getMimeType();
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&readerLock);
} catch (exception &e) { } catch (exception& e) {
mimeType = "application/octet-stream"; mimeType = "application/octet-stream";
} }
@ -506,44 +546,49 @@ struct MHD_Response* handle_content(struct MHD_Connection * connection,
zim::Blob raw_content = article.getData(); zim::Blob raw_content = article.getData();
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&readerLock);
if (mimeType.find("text/") != string::npos || if (mimeType.find("text/") != string::npos
mimeType.find("application/javascript") != string::npos || || mimeType.find("application/javascript") != string::npos
mimeType.find("application/json") != string::npos) || mimeType.find("application/json") != string::npos) {
{
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&readerLock);
content = string(raw_content.data(), raw_content.size()); content = string(raw_content.data(), raw_content.size());
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&readerLock);
/* Special rewrite URL in case of ZIM file use intern *asbolute* url like /A/Kiwix */ /* Special rewrite URL in case of ZIM file use intern *asbolute* url like
* /A/Kiwix */
if (mimeType.find("text/html") != string::npos) { if (mimeType.find("text/html") != string::npos) {
baseUrl = "/" + std::string(1, article.getNamespace()) + "/" + article.getUrl(); baseUrl = "/" + std::string(1, article.getNamespace()) + "/"
content = replaceRegex(content, "$1$2" + humanReadableBookId + "/$3/", + article.getUrl();
"(href|src)(=[\"|\']{0,1}/)([A-Z|\\-])/");
content = replaceRegex(content, "$1$2" + humanReadableBookId + "/$3/",
"(@import[ ]+)([\"|\']{0,1}/)([A-Z|\\-])/");
content = replaceRegex(content, content = replaceRegex(content,
"$1$2" + humanReadableBookId + "/$3/",
"(href|src)(=[\"|\']{0,1}/)([A-Z|\\-])/");
content = replaceRegex(content,
"$1$2" + humanReadableBookId + "/$3/",
"(@import[ ]+)([\"|\']{0,1}/)([A-Z|\\-])/");
content = replaceRegex(
content,
"<head><base href=\"/" + humanReadableBookId + baseUrl + "\" />", "<head><base href=\"/" + humanReadableBookId + baseUrl + "\" />",
"<head>"); "<head>");
introduceTaskbar(content, humanReadableBookId); introduceTaskbar(content, humanReadableBookId);
} else if (mimeType.find("text/css") != string::npos) { } else if (mimeType.find("text/css") != string::npos) {
content = replaceRegex(content, "$1$2" + humanReadableBookId + "/$3/", content = replaceRegex(content,
"$1$2" + humanReadableBookId + "/$3/",
"(url|URL)(\\([\"|\']{0,1}/)([A-Z|\\-])/"); "(url|URL)(\\([\"|\']{0,1}/)([A-Z|\\-])/");
} }
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType); bool deflated
return build_response(content.data(), content.size(), "", mimeType, deflated, true); = acceptEncodingDeflate && compress_content(content, mimeType);
} return build_response(
else content.data(), content.size(), "", mimeType, deflated, true);
{ } else {
return build_callback_response_from_blob(raw_content, mimeType); return build_callback_response_from_blob(raw_content, mimeType);
} }
} }
static static struct MHD_Response* handle_default(
struct MHD_Response* handle_default(struct MHD_Connection * connection, struct MHD_Connection* connection,
int& httpResponseCode, int& httpResponseCode,
kiwix::Reader *reader, kiwix::Reader* reader,
kiwix::Searcher *searcher, kiwix::Searcher* searcher,
const std::string& urlStr, const std::string& urlStr,
const std::string& humanReadableBookId, const std::string& humanReadableBookId,
bool acceptEncodingDeflate) bool acceptEncodingDeflate)
@ -555,23 +600,25 @@ struct MHD_Response* handle_default(struct MHD_Connection * connection,
std::string mimeType = "text/html; charset=utf-8"; std::string mimeType = "text/html; charset=utf-8";
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType); bool deflated = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(content.data(), content.size(), "", mimeType, deflated, true); return build_response(
content.data(), content.size(), "", mimeType, deflated, true);
} }
static int accessHandlerCallback(void *cls, static int accessHandlerCallback(void* cls,
struct MHD_Connection * connection, struct MHD_Connection* connection,
const char * url, const char* url,
const char * method, const char* method,
const char * version, const char* version,
const char * upload_data, const char* upload_data,
size_t * upload_data_size, size_t* upload_data_size,
void ** ptr) void** ptr)
{ {
/* Unexpected method */ /* Unexpected method */
if (0 != strcmp(method, "GET") && 0 != strcmp(method, "POST")) if (0 != strcmp(method, "GET") && 0 != strcmp(method, "POST"))
return MHD_NO; return MHD_NO;
/* The first time only the headers are valid, do not respond in the first round... */ /* The first time only the headers are valid, do not respond in the first
* round... */
static int dummy; static int dummy;
if (&dummy != *ptr) { if (&dummy != *ptr) {
*ptr = &dummy; *ptr = &dummy;
@ -587,41 +634,53 @@ static int accessHandlerCallback(void *cls,
} }
/* Check if the response can be compressed */ /* Check if the response can be compressed */
const char* acceptEncodingHeaderValue = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, MHD_HTTP_HEADER_ACCEPT_ENCODING); const char* acceptEncodingHeaderValue = MHD_lookup_connection_value(
const bool acceptEncodingDeflate = acceptEncodingHeaderValue && string(acceptEncodingHeaderValue).find("deflate") != string::npos; connection, MHD_HEADER_KIND, MHD_HTTP_HEADER_ACCEPT_ENCODING);
const bool acceptEncodingDeflate
= acceptEncodingHeaderValue
&& string(acceptEncodingHeaderValue).find("deflate") != string::npos;
/* Check if range is requested. [TODO] Handle this somehow */ /* Check if range is requested. [TODO] Handle this somehow */
const char* acceptRangeHeaderValue = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, MHD_HTTP_HEADER_RANGE); const char* acceptRangeHeaderValue = MHD_lookup_connection_value(
connection, MHD_HEADER_KIND, MHD_HTTP_HEADER_RANGE);
const bool acceptRange = acceptRangeHeaderValue != NULL; const bool acceptRange = acceptRangeHeaderValue != NULL;
/* Prepare the variables */ /* Prepare the variables */
struct MHD_Response *response; struct MHD_Response* response;
int httpResponseCode = MHD_HTTP_OK; int httpResponseCode = MHD_HTTP_OK;
std::string urlStr = string(url); std::string urlStr = string(url);
/* Get searcher and reader */ /* Get searcher and reader */
std::string humanReadableBookId = ""; std::string humanReadableBookId = "";
if (!(urlStr.size() > 5 && urlStr.substr(0, 6) == "/skin/")) { if (!(urlStr.size() > 5 && urlStr.substr(0, 6) == "/skin/")) {
if (!strcmp(url, "/search") || !strcmp(url, "/suggest") || !strcmp(url, "/random")) { if (!strcmp(url, "/search") || !strcmp(url, "/suggest")
const char* tmpGetValue = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "content"); || !strcmp(url, "/random")) {
const char* tmpGetValue = MHD_lookup_connection_value(
connection, MHD_GET_ARGUMENT_KIND, "content");
humanReadableBookId = (tmpGetValue != NULL ? string(tmpGetValue) : ""); humanReadableBookId = (tmpGetValue != NULL ? string(tmpGetValue) : "");
} else { } else {
humanReadableBookId = urlStr.substr(1, urlStr.find("/", 1) != string::npos ? humanReadableBookId = urlStr.substr(1,
urlStr.find("/", 1) - 1 : urlStr.size() - 2); urlStr.find("/", 1) != string::npos
? urlStr.find("/", 1) - 1
: urlStr.size() - 2);
if (!humanReadableBookId.empty()) { if (!humanReadableBookId.empty()) {
urlStr = urlStr.substr(urlStr.find("/", 1) != string::npos ? urlStr = urlStr.substr(urlStr.find("/", 1) != string::npos
urlStr.find("/", 1) : humanReadableBookId.size()); ? urlStr.find("/", 1)
: humanReadableBookId.size());
} }
} }
} }
pthread_mutex_lock(&mapLock); pthread_mutex_lock(&mapLock);
kiwix::Searcher *searcher = searchers.find(humanReadableBookId) != searchers.end() ? kiwix::Searcher* searcher
searchers.find(humanReadableBookId)->second : NULL; = searchers.find(humanReadableBookId) != searchers.end()
kiwix::Reader *reader = readers.find(humanReadableBookId) != readers.end() ? ? searchers.find(humanReadableBookId)->second
readers.find(humanReadableBookId)->second : NULL; : NULL;
kiwix::Reader* reader = readers.find(humanReadableBookId) != readers.end()
? readers.find(humanReadableBookId)->second
: NULL;
if (reader == NULL) { if (reader == NULL) {
humanReadableBookId=""; humanReadableBookId = "";
} }
pthread_mutex_unlock(&mapLock); pthread_mutex_unlock(&mapLock);
@ -692,16 +751,15 @@ static int accessHandlerCallback(void *cls,
} }
/* Queue the response */ /* Queue the response */
int ret = MHD_queue_response(connection, int ret = MHD_queue_response(connection, httpResponseCode, response);
httpResponseCode,
response);
MHD_destroy_response(response); MHD_destroy_response(response);
return ret; return ret;
} }
int main(int argc, char **argv) { int main(int argc, char** argv)
struct MHD_Daemon *daemon; {
struct MHD_Daemon* daemon;
vector<string> zimPathes; vector<string> zimPathes;
string libraryPath; string libraryPath;
string indexPath; string indexPath;
@ -714,8 +772,8 @@ int main(int argc, char **argv) {
unsigned int PPID = 0; unsigned int PPID = 0;
kiwix::Manager libraryManager; kiwix::Manager libraryManager;
static struct option long_options[] = { static struct option long_options[]
{"daemon", no_argument, 0, 'd'}, = {{"daemon", no_argument, 0, 'd'},
{"verbose", no_argument, 0, 'v'}, {"verbose", no_argument, 0, 'v'},
{"library", no_argument, 0, 'l'}, {"library", no_argument, 0, 'l'},
{"nolibrarybutton", no_argument, 0, 'm'}, {"nolibrarybutton", no_argument, 0, 'm'},
@ -724,16 +782,15 @@ int main(int argc, char **argv) {
{"attachToProcess", required_argument, 0, 'a'}, {"attachToProcess", required_argument, 0, 'a'},
{"port", required_argument, 0, 'p'}, {"port", required_argument, 0, 'p'},
{"interface", required_argument, 0, 'f'}, {"interface", required_argument, 0, 'f'},
{0, 0, 0, 0} {0, 0, 0, 0}};
};
/* Argument parsing */ /* Argument parsing */
while (true) { while (true) {
int option_index = 0; int option_index = 0;
int c = getopt_long(argc, argv, "mndvli:a:p:f:", long_options, &option_index); int c
= getopt_long(argc, argv, "mndvli:a:p:f:", long_options, &option_index);
if (c != -1) { if (c != -1) {
switch (c) { switch (c) {
case 'd': case 'd':
daemonFlag = true; daemonFlag = true;
@ -766,11 +823,10 @@ int main(int argc, char **argv) {
} }
} else { } else {
if (optind <= argc) { if (optind <= argc) {
if (libraryFlag) if (libraryFlag) {
{
libraryPath = argv[optind++]; libraryPath = argv[optind++];
} else { } else {
while ( optind < argc ) while (optind < argc)
zimPathes.push_back(std::string(argv[optind++])); zimPathes.push_back(std::string(argv[optind++]));
} }
break; break;
@ -780,13 +836,21 @@ int main(int argc, char **argv) {
/* Print usage)) if necessary */ /* Print usage)) if necessary */
if (zimPathes.empty() && libraryPath.empty()) { if (zimPathes.empty() && libraryPath.empty()) {
cerr << "Usage: kiwix-serve [--index=INDEX_PATH] [--port=PORT] [--verbose] [--nosearchbar] [--nolibrarybutton] [--daemon] [--attachToProcess=PID] [--interface=IF_NAME] ZIM_PATH+" << endl; cerr << "Usage: kiwix-serve [--index=INDEX_PATH] [--port=PORT] [--verbose] "
cerr << " kiwix-serve --library [--port=PORT] [--verbose] [--daemon] [--nosearchbar] [--nolibrarybutton] [--attachToProcess=PID] [--interface=IF_NAME] LIBRARY_PATH" << endl; "[--nosearchbar] [--nolibrarybutton] [--daemon] "
cerr << "\n If you set more than one ZIM_PATH, you cannot set a INDEX_PATH." << endl; "[--attachToProcess=PID] [--interface=IF_NAME] ZIM_PATH+"
<< endl;
cerr << " kiwix-serve --library [--port=PORT] [--verbose] [--daemon] "
"[--nosearchbar] [--nolibrarybutton] [--attachToProcess=PID] "
"[--interface=IF_NAME] LIBRARY_PATH"
<< endl;
cerr << "\n If you set more than one ZIM_PATH, you cannot set a "
"INDEX_PATH."
<< endl;
exit(1); exit(1);
} }
if ( (zimPathes.size() > 1) && !indexPath.empty() ) { if ((zimPathes.size() > 1) && !indexPath.empty()) {
cerr << "You cannot set a indexPath if you also set several zimPathes"; cerr << "You cannot set a indexPath if you also set several zimPathes";
exit(1); exit(1);
} }
@ -796,33 +860,41 @@ int main(int argc, char **argv) {
vector<string> libraryPaths = kiwix::split(libraryPath, ";"); vector<string> libraryPaths = kiwix::split(libraryPath, ";");
vector<string>::iterator itr; vector<string>::iterator itr;
for ( itr = libraryPaths.begin(); itr != libraryPaths.end(); ++itr ) { for (itr = libraryPaths.begin(); itr != libraryPaths.end(); ++itr) {
if (!itr->empty()) { if (!itr->empty()) {
bool retVal = false; bool retVal = false;
try { try {
string libraryPath = isRelativePath(*itr) ? computeAbsolutePath(removeLastPathElement(getExecutablePath(), true, false), *itr) : *itr; string libraryPath
= isRelativePath(*itr)
? computeAbsolutePath(removeLastPathElement(
getExecutablePath(), true, false),
*itr)
: *itr;
retVal = libraryManager.readFile(libraryPath, true); retVal = libraryManager.readFile(libraryPath, true);
} catch (...) { } catch (...) {
retVal = false; retVal = false;
} }
if (!retVal) { if (!retVal) {
cerr << "Unable to open the XML library file '" << *itr << "'." << endl; cerr << "Unable to open the XML library file '" << *itr << "'."
<< endl;
exit(1); exit(1);
} }
} }
} }
/* Check if the library is not empty (or only remote books)*/ /* Check if the library is not empty (or only remote books)*/
if (libraryManager.getBookCount(true, false)==0) { if (libraryManager.getBookCount(true, false) == 0) {
cerr << "The XML library file '" << libraryPath << "' is empty (or has only remote books)." << endl; cerr << "The XML library file '" << libraryPath
<< "' is empty (or has only remote books)." << endl;
} }
} else { } else {
std::vector<std::string>::iterator it; std::vector<std::string>::iterator it;
for (it = zimPathes.begin(); it != zimPathes.end(); it++) { for (it = zimPathes.begin(); it != zimPathes.end(); it++) {
if (!libraryManager.addBookFromPath(*it, *it, "", false)) { if (!libraryManager.addBookFromPath(*it, *it, "", false)) {
cerr << "Unable to add the ZIM file '" << *it << "' to the internal library." << endl; cerr << "Unable to add the ZIM file '" << *it
<< "' to the internal library." << endl;
exit(1); exit(1);
} }
} }
@ -844,7 +916,7 @@ int main(int argc, char **argv) {
indexPath = currentBook.indexPathAbsolute; indexPath = currentBook.indexPathAbsolute;
/* Instanciate the ZIM file handler */ /* Instanciate the ZIM file handler */
kiwix::Reader *reader = NULL; kiwix::Reader* reader = NULL;
try { try {
reader = new kiwix::Reader(zimPath); reader = new kiwix::Reader(zimPath);
zimFileOk = true; zimFileOk = true;
@ -865,13 +937,14 @@ int main(int argc, char **argv) {
if (!indexPath.empty()) { if (!indexPath.empty()) {
try { try {
kiwix::Searcher *searcher = new kiwix::Searcher(indexPath, reader); kiwix::Searcher* searcher = new kiwix::Searcher(indexPath, reader);
searcher->setProtocolPrefix("/"); searcher->setProtocolPrefix("/");
searcher->setSearchProtocolPrefix("/search?"); searcher->setSearchProtocolPrefix("/search?");
searcher->setContentHumanReadableId(humanReadableId); searcher->setContentHumanReadableId(humanReadableId);
searchers[humanReadableId] = searcher; searchers[humanReadableId] = searcher;
} catch (...) { } catch (...) {
cerr << "Unable to open the search index '" << indexPath << "'." << endl; cerr << "Unable to open the search index '" << indexPath << "'."
<< endl;
} }
} }
} }
@ -879,12 +952,15 @@ int main(int argc, char **argv) {
} }
/* Compute the Welcome HTML */ /* Compute the Welcome HTML */
string welcomeBooksHtml = "" string welcomeBooksHtml
"<div class='book__list'>"; = ""
"<div class='book__list'>";
for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) { for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) {
libraryManager.getBookById(*itr, currentBook); libraryManager.getBookById(*itr, currentBook);
if (!currentBook.path.empty() && readers.find(currentBook.getHumanReadableIdFromPath()) != readers.end()) { if (!currentBook.path.empty()
&& readers.find(currentBook.getHumanReadableIdFromPath())
!= readers.end()) {
welcomeBooksHtml += "" welcomeBooksHtml += ""
"<a href='/" + currentBook.getHumanReadableIdFromPath() + "/'>" "<a href='/" + currentBook.getHumanReadableIdFromPath() + "/'>"
"<div class='book'>" "<div class='book'>"
@ -902,7 +978,8 @@ int main(int argc, char **argv) {
welcomeBooksHtml += "" welcomeBooksHtml += ""
"</div>"; "</div>";
welcomeHTML = replaceRegex(RESOURCE::home_html_tmpl, welcomeBooksHtml, "__BOOKS__"); welcomeHTML
= replaceRegex(RESOURCE::home_html_tmpl, welcomeBooksHtml, "__BOOKS__");
#ifndef _WIN32 #ifndef _WIN32
/* Fork if necessary */ /* Fork if necessary */
@ -954,9 +1031,8 @@ int main(int argc, char **argv) {
extMimeTypes["vtt"] = "text/vtt"; extMimeTypes["vtt"] = "text/vtt";
/* Start the HTTP daemon */ /* Start the HTTP daemon */
void *page = NULL; void* page = NULL;
if (interface.length() > 0) { if (interface.length() > 0) {
#ifndef _WIN32 #ifndef _WIN32
/* TBD IPv6 support */ /* TBD IPv6 support */
@ -966,16 +1042,16 @@ int main(int argc, char **argv) {
/* Search all available interfaces */ /* Search all available interfaces */
if (getifaddrs(&ifaddr) == -1) { if (getifaddrs(&ifaddr) == -1) {
cerr << "Getifaddrs() failed while searching for '" << interface << "'" << endl; cerr << "Getifaddrs() failed while searching for '" << interface << "'"
<< endl;
exit(1); exit(1);
} }
/* Init 'sockAddr' with zeros */ /* Init 'sockAddr' with zeros */
memset(&sockAddr,0, sizeof(sockAddr)); memset(&sockAddr, 0, sizeof(sockAddr));
/* Try to find interfaces in the list of available interfaces */ /* Try to find interfaces in the list of available interfaces */
for (ifa = ifaddr, n = 0; ifa != NULL; ifa = ifa->ifa_next, n++) { for (ifa = ifaddr, n = 0; ifa != NULL; ifa = ifa->ifa_next, n++) {
/* Ignore if no IP attributed to the interface */ /* Ignore if no IP attributed to the interface */
if (ifa->ifa_addr == NULL) if (ifa->ifa_addr == NULL)
continue; continue;
@ -986,7 +1062,8 @@ int main(int argc, char **argv) {
if (strcasecmp(ifa->ifa_name, interface.c_str()) == 0) { if (strcasecmp(ifa->ifa_name, interface.c_str()) == 0) {
sockAddr.sin_family = family; sockAddr.sin_family = family;
sockAddr.sin_port = htons(serverPort); sockAddr.sin_port = htons(serverPort);
sockAddr.sin_addr.s_addr = ((struct sockaddr_in*) ifa->ifa_addr)->sin_addr.s_addr; sockAddr.sin_addr.s_addr
= ((struct sockaddr_in*)ifa->ifa_addr)->sin_addr.s_addr;
break; break;
} }
} }
@ -1026,7 +1103,10 @@ int main(int argc, char **argv) {
} }
if (daemon == NULL) { if (daemon == NULL) {
cerr << "Unable to instanciate the HTTP daemon. The port " << serverPort << " is maybe already occupied or need more permissions to be open. Please try as root or with a port number higher or equal to 1024." << endl; cerr << "Unable to instanciate the HTTP daemon. The port " << serverPort
<< " is maybe already occupied or need more permissions to be open. "
"Please try as root or with a port number higher or equal to 1024."
<< endl;
exit(1); exit(1);
} }
@ -1044,10 +1124,10 @@ int main(int argc, char **argv) {
struct kinfo_proc kp; struct kinfo_proc kp;
size_t len = sizeof(kp); size_t len = sizeof(kp);
mib[0]=CTL_KERN; mib[0] = CTL_KERN;
mib[1]=KERN_PROC; mib[1] = KERN_PROC;
mib[2]=KERN_PROC_PID; mib[2] = KERN_PROC_PID;
mib[3]=PPID; mib[3] = PPID;
int ret = sysctl(mib, MIBSIZE, &kp, &len, NULL, 0); int ret = sysctl(mib, MIBSIZE, &kp, &len, NULL, 0);
if (ret != -1 && len > 0) { if (ret != -1 && len > 0) {