kiwix/kiwix-tools
1
0
Fork 0
mirror of https://github.com/kiwix/kiwix-tools.git synced 2025-09-23 03:52:35 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #49 from kiwix/globalSearch

Browse Source 
Global search
This commit is contained in:
Matthieu Gautier 2017-07-19 09:54:31 +02:00 committed by GitHub
commit 5989146931
3 changed files with 72 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
src/server/kiwix-serve.cpp
View File

@ -92,10 +92,10 @@ static bool verboseFlag = false;
static std::map<std::string, std::string> extMimeTypes; static std::map<std::string, std::string> extMimeTypes;
static std::map<std::string, kiwix::Reader*> readers; static std::map<std::string, kiwix::Reader*> readers;
static std::map<std::string, kiwix::Searcher*> searchers; static std::map<std::string, kiwix::Searcher*> searchers;
static pthread_mutex_t readerLock = PTHREAD_MUTEX_INITIALIZER; static kiwix::Searcher* globalSearcher = nullptr;
static pthread_mutex_t zimLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t mapLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t mapLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t welcomeLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t welcomeLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t searcherLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t compressorLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t compressorLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t verboseFlagLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t verboseFlagLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t mimeTypeLock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t mimeTypeLock = PTHREAD_MUTEX_INITIALIZER;
@ -138,11 +138,20 @@ void introduceTaskbar(string& content, const string& humanReadableBookId)
? " #kiwix_serve_taskbar_library_button { display: none }" ? " #kiwix_serve_taskbar_library_button { display: none }"
: "") : "")
+ "</style>"); + "</style>");
content = appendToFirstOccurence( if ( humanReadableBookId.empty() ) {
content, content = appendToFirstOccurence(
"<body[^>]*>", content,
replaceRegex( "<body[^>]*>",
RESOURCE::taskbar_html_part, humanReadableBookId, "__CONTENT__")); RESOURCE::global_taskbar_html_part);
} else {
content = appendToFirstOccurence(
content,
"<body[^>]*>",
replaceRegex(
RESOURCE::taskbar_html_part,
humanReadableBookId,
"__CONTENT__"));
}
} }
} }
@ -259,31 +268,31 @@ ssize_t callback_reader_from_blob(void* cls,
size_t max) size_t max)
{ {
zim::Blob* blob = static_cast<zim::Blob*>(cls); zim::Blob* blob = static_cast<zim::Blob*>(cls);
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&zimLock);
size_t max_size_to_set = min<size_t>(max, blob->size() - pos); size_t max_size_to_set = min<size_t>(max, blob->size() - pos);
if (max_size_to_set <= 0) { if (max_size_to_set <= 0) {
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
return MHD_CONTENT_READER_END_WITH_ERROR; return MHD_CONTENT_READER_END_WITH_ERROR;
} }
memcpy(buf, blob->data() + pos, max_size_to_set); memcpy(buf, blob->data() + pos, max_size_to_set);
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
return max_size_to_set; return max_size_to_set;
} }
void callback_free_blob(void* cls) void callback_free_blob(void* cls)
{ {
zim::Blob* blob = static_cast<zim::Blob*>(cls); zim::Blob* blob = static_cast<zim::Blob*>(cls);
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&zimLock);
delete blob; delete blob;
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
} }
static struct MHD_Response* build_callback_response_from_blob( static struct MHD_Response* build_callback_response_from_blob(
zim::Blob& blob, const std::string& mimeType) zim::Blob& blob, const std::string& mimeType)
{ {
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&zimLock);
zim::Blob* p_blob = new zim::Blob(blob); zim::Blob* p_blob = new zim::Blob(blob);
struct MHD_Response* response struct MHD_Response* response
= MHD_create_response_from_callback(blob.size(), = MHD_create_response_from_callback(blob.size(),
@ -291,7 +300,7 @@ static struct MHD_Response* build_callback_response_from_blob(
callback_reader_from_blob, callback_reader_from_blob,
p_blob, p_blob,
callback_free_blob); callback_free_blob);
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
/* Tell the client that byte ranges are accepted */ /* Tell the client that byte ranges are accepted */
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
@ -333,6 +342,7 @@ static struct MHD_Response* handle_suggest(
std::cout << "Searching suggestions for: \"" << term << "\"" << endl; std::cout << "Searching suggestions for: \"" << term << "\"" << endl;
} }
pthread_mutex_lock(&zimLock);
/* Get the suggestions */ /* Get the suggestions */
content = "["; content = "[";
reader->searchSuggestionsSmart(term, maxSuggestionCount); reader->searchSuggestionsSmart(term, maxSuggestionCount);
@ -343,6 +353,7 @@ static struct MHD_Response* handle_suggest(
+ "\"}"; + "\"}";
suggestionCount++; suggestionCount++;
} }
pthread_mutex_unlock(&zimLock);
/* Propose the fulltext search if possible */ /* Propose the fulltext search if possible */
if (searcher != NULL) { if (searcher != NULL) {
@ -399,12 +410,12 @@ static struct MHD_Response* handle_search(
std::vector<std::string> variants = reader->getTitleVariants(patternString); std::vector<std::string> variants = reader->getTitleVariants(patternString);
std::vector<std::string>::iterator variantsItr = variants.begin(); std::vector<std::string>::iterator variantsItr = variants.begin();
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&zimLock);
while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) { while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
reader->getPageUrlFromTitle(*variantsItr, patternCorrespondingUrl); reader->getPageUrlFromTitle(*variantsItr, patternCorrespondingUrl);
variantsItr++; variantsItr++;
} }
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
/* If article found then redirect directly to it */ /* If article found then redirect directly to it */
if (!patternCorrespondingUrl.empty()) { if (!patternCorrespondingUrl.empty()) {
@ -416,7 +427,7 @@ static struct MHD_Response* handle_search(
} }
/* Make the search */ /* Make the search */
if (patternCorrespondingUrl.empty() && searcher != NULL) { if (searcher != NULL) {
const char* start = MHD_lookup_connection_value( const char* start = MHD_lookup_connection_value(
connection, MHD_GET_ARGUMENT_KIND, "start"); connection, MHD_GET_ARGUMENT_KIND, "start");
const char* end const char* end
@ -425,16 +436,14 @@ static struct MHD_Response* handle_search(
unsigned int endNumber = end != NULL ? atoi(end) : 25; unsigned int endNumber = end != NULL ? atoi(end) : 25;
/* Get the results */ /* Get the results */
pthread_mutex_lock(&searcherLock); pthread_mutex_lock(&zimLock);
pthread_mutex_lock(&readerLock);
try { try {
searcher->search(patternString, startNumber, endNumber, isVerbose()); searcher->search(patternString, startNumber, endNumber, isVerbose());
content = searcher->getHtml(); content = searcher->getHtml();
} catch (const std::exception& e) { } catch (const std::exception& e) {
std::cerr << e.what() << std::endl; std::cerr << e.what() << std::endl;
} }
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
pthread_mutex_unlock(&searcherLock);
} else { } else {
content = "<!DOCTYPE html>\n<html><head><meta content=\"text/html;charset=UTF-8\" http-equiv=\"content-type\" /><title>Fulltext search unavailable</title></head><body><h1>Not Found</h1><p>There is no article with the title <b>\"" + kiwix::encodeDiples(patternString) + "\"</b> and the fulltext search engine is not available for this content.</p></body></html>"; content = "<!DOCTYPE html>\n<html><head><meta content=\"text/html;charset=UTF-8\" http-equiv=\"content-type\" /><title>Fulltext search unavailable</title></head><body><h1>Not Found</h1><p>There is no article with the title <b>\"" + kiwix::encodeDiples(patternString) + "\"</b> and the fulltext search engine is not available for this content.</p></body></html>";
httpResponseCode = MHD_HTTP_NOT_FOUND; httpResponseCode = MHD_HTTP_NOT_FOUND;
@ -465,9 +474,9 @@ static struct MHD_Response* handle_random(
std::string httpRedirection; std::string httpRedirection;
httpResponseCode = MHD_HTTP_FOUND; httpResponseCode = MHD_HTTP_FOUND;
if (reader != NULL) { if (reader != NULL) {
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&zimLock);
std::string randomUrl = reader->getRandomPageUrl(); std::string randomUrl = reader->getRandomPageUrl();
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
httpRedirection httpRedirection
= "/" + humanReadableBookId + "/" + kiwix::urlEncode(randomUrl); = "/" + humanReadableBookId + "/" + kiwix::urlEncode(randomUrl);
} }
@ -489,7 +498,7 @@ static struct MHD_Response* handle_content(
bool found = false; bool found = false;
zim::Article article; zim::Article article;
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&zimLock);
try { try {
found = reader->getArticleObjectByDecodedUrl(urlStr, article); found = reader->getArticleObjectByDecodedUrl(urlStr, article);
@ -508,7 +517,7 @@ static struct MHD_Response* handle_content(
std::cerr << e.what() << std::endl; std::cerr << e.what() << std::endl;
found = false; found = false;
} }
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
if (!found) { if (!found) {
if (isVerbose()) if (isVerbose())
@ -530,9 +539,9 @@ static struct MHD_Response* handle_content(
} }
try { try {
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&zimLock);
mimeType = article.getMimeType(); mimeType = article.getMimeType();
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
} catch (exception& e) { } catch (exception& e) {
mimeType = "application/octet-stream"; mimeType = "application/octet-stream";
} }
@ -542,16 +551,16 @@ static struct MHD_Response* handle_content(
cout << "mimeType: " << mimeType << endl; cout << "mimeType: " << mimeType << endl;
} }
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&zimLock);
zim::Blob raw_content = article.getData(); zim::Blob raw_content = article.getData();
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
if (mimeType.find("text/") != string::npos if (mimeType.find("text/") != string::npos
|| mimeType.find("application/javascript") != string::npos || mimeType.find("application/javascript") != string::npos
|| mimeType.find("application/json") != string::npos) { || mimeType.find("application/json") != string::npos) {
pthread_mutex_lock(&readerLock); pthread_mutex_lock(&zimLock);
content = string(raw_content.data(), raw_content.size()); content = string(raw_content.data(), raw_content.size());
pthread_mutex_unlock(&readerLock); pthread_mutex_unlock(&zimLock);
/* Special rewrite URL in case of ZIM file use intern *asbolute* url like /* Special rewrite URL in case of ZIM file use intern *asbolute* url like
* /A/Kiwix */ * /A/Kiwix */
@ -675,7 +684,7 @@ static int accessHandlerCallback(void* cls,
kiwix::Searcher* searcher kiwix::Searcher* searcher
= searchers.find(humanReadableBookId) != searchers.end() = searchers.find(humanReadableBookId) != searchers.end()
? searchers.find(humanReadableBookId)->second ? searchers.find(humanReadableBookId)->second
: NULL; : globalSearcher;
kiwix::Reader* reader = readers.find(humanReadableBookId) != readers.end() kiwix::Reader* reader = readers.find(humanReadableBookId) != readers.end()
? readers.find(humanReadableBookId)->second ? readers.find(humanReadableBookId)->second
: NULL; : NULL;
@ -907,6 +916,9 @@ int main(int argc, char** argv)
vector<string> booksIds = libraryManager.getBooksIds(); vector<string> booksIds = libraryManager.getBooksIds();
vector<string>::iterator itr; vector<string>::iterator itr;
kiwix::Book currentBook; kiwix::Book currentBook;
globalSearcher = new kiwix::Searcher();
globalSearcher->setProtocolPrefix("/");
globalSearcher->setSearchProtocolPrefix("/search?");
for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) { for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) {
bool zimFileOk = false; bool zimFileOk = false;
libraryManager.getBookById(*itr, currentBook); libraryManager.getBookById(*itr, currentBook);
@ -928,14 +940,12 @@ int main(int argc, char** argv)
string humanReadableId = currentBook.getHumanReadableIdFromPath(); string humanReadableId = currentBook.getHumanReadableIdFromPath();
readers[humanReadableId] = reader; readers[humanReadableId] = reader;
/* Try to instanciate the zim index. if ( reader->hasFulltextIndex()) {
* If there is no specified indexPath, try to open the kiwix::Searcher* searcher = new kiwix::Searcher();
* embedded index in the zim. */ searcher->add_reader(reader, humanReadableId);
if (indexPath.empty() && reader->hasFulltextIndex()) { globalSearcher->add_reader(reader, humanReadableId);
indexPath = zimPath; searchers[humanReadableId] = searcher;
} } else if ( !indexPath.empty() ) {
if (!indexPath.empty()) {
try { try {
kiwix::Searcher* searcher = new kiwix::Searcher(indexPath, reader); kiwix::Searcher* searcher = new kiwix::Searcher(indexPath, reader);
searcher->setProtocolPrefix("/"); searcher->setProtocolPrefix("/");
@ -981,6 +991,8 @@ int main(int argc, char** argv)
welcomeHTML welcomeHTML
= replaceRegex(RESOURCE::home_html_tmpl, welcomeBooksHtml, "__BOOKS__"); = replaceRegex(RESOURCE::home_html_tmpl, welcomeBooksHtml, "__BOOKS__");
introduceTaskbar(welcomeHTML, "");
#ifndef _WIN32 #ifndef _WIN32
/* Fork if necessary */ /* Fork if necessary */
if (daemonFlag) { if (daemonFlag) {
@ -1001,10 +1013,9 @@ int main(int argc, char** argv)
#endif #endif
/* Mutex init */ /* Mutex init */
pthread_mutex_init(&readerLock, NULL); pthread_mutex_init(&zimLock, NULL);
pthread_mutex_init(&mapLock, NULL); pthread_mutex_init(&mapLock, NULL);
pthread_mutex_init(&welcomeLock, NULL); pthread_mutex_init(&welcomeLock, NULL);
pthread_mutex_init(&searcherLock, NULL);
pthread_mutex_init(&compressorLock, NULL); pthread_mutex_init(&compressorLock, NULL);
pthread_mutex_init(&verboseFlagLock, NULL); pthread_mutex_init(&verboseFlagLock, NULL);
pthread_mutex_init(&mimeTypeLock, NULL); pthread_mutex_init(&mimeTypeLock, NULL);
@ -1143,12 +1154,13 @@ int main(int argc, char** argv)
kiwix::sleep(1000); kiwix::sleep(1000);
} while (waiting); } while (waiting);
delete globalSearcher;
/* Stop the daemon */ /* Stop the daemon */
MHD_stop_daemon(daemon); MHD_stop_daemon(daemon);
/* Mutex destroy */ /* Mutex destroy */
pthread_mutex_destroy(&readerLock); pthread_mutex_destroy(&zimLock);
pthread_mutex_destroy(&searcherLock);
pthread_mutex_destroy(&compressorLock); pthread_mutex_destroy(&compressorLock);
pthread_mutex_destroy(&mapLock); pthread_mutex_destroy(&mapLock);
pthread_mutex_destroy(&welcomeLock); pthread_mutex_destroy(&welcomeLock);

15
static/server/global_taskbar.html.part Normal file
View File

@ -0,0 +1,15 @@
<link type="text/css" href="/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
<link type="text/css" href="/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
<span class="kiwix">
<span id="kiwixtoolbar" class="ui-widget-header">
<div class="kiwix_centered">
<div class="kiwix_searchform">
<form class="kiwixsearch" method="GET" action="/search" id="kiwixsearchform">
<input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text">
<input type="submit" value="Search">
</form>
</div>
</div>
</span>
</span>
<div style="display: block; height: 5em;"></div>

1
static/server/resources_list.txt
View File

@ -21,3 +21,4 @@ home.html.tmpl
include.html.part include.html.part
taskbar.css taskbar.css
taskbar.html.part taskbar.html.part
global_taskbar.html.part