Merge commit 'f596570027141d30117c086d3833d426fb4d2b25'

build-android-with-native.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 ''' Compiles Kiwix dependencies for Android
 
@@ -11,13 +11,14 @@ import re
 import sys
 import copy
 import shutil
+import urllib
 from xml.dom.minidom import parse
 from subprocess import call, check_output
 
 # target platform to compile for
 # list of available toolchains in <NDK_PATH>/toolchains
 # arm-linux-androideabi, mipsel-linux-android, x86, llvm
-ALL_ARCHS = ['arm-linux-androideabi', 'mipsel-linux-android', 'x86']
+ALL_ARCHS = ['arm-linux-androideabi', 'mipsel-linux-android', 'x86', 'aarch64-linux-android']
 
 
 def find_package():
@@ -35,6 +36,8 @@ USAGE = '''Usage:  {arg0} [--option]
     --lzma          Compile liblzma
     --icu           Compile libicu
     --zim           Compile libzim
+    --xapian        Compile libxapian
+    --glassify      Compile glassify binary
     --kiwix         Compile libkiwix
     --strip         Strip libkiwix.so
     --locales       Create the locales.txt file
@@ -45,7 +48,7 @@ USAGE = '''Usage:  {arg0} [--option]
 
     --on=ARCH       Disable steps on all archs and cherry pick the ones wanted.
                     Multiple --on=ARCH can be specified.
-                    ARCH in 'armeabi', 'mips', 'x86'. '''
+                    ARCH in 'armeabi', 'mips', 'x86', 'arm64-v8a'. '''
 
 
 def init_with_args(args):
@@ -56,8 +59,9 @@ def init_with_args(args):
 
     # default is executing all the steps
     create_toolchain = compile_liblzma = compile_libicu = \
-        compile_libzim = compile_libkiwix = strip_libkiwix = \
+        compile_libzim = compile_libkiwix = compile_libxapian = strip_libkiwix = \
         compile_apk = locales_txt = clean = True
+    compile_glassify = False # dont want to compile this everytime
     archs = ALL_ARCHS
 
     options = [a.lower() for a in args[1:]]
@@ -84,7 +88,7 @@ def init_with_args(args):
                                   if rarch == v][0])
                 except:
                     pass
-                doptions.pop(idx)
+                #doptions.pop(idx)
         # recreate options list from other items
         options = [v for v in doptions.values() if not v.startswith('--on=')]
 
@@ -92,8 +96,8 @@ def init_with_args(args):
         # we received options.
         # consider we only want the specified steps
         create_toolchain = compile_liblzma = compile_libicu = compile_libzim = \
-            compile_libkiwix = strip_libkiwix = \
+            compile_libkiwix = compile_libxapian = strip_libkiwix = \
-            compile_apk = locales_txt = clean = False
+            compile_apk = locales_txt = clean = compile_glassify = False
 
         for option in options:
             if 'toolchain' in option:
@@ -106,8 +110,12 @@ def init_with_args(args):
                 compile_libzim = True
             if 'kiwix' in option:
                 compile_libkiwix = True
+            if 'xapian' in option:
+                compile_libxapian = True
             if 'strip' in option:
                 strip_libkiwix = True
+            if 'glassify' in option:
+                compile_glassify = True
             if 'apk' in option:
                 compile_apk = True
             if 'locales' in option:
@@ -116,7 +124,7 @@ def init_with_args(args):
                 clean = True
 
     return (create_toolchain, compile_liblzma, compile_libicu, compile_libzim,
-            compile_libkiwix, strip_libkiwix, compile_apk, locales_txt,
+            compile_libkiwix, compile_libxapian, strip_libkiwix, compile_apk, compile_glassify, locales_txt,
             clean, archs)
 
 # store the OS's environment PATH as we'll mess with it
@@ -132,10 +140,12 @@ PARENT_PATH = os.path.dirname(CURRENT_PATH)
 # different names of folder path for accessing files
 ARCHS_FULL_NAMES = {
     'arm-linux-androideabi': 'arm-linux-androideabi',
+    'aarch64-linux-android': 'aarch64-linux-android',
     'mipsel-linux-android': 'mipsel-linux-android',
     'x86': 'i686-linux-android'}
 ARCHS_SHORT_NAMES = {
     'arm-linux-androideabi': 'armeabi',
+    'aarch64-linux-android' : 'arm64-v8a',
     'mipsel-linux-android': 'mips',
     'x86': 'x86'}
 
@@ -146,13 +156,13 @@ SYSTEMS = {'Linux': 'linux', 'Darwin': 'mac'}
 
 # find out what to execute based on command line arguments
 CREATE_TOOLCHAIN, COMPILE_LIBLZMA, COMPILE_LIBICU, COMPILE_LIBZIM, \
-    COMPILE_LIBKIWIX, STRIP_LIBKIWIX, COMPILE_APK, \
+    COMPILE_LIBKIWIX, COMPILE_LIBXAPIAN, STRIP_LIBKIWIX, COMPILE_APK, \
-    LOCALES_TXT, CLEAN, ARCHS = init_with_args(sys.argv)
+    COMPILE_GLASSIFY, LOCALES_TXT, CLEAN, ARCHS = init_with_args(sys.argv)
 
 # compiler version to use
 # list of available toolchains in <NDK_PATH>/toolchains
 # 4.4.3, 4.6, 4.7, clang3.1, clang3.2
-COMPILER_VERSION = '4.8'
+COMPILER_VERSION = '4.9'
 
 # location of Android NDK
 NDK_PATH = os.environ.get('NDK_PATH',
@@ -287,6 +297,10 @@ for arch in ARCHS:
     platform = os.path.join(PLATFORM_PREFIX, arch)
 
     # prepare the toolchain
+    if "aarch64" in arch_full and "14" in NDK_PLATFORM:
+        NDK_PLATFORM = "android-21"
+    else:
+        NDK_PLATFORM = os.environ.get('NDK_PLATFORM', 'android-14')
     toolchain = '%(arch)s-%(version)s' % {'arch': arch,
                                           'version': COMPILER_VERSION}
     toolchain_cmd = ('%(NDK_PATH)s/build/tools/make-standalone-toolchain.sh '
@@ -319,6 +333,13 @@ for arch in ARCHS:
         syscall('ln -sf %(src)s %(dest)s/'
                 % {'src': ln_src, 'dest': dest})
 
+        if not os.path.exists(os.path.join(platform, arch_full, 'bin', 'gcc')):
+            for target in ["gcc", "g++", "c++"]:
+                syscall('ln -sf %(src)s %(dest)s'
+                    % {'src': os.path.join(platform, 'bin', '%s-%s'
+                    % (arch_full, target)), 'dest': os.path.join(platform,
+                    arch_full, 'bin', target)})
+
     # check that the step went well
     if CREATE_TOOLCHAIN or COMPILE_LIBLZMA or COMPILE_LIBZIM or \
        COMPILE_LIBKIWIX or STRIP_LIBKIWIX:
@@ -395,6 +416,104 @@ for arch in ARCHS:
                            "has not been created for {} and is not present."
                            .format(platform))
 
+    # compile xapian
+    if COMPILE_LIBXAPIAN:
+        # fetch xapian, e2fsprogs, zlib
+        os.chdir(os.path.join(curdir, '../src', 'dependencies'))
+        if not os.path.exists("e2fsprogs-1.42"):
+            syscall('make e2fsprogs-1.42')
+        if not os.path.exists("xapian-core-1.3.4"):
+            print("Fetching recent xapian...")
+            urllib.urlretrieve('http://oligarchy.co.uk/xapian/1.3.4/xapian-core-1.3.4.tar.xz', 'xapian-core-1.3.4.tar.xz') # for glass support
+            change_env(ORIGINAL_ENVIRON)
+            syscall('tar xvf xapian-core-1.3.4.tar.xz')
+            change_env(new_environ)
+            change_env(OPTIMIZATION_ENV)
+
+        if not os.path.exists("zlib-1.2.8"):
+            syscall('make zlib-1.2.8')
+        os.chdir('zlib-1.2.8')
+        if os.path.exists("Makefile"):
+            syscall('make clean')
+        syscall('./configure')
+        syscall('make')
+        shutil.copy('libz.a', os.path.join(platform, 'lib', 'gcc', arch_full, COMPILER_VERSION, 'libz.a'))
+        os.chdir('../e2fsprogs-1.42')
+        print("Fetching latest compile.sub...")
+        shutil.copy(os.path.join("..", "xapian-core-1.3.4", "config.guess"), os.path.join("config", "config.guess"))
+        shutil.copy(os.path.join("..", "xapian-core-1.3.4", "config.sub"), os.path.join("config", "config.sub"))
+#        urllib.urlretrieve('http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD', 'config/config.guess')
+#        urllib.urlretrieve('http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD', 'config/config.sub')
+        if os.path.exists("Makefile"):
+            syscall('make clean')
+        syscall('./configure --host=%s --prefix=%s' % (arch_full, platform))
+        os.chdir('util')
+        change_env(ORIGINAL_ENVIRON)
+        syscall('gcc subst.c -o subst')
+
+        change_env(new_environ)
+        change_env(OPTIMIZATION_ENV)
+
+        os.chdir('..')
+        os.chdir('lib/uuid')
+        syscall('make')
+        try:
+            os.makedirs(os.path.join(platform, 'include', 'c++', COMPILER_VERSION, 'uuid'))
+        except:
+            pass
+        shutil.copy('uuid.h', os.path.join(platform, 'include', 'c++', COMPILER_VERSION, 'uuid', 'uuid.h'))
+        shutil.copy('libuuid.a', os.path.join(platform, 'lib', 'gcc', arch_full, COMPILER_VERSION, 'libuuid.a'))
+        shutil.copy('libuuid.a', os.path.join(platform, 'lib', 'libuuid.a'))
+        os.chdir('../../../xapian-core-1.3.4')
+        if os.path.exists("Makefile"):
+            syscall('make clean')
+
+        syscall('./configure --host=%s --disable-shared --enable-largefile' % arch_full)
+        f = open("config.h", "r")
+        old_contents = f.readlines()
+        f.close()
+        contents = []
+        i = 0
+        while i < len(old_contents):
+            if "HAVE_DECL_SYS_NERR" in old_contents[i]:
+                contents.append("#define HAVE_DECL_SYS_NERR 0\n")
+            else:
+                contents.append(old_contents[i])
+            i = i + 1
+        f = open("config.h", "w")
+        contents = "".join(contents)
+        f.write(contents)
+        f.close()
+
+        f = open(os.path.join(platform, "sysroot", "usr", "include", "fcntl.h"), "r")
+        old_contents = f.readlines()
+        f.close()
+        contents = []
+        i = 0
+        while i < len(old_contents):
+            if not "__creat_too_many_args" in old_contents[i]:
+                contents.append(old_contents[i])
+            i = i + 1
+        f = open(os.path.join(platform, "sysroot", "usr", "include", "fcntl.h"), "w")
+        contents = "".join(contents)
+        f.write(contents)
+        f.close()
+
+        try:
+            shutil.copytree(os.path.join('include', 'xapian'), os.path.join(platform, 'include', 'c++', COMPILER_VERSION, 'xapian'))
+            shutil.copy(os.path.join('include', 'xapian.h'), os.path.join(platform, 'include', 'c++', COMPILER_VERSION, 'xapian.h'))
+        except:
+            pass
+
+        syscall('make')
+        shutil.copy(os.path.join(curdir, '..', 'src', 'dependencies', 'xapian-core-1.3.4', '.libs', 'libxapian-1.3.a'), os.path.join(platform, 'lib', 'libxapian.a'))
+
+    # check that the step went well
+    if COMPILE_LIBXAPIAN or COMPILE_LIBKIWIX:
+        if not os.path.exists(os.path.join(platform, 'lib', 'libxapian.a')):
+            failed_on_step('The libxapian.a archive file has not been created '
+                           'and is not present.')
+
     # create libzim.a
     os.chdir(curdir)
     platform_includes = ['%(platform)s/include/c++/%(gccver)s/'
@@ -495,6 +614,9 @@ for arch in ARCHS:
                 # '%(platform)s/lib/libiculx.a '
                 # '%(platform)s/lib/libicui18n.a '
                 '%(platform)s/lib/libicudata.a '
+                '%(platform)s/lib/libxapian.a '
+                '%(platform)s/lib/gcc/%(arch_full)s/%(gccver)s/libuuid.a '
+                '%(platform)s/lib/gcc/%(arch_full)s/%(gccver)s/libz.a '
                 '-L%(platform)s/%(arch_full)s/lib '
                 '%(NDK_PATH)s/sources/cxx-stl/gnu-libstdc++/%(gccver)s'
                 '/libs/%(arch_short)s/libgnustl_static.a '
@@ -538,10 +660,22 @@ for arch in ARCHS:
                    'arch_full': arch_full,
                    'arch_short': arch_short,
                    'curdir': curdir})
+    if COMPILE_GLASSIFY:
+        os.chdir(curdir)
+        syscall('g++ glassify.cc ../src/dependencies/xapian-core-1.3.4/.libs/libxapian-1.3.a -o glassify_%s -lz -luuid -lrt -I../src/dependencies/xapian-core-1.3.4/include' % arch_short)
 
     os.chdir(curdir)
     change_env(ORIGINAL_ENVIRON)
 
+# recompile xapian for build system arch to compile glassify
+if COMPILE_GLASSIFY:
+    os.chdir(os.path.join(curdir, '..', 'src', 'dependencies', 'xapian-core-1.3.4'))
+    syscall('make clean')
+    syscall('./configure')
+    syscall('make')
+    os.chdir(curdir)
+    syscall('g++ glassify.cc ../src/dependencies/xapian-core-1.3.4/.libs/libxapian-1.3.a -o glassify -lz -luuid -lrt -I../src/dependencies/xapian-core-1.3.4/include')
+
 if LOCALES_TXT:
 
     os.chdir(curdir)

glassify.cc

@@ -0,0 +1,188 @@
+#include <xapian.h>
+
+#include <iomanip>
+#include <iostream>
+
+#include <cmath> // For log10().
+#include <cstdlib> // For exit().
+#include <cstring> // For strcmp() and strrchr().
+#include <string>
+#include <fcntl.h>
+#include <stdio.h>
+#include <ftw.h>
+#include <unistd.h>
+
+using namespace std;
+
+#define PROG_NAME "glassify"
+#define PROG_DESC "Perform a document-by-document copy of one or more Xapian databases and make it a single file"
+
+static void
+show_usage(int rc)
+{
+    cout << "Usage: " PROG_NAME " SOURCE_DATABASE... DESTINATION_DATABASE\n\n"
+"Options:\n"
+"  --no-renumber    Preserve the numbering of document ids (useful if you have\n"
+"                   external references to them, or have set them to match\n"
+"                   unique ids from an external source).  If multiple source\n"
+"                   databases are specified and the same docid occurs in more\n"
+"                   one, the last occurrence will be the one which ends up in\n"
+"                   the destination database.\n"
+"  --help           display this help and exit\n"
+"  --version        output version information and exit" << endl;
+    exit(rc);
+}
+
+void compact(const char* in, const char* out) try {
+    Xapian::Database indb(in);
+    int fd = open(out, O_CREAT|O_RDWR, 0666);
+    if (fd != -1) {
+        indb.compact(fd);
+        cout << "Done!" << endl;
+        return;
+    }
+    cout << "Some error happened..." << endl;
+} catch (const Xapian::Error &e) {
+    cout << e.get_description().c_str() << endl;
+}
+
+int unlinker(const char *fpth, const struct stat *sb, int t, struct FTW *fb) {
+    int rv = remove(fpth);
+    if (rv)
+        perror(fpth);
+    return rv;
+}
+
+int cleaner(const char *path) {
+    return nftw(path, unlinker, 64, FTW_DEPTH | FTW_PHYS);
+}
+
+int
+main(int argc, char **argv)
+try {
+    bool renumber = true;
+    if (argc > 1 && argv[1][0] == '-') {
+	if (strcmp(argv[1], "--help") == 0) {
+	    cout << PROG_NAME " - " PROG_DESC "\n\n";
+	    show_usage(0);
+	}
+	if (strcmp(argv[1], "--version") == 0) {
+	    cout << PROG_NAME << endl;
+	    exit(0);
+	}
+	if (strcmp(argv[1], "--no-renumber") == 0) {
+	    renumber = false;
+	    argv[1] = argv[0];
+	    ++argv;
+	    --argc;
+	}
+    }
+
+    // We expect two or more arguments: at least one source database path
+    // followed by the destination database path.
+    if (argc < 3) show_usage(1);
+
+    // Create the destination database, using DB_CREATE so that we don't
+    // try to overwrite or update an existing database in case the user
+    // got the command line argument order wrong.
+    string dest_str = string(argv[argc - 1]);
+    dest_str += ".tmp";
+    const char *dest = dest_str.c_str();
+    Xapian::WritableDatabase db_out(dest, Xapian::DB_CREATE|Xapian::DB_BACKEND_GLASS);
+
+    for (int i = 1; i < argc - 1; ++i) {
+	char * src = argv[i];
+	if (*src) {
+	    // Remove any trailing directory separator.
+	    char & ch = src[strlen(src) - 1];
+	    if (ch == '/' || ch == '\\') ch = '\0';
+	}
+
+	// Open the source database.
+	Xapian::Database db_in(src);
+
+	// Find the leaf-name of the database path for reporting progress.
+	const char * leaf = strrchr(src, '/');
+#if defined __WIN32__ || defined __OS2__
+	if (!leaf) leaf = strrchr(src, '\\');
+#endif
+	if (leaf) ++leaf; else leaf = src;
+
+	// Iterate over all the documents in db_in, copying each to db_out.
+	Xapian::doccount dbsize = db_in.get_doccount();
+	if (dbsize == 0) {
+	    cout << leaf << ": empty!" << endl;
+	} else {
+	    // Calculate how many decimal digits there are in dbsize.
+	    int width = static_cast<int>(log10(double(dbsize))) + 1;
+
+	    Xapian::doccount c = 0;
+	    Xapian::PostingIterator it = db_in.postlist_begin(string());
+	    while (it != db_in.postlist_end(string())) {
+		Xapian::docid did = *it;
+		if (renumber) {
+		    db_out.add_document(db_in.get_document(did));
+		} else {
+		    db_out.replace_document(did, db_in.get_document(did));
+		}
+
+		// Update for the first 10, and then every 13th document
+		// counting back from the end (this means that all the
+		// digits "rotate" and the counter ends up on the exact
+		// total.
+		++c;
+		if (c <= 10 || (dbsize - c) % 13 == 0) {
+		    cout << '\r' << leaf << ": ";
+		    cout << setw(width) << c << '/' << dbsize << flush;
+		}
+
+		++it;
+	    }
+
+	    cout << endl;
+	}
+
+	cout << "Copying spelling data..." << flush;
+	Xapian::TermIterator spellword = db_in.spellings_begin();
+	while (spellword != db_in.spellings_end()) {
+	    db_out.add_spelling(*spellword, spellword.get_termfreq());
+	    ++spellword;
+	}
+	cout << " done." << endl;
+
+	cout << "Copying synonym data..." << flush;
+	Xapian::TermIterator synkey = db_in.synonym_keys_begin();
+	while (synkey != db_in.synonym_keys_end()) {
+	    string key = *synkey;
+	    Xapian::TermIterator syn = db_in.synonyms_begin(key);
+	    while (syn != db_in.synonyms_end(key)) {
+		db_out.add_synonym(key, *syn);
+		++syn;
+	    }
+	    ++synkey;
+	}
+	cout << " done." << endl;
+
+	cout << "Copying user metadata..." << flush;
+	Xapian::TermIterator metakey = db_in.metadata_keys_begin();
+	while (metakey != db_in.metadata_keys_end()) {
+	    string key = *metakey;
+	    db_out.set_metadata(key, db_in.get_metadata(key));
+	    ++metakey;
+	}
+	cout << " done." << endl;
+    }
+
+    cout << "Committing..." << flush;
+    // Commit explicitly so that any error is reported.
+    db_out.commit();
+    cout << " done." << endl;
+    cout << "Turning into single file..." << endl;
+    compact(dest, argv[argc - 1]);
+    cout << "All finished. Cleaning up..." << endl;
+    cleaner(dest);
+    cout << "Done!" << endl;
+} catch (const Xapian::Error & e) {
+    cerr << '\n' << argv[0] << ": " << e.get_description() << endl;
+    exit(1);
+}

kiwix.c

@@ -13,6 +13,8 @@
 #include <android/log.h>
 #define  LOGI(...)  __android_log_print(ANDROID_LOG_INFO, "kiwix", __VA_ARGS__)
 
+#include <xapian.h>
+
 /* global variables */
 kiwix::Reader *reader = NULL;
 
@@ -323,3 +325,56 @@ JNIEXPORT void JNICALL Java_org_kiwix_kiwixmobile_JNIKiwix_setDataDirectory
   }
   pthread_mutex_unlock(&readerLock);
 }
+
+const char* executeQuery(const char* dbLoc, const char* qu, bool partial) try {
+    Xapian::Database db(dbLoc);
+
+    // Start an enquire session.
+    Xapian::Enquire enquire(db);
+
+    std::string query_string(qu);
+    std::string reply("");
+
+    // Parse the query string to produce a Xapian::Query object.
+    Xapian::QueryParser qp;
+    Xapian::Stem stemmer("english");
+    qp.set_stemmer(stemmer);
+    qp.set_database(db);
+    qp.set_stemming_strategy(Xapian::QueryParser::STEM_ALL);
+    Xapian::Query query;
+
+    if (partial)
+        query = qp.parse_query(query_string, Xapian::QueryParser::FLAG_PARTIAL);
+    else
+        query = qp.parse_query(query_string);
+
+    // Find the top 20 results for the query.
+    enquire.set_query(query);
+    Xapian::MSet matches = enquire.get_mset(0, 20);
+
+    for (Xapian::MSetIterator i = matches.begin(); i != matches.end(); ++i) {
+        reply += i.get_document().get_data();
+        reply += "\n";
+    }
+    return reply.c_str();
+}  catch (const Xapian::Error &e) {
+    //return e.get_description().c_str();
+    return "";
+}
+
+JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixmobile_JNIKiwix_indexedQuery
+        (JNIEnv *env, jclass thiz, jstring db, jstring qu) {
+    const char* d = env->GetStringUTFChars(db, 0);
+    const char* q = env->GetStringUTFChars(qu, 0);
+    const char* result = executeQuery(d, q, false);
+    return env->NewStringUTF(result);
+}
+
+JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixmobile_JNIKiwix_indexedQueryPartial
+             (JNIEnv *env,jclass thiz, jstring db, jstring qu) {
+    const char* d = env->GetStringUTFChars(db, 0);
+    const char* q = env->GetStringUTFChars(qu, 0);
+    const char* result = executeQuery(d, q, true);
+    return env->NewStringUTF(result);
+}
+

src/org/kiwix/kiwixmobile/JNIKiwix.java

@@ -64,6 +64,10 @@ public class JNIKiwix {
   public native boolean getRandomPage(JNIKiwixString url);
 
   public native void setDataDirectory(String icuDataDir);
+
+  public static native String indexedQuery(String db, String query);
+
+  public static native String indexedQueryPartial(String db, String query);
 }
 
 class JNIKiwixString {

src/org/kiwix/kiwixmobile/KiwixMobileActivity.java

@@ -909,8 +909,15 @@ public class KiwixMobileActivity extends AppCompatActivity
         break;
       case REQUEST_FILE_SEARCH:
         if (resultCode == RESULT_OK) {
-          String title = data.getStringExtra(TAG_FILE_SEARCHED);
+            String title = data.getStringExtra(TAG_FILE_SEARCHED).replace("<b>", "").replace("</b>", "");
-          String articleUrl = ZimContentProvider.getPageUrlFromTitle(title);
+            String articleUrl = "";
+
+            if(title.startsWith("A/")) {
+                articleUrl = title;
+            } else articleUrl = ZimContentProvider.getPageUrlFromTitle(title);
+
+            //System.out.println("Opening "+articleUrl + " (" + title + ")");
+
             openArticle(articleUrl);
         }
         break;

src/org/kiwix/kiwixmobile/SearchActivity.java

@@ -80,7 +80,7 @@ public class SearchActivity extends AppCompatActivity implements AdapterView.OnI
 
   @Override
   public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
-    String title = mAdapter.getItem(position);
+    String title = mAdapter.getItemRaw(position);
     sendMessage(title);
   }
 

src/org/kiwix/kiwixmobile/views/AutoCompleteAdapter.java

@@ -1,11 +1,22 @@
 package org.kiwix.kiwixmobile.views;
 
 import android.content.Context;
+import android.text.Html;
+import android.text.TextUtils;
+import android.view.View;
+import android.view.ViewGroup;
 import android.widget.ArrayAdapter;
 import android.widget.Filter;
 import android.widget.Filterable;
+import android.widget.TextView;
+import org.kiwix.kiwixmobile.JNIKiwix;
+
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.kiwix.kiwixmobile.ZimContentProvider;
 
 public class AutoCompleteAdapter extends ArrayAdapter<String> implements Filterable {
@@ -25,8 +36,27 @@ public class AutoCompleteAdapter extends ArrayAdapter<String> implements Filtera
     return mData.size();
   }
 
+  @Override
+  public View getView(int position, View convertView, ViewGroup parent) {
+    View row = super.getView(position, convertView, parent);
+
+    TextView tv = (TextView) row.findViewById(android.R.id.text1);
+    tv.setText(Html.fromHtml(getItem(position)));
+
+    return row;
+  }
+
   @Override
   public String getItem(int index) {
+      String a = mData.get(index);
+      if(a.endsWith(".html")) {
+          String trim = a.substring(2);
+          trim = trim.substring(0, trim.length() - 5);
+          return trim.replace("_", " ");
+      } else return a;
+  }
+
+  public String getItemRaw(int index) {
     return mData.get(index);
   }
 
@@ -37,6 +67,17 @@ public class AutoCompleteAdapter extends ArrayAdapter<String> implements Filtera
 
   class KiwixFilter extends Filter {
 
+    private void addToList(List data, String result, String prefix) {
+        // highlight by word
+        String[] highlight = prefix.split(" ");
+        String toAdd = result.substring(0, result.length()-5).substring(2);
+        for (String todo : highlight)
+            if(todo.length() > 0)
+                toAdd = toAdd.replaceAll("(?i)(" + Pattern.quote(todo)+")", "<b>$1</b>");
+        // add to list
+        data.add("A/"+toAdd+".html");
+    }
+
     @Override
     protected FilterResults performFiltering(CharSequence constraint) {
         FilterResults filterResults = new FilterResults();
@@ -44,16 +85,75 @@ public class AutoCompleteAdapter extends ArrayAdapter<String> implements Filtera
         if (constraint != null) {
             // A class that queries a web API, parses the data and returns an ArrayList<Style>
             try {
-          String prefix = constraint.toString();
+                final String prefix = constraint.toString();
+                /*ZimContentProvider.searchSuggestions(prefix, 200);
 
-          ZimContentProvider.searchSuggestions(prefix, 200);
                 String suggestion;
 
                 data.clear();
                 while ((suggestion = ZimContentProvider.getNextSuggestion()) != null) {
                     data.add(suggestion);
+                    //System.out.println(suggestion);
+                }*/
+
+                String[] ps = prefix.split(" ");
+                String[] rs = new String[ps.length];
+                for (int i = 0; i < ps.length; i++) {
+                    rs[i] = (ps[i].length() > 1 ? Character.toUpperCase(ps[i].charAt(0)) + ps[i].substring(1) : ps[i].toUpperCase());
+                }
+                String qStr = TextUtils.join(" ", rs);
+                qStr.replace("us ", "U.S. ");
+                //System.out.println("Q: "+qStr);
+                //System.out.println(ZimContentProvider.getZimFile() + ".idx");
+
+                String[] result = JNIKiwix.indexedQuery(ZimContentProvider.getZimFile() + ".idx", qStr).split("\n");
+                //System.out.println(result.length);
+
+                if (result.length < 2 && result[0].trim().equals("")) {
+                    result = JNIKiwix.indexedQueryPartial(ZimContentProvider.getZimFile() + ".idx", qStr).split("\n");
+                }
+
+                if (!result[0].trim().equals("")) {
+                    data.clear();
+                    System.out.println(result.length);
+                    List<String> alreadyAdded = new ArrayList<String>();
+                    ZimContentProvider.searchSuggestions(qStr, 5);
+                    String ttl = ZimContentProvider.getPageUrlFromTitle(prefix);
+                    if (ttl != null) {
+                        addToList(data, ttl, prefix);
+                        alreadyAdded.add(ttl);
+                    }
+                    for(int i = 0; i < 3; i++){
+                        String sug = ZimContentProvider.getNextSuggestion();
+                        if(sug != null && sug.length() > 0) {
+                            ttl = ZimContentProvider.getPageUrlFromTitle(sug);
+                            if(!alreadyAdded.contains(ttl)) {
+                                addToList(data, ttl, prefix);
+                                alreadyAdded.add(ttl);
+                            }
+                        }
+                    }
+                    for (int i = 0; i < result.length; i++) {
+                        if(!alreadyAdded.contains(result[i])) {
+                            addToList(data, result[i], prefix);
+                            System.out.println(result[i]);
+                        }
+                    }
+                } else {
+                    // fallback to legacy search method if index not found
+                    ZimContentProvider.searchSuggestions(prefix, 200);
+                    //System.out.println("legacy");
+
+                    String suggestion;
+
+                    data.clear();
+                    while ((suggestion = ZimContentProvider.getNextSuggestion()) != null) {
+                        data.add(suggestion);
+                        //System.out.println(suggestion);
+                    }
                 }
             } catch (Exception e) {
+                e.printStackTrace();
             }
 
             // Now assign the values and count to the FilterResults object