kiwix/kiwix-android
1
0
Fork 0
mirror of https://github.com/kiwix/kiwix-android.git synced 2025-08-03 10:46:53 -04:00
Code Issues Packages Projects Releases Wiki Activity

Compile against xapian 1.4 and remove useless glassify

Browse Source
This commit is contained in:
Kelson 2016-08-14 01:23:06 +02:00
parent b583aa3e15
commit 417c2e2df8
2 changed files with 8 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
build-android-with-native.py
View File

@ -48,7 +48,6 @@ USAGE = '''Usage: {arg0} [--option]
--icu Compile libicu
--zim Compile libzim
--xapian Compile libxapian
--glassify Compile glassify binary
--kiwix Compile libkiwix
--strip Strip libkiwix.so
--locales Create the locales.txt file
@ -72,7 +71,6 @@ def init_with_args(args):
create_toolchain = compile_liblzma = compile_libicu = \
compile_libzim = compile_libkiwix = compile_libxapian = strip_libkiwix = \
compile_apk = locales_txt = clean = True
compile_glassify = False # dont want to compile this everytime
archs = ALL_ARCHS
options = [a.lower() for a in args[1:]]
@ -111,7 +109,7 @@ def init_with_args(args):
# consider we only want the specified steps
create_toolchain = compile_liblzma = compile_libicu = compile_libzim = \
compile_libkiwix = compile_libxapian = strip_libkiwix = \
compile_apk = locales_txt = clean = compile_glassify = False
compile_apk = locales_txt = clean = False
for option in options:
if 'toolchain' in option:
@ -128,8 +126,6 @@ def init_with_args(args):
compile_libxapian = True
if 'strip' in option:
strip_libkiwix = True
if 'glassify' in option:
compile_glassify = True
if 'apk' in option:
compile_apk = True
if 'locales' in option:
@ -138,7 +134,7 @@ def init_with_args(args):
clean = True
return (create_toolchain, compile_liblzma, compile_libicu, compile_libzim,
compile_libkiwix, compile_libxapian, strip_libkiwix, compile_apk, compile_glassify, locales_txt,
compile_libkiwix, compile_libxapian, strip_libkiwix, compile_apk, locales_txt,
clean, archs)
# store the OS's environment PATH as we'll mess with it
@ -174,7 +170,7 @@ SYSTEMS = {'Linux': 'linux', 'Darwin': 'mac'}
# find out what to execute based on command line arguments
CREATE_TOOLCHAIN, COMPILE_LIBLZMA, COMPILE_LIBICU, COMPILE_LIBZIM, \
COMPILE_LIBKIWIX, COMPILE_LIBXAPIAN, STRIP_LIBKIWIX, COMPILE_APK, \
COMPILE_GLASSIFY, LOCALES_TXT, CLEAN, ARCHS = init_with_args(sys.argv)
LOCALES_TXT, CLEAN, ARCHS = init_with_args(sys.argv)
# compiler version to use
@ -220,7 +216,7 @@ LIBICU_INCLUDES = [os.path.join(LIBICU_SRC, 'i18n'),
# root folder for libzim
LIBZIM_SRC = os.path.join(os.path.dirname(CURRENT_PATH),
'src', 'dependencies', 'zimlib-1.2')
'src', 'dependencies', 'zimlib-1.3')
# headers for libzim
LIBZIM_INCLUDES = [os.path.join(LIBZIM_SRC, 'include')]
@ -426,19 +422,11 @@ for arch in ARCHS:
# compile xapian
if COMPILE_LIBXAPIAN:
# fetch xapian, e2fsprogs, zlib
os.chdir(os.path.join(curdir, '../src', 'dependencies'))
if not os.path.exists("e2fsprogs-1.42"):
syscall('make e2fsprogs-1.42')
# TODO: this needs to be handle uniquely in src/dependencies (at the time xapian 1.3 is stable)
if not os.path.exists("xapian-core-1.3.4"):
print("Fetching recent xapian...")
urllib.urlretrieve('http://download.kiwix.org/dev/xapian-core-1.3.4.tar.gz', 'xapian-core-1.3.4.tar.xz') # for glass support
change_env(ORIGINAL_ENVIRON)
syscall('tar xvf xapian-core-1.3.4.tar.xz')
change_env(new_environ)
change_env(OPTIMIZATION_ENV)
if not os.path.exists("zlib-1.2.8"):
syscall('make zlib-1.2.8')
os.chdir('zlib-1.2.8')
@ -449,10 +437,8 @@ for arch in ARCHS:
shutil.copy('libz.a', os.path.join(platform, 'lib', 'gcc', arch_full, COMPILER_VERSION, 'libz.a'))
os.chdir('../e2fsprogs-1.42')
print("Fetching latest compile.sub...")
shutil.copy(os.path.join("..", "xapian-core-1.3.4", "config.guess"), os.path.join("config", "config.guess"))
shutil.copy(os.path.join("..", "xapian-core-1.3.4", "config.sub"), os.path.join("config", "config.sub"))
# urllib.urlretrieve('http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD', 'config/config.guess')
# urllib.urlretrieve('http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD', 'config/config.sub')
shutil.copy(os.path.join("..", "xapian-core-1.4.0", "config.guess"), os.path.join("config", "config.guess"))
shutil.copy(os.path.join("..", "xapian-core-1.4.0", "config.sub"), os.path.join("config", "config.sub"))
if os.path.exists("Makefile"):
syscall('make clean')
syscall('./configure --host=%s --prefix=%s' % (arch_full, platform))
@ -473,7 +459,7 @@ for arch in ARCHS:
shutil.copy('uuid.h', os.path.join(platform, 'include', 'c++', COMPILER_VERSION, 'uuid', 'uuid.h'))
shutil.copy('libuuid.a', os.path.join(platform, 'lib', 'gcc', arch_full, COMPILER_VERSION, 'libuuid.a'))
shutil.copy('libuuid.a', os.path.join(platform, 'lib', 'libuuid.a'))
os.chdir('../../../xapian-core-1.3.4')
os.chdir('../../../xapian-core-1.4.0')
if os.path.exists("Makefile"):
syscall('make clean')
@ -515,7 +501,6 @@ for arch in ARCHS:
pass
syscall('make')
shutil.copy(os.path.join(curdir, '..', 'src', 'dependencies', 'xapian-core-1.3.4', '.libs', 'libxapian-1.3.a'), os.path.join(platform, 'lib', 'libxapian.a'))
os.chdir(curdir)
# check that the step went well
@ -670,22 +655,10 @@ for arch in ARCHS:
'arch_full': arch_full,
'arch_short': arch_short,
'curdir': curdir})
if COMPILE_GLASSIFY:
os.chdir(curdir)
syscall('g++ glassify.cc ../src/dependencies/xapian-core-1.3.4/.libs/libxapian-1.3.a -o glassify_%s -lz -luuid -lrt -I../src/dependencies/xapian-core-1.3.4/include' % arch_short)
os.chdir(curdir)
change_env(ORIGINAL_ENVIRON)
# recompile xapian for build system arch to compile glassify
if COMPILE_GLASSIFY:
os.chdir(os.path.join(curdir, '..', 'src', 'dependencies', 'xapian-core-1.3.4'))
syscall('make clean')
syscall('./configure')
syscall('make')
os.chdir(curdir)
syscall('g++ glassify.cc ../src/dependencies/xapian-core-1.3.4/.libs/libxapian-1.3.a -o glassify -lz -luuid -lrt -I../src/dependencies/xapian-core-1.3.4/include')
if LOCALES_TXT:
os.chdir(curdir)

188
glassify.cc
View File

@ -1,188 +0,0 @@
#include <xapian.h>
#include <iomanip>
#include <iostream>
#include <cmath> // For log10().
#include <cstdlib> // For exit().
#include <cstring> // For strcmp() and strrchr().
#include <string>
#include <fcntl.h>
#include <stdio.h>
#include <ftw.h>
#include <unistd.h>
using namespace std;
#define PROG_NAME "glassify"
#define PROG_DESC "Perform a document-by-document copy of one or more Xapian databases and make it a single file"
static void
show_usage(int rc)
{
cout << "Usage: " PROG_NAME " SOURCE_DATABASE... DESTINATION_DATABASE\n\n"
"Options:\n"
" --no-renumber Preserve the numbering of document ids (useful if you have\n"
" external references to them, or have set them to match\n"
" unique ids from an external source). If multiple source\n"
" databases are specified and the same docid occurs in more\n"
" one, the last occurrence will be the one which ends up in\n"
" the destination database.\n"
" --help display this help and exit\n"
" --version output version information and exit" << endl;
exit(rc);
}
void compact(const char* in, const char* out) try {
Xapian::Database indb(in);
int fd = open(out, O_CREAT|O_RDWR, 0666);
if (fd != -1) {
indb.compact(fd);
cout << "Done!" << endl;
return;
}
cout << "Some error happened..." << endl;
} catch (const Xapian::Error &e) {
cout << e.get_description().c_str() << endl;
}
int unlinker(const char *fpth, const struct stat *sb, int t, struct FTW *fb) {
int rv = remove(fpth);
if (rv)
perror(fpth);
return rv;
}
int cleaner(const char *path) {
return nftw(path, unlinker, 64, FTW_DEPTH | FTW_PHYS);
}
int
main(int argc, char **argv)
try {
bool renumber = true;
if (argc > 1 && argv[1][0] == '-') {
if (strcmp(argv[1], "--help") == 0) {
cout << PROG_NAME " - " PROG_DESC "\n\n";
show_usage(0);
}
if (strcmp(argv[1], "--version") == 0) {
cout << PROG_NAME << endl;
exit(0);
}
if (strcmp(argv[1], "--no-renumber") == 0) {
renumber = false;
argv[1] = argv[0];
++argv;
--argc;
}
}
// We expect two or more arguments: at least one source database path
// followed by the destination database path.
if (argc < 3) show_usage(1);
// Create the destination database, using DB_CREATE so that we don't
// try to overwrite or update an existing database in case the user
// got the command line argument order wrong.
string dest_str = string(argv[argc - 1]);
dest_str += ".tmp";
const char *dest = dest_str.c_str();
Xapian::WritableDatabase db_out(dest, Xapian::DB_CREATE|Xapian::DB_BACKEND_GLASS);
for (int i = 1; i < argc - 1; ++i) {
char * src = argv[i];
if (*src) {
// Remove any trailing directory separator.
char & ch = src[strlen(src) - 1];
if (ch == '/' || ch == '\\') ch = '\0';
}
// Open the source database.
Xapian::Database db_in(src);
// Find the leaf-name of the database path for reporting progress.
const char * leaf = strrchr(src, '/');
#if defined __WIN32__ || defined __OS2__
if (!leaf) leaf = strrchr(src, '\\');
#endif
if (leaf) ++leaf; else leaf = src;
// Iterate over all the documents in db_in, copying each to db_out.
Xapian::doccount dbsize = db_in.get_doccount();
if (dbsize == 0) {
cout << leaf << ": empty!" << endl;
} else {
// Calculate how many decimal digits there are in dbsize.
int width = static_cast<int>(log10(double(dbsize))) + 1;
Xapian::doccount c = 0;
Xapian::PostingIterator it = db_in.postlist_begin(string());
while (it != db_in.postlist_end(string())) {
Xapian::docid did = *it;
if (renumber) {
db_out.add_document(db_in.get_document(did));
} else {
db_out.replace_document(did, db_in.get_document(did));
}
// Update for the first 10, and then every 13th document
// counting back from the end (this means that all the
// digits "rotate" and the counter ends up on the exact
// total.
++c;
if (c <= 10 || (dbsize - c) % 13 == 0) {
cout << '\r' << leaf << ": ";
cout << setw(width) << c << '/' << dbsize << flush;
}
++it;
}
cout << endl;
}
cout << "Copying spelling data..." << flush;
Xapian::TermIterator spellword = db_in.spellings_begin();
while (spellword != db_in.spellings_end()) {
db_out.add_spelling(*spellword, spellword.get_termfreq());
++spellword;
}
cout << " done." << endl;
cout << "Copying synonym data..." << flush;
Xapian::TermIterator synkey = db_in.synonym_keys_begin();
while (synkey != db_in.synonym_keys_end()) {
string key = *synkey;
Xapian::TermIterator syn = db_in.synonyms_begin(key);
while (syn != db_in.synonyms_end(key)) {
db_out.add_synonym(key, *syn);
++syn;
}
++synkey;
}
cout << " done." << endl;
cout << "Copying user metadata..." << flush;
Xapian::TermIterator metakey = db_in.metadata_keys_begin();
while (metakey != db_in.metadata_keys_end()) {
string key = *metakey;
db_out.set_metadata(key, db_in.get_metadata(key));
++metakey;
}
cout << " done." << endl;
}
cout << "Committing..." << flush;
// Commit explicitly so that any error is reported.
db_out.commit();
cout << " done." << endl;
cout << "Turning into single file..." << endl;
compact(dest, argv[argc - 1]);
cout << "All finished. Cleaning up..." << endl;
cleaner(dest);
cout << "Done!" << endl;
} catch (const Xapian::Error & e) {
cerr << '\n' << argv[0] << ": " << e.get_description() << endl;
exit(1);
}