Added native code cross-compilation script for Android

create_libkiwix.so.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+
+''' Compiles Kiwix dependencies for Android
+
+    . Compile liblzma
+    . Compile libzim
+    . Compile libkiwix '''
+
+import os
+import sys
+import copy
+import shutil
+from subprocess import call, check_output
+
+# switchs for debugging purposes ; please ignore.
+CREATE_TOOLCHAIN = True
+COMPILE_LIBLZMA = True
+COMPILE_LIBZIM = True
+COMPILE_LIBKIWIX = True
+STRIP_LIBKIWIX = False
+
+# store the OS's environment PATH as we'll mess with it
+# ORIGINAL_ENVIRON_PATH = os.environ.get('PATH')
+ORIGINAL_ENVIRON = copy.deepcopy(os.environ)
+
+# the directory of this file for relative referencing
+CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
+
+PLATFORMS = {
+    'arm-linux-androideabi': 'arm-linux-androideabi',
+    'mipsel-linux-android': 'mipsel-linux-android',
+    'x86': 'i686-linux-android'}
+
+# store host machine name
+UNAME = check_output(['uname', '-s']).strip()
+
+# target platform to compile for
+# list of available toolchains in <NDK_PATH>/toolchains
+# arm-linux-androideabi, mipsel-linux-android, x86, llvm
+ARCHS = ('arm-linux-androideabi', 'mipsel-linux-android', 'x86')
+
+# compiler version to use
+# list of available toolchains in <NDK_PATH>/toolchains
+# 4.4.3, 4.6, 4.7, clang3.1, clang3.2
+COMPILER_VERSION = '4.6'  # /!\ doesn't work with 4.7
+
+# location of Android NDK
+NDK_PATH = os.environ.get('NDK_PATH',
+                          os.path.join(os.path.dirname(CURRENT_PATH),
+                                       'src', 'dependencies',
+                                       'android-ndk-r8e'))
+
+# Target Android EABI/version to compile for.
+# list of available platforms in <NDK_PATH>/platforms
+# android-14, android-3, android-4, android-5, android-8, android-9
+NDK_PLATFORM = os.environ.get('NDK_PLATFORM', 'android-9')
+
+# will contain the different prepared toolchains for a specific build
+PLATFORM_PREFIX = os.environ.get('PLATFORM_PREFIX',
+                                 os.path.join(CURRENT_PATH, 'platforms'))
+if not os.path.exists(PLATFORM_PREFIX):
+    os.makedirs(PLATFORM_PREFIX)
+
+# root folder for liblzma
+LIBLZMA_SRC = os.path.join(os.path.dirname(CURRENT_PATH),
+                           'src', 'dependencies', 'xz-5.0.4')
+
+# headers for liblzma
+LIBLZMA_INCLUDES = [os.path.join(LIBLZMA_SRC, 'src', 'liblzma', 'api')]
+
+# root folder for libzim
+LIBZIM_SRC = os.path.join(os.path.dirname(CURRENT_PATH),
+                          'src', 'zimlib')
+
+# headers for libzim
+LIBZIM_INCLUDES = [os.path.join(LIBZIM_SRC, 'include')]
+
+# source files for building libzim
+LIBZIM_SOURCE_FILES = ('article.cpp', 'articlesearch.cpp', 'cluster.cpp',
+                       'dirent.cpp', 'file.cpp', 'fileheader.cpp',
+                       'fileimpl.cpp', 'indexarticle.cpp', 'ptrstream.cpp',
+                       'search.cpp', 'template.cpp', 'unicode.cpp', 'uuid.cpp',
+                       'zintstream.cpp', 'envvalue.cpp', 'lzmastream.cpp',
+                       'unlzmastream.cpp', 'fstream.cpp', 'md5.cpp',
+                       'md5stream.cpp')
+
+# root folder for libkiwix
+LIBKIWIX_SRC = os.path.join(os.path.dirname(CURRENT_PATH),
+                            'src', 'common')
+
+# list of path that should already be set
+REQUIRED_PATHS = (NDK_PATH, PLATFORM_PREFIX,
+                  LIBLZMA_SRC, LIBZIM_SRC, LIBKIWIX_SRC)
+
+def fail_on_missing(path):
+    ''' check existence of path and error msg + exit if it fails '''
+    if not os.path.exists(path):
+        print(u"Required PATH is missing or misdefined: %s.\n"
+              u"Check that you have installed the Android NDK properly "
+              u"and run 'make' in 'src/dependencies'" % path)
+        sys.exit(1)
+
+def syscall(args, in_bash=False):
+    ''' make a system call via bash '''
+    args = args.split()
+    if in_bash:
+        args = ['/bin/bash'] + args
+    print(u"-----------\n" + u" ".join(args) + u"\n-----------")
+    call(args, shell=False)
+
+def change_env(values):
+    ''' update a set of environment variables '''
+    for k, v in values.items():
+        os.environ[k] = v
+
+# check that required paths are in place before we start
+for path in REQUIRED_PATHS:
+    fail_on_missing(path)
+
+for arch in ARCHS:
+    # second name of the platform ; used as subfolder in platform/
+    arch_full = PLATFORMS.get(arch)
+
+    # store where we are so we can go back
+    curdir = os.getcwd()
+
+    # platform contains the toolchain
+    platform = os.path.join(PLATFORM_PREFIX, arch)
+
+    # prepare the toolchain
+    toolchain = '%(arch)s-%(version)s' % {'arch': arch,
+                                          'version': COMPILER_VERSION}
+    toolchain_cmd = ('%(NDK_PATH)s/build/tools/make-standalone-toolchain.sh '
+                     '--toolchain=%(toolchain)s '
+                     '--platform=%(NDK_PLATFORM)s '
+                     '--install-dir=%(PLATFORM_PREFIX)s'
+                     % {'NDK_PATH': NDK_PATH,
+                        'NDK_PLATFORM': NDK_PLATFORM,
+                        'toolchain': toolchain,
+                        'PLATFORM_PREFIX': platform})
+    # required for compilation on an OSX host
+    if UNAME == 'Darwin':
+        toolchain_cmd += ' --system=darwin-x86_64'
+
+    if CREATE_TOOLCHAIN:
+        # copies the precompiled toolchain for the platform:
+        # includes gcc, headers and tools.
+        syscall(toolchain_cmd, in_bash=True)
+
+        # add a symlink for liblto_plugin.so to work
+        # could not find how to direct gcc to the right folder
+        ln_src = '%(platform)s/libexec' % {'platform': platform}
+        dest = '%(platform)s/%(arch_full)s' % {'platform': platform,
+                                               'arch_full': arch_full}
+        syscall('ln -s %(src)s %(dest)s/'
+                % {'src': ln_src, 'dest': dest})
+
+    # change the PATH for compilation to use proper tools
+    new_environ = {'PATH': ('%(platform)s/bin:%(platform)s/%(arch_full)s'
+                            '/bin:%(platform)s/libexec/gcc/%(arch_full)s/'
+                            '%(gccver)s/:%(orig)s'
+                          % {'platform': platform,
+                             'orig': ORIGINAL_ENVIRON['PATH'],
+                             'arch_full': arch_full,
+                             'gccver': COMPILER_VERSION}),
+                   'CFLAGS': ' -fPIC '
+                    }
+    change_env(new_environ)
+
+    # compile liblzma.a, liblzma.so
+    os.chdir(LIBLZMA_SRC)
+    configure_cmd = ('./configure --host=%(arch)s --prefix=%(platform)s '
+                     '--disable-assembler --enable-shared --enable-static'
+                     % {'arch': arch_full,
+                        'platform': platform})
+    if COMPILE_LIBLZMA:
+        # configure, compile, copy and clean liblzma from official sources.
+        # even though we need only static, we conpile also shared so it
+        # switches the -fPIC properly.
+        syscall(configure_cmd, in_bash=True)
+        syscall('make clean')
+        syscall('make')
+        syscall('make install')
+        syscall('make clean')
+
+    # create libzim.a
+    os.chdir(curdir)
+    platform_includes = [
+                        '%(platform)s/include/c++/%(gccver)s/'
+                         % {'platform': platform, 'gccver': COMPILER_VERSION},
+
+                         '%(platform)s/include/c++/%(gccver)s/%(arch_full)s'
+                         % {'platform': platform, 'gccver': COMPILER_VERSION,
+                            'arch_full': arch_full},
+
+                         '%(platform)s/sysroot/usr/include/'
+                         % {'platform': platform},
+
+                         '%(platform)s/lib/gcc/%(arch_full)s/'
+                         '%(gccver)s/include'
+                         % {'platform': platform, 'arch_full': arch_full,
+                            'gccver': COMPILER_VERSION},
+
+                         '%(platform)s/lib/gcc/%(arch_full)s/%(gccver)s'
+                         '/include-fixed'
+                         % {'platform': platform, 'arch_full': arch_full,
+                            'gccver': COMPILER_VERSION},
+
+                         '%(platform)s/sysroot/usr/include/linux/'
+                         % {'platform': platform}
+                         ]
+
+    src_dir = os.path.join(LIBZIM_SRC, 'src')
+    compile_cmd = ('g++ -fPIC -c -D_FILE_OFFSET_BITS=64 '
+                   '-D_LARGEFILE64_SOURCE '
+                   '-B%(platform)s/sysroot '
+                   '%(source_files)s -I%(include_paths)s '
+                   % {'platform': platform,
+                      'arch_full': arch_full,
+                      'gccver': COMPILER_VERSION,
+                      'source_files': ' '.join([os.path.join(src_dir, src)
+                                                for src
+                                                in LIBZIM_SOURCE_FILES]),
+                      'include_paths': ' -I'.join(LIBLZMA_INCLUDES
+                                                  + LIBZIM_INCLUDES
+                                                  + platform_includes)})
+    link_cmd = ('ar rvs libzim.a '
+                '%(obj_files)s '
+                % {'obj_files': ' '.join([n.replace('.cpp', '.o')
+                                          for n in LIBZIM_SOURCE_FILES])})
+
+    if COMPILE_LIBZIM:
+        syscall(compile_cmd)
+        syscall(link_cmd)
+
+        libzim_file = os.path.join(curdir, 'libzim.a')
+        shutil.copy(libzim_file, os.path.join(platform, 'lib'))
+        os.remove(libzim_file)
+
+        for src in LIBZIM_SOURCE_FILES:
+            os.remove(src.replace('.cpp', '.o'))
+
+    # compile JNI header
+    syscall('javac JNIKiwix.java')
+    syscall('javah -jni JNIKiwix')
+
+    # create libkiwix.so
+    os.chdir(curdir)
+    compile_cmd = ('g++ -fPIC -c -B%(platform)s/sysroot '
+                   'kiwix.c %(kwsrc)s/kiwix/reader.cpp %(kwsrc)s'
+                   '/stringTools.cpp '
+                   '-I%(include_paths)s '
+                   % {'platform': platform,
+                      'arch_full': arch_full,
+                      'gccver': COMPILER_VERSION,
+                      'kwsrc': LIBKIWIX_SRC,
+                      'include_paths': ' -I'.join(LIBLZMA_INCLUDES
+                                                  + LIBZIM_INCLUDES
+                                                  + platform_includes
+                                                  + [LIBKIWIX_SRC,
+                                                     os.path.join(LIBZIM_SRC,
+                                                                  'include')])
+                      })
+
+    link_cmd = ('g++ -fPIC -shared -B%(platform)s/sysroot '
+                '--sysroot %(platform)s/sysroot '
+                '-nostdlib '
+                'kiwix.o reader.o stringTools.o '
+                '%(platform)s/lib/libzim.a %(platform)s/lib/liblzma.a '
+                '-L%(platform)s/%(arch_full)s/lib '
+                # '-lgnustl_shared -llog -landroid -lstdc++ '
+                '%(platform)s/lib/gcc/%(arch_full)s/%(gccver)s/libgcc.a '
+                '-o %(platform)s/lib/libkiwix.so'
+                % {'kwsrc': LIBKIWIX_SRC,
+                   'platform': platform,
+                   'arch_full': arch_full,
+                   'gccver': COMPILER_VERSION})
+
+    if COMPILE_LIBKIWIX:
+        syscall(compile_cmd)
+        syscall(link_cmd)
+
+        for obj in ('kiwix.o', 'reader.o', 'stringTools.o'):
+            os.remove(obj)
+
+    if STRIP_LIBKIWIX:
+        syscall('%(platform)s/%(arch_full)s/bin/strip '
+                '%(platform)s/lib/libkiwix.so'
+                % {'platform': platform,
+                   'arch_full': arch_full})
+
+    os.chdir(curdir)
+    change_env(ORIGINAL_ENVIRON)

devel_build-libkiwix.so.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# this script is just for testing purposes
+# it helps generate a libkiwix.so on a dev linux machine to assist
+# with JNI debugging.
+
+# change the following to point to absolute of kiwix repo.
+KIWIX_ROOT=`pwd`/../
+
+cd $KIWIX_ROOT
+
+# compilation de liblzma.a
+cd src/dependencies/xz-5.0.4
+CFLAGS=" -fPIC " ./configure --disable-assembler --enable-shared --enable-static
+make
+cp src/liblzma/.libs/liblzma.a $KIWIX_ROOT/android/
+make clean
+
+# compile libzim.a
+cd $KIWIX_ROOT/src/zimlib/src
+g++ -fPIC -c -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE article.cpp articlesearch.cpp cluster.cpp dirent.cpp file.cpp fileheader.cpp fileimpl.cpp indexarticle.cpp ptrstream.cpp search.cpp template.cpp unicode.cpp uuid.cpp zintstream.cpp envvalue.cpp lzmastream.cpp unlzmastream.cpp fstream.cpp md5.cpp md5stream.cpp -I. -I../include/ -I../../dependencies/xz-5.0.4/src/liblzma/api/
+ar rvs libzim.a article.o articlesearch.o cluster.o dirent.o file.o fileheader.o fileimpl.o indexarticle.o ptrstream.o search.o template.o unicode.o uuid.o zintstream.o envvalue.o lzmastream.o unlzmastream.o fstream.o md5.o md5stream.o
+cp libzim.a $KIWIX_ROOT/android/
+rm *.o
+
+# compile libkiwix
+cd $KIWIX_ROOT/android/
+rm *.o
+g++ -fPIC -c kiwix.c $KIWIX_ROOT/src/common/kiwix/reader.cpp $KIWIX_ROOT/src/common/stringTools.cpp -I$KIWIX_ROOT/src/zimlib/include -I../../dependencies/xz-5.0.4/src/liblzma/api/ -I$KIWIX_ROOT/src/common -I/usr/lib/jvm/java-7-openjdk-amd64/include/
+g++ -fPIC -shared kiwix.o reader.o stringTools.o libzim.a liblzma.a /usr/lib/gcc/x86_64-linux-gnu/4.7/libgcc.a -o libkiwix.so
+
+ls -lh libkiwix.so