#!/usr/bin/env python ''' Compiles Kiwix dependencies for Android . Compile liblzma . Compile libzim . Compile libkiwix ''' import os import sys import copy import shutil from subprocess import call, check_output # target platform to compile for # list of available toolchains in /toolchains # arm-linux-androideabi, mipsel-linux-android, x86, llvm ALL_ARCHS = ['arm-linux-androideabi', 'mipsel-linux-android', 'x86'] USAGE = '''Usage: %s [--option] Without option, all steps are executed on all archs. --toolchain Creates the toolchain --lzma Compile liblzma --icu Compile libicu --zim Compile libzim --kiwix Compile libkiwix --strip Strip libkiwix.so --apk Create an APK file Note that the '--' prefix is optionnal. --on=ARCH Disable steps on all archs and cherry pick the ones wanted. Multiple --on=ARCH can be specified. ARCH in 'armeabi', 'mips', 'x86'. ''' def init_with_args(args): def display_usage(): print(USAGE % args[0]) sys.exit(0) # default is executing all the steps create_toolchain = compile_liblzma = compile_libicu = \ compile_libzim = compile_libkiwix = strip_libkiwix = \ compile_apk = True archs = ALL_ARCHS options = [a.lower() for a in args[1:]] # print usage if help is requested for help_str in ('-h', '--help'): if options.count(help_str): display_usage() # do we have an --on= flag? if '--on=' in u' '.join(args): # yes, so we clear the arch list and build from request archs = [] # store options on a dict so we can safely remove as we process doptions = {} for idx, param in enumerate(options): doptions[idx] = param # add found arch to list of archs for idx, param in doptions.items(): if param.startswith('--on='): try: rarch = param.split('=', 1)[1] archs.append([k for k, v in ARCHS_SHORT_NAMES.items() if rarch == v][0]) except: pass doptions.pop(idx) # recreate options list from other items options = [v for v in doptions.values() if not v.startswith('--on=')] if len(options): # we received options. # consider we only want the specified steps create_toolchain = compile_liblzma = compile_libicu = compile_libzim = \ compile_libkiwix = strip_libkiwix = compile_apk = False for option in options: if 'toolchain' in option: create_toolchain = True if 'lzma' in option: compile_liblzma = True if 'icu' in option: compile_libicu = True if 'zim' in option: compile_libzim = True if 'kiwix' in option: compile_libkiwix = True if 'strip' in option: strip_libkiwix = True if 'apk' in option: compile_apk = True return (create_toolchain, compile_liblzma, compile_libicu, compile_libzim, compile_libkiwix, strip_libkiwix, compile_apk, archs) # store the OS's environment PATH as we'll mess with it # ORIGINAL_ENVIRON_PATH = os.environ.get('PATH') ORIGINAL_ENVIRON = copy.deepcopy(os.environ) # the directory of this file for relative referencing CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) # different names of folder path for accessing files ARCHS_FULL_NAMES = { 'arm-linux-androideabi': 'arm-linux-androideabi', 'mipsel-linux-android': 'mipsel-linux-android', 'x86': 'i686-linux-android'} ARCHS_SHORT_NAMES = { 'arm-linux-androideabi': 'armeabi', 'mipsel-linux-android': 'mips', 'x86': 'x86'} # store host machine name UNAME = check_output(['uname', '-s']).strip() UARCH = check_output(['uname', '-m']).strip() SYSTEMS = {'Linux': 'linux', 'Darwin': 'mac'} # find out what to execute based on command line arguments CREATE_TOOLCHAIN, COMPILE_LIBLZMA, COMPILE_LIBICU, COMPILE_LIBZIM, \ COMPILE_LIBKIWIX, STRIP_LIBKIWIX, COMPILE_APK, ARCHS = init_with_args(sys.argv) # compiler version to use # list of available toolchains in /toolchains # 4.4.3, 4.6, 4.7, clang3.1, clang3.2 COMPILER_VERSION = '4.6' # /!\ doesn't work with 4.7 # location of Android NDK NDK_PATH = os.environ.get('NDK_PATH', os.path.join(os.path.dirname(CURRENT_PATH), 'src', 'dependencies', 'android-ndk-r8e')) SDK_PATH = os.environ.get('ANDROID_HOME', os.path.join(os.path.dirname(CURRENT_PATH), 'src', 'dependencies', 'android-sdk', 'sdk')) # Target Android EABI/version to compile for. # list of available platforms in /platforms # android-14, android-3, android-4, android-5, android-8, android-9 NDK_PLATFORM = os.environ.get('NDK_PLATFORM', 'android-14') # will contain the different prepared toolchains for a specific build PLATFORM_PREFIX = os.environ.get('PLATFORM_PREFIX', os.path.join(CURRENT_PATH, 'platforms')) if not os.path.exists(PLATFORM_PREFIX): os.makedirs(PLATFORM_PREFIX) # root folder for liblzma LIBLZMA_SRC = os.path.join(os.path.dirname(CURRENT_PATH), 'src', 'dependencies', 'xz') # headers for liblzma LIBLZMA_INCLUDES = [os.path.join(LIBLZMA_SRC, 'src', 'liblzma', 'api')] # root folder for libicu LIBICU_SRC = os.path.join(os.path.dirname(CURRENT_PATH), 'src', 'dependencies', 'icu', 'source') # headers for libicu LIBICU_INCLUDES = [os.path.join(LIBICU_SRC, 'i18n'), os.path.join(LIBICU_SRC, 'common')] # root folder for libzim LIBZIM_SRC = os.path.join(os.path.dirname(CURRENT_PATH), 'src', 'dependencies', 'zimlib-1.1') # headers for libzim LIBZIM_INCLUDES = [os.path.join(LIBZIM_SRC, 'include')] # source files for building libzim LIBZIM_SOURCE_FILES = ('article.cpp', 'articlesearch.cpp', 'cluster.cpp', 'dirent.cpp', 'file.cpp', 'fileheader.cpp', 'fileimpl.cpp', 'indexarticle.cpp', 'ptrstream.cpp', 'search.cpp', 'template.cpp', 'unicode.cpp', 'uuid.cpp', 'zintstream.cpp', 'envvalue.cpp', 'lzmastream.cpp', 'unlzmastream.cpp', 'fstream.cpp', 'md5.cpp', 'md5stream.cpp') # root folder for libkiwix LIBKIWIX_SRC = os.path.join(os.path.dirname(CURRENT_PATH), 'src', 'common') OPTIMIZATION_ENV = {'CXXFLAGS': ' -D__OPTIMIZE__ -fno-strict-aliasing ' ' -DU_HAVE_NL_LANGINFO_CODESET=0 -DU_STATIC_IMPLEMENTATION -DU_HAVE_STD_STRING -DU_TIMEZONE=0', 'NDK_DEBUG': '0'} # list of path that should already be set REQUIRED_PATHS = (NDK_PATH, PLATFORM_PREFIX, LIBLZMA_SRC, LIBZIM_SRC, LIBKIWIX_SRC) # list of paths for libicu ICU_TMP = PLATFORM_PREFIX + '/tmp/' ICU_TMP_HOST = ICU_TMP + 'host/' ICU_TMP_TARGET = ICU_TMP + 'target/' def fail_on_missing(path): ''' check existence of path and error msg + exit if it fails ''' if not os.path.exists(path): print(u"Required PATH is missing or misdefined: %s.\n" u"Check that you have installed the Android NDK properly " u"and run 'make' in 'src/dependencies'" % path) sys.exit(1) def syscall(args, shell=False, with_print=True): ''' make a system call ''' args = args.split() if with_print: print(u"-----------\n" + u" ".join(args) + u"\n-----------") if shell: args = ' '.join(args) call(args, shell=shell) def change_env(values): ''' update a set of environment variables ''' for k, v in values.items(): os.environ[k] = v syscall('export %s="%s"' % (k, v), shell=True, with_print=False) def failed_on_step(error_msg): print('[ERROR] %s. Aborting.' % error_msg) sys.exit(1) # check that required paths are in place before we start for path in REQUIRED_PATHS: fail_on_missing(path) # store where we are so we can go back curdir = os.getcwd() # Prepare the libicu cross-compilation if COMPILE_LIBICU: if (not os.path.exists(ICU_TMP)): os.mkdir(ICU_TMP); if (not os.path.exists(ICU_TMP_HOST)): os.mkdir(ICU_TMP_HOST); if (not os.path.exists(ICU_TMP_TARGET)): os.mkdir(ICU_TMP_TARGET); os.chdir(ICU_TMP_HOST) syscall(LIBICU_SRC + '/configure', shell=True) syscall('make', shell=True) os.chdir(os.getcwd()) for arch in ARCHS: # second name of the platform ; used as subfolder in platform/ arch_full = ARCHS_FULL_NAMES.get(arch) arch_short = ARCHS_SHORT_NAMES.get(arch) # platform contains the toolchain platform = os.path.join(PLATFORM_PREFIX, arch) # prepare the toolchain toolchain = '%(arch)s-%(version)s' % {'arch': arch, 'version': COMPILER_VERSION} toolchain_cmd = ('%(NDK_PATH)s/build/tools/make-standalone-toolchain.sh ' '--toolchain=%(toolchain)s ' '--platform=%(NDK_PLATFORM)s ' '--install-dir=%(PLATFORM_PREFIX)s' % {'NDK_PATH': NDK_PATH, 'NDK_PLATFORM': NDK_PLATFORM, 'toolchain': toolchain, 'PLATFORM_PREFIX': platform}) # required for compilation on an OSX host if UNAME == 'Darwin': toolchain_cmd += ' --system=darwin-x86_64' elif UNAME == 'Linux': if UARCH == 'i686': toolchain_cmd += ' --system=linux-x86' else: toolchain_cmd += ' --system=linux-x86_64' if CREATE_TOOLCHAIN: # copies the precompiled toolchain for the platform: # includes gcc, headers and tools. syscall(toolchain_cmd, shell=True) # add a symlink for liblto_plugin.so to work # could not find how to direct gcc to the right folder ln_src = '%(platform)s/libexec' % {'platform': platform} dest = '%(platform)s/%(arch_full)s' % {'platform': platform, 'arch_full': arch_full} syscall('ln -sf %(src)s %(dest)s/' % {'src': ln_src, 'dest': dest}) # check that the step went well if CREATE_TOOLCHAIN or COMPILE_LIBLZMA or COMPILE_LIBZIM or \ COMPILE_LIBKIWIX or STRIP_LIBKIWIX: if (not os.path.exists(os.path.join(platform, arch_full, 'bin', 'gcc')) or not os.path.exists(os.path.join(platform, arch_full, 'libexec'))): failed_on_step('The toolchain was not ' 'copied properly and is not present.') # change the PATH for compilation to use proper tools new_environ = {'PATH': ('%(platform)s/bin:%(platform)s/%(arch_full)s' '/bin:%(platform)s/libexec/gcc/%(arch_full)s/' '%(gccver)s/:%(sdka)s:%(sdkb)s/:%(orig)s' % {'platform': platform, 'orig': ORIGINAL_ENVIRON['PATH'], 'arch_full': arch_full, 'gccver': COMPILER_VERSION, 'sdka': os.path.join(SDK_PATH, 'platform-tools'), 'sdkb': os.path.join(SDK_PATH, 'tools')}), 'CFLAGS': ' -fPIC -D_FILE_OFFSET_BITS=64 ', 'ANDROID_HOME': SDK_PATH} change_env(new_environ) change_env(OPTIMIZATION_ENV) # check that the path has been changed if not platform in os.environ.get('PATH'): failed_on_step('The PATH environment variable was not set properly.') # compile liblzma.a, liblzma.so os.chdir(LIBLZMA_SRC) configure_cmd = ('./configure --host=%(arch)s --prefix=%(platform)s ' '--disable-assembler --enable-shared --enable-static ' '--enable-largefile' % {'arch': arch_full, 'platform': platform}) if COMPILE_LIBLZMA: # configure, compile, copy and clean liblzma from official sources. # even though we need only static, we conpile also shared so it # switches the -fPIC properly. syscall(configure_cmd, shell=True) syscall('make clean', shell=True) syscall('make', shell=True) syscall('make install', shell=True) syscall('make clean', shell=True) # check that the step went well if COMPILE_LIBLZMA or COMPILE_LIBZIM or COMPILE_LIBKIWIX: if not os.path.exists(os.path.join(platform, 'lib', 'liblzma.a')): failed_on_step('The liblzma.a archive file has not been created ' 'and is not present.') # compile libicu.a, libicu.so os.chdir(ICU_TMP_TARGET) configure_cmd = ( LIBICU_SRC + '/configure --host=%(arch)s --enable-static ' '--prefix=%(platform)s --with-cross-build=%(icu)s --disable-shared --enable-static ' % {'arch': arch_full, 'platform': platform, 'icu': ICU_TMP_HOST}) if COMPILE_LIBICU: # configure, compile, copy and clean libicu from official sources. # even though we need only static, we conpile also shared so it # switches the -fPIC properly. syscall(configure_cmd, shell=True) syscall('make clean', shell=True) syscall('make VERBOSE=1', shell=True) syscall('make install', shell=True) syscall('make clean', shell=True) # check that the step went well if COMPILE_LIBICU or COMPILE_LIBKIWIX: if not os.path.exists(os.path.join(platform, 'lib', 'libicui18n.a')): failed_on_step('The libicu.a archive file has not been created for ' + platform + ' and is not present.') # create libzim.a os.chdir(curdir) platform_includes = ['%(platform)s/include/c++/%(gccver)s/' % {'platform': platform, 'gccver': COMPILER_VERSION}, '%(platform)s/include/c++/%(gccver)s/%(arch_full)s' % {'platform': platform, 'gccver': COMPILER_VERSION, 'arch_full': arch_full}, '%(platform)s/sysroot/usr/include/' % {'platform': platform}, '%(platform)s/lib/gcc/%(arch_full)s/' '%(gccver)s/include' % {'platform': platform, 'arch_full': arch_full, 'gccver': COMPILER_VERSION}, '%(platform)s/lib/gcc/%(arch_full)s/%(gccver)s' '/include-fixed' % {'platform': platform, 'arch_full': arch_full, 'gccver': COMPILER_VERSION}, '%(platform)s/sysroot/usr/include/linux/' % {'platform': platform} ] src_dir = os.path.join(LIBZIM_SRC, 'src') compile_cmd = ('g++ -fPIC -c -D_FILE_OFFSET_BITS=64 -DHAVE_LSEEK64 ' '-D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE ' '-B%(platform)s/sysroot ' '%(source_files)s -I%(include_paths)s ' % {'platform': platform, 'arch_full': arch_full, 'gccver': COMPILER_VERSION, 'source_files': ' '.join([os.path.join(src_dir, src) for src in LIBZIM_SOURCE_FILES]), 'include_paths': ' -I'.join(LIBLZMA_INCLUDES + LIBICU_INCLUDES + LIBZIM_INCLUDES + platform_includes)}) link_cmd = ('ar rvs libzim.a ' '%(obj_files)s ' % {'obj_files': ' '.join([n.replace('.cpp', '.o') for n in LIBZIM_SOURCE_FILES])}) if COMPILE_LIBZIM: syscall(compile_cmd) syscall(link_cmd) libzim_file = os.path.join(curdir, 'libzim.a') shutil.copy(libzim_file, os.path.join(platform, 'lib')) os.remove(libzim_file) for src in LIBZIM_SOURCE_FILES: os.remove(src.replace('.cpp', '.o')) # check that the step went well if COMPILE_LIBZIM or COMPILE_LIBKIWIX: if not os.path.exists(os.path.join(platform, 'lib', 'libzim.a')): failed_on_step('The libzim.a archive file has not been created ' 'and is not present.') # create libkiwix.so os.chdir(curdir) compile_cmd = ('g++ -fPIC -c -B%(platform)s/sysroot ' '-DU_HAVE_STD_STRING ' '-D_FILE_OFFSET_BITS=64 ' '-D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE ' '-DANDROID_NDK ' 'kiwix.c %(kwsrc)s/kiwix/reader.cpp ' '%(kwsrc)s/stringTools.cpp ' '%(kwsrc)s/pathTools.cpp ' '-I%(include_paths)s ' % {'platform': platform, 'arch_full': arch_full, 'gccver': COMPILER_VERSION, 'kwsrc': LIBKIWIX_SRC, 'include_paths': ' -I'.join(LIBLZMA_INCLUDES + LIBICU_INCLUDES + LIBZIM_INCLUDES + platform_includes + [LIBKIWIX_SRC, os.path.join(LIBZIM_SRC, 'include'), os.path.join(curdir, 'src')]) }) link_cmd = ('g++ -fPIC -shared -B%(platform)s/sysroot ' '--sysroot %(platform)s/sysroot ' '-nostdlib ' 'kiwix.o reader.o stringTools.o pathTools.o ' '%(platform)s/lib/gcc/%(arch_full)s/%(gccver)s/crtbegin.o ' '%(platform)s/lib/libzim.a %(platform)s/lib/liblzma.a ' # '%(platform)s/lib/libicutu.a ' # '%(platform)s/lib/libicuio.a ' '%(platform)s/lib/libicuuc.a ' # '%(platform)s/lib/libicule.a ' # '%(platform)s/lib/libiculx.a ' # '%(platform)s/lib/libicui18n.a ' '%(platform)s/lib/libicudata.a ' '-L%(platform)s/%(arch_full)s/lib ' '%(NDK_PATH)s/sources/cxx-stl/gnu-libstdc++/%(gccver)s' '/libs/%(arch_short)s/libgnustl_static.a ' '-llog -landroid -lstdc++ -lc -lm -ldl ' '%(platform)s/lib/gcc/%(arch_full)s/%(gccver)s/libgcc.a ' '-o %(curdir)s/libs/%(arch_short)s/libkiwix.so' % {'kwsrc': LIBKIWIX_SRC, 'platform': platform, 'arch_full': arch_full, 'arch_short': arch_short, 'curdir': curdir, 'gccver': COMPILER_VERSION, 'NDK_PATH': NDK_PATH}) if COMPILE_LIBKIWIX: # compile JNI header os.chdir(os.path.join(curdir, 'src', 'org', 'kiwix', 'kiwixmobile')) syscall('javac JNIKiwix.java') os.chdir(os.path.join(curdir, 'src')) syscall('javah -jni org.kiwix.kiwixmobile.JNIKiwix') os.chdir(curdir) syscall(compile_cmd) syscall(link_cmd) for obj in ('kiwix.o', 'reader.o', 'stringTools.o', 'pathTools.o', 'src/org_kiwix_kiwixmobile_JNIKiwix.h'): os.remove(obj) # check that the step went well if COMPILE_LIBKIWIX or STRIP_LIBKIWIX or COMPILE_APK: if not os.path.exists(os.path.join('libs', arch_short, 'libkiwix.so')): failed_on_step('The libkiwix.so shared lib has not been created ' 'and is not present.') if STRIP_LIBKIWIX: syscall('%(platform)s/%(arch_full)s/bin/strip ' '%(curdir)s/libs/%(arch_short)s/libkiwix.so' % {'platform': platform, 'arch_full': arch_full, 'arch_short': arch_short, 'curdir': curdir}) os.chdir(curdir) change_env(ORIGINAL_ENVIRON) if COMPILE_APK: syscall('rm -f bin/*.apk', shell=True) syscall('ant debug') syscall('ls -lh bin/*.apk', shell=True) # check that the step went well if COMPILE_APK: if not os.path.exists(os.path.join('bin', 'Kiwix-debug.apk')): failed_on_step('The Kiwix-debug.apk package has not been created ' 'and is not present.')