From 3317a36e3157c893753fc4c6375602262fdab9c2 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Thu, 4 Apr 2013 00:59:13 +0200 Subject: [PATCH] Added native code cross-compilation script for Android --- create_libkiwix.so.py | 293 +++++++++++++++++++++++++++++++++++++ devel_build-libkiwix.so.sh | 32 ++++ 2 files changed, 325 insertions(+) create mode 100755 create_libkiwix.so.py create mode 100755 devel_build-libkiwix.so.sh diff --git a/create_libkiwix.so.py b/create_libkiwix.so.py new file mode 100755 index 000000000..d42e66965 --- /dev/null +++ b/create_libkiwix.so.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python + +''' Compiles Kiwix dependencies for Android + + . Compile liblzma + . Compile libzim + . Compile libkiwix ''' + +import os +import sys +import copy +import shutil +from subprocess import call, check_output + +# switchs for debugging purposes ; please ignore. +CREATE_TOOLCHAIN = True +COMPILE_LIBLZMA = True +COMPILE_LIBZIM = True +COMPILE_LIBKIWIX = True +STRIP_LIBKIWIX = False + +# store the OS's environment PATH as we'll mess with it +# ORIGINAL_ENVIRON_PATH = os.environ.get('PATH') +ORIGINAL_ENVIRON = copy.deepcopy(os.environ) + +# the directory of this file for relative referencing +CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) + +PLATFORMS = { + 'arm-linux-androideabi': 'arm-linux-androideabi', + 'mipsel-linux-android': 'mipsel-linux-android', + 'x86': 'i686-linux-android'} + +# store host machine name +UNAME = check_output(['uname', '-s']).strip() + +# target platform to compile for +# list of available toolchains in /toolchains +# arm-linux-androideabi, mipsel-linux-android, x86, llvm +ARCHS = ('arm-linux-androideabi', 'mipsel-linux-android', 'x86') + +# compiler version to use +# list of available toolchains in /toolchains +# 4.4.3, 4.6, 4.7, clang3.1, clang3.2 +COMPILER_VERSION = '4.6' # /!\ doesn't work with 4.7 + +# location of Android NDK +NDK_PATH = os.environ.get('NDK_PATH', + os.path.join(os.path.dirname(CURRENT_PATH), + 'src', 'dependencies', + 'android-ndk-r8e')) + +# Target Android EABI/version to compile for. +# list of available platforms in /platforms +# android-14, android-3, android-4, android-5, android-8, android-9 +NDK_PLATFORM = os.environ.get('NDK_PLATFORM', 'android-9') + +# will contain the different prepared toolchains for a specific build +PLATFORM_PREFIX = os.environ.get('PLATFORM_PREFIX', + os.path.join(CURRENT_PATH, 'platforms')) +if not os.path.exists(PLATFORM_PREFIX): + os.makedirs(PLATFORM_PREFIX) + +# root folder for liblzma +LIBLZMA_SRC = os.path.join(os.path.dirname(CURRENT_PATH), + 'src', 'dependencies', 'xz-5.0.4') + +# headers for liblzma +LIBLZMA_INCLUDES = [os.path.join(LIBLZMA_SRC, 'src', 'liblzma', 'api')] + +# root folder for libzim +LIBZIM_SRC = os.path.join(os.path.dirname(CURRENT_PATH), + 'src', 'zimlib') + +# headers for libzim +LIBZIM_INCLUDES = [os.path.join(LIBZIM_SRC, 'include')] + +# source files for building libzim +LIBZIM_SOURCE_FILES = ('article.cpp', 'articlesearch.cpp', 'cluster.cpp', + 'dirent.cpp', 'file.cpp', 'fileheader.cpp', + 'fileimpl.cpp', 'indexarticle.cpp', 'ptrstream.cpp', + 'search.cpp', 'template.cpp', 'unicode.cpp', 'uuid.cpp', + 'zintstream.cpp', 'envvalue.cpp', 'lzmastream.cpp', + 'unlzmastream.cpp', 'fstream.cpp', 'md5.cpp', + 'md5stream.cpp') + +# root folder for libkiwix +LIBKIWIX_SRC = os.path.join(os.path.dirname(CURRENT_PATH), + 'src', 'common') + +# list of path that should already be set +REQUIRED_PATHS = (NDK_PATH, PLATFORM_PREFIX, + LIBLZMA_SRC, LIBZIM_SRC, LIBKIWIX_SRC) + +def fail_on_missing(path): + ''' check existence of path and error msg + exit if it fails ''' + if not os.path.exists(path): + print(u"Required PATH is missing or misdefined: %s.\n" + u"Check that you have installed the Android NDK properly " + u"and run 'make' in 'src/dependencies'" % path) + sys.exit(1) + +def syscall(args, in_bash=False): + ''' make a system call via bash ''' + args = args.split() + if in_bash: + args = ['/bin/bash'] + args + print(u"-----------\n" + u" ".join(args) + u"\n-----------") + call(args, shell=False) + +def change_env(values): + ''' update a set of environment variables ''' + for k, v in values.items(): + os.environ[k] = v + +# check that required paths are in place before we start +for path in REQUIRED_PATHS: + fail_on_missing(path) + +for arch in ARCHS: + # second name of the platform ; used as subfolder in platform/ + arch_full = PLATFORMS.get(arch) + + # store where we are so we can go back + curdir = os.getcwd() + + # platform contains the toolchain + platform = os.path.join(PLATFORM_PREFIX, arch) + + # prepare the toolchain + toolchain = '%(arch)s-%(version)s' % {'arch': arch, + 'version': COMPILER_VERSION} + toolchain_cmd = ('%(NDK_PATH)s/build/tools/make-standalone-toolchain.sh ' + '--toolchain=%(toolchain)s ' + '--platform=%(NDK_PLATFORM)s ' + '--install-dir=%(PLATFORM_PREFIX)s' + % {'NDK_PATH': NDK_PATH, + 'NDK_PLATFORM': NDK_PLATFORM, + 'toolchain': toolchain, + 'PLATFORM_PREFIX': platform}) + # required for compilation on an OSX host + if UNAME == 'Darwin': + toolchain_cmd += ' --system=darwin-x86_64' + + if CREATE_TOOLCHAIN: + # copies the precompiled toolchain for the platform: + # includes gcc, headers and tools. + syscall(toolchain_cmd, in_bash=True) + + # add a symlink for liblto_plugin.so to work + # could not find how to direct gcc to the right folder + ln_src = '%(platform)s/libexec' % {'platform': platform} + dest = '%(platform)s/%(arch_full)s' % {'platform': platform, + 'arch_full': arch_full} + syscall('ln -s %(src)s %(dest)s/' + % {'src': ln_src, 'dest': dest}) + + # change the PATH for compilation to use proper tools + new_environ = {'PATH': ('%(platform)s/bin:%(platform)s/%(arch_full)s' + '/bin:%(platform)s/libexec/gcc/%(arch_full)s/' + '%(gccver)s/:%(orig)s' + % {'platform': platform, + 'orig': ORIGINAL_ENVIRON['PATH'], + 'arch_full': arch_full, + 'gccver': COMPILER_VERSION}), + 'CFLAGS': ' -fPIC ' + } + change_env(new_environ) + + # compile liblzma.a, liblzma.so + os.chdir(LIBLZMA_SRC) + configure_cmd = ('./configure --host=%(arch)s --prefix=%(platform)s ' + '--disable-assembler --enable-shared --enable-static' + % {'arch': arch_full, + 'platform': platform}) + if COMPILE_LIBLZMA: + # configure, compile, copy and clean liblzma from official sources. + # even though we need only static, we conpile also shared so it + # switches the -fPIC properly. + syscall(configure_cmd, in_bash=True) + syscall('make clean') + syscall('make') + syscall('make install') + syscall('make clean') + + # create libzim.a + os.chdir(curdir) + platform_includes = [ + '%(platform)s/include/c++/%(gccver)s/' + % {'platform': platform, 'gccver': COMPILER_VERSION}, + + '%(platform)s/include/c++/%(gccver)s/%(arch_full)s' + % {'platform': platform, 'gccver': COMPILER_VERSION, + 'arch_full': arch_full}, + + '%(platform)s/sysroot/usr/include/' + % {'platform': platform}, + + '%(platform)s/lib/gcc/%(arch_full)s/' + '%(gccver)s/include' + % {'platform': platform, 'arch_full': arch_full, + 'gccver': COMPILER_VERSION}, + + '%(platform)s/lib/gcc/%(arch_full)s/%(gccver)s' + '/include-fixed' + % {'platform': platform, 'arch_full': arch_full, + 'gccver': COMPILER_VERSION}, + + '%(platform)s/sysroot/usr/include/linux/' + % {'platform': platform} + ] + + src_dir = os.path.join(LIBZIM_SRC, 'src') + compile_cmd = ('g++ -fPIC -c -D_FILE_OFFSET_BITS=64 ' + '-D_LARGEFILE64_SOURCE ' + '-B%(platform)s/sysroot ' + '%(source_files)s -I%(include_paths)s ' + % {'platform': platform, + 'arch_full': arch_full, + 'gccver': COMPILER_VERSION, + 'source_files': ' '.join([os.path.join(src_dir, src) + for src + in LIBZIM_SOURCE_FILES]), + 'include_paths': ' -I'.join(LIBLZMA_INCLUDES + + LIBZIM_INCLUDES + + platform_includes)}) + link_cmd = ('ar rvs libzim.a ' + '%(obj_files)s ' + % {'obj_files': ' '.join([n.replace('.cpp', '.o') + for n in LIBZIM_SOURCE_FILES])}) + + if COMPILE_LIBZIM: + syscall(compile_cmd) + syscall(link_cmd) + + libzim_file = os.path.join(curdir, 'libzim.a') + shutil.copy(libzim_file, os.path.join(platform, 'lib')) + os.remove(libzim_file) + + for src in LIBZIM_SOURCE_FILES: + os.remove(src.replace('.cpp', '.o')) + + # compile JNI header + syscall('javac JNIKiwix.java') + syscall('javah -jni JNIKiwix') + + # create libkiwix.so + os.chdir(curdir) + compile_cmd = ('g++ -fPIC -c -B%(platform)s/sysroot ' + 'kiwix.c %(kwsrc)s/kiwix/reader.cpp %(kwsrc)s' + '/stringTools.cpp ' + '-I%(include_paths)s ' + % {'platform': platform, + 'arch_full': arch_full, + 'gccver': COMPILER_VERSION, + 'kwsrc': LIBKIWIX_SRC, + 'include_paths': ' -I'.join(LIBLZMA_INCLUDES + + LIBZIM_INCLUDES + + platform_includes + + [LIBKIWIX_SRC, + os.path.join(LIBZIM_SRC, + 'include')]) + }) + + link_cmd = ('g++ -fPIC -shared -B%(platform)s/sysroot ' + '--sysroot %(platform)s/sysroot ' + '-nostdlib ' + 'kiwix.o reader.o stringTools.o ' + '%(platform)s/lib/libzim.a %(platform)s/lib/liblzma.a ' + '-L%(platform)s/%(arch_full)s/lib ' + # '-lgnustl_shared -llog -landroid -lstdc++ ' + '%(platform)s/lib/gcc/%(arch_full)s/%(gccver)s/libgcc.a ' + '-o %(platform)s/lib/libkiwix.so' + % {'kwsrc': LIBKIWIX_SRC, + 'platform': platform, + 'arch_full': arch_full, + 'gccver': COMPILER_VERSION}) + + if COMPILE_LIBKIWIX: + syscall(compile_cmd) + syscall(link_cmd) + + for obj in ('kiwix.o', 'reader.o', 'stringTools.o'): + os.remove(obj) + + if STRIP_LIBKIWIX: + syscall('%(platform)s/%(arch_full)s/bin/strip ' + '%(platform)s/lib/libkiwix.so' + % {'platform': platform, + 'arch_full': arch_full}) + + os.chdir(curdir) + change_env(ORIGINAL_ENVIRON) diff --git a/devel_build-libkiwix.so.sh b/devel_build-libkiwix.so.sh new file mode 100755 index 000000000..99956f709 --- /dev/null +++ b/devel_build-libkiwix.so.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# this script is just for testing purposes +# it helps generate a libkiwix.so on a dev linux machine to assist +# with JNI debugging. + +# change the following to point to absolute of kiwix repo. +KIWIX_ROOT=`pwd`/../ + +cd $KIWIX_ROOT + +# compilation de liblzma.a +cd src/dependencies/xz-5.0.4 +CFLAGS=" -fPIC " ./configure --disable-assembler --enable-shared --enable-static +make +cp src/liblzma/.libs/liblzma.a $KIWIX_ROOT/android/ +make clean + +# compile libzim.a +cd $KIWIX_ROOT/src/zimlib/src +g++ -fPIC -c -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE article.cpp articlesearch.cpp cluster.cpp dirent.cpp file.cpp fileheader.cpp fileimpl.cpp indexarticle.cpp ptrstream.cpp search.cpp template.cpp unicode.cpp uuid.cpp zintstream.cpp envvalue.cpp lzmastream.cpp unlzmastream.cpp fstream.cpp md5.cpp md5stream.cpp -I. -I../include/ -I../../dependencies/xz-5.0.4/src/liblzma/api/ +ar rvs libzim.a article.o articlesearch.o cluster.o dirent.o file.o fileheader.o fileimpl.o indexarticle.o ptrstream.o search.o template.o unicode.o uuid.o zintstream.o envvalue.o lzmastream.o unlzmastream.o fstream.o md5.o md5stream.o +cp libzim.a $KIWIX_ROOT/android/ +rm *.o + +# compile libkiwix +cd $KIWIX_ROOT/android/ +rm *.o +g++ -fPIC -c kiwix.c $KIWIX_ROOT/src/common/kiwix/reader.cpp $KIWIX_ROOT/src/common/stringTools.cpp -I$KIWIX_ROOT/src/zimlib/include -I../../dependencies/xz-5.0.4/src/liblzma/api/ -I$KIWIX_ROOT/src/common -I/usr/lib/jvm/java-7-openjdk-amd64/include/ +g++ -fPIC -shared kiwix.o reader.o stringTools.o libzim.a liblzma.a /usr/lib/gcc/x86_64-linux-gnu/4.7/libgcc.a -o libkiwix.so + +ls -lh libkiwix.so \ No newline at end of file