Initial version of CustomApp generator

Generator script takes a JSON descriptor file.
CustomApp behavior is trigger on the Constants.IS_CUSTOM_APP switch
CustomApp looks for a companion file as ZIM and loads it up.
If not present, exits inviting to uninstall and reinstall from store.

build-android-with-native.py

@@ -18,6 +18,8 @@ from subprocess import call, check_output
 # arm-linux-androideabi, mipsel-linux-android, x86, llvm
 ALL_ARCHS = ['arm-linux-androideabi', 'mipsel-linux-android', 'x86']
 
+PACKAGE = 'org.kiwix.kiwixmobile'
+
 USAGE = '''Usage:  {arg0} [--option]
 
     Without option, all steps are executed on all archs.
@@ -36,7 +38,9 @@ USAGE = '''Usage:  {arg0} [--option]
 
     --on=ARCH       Disable steps on all archs and cherry pick the ones wanted.
                     Multiple --on=ARCH can be specified.
-                    ARCH in 'armeabi', 'mips', 'x86'. '''
+                    ARCH in 'armeabi', 'mips', 'x86'.
+
+    --package=org.kiwix.kiwixmobile '''
 
 
 def init_with_args(args):
@@ -79,6 +83,13 @@ def init_with_args(args):
         # recreate options list from other items
         options = [v for v in doptions.values() if not v.startswith('--on=')]
 
+    # do we have an --package= flag?
+    if '--package=' in u' '.join(args):
+        for option in options:
+            if option.startswith('--package'):
+                global PACKAGE
+                PACKAGE = option.split('=', 1)[-1]
+
     if len(options):
         # we received options.
         # consider we only want the specified steps
@@ -505,17 +516,17 @@ for arch in ARCHS:
     if COMPILE_LIBKIWIX:
 
         # compile JNI header
-        os.chdir(os.path.join(curdir, 'src', 'org', 'kiwix', 'kiwixmobile'))
+        os.chdir(os.path.join(curdir, 'src', *PACKAGE.split('.')))
         syscall('javac JNIKiwix.java')
         os.chdir(os.path.join(curdir, 'src'))
-        syscall('javah -jni org.kiwix.kiwixmobile.JNIKiwix')
+        syscall('javah -jni {package}.JNIKiwix'.format(package=PACKAGE))
         os.chdir(curdir)
 
         syscall(compile_cmd)
         syscall(link_cmd)
 
         for obj in ('kiwix.o', 'reader.o', 'stringTools.o', 'pathTools.o',
-                    'src/org_kiwix_kiwixmobile_JNIKiwix.h'):
+                    'src/{}_JNIKiwix.h'.format("_".join(PACKAGE.split('.')))):
             os.remove(obj)
 
     # check that the step went well

gen-custom-android-build.py

@@ -0,0 +1,339 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim: ai ts=4 sts=4 et sw=4 nu
+
+from __future__ import (unicode_literals, absolute_import,
+                        division, print_function)
+import sys
+import os
+import copy
+import json
+import shutil
+import logging
+import StringIO
+import tempfile
+import urllib2
+from subprocess import call
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+# the directory of this file for relative referencing
+CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
+
+# the parent directory of this file for relative referencing
+PARENT_PATH = os.path.dirname(CURRENT_PATH)
+
+ANDROID_PATH = tempfile.mkdtemp(prefix='android-custom-', dir=PARENT_PATH)
+
+DEFAULT_JSDATA = {
+    # mandatory fields
+    # 'app_name': "Kiwix Custom App",
+    # 'package': "org.kiwix.zim.custom",
+    # 'version_name': "1.0",
+    # 'zim_file': "wikipedia_bm.zim",
+    # 'license': None,
+
+    # main icon source & store icon
+    'ic_launcher': os.path.join('android',
+                                'Kiwix_icon_transparent_512x512.png'),
+
+    # store listing
+    'feature_image': None,
+    'phone_screenshot': None,
+    'tablet7_screenshot': None,
+    'tablet10_screenshot': None,
+    'category': '',
+    'rating': 'everyone',
+    'website': "http://kiwix.org",
+    'email': "kelson@kiwix.org",
+
+    # help page content
+    'support_email': "kelson@kiwix.org"
+}
+
+SIZE_MATRIX = {
+    'xhdpi': 96,
+    'mdpi': 72,
+    'ldpi': 48,
+    'hdpi': 72,
+}
+
+PERMISSIONS = [
+    'com.android.vending.CHECK_LICENSE',  # access Google Play Licensing
+    'android.permission.WAKE_LOCK',  # keep CPU alive while downloading files
+    'android.permission.ACCESS_WIFI_STATE'  # check whether Wi-Fi is enabled
+]
+
+# external dependencies (make sure we're all set up!)
+try:
+    import requests
+    from bs4 import BeautifulSoup
+    # check for convert (imagemagick)
+except ImportError:
+    logger.error("Missing dependency: Unable to import requests.\n"
+                 "Please install requests with "
+                 "`pip install requests BeautifulSoup4 lxml` "
+                 "either on your machine or in a virtualenv.")
+    sys.exit(1)
+
+# JSON fields that are mandatory to build
+required_fields = ('app_name', 'package', 'version_name', 'version_code',
+                   'zim_file')
+
+
+def usage(arg0, exit=None):
+    print("Usage: {} <json_file>".format(arg0))
+    if exit is not None:
+        sys.exit(exit)
+
+
+def syscall(args, shell=False, with_print=True):
+    ''' make a system call '''
+    args = args.split()
+    if with_print:
+        print(u"-----------\n" + u" ".join(args) + u"\n-----------")
+
+    if shell:
+        args = ' '.join(args)
+    call(args, shell=shell)
+
+
+def get_remote_content(url):
+    req = requests.get(url)
+    try:
+        req.raise_for_status()
+    except Exception as e:
+        logger.error("Failed to load data at `{}`".format(url))
+        logger.exception(e)
+        sys.exit(1)
+    return StringIO.StringIO(req.text)
+
+
+def get_local_content(path):
+    if not os.path.exists(path) or not os.path.isfile(path):
+        logger.error("Unable to find JSON file `{}`".format(path))
+        sys.exit(1)
+
+    try:
+        fd = open(path, 'r')
+    except Exception as e:
+        logger.error("Unable to open file `{}`".format(path))
+        logger.exception(e)
+        sys.exit(1)
+    return fd
+
+
+def is_remote_path(path):
+    return path.startswith('http:')
+
+
+def get_local_remote_fd(path):
+    if is_remote_path(path):
+        return get_remote_content(path)
+    else:
+        return get_local_content(path)
+
+
+def copy_to(src, dst):
+    if is_remote_path(src):
+        local = tempfile.NamedTemporaryFile(delete=False)
+        local.write(get_remote_content(src))
+        local.close()
+        src = local
+    shutil.copy(src, dst)
+
+
+def get_remote_url_size(url):
+    try:
+        return int(urllib2.urlopen(url).info().getheaders("Content-Length")[0])
+    except:
+        return None
+
+
+def download_remote_file(url, path):
+    req = requests.get(url)
+    req.raise_for_status()
+    with open(path, 'w') as f:
+        f.write(req.text)
+
+
+def get_file_size(path):
+    if is_remote_path(path):
+        url = path
+        size = get_remote_url_size(url)
+        if size is not None:
+            return size
+        path = "fetched-zim_{}".format(url.rsplit('/', 1))
+        download_remote_file(url, path)
+    return os.stat(path).st_size
+
+
+def flushxml(dom, rootNodeName, fpath, head=True):
+    head = '<?xml version="1.0" encoding="utf-8"?>\n' if head else ''
+    with open(fpath, 'w') as f:
+        f.write("{head}{content}"
+                .format(head=head,
+                        content=dom.find(rootNodeName).encode()))
+
+
+def main(args):
+
+    # ensure we were provided a Json argument
+    if len(args) < 2:
+        usage(args[0], 1)
+
+    jspath = args[1]
+
+    fd = get_local_remote_fd(jspath)
+
+    # parse the json file
+    jsdata = copy.copy(DEFAULT_JSDATA)
+    try:
+        jsdata.update(json.load(fd))
+    except Exception as e:
+        logger.error("Unable to parse JSON file `{}`. Might be malformed."
+                     .format(jspath))
+        logger.exception(e)
+        sys.exit(1)
+
+    # ensure required properties are present
+    for key in required_fields:
+        if key not in jsdata.keys():
+            logger.error("Required field `{}` is missing from JSON file."
+                         .format(key))
+            logger.error("Required fields are: {}"
+                         .format(", ".join(required_fields)))
+            sys.exit(1)
+
+    # ensure ZIM file is present and find file size
+    jsdata.update({'zim_size': str(get_file_size(jsdata.get('zim_file')))})
+
+    # greetings
+    logger.info("Your are now building {app_name} version {version_name} "
+                "at {path}"
+                .format(app_name=jsdata.get('app_name'),
+                        version_name=jsdata.get('version_name'),
+                        path=ANDROID_PATH))
+
+    # move to PARENT_PATH (Kiwix main root) to avoid relative remove hell
+    os.chdir(PARENT_PATH)
+
+    # remove android folder if exists
+    shutil.rmtree(ANDROID_PATH)
+
+    # copy the whole android tree
+    shutil.copytree(os.path.join(PARENT_PATH, 'android'),
+                    ANDROID_PATH, symlinks=True)
+
+    # move to the newly-created android tree
+    os.chdir(ANDROID_PATH)
+
+    # copy launcher icons
+    copy_to(jsdata.get('ic_launcher'), os.path.join(ANDROID_PATH,
+                                                    'ic_launcher_512.png'))
+    # create multiple size icons
+    for density, pixels in SIZE_MATRIX.items():
+        syscall("convert {inf} -resize {p}x{p} {outf}"
+                .format(inf=os.path.join(ANDROID_PATH, 'ic_launcher_512.png'),
+                        p=pixels,
+                        outf=os.path.join(ANDROID_PATH, 'res',
+                                          'drawable-{}'.format(density),
+                                          'kiwix_icon.png')))
+
+    # copy and rewrite res/values/branding.xml
+    branding_xml = os.path.join(ANDROID_PATH, 'res', 'values', 'branding.xml')
+    soup = soup = BeautifulSoup(open(branding_xml, 'r'),
+                                'xml', from_encoding='utf-8')
+    for elem in soup.findAll('string'):
+        elem.string.replace_with(
+            elem.text.replace('Kiwix', jsdata.get('app_name')))
+    flushxml(soup, 'resources', branding_xml)
+
+    # copy and rewrite src/org/kiwix/kiwimobile/settings/Constants.java
+    shutil.copy(os.path.join(ANDROID_PATH, 'templates', 'Constants.java'),
+                os.path.join(ANDROID_PATH, 'src', 'org', 'kiwix',
+                             'kiwixmobile', 'settings', 'Constants.java'))
+    cpath = os.path.join(ANDROID_PATH, 'src', 'org', 'kiwix',
+                         'kiwixmobile', 'settings', 'Constants.java')
+    content = open(cpath, 'r').read()
+    for key, value in jsdata.items():
+        content = content.replace('~{key}~'.format(key=key), value or '')
+    with open(cpath, 'w') as f:
+        f.write(content)
+
+    # Parse and edit res/menu/main.xml
+    menu_xml = os.path.join(ANDROID_PATH, 'res', 'menu', 'main.xml')
+    soup = soup = BeautifulSoup(open(menu_xml, 'r'),
+                                'xml', from_encoding='utf-8')
+    for elem in soup.findAll('item'):
+        if elem.get('android:id') == '@+id/menu_openfile':
+            elem['android:showAsAction'] = "never"
+            elem['android:visible'] = "false"
+    flushxml(soup, 'menu', menu_xml, head=False)
+
+    # Parse and edit AndroidManifest.xml
+    manif_xml = os.path.join(ANDROID_PATH, 'AndroidManifest.xml')
+    soup = soup = BeautifulSoup(open(manif_xml, 'r'),
+                                'xml', from_encoding='utf-8')
+    # change package
+    manifest = soup.find('manifest')
+    manifest['package'] = jsdata.get('package')
+    # change versionCode & versionName
+    manifest['android:versionCode'] = jsdata.get('version_code')
+    manifest['android:versionName'] = jsdata.get('version_name')
+
+    # remove file opening intents
+    for intent in soup.findAll('intent-filter'):
+        if not intent.find("action")['android:name'].endswith('.VIEW'):
+            # only remove VIEW intents (keep LAUNCHER and GET_CONTENT)
+            continue
+        intent.replace_with('')
+    flushxml(soup, 'manifest', manif_xml)
+    # move kiwixmobile to proper package name
+    package_tail = jsdata.get('package').split('.')[-1]
+    shutil.move(
+        os.path.join(ANDROID_PATH, 'src', 'org', 'kiwix', 'kiwixmobile'),
+        os.path.join(ANDROID_PATH, 'src', 'org', 'kiwix', package_tail))
+    # replace package in every file
+    for dirpath, dirnames, filenames in os.walk(ANDROID_PATH):
+        for filename in filenames:
+            if filename.endswith('.java') or \
+                    filename in ('AndroidManifest.xml', 'main.xml'):
+                fpath = os.path.join(dirpath, filename)
+                content = open(fpath, 'r').read()
+                with open(fpath, 'w') as f:
+                    f.write(
+                        content.replace('org.kiwix.kiwixmobile',
+                                        jsdata.get('package'))
+                               .replace('org.kiwix.zim.base',
+                                        'org.kiwix.zim.{}'
+                                        .format(package_tail)))
+    # rewrite kiwix.c for JNI
+    fpath = os.path.join(ANDROID_PATH, 'kiwix.c')
+    content = open(fpath, 'r').read()
+    with open(fpath, 'w') as f:
+        f.write(content.replace('org_kiwix_kiwixmobile',
+                                "_".join(jsdata.get('package').split('.'))))
+
+    # compile KiwixAndroid
+    syscall('./build-android-with-native.py '
+            '--toolchain '
+            '--lzma '
+            '--icu '
+            '--zim '
+            '--kiwix '
+            '--strip '
+            '--locales '
+            '--apk '
+            '--clean '
+            '--package={}'
+            .format(jsdata.get('package')))  # --apk --clean')
+
+    # copy APK somewhere
+
+    # delete temp folder
+    # shutil.rmtree(ANDROID_PATH)
+
+if __name__ == '__main__':
+    main(sys.argv)

res/values/strings.xml

@@ -57,5 +57,7 @@
     <string name="pref_language_chooser">Choose a language</string>
     <string name="error_internet_connection_failed">Couldn\'t connect to the internet.</string>
     <string name="tts_lang_not_supported">The language of this page is not supported, or appropriate language data was not installed. The article may not be properly read.</string>
-    <string name="no_reader_application_installed">"Could not find an installed application for this type of file"</string>
+    <string name="no_reader_application_installed">Could not find an installed application for this type of file</string>
+    <string name="customapp_missing_content">Your application is corrupted.\nThis might happen when you remove files on the SD Card.\nYou need to uninstall then reinstall the App from the Play Store.</string>
+    <string name="go_to_play_store">Go to Play Store</string>
 </resources>

src/org/kiwix/kiwixmobile/FileUtils.java

@@ -3,6 +3,8 @@ package org.kiwix.kiwixmobile;
 import android.content.Context;
 import android.os.Environment;
 
+import org.kiwix.kiwixmobile.settings.Constants;
+
 import java.io.File;
 
 public class FileUtils {
@@ -25,4 +27,56 @@ public class FileUtils {
         }
 
     }
+
+    /**
+     * Returns the file name (without full path) for an Expansion APK file from
+     * the given context.
+     *
+     * @param mainFile true for main file, false for patch file
+     * @return String the file name of the expansion file
+     */
+    public static String getExpansionAPKFileName(boolean mainFile) {
+        return (mainFile ? "main." : "patch.") + Constants.CUSTOM_APP_VERSION_CODE + "." + Constants.CUSTOM_APP_ID + ".obb";
+    }
+
+    /**
+     * Returns the filename (where the file should be saved) from info about a
+     * download
+     */
+    static public String generateSaveFileName(String fileName) {
+        return getSaveFilePath() + File.separator + fileName;
+    }
+
+    static public String getSaveFilePath() {
+        String obbFolder = File.separator + "Android" + File.separator + "obb" + File.separator;
+        File root = Environment.getExternalStorageDirectory();
+        return root.toString() + obbFolder + Constants.CUSTOM_APP_ID;
+    }
+
+    /**
+     * Helper function to ascertain the existence of a file and return
+     * true/false appropriately
+     *
+     * @param fileName the name (sans path) of the file to query
+     * @param fileSize the size that the file must match
+     * @param deleteFileOnMismatch if the file sizes do not match, delete the
+     *            file
+     * @return true if it does exist, false otherwise
+     */
+    static public boolean doesFileExist(String fileName, long fileSize, boolean deleteFileOnMismatch) {
+        // the file may have been delivered by Market --- let's make sure
+        // it's the size we expect
+        File fileForNewFile = new File(generateSaveFileName(fileName));
+        if (fileForNewFile.exists()) {
+            if (fileForNewFile.length() == fileSize) {
+                return true;
+            }
+            if (deleteFileOnMismatch) {
+                // delete the file --- we won't be able to resume
+                // because we cannot confirm the integrity of the file
+                fileForNewFile.delete();
+            }
+        }
+        return false;
+    }
 }

src/org/kiwix/kiwixmobile/KiwixMobileFragment.java

@@ -321,6 +321,38 @@ public class KiwixMobileFragment extends SherlockFragment {
                 openZimFile(new File(zimFile), false);
                 // Alternative would be to restore webView state. But more effort to implement, and actually
                 // fits better normal android behavior if after closing app ("back" button) state is not maintained.
+            } else {
+                if (Constants.IS_CUSTOM_APP) {
+                    Log.d(TAG_KIWIX, "Kiwix Custom App starting for the first time. Check Companion ZIM.");
+
+                    //Context context = this.getActivity().getApplicationContext();
+                    String fileName = FileUtils.getExpansionAPKFileName(true);
+                    Log.d(TAG_KIWIX, "Looking for: " + fileName + " -- filesize: " + Constants.ZIM_FILE_SIZE);
+                    if (!FileUtils.doesFileExist(fileName, Constants.ZIM_FILE_SIZE, false)) {
+                        Log.d(TAG_KIWIX, "... doesn't exist.");
+
+                        AlertDialog.Builder zimFileMissingBuilder = new AlertDialog.Builder(this.getActivity());
+                        zimFileMissingBuilder.setTitle(R.string.app_name);
+                        zimFileMissingBuilder.setMessage(R.string.customapp_missing_content);
+                        zimFileMissingBuilder.setIcon(R.drawable.kiwix_icon);
+                        final Activity activity = this.getActivity();
+                        zimFileMissingBuilder.setPositiveButton(getString(R.string.go_to_play_store),
+                                new DialogInterface.OnClickListener() {
+                                    public void onClick(DialogInterface dialog, int which) {
+                                        String market_uri = "market://details?id=" + Constants.CUSTOM_APP_ID;
+                                        Intent intent = new Intent(Intent.ACTION_VIEW);
+                                        intent.setData(Uri.parse(market_uri));
+                                        startActivity(intent);
+                                        activity.finish();
+                                        System.exit(0);
+                                    }
+                                });
+                        zimFileMissingBuilder.setCancelable(false);
+                        AlertDialog zimFileMissingDialog = zimFileMissingBuilder.create();
+                        zimFileMissingDialog.show();
+                    } else {
+                        openZimFile(new File(FileUtils.generateSaveFileName(fileName)), true);
+                    }
                 } else {
                     Log.d(TAG_KIWIX,
                             " Kiwix normal start, no zimFile loaded last time  -> display welcome page");
@@ -328,6 +360,7 @@ public class KiwixMobileFragment extends SherlockFragment {
                 }
             }
         }
+    }
 
     private void setUpArticleSearchTextView(Bundle savedInstanceState) {
 

src/org/kiwix/kiwixmobile/settings/Constants.java

@@ -2,6 +2,12 @@ package org.kiwix.kiwixmobile.settings;
 
 public class Constants {
     public static final boolean IS_CUSTOM_APP = false;
-    public static final String CUSTOM_APP_ID = null;
-    public static final String CUSTOM_APP_SUPPORT_EMAIL = "kelson@kiwix.org";
+    public static final String CUSTOM_APP_ID = "~package~";
+    public static final long ZIM_FILE_SIZE = 0;
+    public static final String CUSTOM_APP_VERSION_NAME = "~version_name~";
+    public static final int CUSTOM_APP_VERSION_CODE = 2;
+    public static final String CUSTOM_APP_LICENSE = "~license~";
+    public static final String CUSTOM_APP_WEBSITE = "~website~";
+    public static final String CUSTOM_APP_EMAIL = "~support_email~";
+    public static final String CUSTOM_APP_SUPPORT_EMAIL = "~support_email~";
 }

templates/Constants.java

@@ -0,0 +1,13 @@
+package org.kiwix.kiwixmobile.settings;
+
+public class Constants {
+    public static final boolean IS_CUSTOM_APP = true;
+    public static final String CUSTOM_APP_ID = "~package~";
+    public static final long ZIM_FILE_SIZE = ~zim_size~;
+    public static final String CUSTOM_APP_VERSION_NAME = "~version_name~";
+    public static final int CUSTOM_APP_VERSION_CODE = ~version_code~;
+    public static final String CUSTOM_APP_LICENSE = "~license~";
+    public static final String CUSTOM_APP_WEBSITE = "~website~";
+    public static final String CUSTOM_APP_EMAIL = "~support_email~";
+    public static final String CUSTOM_APP_SUPPORT_EMAIL = "~support_email~";
+}