CustomApp now works with embedded zim file

This commit is contained in:
renaud gaudin 2015-06-06 12:26:11 +02:00
parent 05aab203f4
commit 682c4ba318
7 changed files with 332 additions and 102 deletions

View File

@ -11,6 +11,7 @@ import re
import sys
import copy
import shutil
import tempfile
from xml.dom.minidom import parse
from subprocess import call, check_output
@ -541,6 +542,17 @@ for arch in ARCHS:
'arch_short': arch_short,
'curdir': curdir})
if COMPILE_LIBKIWIX:
# create content-libs.jar
tmpd = tempfile.mkdtemp()
for arch in ARCHS:
os.makedirs(os.path.join(tmpd, 'lib', arch))
# shutil.copy(os.path.join('libs', arch, 'libkiwix.so'),
# os.path.join(tmpd, 'lib', arch, 'libkiwix.so'))
os.chdir(tmpd)
syscall('zip -r -0 -y {} lib'
.format(os.path.join(curdir, 'content-libs.jar')))
os.chdir(curdir)
change_env(ORIGINAL_ENVIRON)

View File

@ -21,6 +21,7 @@ dependencies {
compile 'com.android.support:appcompat-v7:22.2.0'
compile 'com.android.support:support-v4:22.2.0'
compile files("$buildDir/native-libs/native-libs.jar")
compile fileTree(dir: '.', include: 'content-libs.jar')
}

View File

@ -2,10 +2,32 @@
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu
''' Generate a custom build of Kiwix for Android working with a single content
The generated App either embed the ZIM file inside (creating large APKs)
or is prepared to make use of a Play Store comapnion file.
APKs uploaded to Play Store are limited to 50MB in size and can have
up to 2 comapnion files of 2GB each.
Note: multiple companion files is not supported currently
~~ needs update to the libzim.
The companion file is stored (by the Play Store) on the SD card.
Large APKs can be distributed outside the Play Store.
Note that the larger the APK, the longer it takes to install.
Also, APKs are downloaded then extracted to the *internal* storage
of the device unless the user specificaly change its settings to
install to SD card.
Standard usage is to launch the script with a single JSON file as argument.
Take a look at JSDATA sample in this script's source code for
required and optional values to include. '''
from __future__ import (unicode_literals, absolute_import,
division, print_function)
import sys
import os
import re
import copy
import json
import shutil
@ -13,19 +35,12 @@ import logging
import StringIO
import tempfile
import urllib2
from collections import OrderedDict
from subprocess import call
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
# the directory of this file for relative referencing
CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
# the parent directory of this file for relative referencing
PARENT_PATH = os.path.dirname(CURRENT_PATH)
ANDROID_PATH = tempfile.mkdtemp(prefix='android-custom-', dir=PARENT_PATH)
DEFAULT_JSDATA = {
# mandatory fields
# 'app_name': "Kiwix Custom App",
@ -33,6 +48,8 @@ DEFAULT_JSDATA = {
# 'version_name': "1.0",
# 'zim_file': "wikipedia_bm.zim",
# 'license': None,
'enforced_lang': None,
'embed_zim': False,
# main icon source & store icon
'ic_launcher': os.path.join('android',
@ -50,8 +67,6 @@ DEFAULT_JSDATA = {
# help page content
'support_email': "kelson@kiwix.org",
'enforced_lang': None
}
SIZE_MATRIX = {
@ -61,11 +76,17 @@ SIZE_MATRIX = {
'hdpi': 72,
}
PERMISSIONS = [
'com.android.vending.CHECK_LICENSE', # access Google Play Licensing
'android.permission.WAKE_LOCK', # keep CPU alive while downloading files
'android.permission.ACCESS_WIFI_STATE' # check whether Wi-Fi is enabled
]
# JSON fields that are mandatory to build
REQUIRED_FIELDS = ('app_name', 'package', 'version_name', 'version_code',
'zim_file')
# the directory of this file for relative referencing
CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
# the parent directory of this file for relative referencing
PARENT_PATH = os.path.dirname(CURRENT_PATH)
ANDROID_PATH = tempfile.mkdtemp(prefix='android-custom-', dir=PARENT_PATH)
# external dependencies (make sure we're all set up!)
try:
@ -79,19 +100,9 @@ except ImportError:
"either on your machine or in a virtualenv.")
sys.exit(1)
# JSON fields that are mandatory to build
required_fields = ('app_name', 'package', 'version_name', 'version_code',
'zim_file')
def usage(arg0, exit=None):
print("Usage: {} <json_file>".format(arg0))
if exit is not None:
sys.exit(exit)
def syscall(args, shell=False, with_print=True):
''' make a system call '''
''' execute an external command. Use shell=True if using bash specifics '''
args = args.split()
if with_print:
print(u"-----------\n" + u" ".join(args) + u"\n-----------")
@ -102,6 +113,7 @@ def syscall(args, shell=False, with_print=True):
def get_remote_content(url):
''' file descriptor from remote file using GET '''
req = requests.get(url)
try:
req.raise_for_status()
@ -113,6 +125,7 @@ def get_remote_content(url):
def get_local_content(path):
''' file descriptor from local file '''
if not os.path.exists(path) or not os.path.isfile(path):
logger.error("Unable to find JSON file `{}`".format(path))
sys.exit(1)
@ -131,6 +144,7 @@ def is_remote_path(path):
def get_local_remote_fd(path):
''' file descriptor for a path (either local or remote) '''
if is_remote_path(path):
return get_remote_content(path)
else:
@ -138,6 +152,7 @@ def get_local_remote_fd(path):
def copy_to(src, dst):
''' copy source content (local or remote) to local file '''
if is_remote_path(src):
local = tempfile.NamedTemporaryFile(delete=False)
local.write(get_remote_content(src))
@ -154,6 +169,7 @@ def get_remote_url_size(url):
def download_remote_file(url, path):
''' download url to path '''
req = requests.get(url)
req.raise_for_status()
with open(path, 'w') as f:
@ -161,6 +177,7 @@ def download_remote_file(url, path):
def get_file_size(path):
''' file size in bytes of a path (either remote or local) '''
if is_remote_path(path):
url = path
size = get_remote_url_size(url)
@ -172,6 +189,7 @@ def get_file_size(path):
def flushxml(dom, rootNodeName, fpath, head=True):
''' write back XML from a BeautifulSoup DOM and root element '''
head = '<?xml version="1.0" encoding="utf-8"?>\n' if head else ''
with open(fpath, 'w') as f:
f.write("{head}{content}"
@ -179,44 +197,16 @@ def flushxml(dom, rootNodeName, fpath, head=True):
content=dom.find(rootNodeName).encode()))
def main(args):
def move_to_android_placeholder():
os.chdir(ANDROID_PATH)
# ensure we were provided a Json argument
if len(args) < 2:
usage(args[0], 1)
jspath = args[1]
def move_to_current_folder():
os.chdir(CURRENT_PATH)
fd = get_local_remote_fd(jspath)
# parse the json file
jsdata = copy.copy(DEFAULT_JSDATA)
try:
jsdata.update(json.load(fd))
except Exception as e:
logger.error("Unable to parse JSON file `{}`. Might be malformed."
.format(jspath))
logger.exception(e)
sys.exit(1)
# ensure required properties are present
for key in required_fields:
if key not in jsdata.keys():
logger.error("Required field `{}` is missing from JSON file."
.format(key))
logger.error("Required fields are: {}"
.format(", ".join(required_fields)))
sys.exit(1)
# ensure ZIM file is present and find file size
jsdata.update({'zim_size': str(get_file_size(jsdata.get('zim_file')))})
# greetings
logger.info("Your are now building {app_name} version {version_name} "
"at {path}"
.format(app_name=jsdata.get('app_name'),
version_name=jsdata.get('version_name'),
path=ANDROID_PATH))
def step_create_android_placeholder(jsdata, **options):
''' copy the android source tree in a different place (yet level equiv.)'''
# move to PARENT_PATH (Kiwix main root) to avoid relative remove hell
os.chdir(PARENT_PATH)
@ -228,10 +218,12 @@ def main(args):
shutil.copytree(os.path.join(PARENT_PATH, 'android'),
ANDROID_PATH, symlinks=True)
# move to the newly-created android tree
os.chdir(ANDROID_PATH)
# copy launcher icons
def step_prepare_launcher_icons(jsdata, **options):
''' generate all-sizes icons from the 512 provided one '''
move_to_android_placeholder()
copy_to(jsdata.get('ic_launcher'), os.path.join(ANDROID_PATH,
'ic_launcher_512.png'))
# create multiple size icons
@ -243,6 +235,12 @@ def main(args):
'drawable-{}'.format(density),
'kiwix_icon.png')))
def step_update_branding_xml(jsdata, **options):
''' change app_name value in branding.xml '''
move_to_android_placeholder()
# copy and rewrite res/values/branding.xml
branding_xml = os.path.join(ANDROID_PATH, 'res', 'values', 'branding.xml')
soup = soup = BeautifulSoup(open(branding_xml, 'r'),
@ -252,18 +250,41 @@ def main(args):
elem.text.replace('Kiwix', jsdata.get('app_name')))
flushxml(soup, 'resources', branding_xml)
def step_gen_constants_java(jsdata, **options):
''' gen Java Source class (final constants) with all JSON values '''
move_to_android_placeholder()
# copy and rewrite src/org/kiwix/kiwimobile/settings/Constants.java
def value_cleaner(val):
if val is None:
return ""
if isinstance(val, bool):
return str(val).lower()
return str(val)
# copy template to actual location
shutil.copy(os.path.join(ANDROID_PATH, 'templates', 'Constants.java'),
os.path.join(ANDROID_PATH, 'src', 'org', 'kiwix',
'kiwixmobile', 'settings', 'Constants.java'))
cpath = os.path.join(ANDROID_PATH, 'src', 'org', 'kiwix',
'kiwixmobile', 'settings', 'Constants.java')
content = open(cpath, 'r').read()
# loop through JSON file keys are replace all values
for key, value in jsdata.items():
content = content.replace('~{key}~'.format(key=key), value or '')
content = content.replace('~{key}~'.format(key=key),
value_cleaner(value))
with open(cpath, 'w') as f:
f.write(content)
def step_update_main_menu_xml(jsdata, **options):
''' remove Open File menu item from main menu '''
move_to_android_placeholder()
# Parse and edit res/menu/main.xml
menu_xml = os.path.join(ANDROID_PATH, 'res', 'menu', 'main.xml')
soup = soup = BeautifulSoup(open(menu_xml, 'r'),
@ -274,10 +295,19 @@ def main(args):
elem['android:visible'] = "false"
flushxml(soup, 'menu', menu_xml, head=False)
def step_update_android_manifest(jsdata, **options):
''' update AndroidManifest.xml to set package, name, version
and remove intents (so that it's not a ZIM file reader) '''
move_to_android_placeholder()
# Parse and edit AndroidManifest.xml
manif_xml = os.path.join(ANDROID_PATH, 'AndroidManifest.xml')
soup = soup = BeautifulSoup(open(manif_xml, 'r'),
'xml', from_encoding='utf-8')
# change package
manifest = soup.find('manifest')
manifest['package'] = jsdata.get('package')
@ -292,11 +322,13 @@ def main(args):
continue
intent.replace_with('')
flushxml(soup, 'manifest', manif_xml)
# move kiwixmobile to proper package name
package_tail = jsdata.get('package').split('.')[-1]
shutil.move(
os.path.join(ANDROID_PATH, 'src', 'org', 'kiwix', 'kiwixmobile'),
os.path.join(ANDROID_PATH, 'src', 'org', 'kiwix', package_tail))
# replace package in every file
for dirpath, dirnames, filenames in os.walk(ANDROID_PATH):
for filename in filenames:
@ -311,6 +343,13 @@ def main(args):
.replace('org.kiwix.zim.base',
'org.kiwix.zim.{}'
.format(package_tail)))
def step_update_kiwix_c(jsdata, **options):
''' rewrite imports in JNI/C to match new package '''
move_to_android_placeholder()
# rewrite kiwix.c for JNI
fpath = os.path.join(ANDROID_PATH, 'kiwix.c')
content = open(fpath, 'r').read()
@ -318,7 +357,13 @@ def main(args):
f.write(content.replace('org_kiwix_kiwixmobile',
"_".join(jsdata.get('package').split('.'))))
# compile KiwixAndroid
def step_compile_libkiwix(jsdata, **options):
''' launch the native libkiwix script without building an APK '''
move_to_android_placeholder()
# compile libkiwix and all dependencies
syscall('./build-android-with-native.py '
'--toolchain '
'--lzma '
@ -326,12 +371,55 @@ def main(args):
'--zim '
'--kiwix '
'--strip '
'--locales '
'--apk '
'--clean '
'--package={}'
.format(jsdata.get('package'))) # --apk --clean')
'--locales ')
def step_embed_zimfile(jsdata, **options):
''' prepare a content-libs.jar file with ZIM file for inclusion in APK '''
move_to_android_placeholder()
if jsdata.get('embed_zim'):
# create content-libs.jar
tmpd = tempfile.mkdtemp()
archs = os.listdir('libs')
for arch in archs:
os.makedirs(os.path.join(tmpd, 'lib', arch))
# shutil.copy(os.path.join('libs', arch, 'libkiwix.so'),
# os.path.join(tmpd, 'lib', arch, 'libkiwix.so'))
copy_to(jsdata.get('zim_file'),
os.path.join(tmpd, 'lib', archs[0], jsdata.get('zim_name')))
for arch in archs[1:]:
os.chdir(os.path.join(tmpd, 'lib', arch))
os.symlink('../{}/{}'.format(archs[0], jsdata.get('zim_name')),
jsdata.get('zim_name'))
os.chdir(tmpd)
syscall('zip -r -0 -y {} lib'
.format(os.path.join(ANDROID_PATH, 'content-libs.jar')))
os.chdir(ANDROID_PATH)
def step_build_apk(jsdata, **options):
''' build the actual APK '''
move_to_android_placeholder()
# compile KiwixAndroid
syscall('./build-android-with-native.py '
'--apk '
'--clean ')
def step_move_apk_to_destination(jsdata, **options):
''' place and rename built APKs to main output directory '''
move_to_current_folder()
# ensure target directory exists (might not if kiwix was not built)
try:
os.makedirs(os.path.join(CURRENT_PATH, 'build', 'outputs', 'apk'))
except OSError:
pass
# move generated APK to satisfy other scripts
for variant in ('debug', 'debug-unaligned', 'release-unsigned'):
shutil.move(os.path.join(ANDROID_PATH, 'build', 'outputs', 'apk',
@ -341,8 +429,121 @@ def main(args):
"{}-{}.apk"
.format(jsdata.get('package'), variant)))
def step_remove_android_placeholder(jsdata, **options):
''' remove created (temp) android placeholder (useless is success) '''
move_to_current_folder()
# delete temp folder
shutil.rmtree(ANDROID_PATH)
def step_list_output_apk(jsdata, **options):
''' ls on the expected APK to check presence and size '''
move_to_current_folder()
syscall('ls -lh build/outputs/apk/{}-*'
.format(jsdata.get('package')), shell=True)
ARGS_MATRIX = OrderedDict([
('setup', step_create_android_placeholder),
('icons', step_prepare_launcher_icons),
('branding', step_update_branding_xml),
('constants', step_gen_constants_java),
('menu', step_update_main_menu_xml),
('manifest', step_update_android_manifest),
('jni', step_update_kiwix_c),
('libkiwix', step_compile_libkiwix),
('embed', step_embed_zimfile),
('build', step_build_apk),
('move', step_move_apk_to_destination),
('clean', step_remove_android_placeholder),
('list', step_list_output_apk),
])
def usage(arg0, exit=None):
usage_txt = "Usage: {} <json_file>".format(arg0)
for idx, step in enumerate(ARGS_MATRIX.keys()):
if idx > 0 and idx % 3 == 0:
usage_txt += "\n\t\t\t\t\t\t"
usage_txt += " [--{}]".format(step)
print(usage_txt)
print("\tjson_file:\t\tmandatory parameter holder (cf. source for sample)")
print("\t--step:\t\t\trun this step. if none specified, all are run.")
if exit is not None:
sys.exit(exit)
def main(jspath, **options):
fd = get_local_remote_fd(jspath)
# parse the json file
jsdata = copy.copy(DEFAULT_JSDATA)
try:
jsdata.update(json.load(fd))
except Exception as e:
logger.error("Unable to parse JSON file `{}`. Might be malformed."
.format(jspath))
logger.exception(e)
sys.exit(1)
# ensure required properties are present
for key in REQUIRED_FIELDS:
if key not in jsdata.keys():
logger.error("Required field `{}` is missing from JSON file."
.format(key))
logger.error("Required fields are: {}"
.format(", ".join(REQUIRED_FIELDS)))
sys.exit(1)
def zim_name_from_path(path):
fname = path.rsplit('/', 1)[-1]
return re.sub(r'[^a-z0-9\_.]+', '_', fname.lower())
# ensure ZIM file is present and find file size
jsdata.update({'zim_size': str(get_file_size(jsdata.get('zim_file')))})
jsdata.update({'zim_name': zim_name_from_path(jsdata.get('zim_file'))})
if jsdata.get('embed_zim'):
jsdata.update({'zim_name': 'libcontent.so'})
# greetings
logger.info("Your are now building {app_name} version {version_name} "
"at {path} for {zim_name}"
.format(app_name=jsdata.get('app_name'),
version_name=jsdata.get('version_name'),
path=ANDROID_PATH,
zim_name=jsdata.get('zim_name')))
# loop through each step and execute if requested by command line
for step_name, step_func in ARGS_MATRIX.items():
if options.get('do_{}'.format(step_name), False):
step_func(jsdata, **options)
if __name__ == '__main__':
main(sys.argv)
# ensure we were provided a JSON file as first argument
if len(sys.argv) < 2:
usage(sys.argv[0], 1)
else:
jspath = sys.argv[1]
args = sys.argv[2:]
if len(args) == 0:
options = OrderedDict([('do_{}'.format(step), True)
for step in ARGS_MATRIX.keys()])
else:
options = OrderedDict()
for arg in args:
step_name = re.sub(r'^\-\-', '', arg)
if step_name not in ARGS_MATRIX.keys():
logger.error("{} not a valid step. Exiting.".format(step_name))
usage(sys.argv[0], 1)
else:
options.update({'do_{}'.format(step_name): True})
main(jspath, **options)

View File

@ -60,11 +60,10 @@ public class FileUtils {
* @param deleteFileOnMismatch if the file sizes do not match, delete the file
* @return true if it does exist, false otherwise
*/
static public boolean doesFileExist(String fileName, long fileSize,
boolean deleteFileOnMismatch) {
static public boolean doesFileExist(String fileName, long fileSize, boolean deleteFileOnMismatch) {
// the file may have been delivered by Market --- let's make sure
// it's the size we expect
File fileForNewFile = new File(generateSaveFileName(fileName));
File fileForNewFile = new File(fileName);
if (fileForNewFile.exists()) {
if (fileForNewFile.length() == fileSize) {
return true;

View File

@ -396,19 +396,38 @@ public class KiwixMobileActivity extends AppCompatActivity
class JsObject {
@JavascriptInterface
public String appName() {
return getResources().getString(R.string.app_name);
}
public boolean isCustomApp() { return Constants.IS_CUSTOM_APP; }
@JavascriptInterface
public String supportEmail() {
return Constants.CUSTOM_APP_SUPPORT_EMAIL;
}
public String appId() { return Constants.CUSTOM_APP_ID; }
@JavascriptInterface
public String appId() {
return Constants.CUSTOM_APP_ID;
}
public boolean hasEmbedZim() { return Constants.CUSTOM_APP_HAS_EMBEDDED_ZIM; }
@JavascriptInterface
public String zimFileName() { return Constants.CUSTOM_APP_ZIM_FILE_NAME; }
@JavascriptInterface
public long zimFileSize() { return Constants.CUSTOM_APP_ZIM_FILE_SIZE; }
@JavascriptInterface
public String versionName() { return Constants.CUSTOM_APP_VERSION_NAME; }
@JavascriptInterface
public int versionCode() { return Constants.CUSTOM_APP_VERSION_CODE; }
@JavascriptInterface
public String website() { return Constants.CUSTOM_APP_WEBSITE; }
@JavascriptInterface
public String email() { return Constants.CUSTOM_APP_EMAIL; }
@JavascriptInterface
public String supportEmail() { return Constants.CUSTOM_APP_SUPPORT_EMAIL; }
@JavascriptInterface
public String enforcedLang() { return Constants.CUSTOM_APP_ENFORCED_LANG; }
}
webView.addJavascriptInterface(new JsObject(), "branding");
webView.loadUrl("file:///android_res/raw/help_custom.html");
@ -1105,11 +1124,16 @@ public class KiwixMobileActivity extends AppCompatActivity
this.startActivity(new Intent(this, this.getClass()));
}
//Context context = this.getApplicationContext();
String fileName = FileUtils.getExpansionAPKFileName(true);
Log.d(TAG_KIWIX, "Looking for: " + fileName + " -- filesize: "
+ Constants.ZIM_FILE_SIZE);
if (!FileUtils.doesFileExist(fileName, Constants.ZIM_FILE_SIZE, false)) {
String filePath;
if (Constants.CUSTOM_APP_HAS_EMBEDDED_ZIM) {
filePath = String.format("/data/data/%s/lib/%s", Constants.CUSTOM_APP_ID, Constants.CUSTOM_APP_ZIM_FILE_NAME);
} else {
String fileName = FileUtils.getExpansionAPKFileName(true);
filePath = FileUtils.generateSaveFileName(fileName);
}
Log.d(TAG_KIWIX, "Looking for: " + filePath + " -- filesize: " + Constants.CUSTOM_APP_ZIM_FILE_SIZE);
if (!FileUtils.doesFileExist(filePath, Constants.CUSTOM_APP_ZIM_FILE_SIZE, false)) {
Log.d(TAG_KIWIX, "... doesn't exist.");
AlertDialog.Builder zimFileMissingBuilder = new AlertDialog.Builder(
@ -1135,7 +1159,7 @@ public class KiwixMobileActivity extends AppCompatActivity
AlertDialog zimFileMissingDialog = zimFileMissingBuilder.create();
zimFileMissingDialog.show();
} else {
openZimFile(new File(FileUtils.generateSaveFileName(fileName)), true);
openZimFile(new File(filePath), true);
}
} else {
Log.d(TAG_KIWIX,

View File

@ -3,22 +3,14 @@ package org.kiwix.kiwixmobile.settings;
public class Constants {
public static final boolean IS_CUSTOM_APP = false;
public static final String CUSTOM_APP_ID = "~package~";
public static final long ZIM_FILE_SIZE = 0;
public static final boolean CUSTOM_APP_HAS_EMBEDDED_ZIM = false;
public static final String CUSTOM_APP_ZIM_FILE_NAME = "~zim_name~";
public static final long CUSTOM_APP_ZIM_FILE_SIZE = 0;
public static final String CUSTOM_APP_VERSION_NAME = "~version_name~";
public static final int CUSTOM_APP_VERSION_CODE = 2;
public static final String CUSTOM_APP_LICENSE = "~license~";
public static final String CUSTOM_APP_WEBSITE = "~website~";
public static final String CUSTOM_APP_EMAIL = "~support_email~";
public static final String CUSTOM_APP_SUPPORT_EMAIL = "~support_email~";
public static final String CUSTOM_APP_ENFORCED_LANG = "";
}

View File

@ -3,10 +3,11 @@ package org.kiwix.kiwixmobile.settings;
public class Constants {
public static final boolean IS_CUSTOM_APP = true;
public static final String CUSTOM_APP_ID = "~package~";
public static final long ZIM_FILE_SIZE = ~zim_size~;
public static final boolean CUSTOM_APP_HAS_EMBEDDED_ZIM = ~embed_zim~;
public static final String CUSTOM_APP_ZIM_FILE_NAME = "~zim_name~";
public static final long CUSTOM_APP_ZIM_FILE_SIZE = ~zim_size~;
public static final String CUSTOM_APP_VERSION_NAME = "~version_name~";
public static final int CUSTOM_APP_VERSION_CODE = ~version_code~;
public static final String CUSTOM_APP_LICENSE = "~license~";
public static final String CUSTOM_APP_WEBSITE = "~website~";
public static final String CUSTOM_APP_EMAIL = "~support_email~";
public static final String CUSTOM_APP_SUPPORT_EMAIL = "~support_email~";