From 7086009a0a7d06c8a4fa8f59f7b1e1816ba49eef Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 10 Jun 2022 10:11:01 +0200 Subject: [PATCH 1/2] We don't need to upload to bintray now. --- .github/scripts/build_release_nightly.py | 30 --------- .github/scripts/upload_to_bintray.py | 83 ------------------------ 2 files changed, 113 deletions(-) delete mode 100755 .github/scripts/upload_to_bintray.py diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index 61277e1..97d679c 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -1,14 +1,11 @@ #!/usr/bin/env python3 import os -import json -import shutil from common import ( run_kiwix_build, main_project_versions, release_versions, - get_postfix, make_archive, create_desktop_image, update_flathub_git, @@ -24,8 +21,6 @@ from common import ( notarize_macos_build, ) -from upload_to_bintray import upload_from_json - if os.environ.get('GITHUB_EVENT_NAME') == 'schedule': RELEASE = False @@ -106,28 +101,3 @@ if RELEASE: # Publish flathub if PLATFORM_TARGET == "flatpak" and "kiwix-desktop" in TARGETS: update_flathub_git() - - if PLATFORM_TARGET == "android" and "libkiwix" in TARGETS: - postfix = get_postfix("libkiwix") - basename = "kiwixlib-{}".format(postfix) - - output_release_dir = ( - HOME / "BUILD_android" / "libkiwix-app" / "kiwixLibAndroid" / "build" - ) - shutil.copy( - str(output_release_dir / "outputs" / "aar" / "kiwixLibAndroid-release.aar"), - str(TMP_DIR / (basename + ".aar")), - ) - shutil.copy( - str(output_release_dir / "pom.xml"), str(TMP_DIR / (basename + ".pom")) - ) - - json_filename = "{}_bintray_info.json".format(basename) - data = { - "version": postfix, - "files": [basename + ext for ext in (".aar", ".pom")], - } - with open(str(TMP_DIR / json_filename), "w") as f: - json.dump(data, f) - - upload_from_json(TMP_DIR / json_filename) diff --git a/.github/scripts/upload_to_bintray.py b/.github/scripts/upload_to_bintray.py deleted file mode 100755 index c2fc8c2..0000000 --- a/.github/scripts/upload_to_bintray.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 - -import os, sys -import json -import requests - -bintray_auth = (os.environ.get('BINTRAY_USER'), os.environ.get('BINTRAY_PASS')) - -def create_version(version): - url = "https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/versions" - payload = { - 'name': version, - 'desc': 'Release of libkiwix' - } - headers = { - 'Content-Type': 'application/json' - } - - r = requests.post(url, data=json.dumps(payload), headers=headers, auth=bintray_auth) - rcode = r.status_code - - if rcode == 409: - print("Bintray version %s already exists, skipping." % version) - return True - - rcode_family = rcode // 100 - if rcode_family in (2, 3): - print("Bintray Version created!") - return True - - print("ERROR : Bintray API response {}".format(rcode)) - return False - - -def upload(version, filepath, artefact): - url_template = "https://api.bintray.com/content/kiwix/kiwix/kiwixlib/{version}/org/kiwix/kiwixlib/kiwixlib/{version}/{artefact}" - parameters = { - 'publish': 1, - 'override': 1 - } - - # Upload the main artefact - url = url_template.format(version=version, artefact=artefact) - with open(filepath, 'rb') as f: - r = requests.put(url, data=f, auth=bintray_auth, params=parameters) - - rcode = r.status_code - rcode_family = rcode // 100 - if rcode_family not in (2, 3): - print("ERROR: Fail to upload artefact") - print(r.text) - return False - - return True - - -def upload_from_json(json_path): - basedir = os.path.dirname(str(json_path)) - with open(str(json_path)) as f: - options = json.load(f) - - if not create_version(options['version']): - raise RuntimeError("Cannot create version") - - for file_ in options['files']: - path = os.path.join(basedir, file_) - if not upload(options['version'], path, file_): - raise RuntimeError("Cannot upload file {}".format(file_)) - - -if __name__ == "__main__": - try: - info_file = sys.argv[1] - except IndexError: - print("Usage {} infofile".format(sys.argv[0])) - sys.exit(-1) - - print("Use info file {}".format(info_file)) - try: - upload_from_json(info_file) - except RuntimeError as e: - sys.exit(str(e)) - From 9c220866ac3496a8f0206d170f19f514c0f73fba Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 10 Jun 2022 10:26:24 +0200 Subject: [PATCH 2/2] Build and release each libkiwix android builds independently. While it is ok to build all libkiwix android builds in one step, the "release system" upload only one archive per platform. So we need 4 platforms to do 4 uploads. As we don't build on "android" platform now, we can clean up our scripts. --- .github/scripts/build_release_nightly.py | 8 +------- .github/scripts/common.py | 19 ++++++------------- .github/scripts/ensure_base_deps.py | 19 ++----------------- .github/workflows/releaseNigthly.yml | 16 ++++++++++++++-- 4 files changed, 23 insertions(+), 39 deletions(-) diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index 97d679c..4fe4f32 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -27,7 +27,7 @@ if os.environ.get('GITHUB_EVENT_NAME') == 'schedule': else: RELEASE = True -if PLATFORM_TARGET == "android": +if PLATFORM_TARGET.startswith("android_"): TARGETS = ("libkiwix",) elif PLATFORM_TARGET.startswith("iOS"): TARGETS = ("libzim", "libkiwix") @@ -54,12 +54,6 @@ if RELEASE: return release_versions.get(project) is not None TARGETS = tuple(filter(release_filter, TARGETS)) -if RELEASE and PLATFORM_TARGET == "android": - # libkiwix need to know the extrapostfix version to correctly generate the pom.xml file. - extra_postfix = release_versions.get('libkiwix') - if extra_postfix: - os.environ['KIWIXLIB_BUILDVERSION'] = str(extra_postfix) - for target in TARGETS: run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=RELEASE) if target == "kiwix-desktop": diff --git a/.github/scripts/common.py b/.github/scripts/common.py index c38c162..2a2e529 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -145,12 +145,7 @@ def run_kiwix_build( command.append("--hide-progress") command.append("--fast-clone") command.append("--assume-packages-installed") - if platform == "android": - command.extend(["--target-platform", "android"]) - for arch in ("arm", "arm64", "x86", "x86_64"): - command.extend(["--android-arch", arch]) - else: - command.extend(["--target-platform", platform]) + command.extend(["--target-platform", platform]) if build_deps_only: command.append("--build-deps-only") if target_only: @@ -245,14 +240,12 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive += HOME.glob("BUILD_*/LOGS") if PLATFORM_TARGET == "native_mixed": files_to_archive += [HOME / "BUILD_native_static" / "INSTALL"] - if PLATFORM_TARGET.startswith("android"): + if PLATFORM_TARGET.startswith("android_"): files_to_archive.append(HOME / "BUILD_neutral" / "INSTALL") - if PLATFORM_TARGET == "android": - for arch in ("arm", "arm64", "x86", "x86_64"): - base_dir = HOME / "BUILD_android_{}".format(arch) - files_to_archive.append(base_dir / "INSTALL") - if (base_dir / "meson_cross_file.txt").exists(): - files_to_archive.append(base_dir / "meson_cross_file.txt") + base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET) + files_to_archive.append(base_dir / "INSTALL") + if (base_dir / "meson_cross_file.txt").exists(): + files_to_archive.append(base_dir / "meson_cross_file.txt") files_to_archive += HOME.glob("BUILD_*/android-ndk*") if (BASE_DIR / "meson_cross_file.txt").exists(): files_to_archive.append(BASE_DIR / "meson_cross_file.txt") diff --git a/.github/scripts/ensure_base_deps.py b/.github/scripts/ensure_base_deps.py index af9c7fa..ffa012e 100755 --- a/.github/scripts/ensure_base_deps.py +++ b/.github/scripts/ensure_base_deps.py @@ -48,25 +48,10 @@ try: f.extractall(str(HOME)) os.remove(str(local_filename)) except URLError: - print_message("Cannot get archive. Build dependencies") - if PLATFORM_TARGET == "android": - for arch in ("arm", "arm64", "x86", "x86_64"): - archive_name = ARCHIVE_NAME_TEMPLATE.format( - os=OS_NAME, - platform="android_{}".format(arch), - version=base_deps_meta_version, - ) - print_message("Getting archive {}", archive_name) - try: - local_filename = download_base_archive(archive_name) - with tarfile.open(local_filename) as f: - f.extractall(str(HOME)) - os.remove(str(local_filename)) - except URLError: - pass - elif PLATFORM_TARGET == "flatpak": + if PLATFORM_TARGET == "flatpak": print_message("Cannot get archive. Move on") else: + print_message("Cannot get archive. Build dependencies") run_kiwix_build("alldependencies", platform=PLATFORM_TARGET) archive_file = make_deps_archive(name=base_dep_archive_name, full=True) upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci") diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index 7e67718..f49cde2 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -43,7 +43,10 @@ jobs: - armhf_static - win32_static - i586_static - - android + - android_arm + - android_arm64 + - android_x86 + - android_x86_64 include: - target: native_static image_variant: bionic @@ -66,7 +69,16 @@ jobs: - target: i586_static image_variant: bionic lib_postfix: '/x86_64-linux-gnu' - - target: android + - target: android_arm + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' + - target: android_arm64 + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' + - target: android_x86 + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' + - target: android_x86_64 image_variant: bionic lib_postfix: '/x86_64-linux-gnu' env: