From d37acc2cbe6a9b9d5db451f2c552d3ec906e3fc7 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 20 Jan 2020 23:01:00 +0100 Subject: [PATCH] Build basedeps archives on GithubAction. Use a specific script to download/build the base deps. Mainly based on script in travis/compile_all.py --- .github/scripts/common.py | 162 ++++++++++++++++++++++++++++ .github/scripts/ensure_base_deps.py | 72 +++++++++++++ .github/workflows/base.yml | 58 ++++++++++ 3 files changed, 292 insertions(+) create mode 100644 .github/scripts/common.py create mode 100755 .github/scripts/ensure_base_deps.py diff --git a/.github/scripts/common.py b/.github/scripts/common.py new file mode 100644 index 0000000..58ea5c6 --- /dev/null +++ b/.github/scripts/common.py @@ -0,0 +1,162 @@ +import os +from os import environ as _environ +from pathlib import Path +from datetime import date +import tarfile +import subprocess +import re + +from kiwixbuild.versions import base_deps_versions + + +PLATFORM_TARGET = _environ["PLATFORM_TARGET"] +OS_NAME = _environ["OS_NAME"] +HOME = Path(os.path.expanduser("~")) + +BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET) +SOURCE_DIR = HOME / "SOURCE" +ARCHIVE_DIR = HOME / "ARCHIVE" +INSTALL_DIR = BASE_DIR / "INSTALL" +TMP_DIR = Path("/tmp") + +# [TODO] +KIWIX_DESKTOP_ONLY = False + +_ref = _environ.get("GITHUB_REF", "").split("/")[-1] +MAKE_RELEASE = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", _ref) is not None + + +def print_message(message, *args, **kwargs): + message = message.format(*args, **kwargs) + message = "{0} {1} {0}".format("-" * 3, message) + print(message, flush=True) + + +MANIFEST_TEMPLATE = """{archive_name} +*************************** + +Dependencies archive for {target} on platform {platform} +Generated at {date} +""" + + +def write_manifest(manifest_file, archive_name, target, platform): + with manifest_file.open(mode="w") as f: + f.write( + MANIFEST_TEMPLATE.format( + archive_name=archive_name, + target=target, + platform=platform, + date=date.today().isoformat(), + ) + ) + + +def run_kiwix_build( + target, + platform, + build_deps_only=False, + target_only=False, + make_release=False, + make_dist=False, +): + command = ["kiwix-build"] + command.append(target) + command.append("--hide-progress") + if platform == "flatpak" or platform.startswith("win32_"): + command.append("--assume-packages-installed") + if target == "kiwix-lib-app" and platform.startswith("android_"): + command.extend(["--target-platform", "android", "--android-arch", platform[8:]]) + elif platform == "android": + command.extend(["--target-platform", "android"]) + for arch in ("arm", "arm64", "x86", "x86_64"): + command.extend(["--android-arch", arch]) + else: + command.extend(["--target-platform", platform]) + if build_deps_only: + command.append("--build-deps-only") + if target_only: + command.append("--build-nodeps") + if make_release: + command.append("--make-release") + if make_dist: + command.append("--make-dist") + print_message( + "Build {} (deps={}, release={}, dist={})", + target, + build_deps_only, + make_release, + make_dist, + ) + subprocess.check_call(command, cwd=str(HOME)) + print_message("Build ended") + + +def upload(file_to_upload, host, dest_path): + command = [ + "scp", + "-i", + _environ.get("SSH_KEY"), + "-o", + "StrictHostKeyChecking=no", + str(file_to_upload), + "{}:{}".format(host, dest_path), + ] + print_message("Sending archive with command {}", command) + subprocess.check_call(command) + + +def make_deps_archive(target=None, name=None, full=False): + archive_name = name or "deps2_{}_{}_{}.tar.xz".format( + OS_NAME, PLATFORM_TARGET, target + ) + print_message("Create archive {}.", archive_name) + files_to_archive = [INSTALL_DIR] + files_to_archive += HOME.glob("BUILD_*/LOGS") + if PLATFORM_TARGET == "native_mixed": + files_to_archive += [HOME / "BUILD_native_static" / "INSTALL"] + if PLATFORM_TARGET.startswith("android"): + files_to_archive.append(HOME / "BUILD_neutral" / "INSTALL") + if PLATFORM_TARGET == "android": + for arch in ("arm", "arm64", "x86", "x86_64"): + base_dir = HOME / "BUILD_android_{}".format(arch) + files_to_archive.append(base_dir / "INSTALL") + if (base_dir / "meson_cross_file.txt").exists(): + files_to_archive.append(base_dir / "meson_cross_file.txt") + files_to_archive += HOME.glob("BUILD_*/android-ndk*") + files_to_archive += HOME.glob("BUILD_*/android-sdk*") + if (BASE_DIR / "meson_cross_file.txt").exists(): + files_to_archive.append(BASE_DIR / "meson_cross_file.txt") + + manifest_file = BASE_DIR / "manifest.txt" + write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET) + files_to_archive.append(manifest_file) + + relative_path = HOME + if full: + files_to_archive += ARCHIVE_DIR.glob(".*_ok") + files_to_archive += BASE_DIR.glob("*/.*_ok") + files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") + files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") + files_to_archive += HOME.glob("BUILD_android*/**/.*_ok") + files_to_archive += SOURCE_DIR.glob("*/.*_ok") + files_to_archive += [ + SOURCE_DIR / "pugixml-{}".format(base_deps_versions["pugixml"]) + ] + files_to_archive += HOME.glob( + "BUILD_*/pugixml-{}".format(base_deps_versions["pugixml"]) + ) + if PLATFORM_TARGET.startswith("armhf"): + files_to_archive += (SOURCE_DIR / "armhf").glob("*") + toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*") + for subdir in toolchains_subdirs: + if not subdir.match("tools"): + files_to_archive.append(subdir) + + archive_file = TMP_DIR / archive_name + with tarfile.open(str(archive_file), "w:xz") as tar: + for name in set(files_to_archive): + print(".{}".format(name), flush=True) + tar.add(str(name), arcname=str(name.relative_to(relative_path))) + + return archive_file diff --git a/.github/scripts/ensure_base_deps.py b/.github/scripts/ensure_base_deps.py new file mode 100755 index 0000000..6d0ccff --- /dev/null +++ b/.github/scripts/ensure_base_deps.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import os +import tarfile +from urllib.request import urlopen +from urllib.error import URLError + +from kiwixbuild.versions import base_deps_meta_version + +from common import ( + print_message, + run_kiwix_build, + upload, + make_deps_archive, + HOME, + PLATFORM_TARGET, + OS_NAME, + MAKE_RELEASE, +) + + +def download_base_archive(base_name): + url = "http://tmp.kiwix.org/ci/{}".format(base_name) + file_path = str(HOME / base_name) + batch_size = 1024 * 1024 * 8 + with urlopen(url) as resource, open(file_path, "wb") as file: + while True: + batch = resource.read(batch_size) + if not batch: + break + print(".", end="", flush=True) + file.write(batch) + return file_path + + +ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}_{release}.tar.xz" +base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( + os=OS_NAME, + platform=PLATFORM_TARGET, + version=base_deps_meta_version, + release="release" if MAKE_RELEASE else "debug", +) + +print_message("Getting archive {}", base_dep_archive_name) +try: + local_filename = download_base_archive(base_dep_archive_name) + with tarfile.open(local_filename) as f: + f.extractall(str(HOME)) + os.remove(str(local_filename)) +except URLError: + print_message("Cannot get archive. Build dependencies") + if PLATFORM_TARGET == "android": + for arch in ("arm", "arm64", "x86", "x86_64"): + archive_name = ARCHIVE_NAME_TEMPLATE.format( + os=OS_NAME, + platform="android_{}".format(arch), + version=base_deps_meta_version, + release="release" if MAKE_RELEASE else "debug", + ) + print_message("Getting archive {}", archive_name) + try: + local_filename = download_base_archive(archive_name) + with tarfile.open(local_filename) as f: + f.extractall(str(HOME)) + os.remove(str(local_filename)) + except URLError: + pass + else: + run_kiwix_build("alldependencies", platform=PLATFORM_TARGET) + archive_file = make_deps_archive(name=base_dep_archive_name, full=True) + upload(archive_file, "ci@tmp.kiwix.org", "/data/tmp/ci") + os.remove(str(archive_file)) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 5ec2a8c..7a1b6b1 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -26,3 +26,61 @@ jobs: docker push ${FULLTAGNAME} fi + Linux: + strategy: + fail-fast: false + matrix: + target: + - native_static + - native_dyn + - android_arm + - android_arm64 + - win32_static + - win32_dyn + include: + - target: native_static + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: native_dyn + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: android_arm + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: android_arm64 + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: win32_static + image_variant: f30 + lib_postfix: '64' + - target: win32_dyn + image_variant: f30 + lib_postfix: '64' + env: + HOME: /home/runner + SSH_KEY: /tmp/id_rsa + runs-on: ubuntu-latest + needs: Docker + container: + image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-25" + steps: + - name: Checkout code + shell: bash + run: | + cd $HOME + git clone https://github.com/${REP} --depth=1 --branch ${GITHUB_REF##*/} + pip3 install --user --no-deps ./${REP##*/} + env: + REP: ${{github.repository}} + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/ensure_base_deps.py + env: + PLATFORM_TARGET: ${{matrix.target}}