diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ae5522c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,149 +0,0 @@ -language: cpp - -dist: xenial -sudo: required -services: -- docker -branches: - only: - - master - - /\d+\.\d+\.\d+$/ - -if: type != push OR tag IS present - -stages: -- build_docker_images -- build -- android_build - -jobs: - include: - - stage: build_docker_images - script: &build_images - - | - if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${VARIANT}-${DOCKER_VERSION}) ] - then - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker build -t kiwix-build_ci -f travis/${VARIANT}_builder.dockerfile . - docker tag kiwix-build_ci kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION} - docker push kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION} - fi - env: VARIANT=xenial - - stage: build_docker_images - script: *build_images - env: VARIANT=bionic - - stage: build_docker_images - script: *build_images - env: VARIANT=focal - - stage: build_docker_images - script: *build_images - env: VARIANT=f30 - - stage: android_build - env: PLATFORM="android" - if: type != pull_request - - env: PLATFORM="native_dyn" - os: osx - - env: PLATFORM="iOS_arm64" - os: osx - - - -before_install: -- PATH=$PATH:${HOME}/bin:${HOME}/.local/bin -- mkdir -p /tmp/private $HOME/EXPORT/BASE /$HOME/.cache $HOME/.gradle/caches -- chmod -R a+wX $HOME/EXPORT $HOME/.cache $HOME/.gradle -- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi -- '[ -n "$encrypted_eba2f7543984_iv" ] && openssl aes-256-cbc - -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv - -in travis/travisci_builder_id_key.enc -out $SSH_KEY -d' -- chmod 600 $SSH_KEY -before_cache: - - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ -cache: - ccache: true - directories: - - $HOME/.cache/pip - - $HOME/.gradle/caches/ - - $HOME/.gradle/wrapper/ - - $HOME/.android/build-cache -install: if [[ $TRAVIS_OS_NAME != "linux" ]]; then travis/install_extra_deps.sh; fi -stage: build -script: -- | - if [[ $TRAVIS_OS_NAME = "linux" ]] - then - VARIANT=xenial - if [[ $DESKTOP_ONLY = 1 ]]; then VARIANT=bionic; fi - if [[ $PLATFORM =~ ^win32_* ]]; then VARIANT=f30; fi - fi -- | - if [[ $TRAVIS_OS_NAME = "linux" ]] - then - docker run --rm \ - -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG \ - --device /dev/fuse --cap-add ALL --privileged \ - --mount=type=bind,src=$HOME/EXPORT,dst=/home/ci_builder/EXPORT \ - --mount=type=bind,src=$HOME/.cache,dst=/home/ci_builder/.cache \ - --mount=type=bind,src=$HOME/.gradle/caches,dst=/home/ci_builder/.gradle/caches \ - --mount=type=bind,src=$TRAVIS_BUILD_DIR,dst=/home/ci_builder/kiwix-build \ - kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION} - fi -- if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi -after_success: -- travis/upload_base_deps.sh -after_failure: -- travis/upload_base_deps.sh -- travis/upload_all_log.sh -deploy: - - provider: script - skip_cleanup: true - script: travis/deploy.sh - on: - tags: true - condition: $TRAVIS_BUILD_STAGE_NAME != "Build_docker_images" - - provider: script - skip_cleanup: true - script: travis/deploy.sh - on: - branch: master - condition: $TRAVIS_BUILD_STAGE_NAME != "Build_docker_images" && $TRAVIS_EVENT_TYPE = cron -env: - global: - - NIGHTLY_DATE=$(date +%Y-%m-%d) - - SSH_KEY=/tmp/private/travisci_builder_id_key - - DOCKER_VERSION=4 - matrix: - - PLATFORM="flatpak" DESKTOP_ONLY=1 - - PLATFORM="native_dyn" - - PLATFORM="native_static" - - PLATFORM="native_mixed" - - PLATFORM="armhf_dyn" - - PLATFORM="armhf_static" - - PLATFORM="win32_dyn" - - PLATFORM="win32_static" - - PLATFORM="i586_dyn" - - PLATFORM="i586_static" - - PLATFORM="native_dyn" DESKTOP_ONLY=1 - - PLATFORM="android_arm" - - PLATFORM="android_arm64" - - PLATFORM="android_x86" - - PLATFORM="android_x86_64" -addons: - ssh_known_hosts: - - download.kiwix.org - - tmp.kiwix.org - - download.openzim.org - -notifications: - irc: - channels: - - "chat.freenode.net#kiwix" - on_success: change - on_failure: always - email: - recipients: - - contact@kiwix.org - - mgautier@kymeria.fr - on_success: change - on_failure: always diff --git a/travis/bionic_builder.dockerfile b/travis/bionic_builder.dockerfile deleted file mode 100644 index daecbf2..0000000 --- a/travis/bionic_builder.dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu:bionic - -ENV LANG C.UTF-8 - -RUN apt update -q && \ - apt install -q -y --no-install-recommends \ -# Base build tools - build-essential automake libtool cmake ccache pkg-config autopoint patch \ - python3-pip python3-setuptools python3-wheel git subversion wget unzip \ - ninja-build \ -# Packaged dependencies - libbz2-dev libmagic-dev uuid-dev zlib1g-dev \ - libmicrohttpd-dev aria2 \ -# Qt packages - libqt5gui5 qtbase5-dev qtwebengine5-dev libqt5svg5-dev qt5-image-formats-plugins qt5-default \ -# To create the appimage of kiwix-desktop - libfuse2 fuse patchelf \ -# Flatpak tools - elfutils flatpak flatpak-builder \ -# Cross win32 compiler - g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \ -# Other tools (to remove) -# vim less grep \ - && \ - apt-get clean -y && \ - rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* - -# Create user -RUN useradd --create-home ci_builder -USER ci_builder -WORKDIR /home/ci_builder -ENV PATH="/home/ci_builder/.local/bin:${PATH}" - -ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build -ENV TRAVIS_OS_NAME linux_bionic - -CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py diff --git a/travis/compile_all.py b/travis/compile_all.py deleted file mode 100755 index 2414b69..0000000 --- a/travis/compile_all.py +++ /dev/null @@ -1,527 +0,0 @@ -#!/usr/bin/env python3 - -import sys, os -import shutil -from os import environ -from pathlib import Path -from datetime import date -import tarfile, zipfile -import subprocess -import re -import json -from urllib.request import urlopen -from urllib.error import URLError - -from kiwixbuild.versions import ( - main_project_versions, - release_versions, - base_deps_versions, - base_deps_meta_version) - -PLATFORM = environ['PLATFORM'] -TRAVIS_OS_NAME = environ['TRAVIS_OS_NAME'] -HOME = Path(os.path.expanduser('~')) -NIGHTLY_DATE = environ['NIGHTLY_DATE'] -KBUILD_SOURCE_DIR = Path(environ['TRAVIS_BUILD_DIR']) -KIWIX_DESKTOP_ONLY = environ.get('DESKTOP_ONLY') == '1' - -BASE_DIR = HOME/"BUILD_{}".format(PLATFORM) -EXPORT_DIR = HOME/'EXPORT' -BASE_EXPORT_DIR = EXPORT_DIR/"BASE" -GIT_EXPORT_DIR = EXPORT_DIR/"GIT" -SOURCE_DIR = HOME/"SOURCE" -ARCHIVE_DIR = HOME/"ARCHIVE" -INSTALL_DIR = BASE_DIR/"INSTALL" -NIGHTLY_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'NIGHTLY_KIWIX_ARCHIVES'/NIGHTLY_DATE -RELEASE_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'RELEASE_KIWIX_ARCHIVES' -NIGHTLY_ZIM_ARCHIVES_DIR = EXPORT_DIR/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE -RELEASE_ZIM_ARCHIVES_DIR = EXPORT_DIR/'RELEASE_ZIM_ARCHIVES' -DIST_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'DIST_KIWIX_ARCHIVES' -DIST_ZIM_ARCHIVES_DIR = EXPORT_DIR/'DIST_ZIM_ARCHIVES' -BINTRAY_ARCHIVES_DIR = EXPORT_DIR/'BINTRAY_ARCHIVES' - -BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else '' - -# We have build everything. Now create archives for public deployement. -EXPORT_FILES = { - 'kiwix-tools': - (INSTALL_DIR/'bin', [f+BIN_EXT for f in ('kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve')]), - 'zim-tools': - (INSTALL_DIR/'bin', [f+BIN_EXT for f in ('zimbench', 'zimcheck', 'zimdump', 'zimsearch', 'zimdiff', 'zimpatch', 'zimsplit')]), - 'zimwriterfs': - (INSTALL_DIR/'bin', ['zimwriterfs'+BIN_EXT]), - 'libzim': - (INSTALL_DIR, ('lib/x86_64-linux-gnu/libzim.so.{}'.format(main_project_versions['libzim']), - 'lib/x86_64-linux-gnu/libzim.so.{}'.format(main_project_versions['libzim'][0]), - 'include/zim/**/*.h')) -} - -FLATPAK_HTTP_GIT_REMOTE = 'https://github.com/flathub/org.kiwix.desktop.git' -FLATPAK_GIT_REMOTE = 'git@github.com:flathub/org.kiwix.desktop.git' - -_date = date.today().isoformat() - -def print_message(message, *args, **kwargs): - message = message.format(*args, **kwargs) - message = "{0} {1} {0}".format('-'*3, message) - print(message, flush=True) - - -def write_manifest(manifest_file, archive_name, target, platform): - with manifest_file.open(mode='w') as f: - f.write('''{archive_name} -*************************** - -Dependencies archive for {target} on platform {platform} -Generated at {date} -'''.format( - archive_name=archive_name, - target=target, - platform=platform, - date=date.today().isoformat())) - - -def run_kiwix_build(target, platform, - build_deps_only=False, - target_only=False, - make_release=False, - make_dist=False): - command = ['kiwix-build'] - command.append(target) - command.append('--hide-progress') - command.append('--fast-clone') - if platform == 'flatpak' or platform.startswith('win32_'): - command.append('--assume-packages-installed') - if target == 'kiwix-lib-app' and platform.startswith('android_'): - command.extend(['--target-platform', 'android', '--android-arch', platform[8:]]) - elif platform == 'android': - command.extend(['--target-platform', 'android']) - for arch in ('arm', 'arm64', 'x86', 'x86_64'): - command.extend(['--android-arch', arch]) - else: - command.extend(['--target-platform', platform]) - if build_deps_only: - command.append('--build-deps-only') - if target_only: - command.append('--build-nodeps') - if make_release: - command.append('--make-release') - if make_dist: - command.append('--make-dist') - print_message("Build {} (deps={}, release={}, dist={})", - target, build_deps_only, make_release, make_dist) - subprocess.check_call(command, cwd=str(HOME)) - print_message("{}", list(BASE_DIR.glob("{}*/**/*.tar.xz".format(target)))) - print_message("Build ended") - - -def create_desktop_image(): - print_message("creating desktop image") - if make_release: - postfix = main_project_versions['kiwix-desktop'] - extra_postfix = release_versions.get('kiwix-desktop') - if extra_postfix is None: - # We should not make archives for release not wanted - return - if extra_postfix: - postfix = "{}-{}".format(postfix, extra_postfix) - archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/'kiwix-desktop' - src_dir = SOURCE_DIR/'kiwix-desktop_release' - else: - postfix = _date - archive_dir = NIGHTLY_KIWIX_ARCHIVES_DIR - src_dir = SOURCE_DIR/'kiwix-desktop' - - if PLATFORM == 'flatpak': - build_path = BASE_DIR/'org.kiwix.desktop.flatpak' - app_name = 'org.kiwix.desktop.{}.flatpak'.format(postfix) - print_message("archive is {}", build_path) - else: - build_path = HOME/'Kiwix-{}-x86_64.AppImage'.format(postfix) - app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix) - command = ['kiwix-build/scripts/create_kiwix-desktop_appImage.sh', - str(INSTALL_DIR), str(src_dir), str(HOME/'AppDir')] - env = dict(os.environ) - env['VERSION'] = postfix - print_message("Build AppImage of kiwix-desktop") - subprocess.check_call(command, cwd=str(HOME), env=env) - - try: - archive_dir.mkdir(parents=True) - except FileExistsError: - pass - - print_message("Copy Build to {}".format(archive_dir/app_name)) - shutil.copy(str(build_path), str(archive_dir/app_name)) - - -def make_archive(project, platform): - base_dir, export_files = EXPORT_FILES[project] - - if make_release: - postfix = main_project_versions[project] - extra_postfix = release_versions.get(project) - if extra_postfix is None: - # We should not make archives for release not wanted - return - if extra_postfix: - postfix = "{}-{}".format(postfix, extra_postfix) - if project in ('kiwix-lib', 'kiwix-tools'): - archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/project - else: - archive_dir = RELEASE_ZIM_ARCHIVES_DIR/project - else: - postfix = _date - if project in ('kiwix-lib', 'kiwix-tools'): - archive_dir = NIGHTLY_KIWIX_ARCHIVES_DIR - else: - archive_dir = NIGHTLY_ZIM_ARCHIVES_DIR - - try: - archive_dir.mkdir(parents=True) - except FileExistsError: - pass - - archive_name = "{}_{}-{}".format(project, platform, postfix) - - files_to_archive = [] - for export_file in export_files: - files_to_archive.extend(base_dir.glob(export_file)) - if platform == "win-i686": - open_archive = lambda a : zipfile.ZipFile(str(a), 'w', compression=zipfile.ZIP_DEFLATED) - archive_add = lambda a, f : a.write(str(f), arcname=str(f.relative_to(base_dir))) - archive_ext = ".zip" - else: - open_archive = lambda a : tarfile.open(str(a), 'w:gz') - archive_add = lambda a, f : a.add(str(f), arcname="{}/{}".format(archive_name, str(f.relative_to(base_dir)))) - archive_ext = ".tar.gz" - - - archive = archive_dir/'{}{}'.format(archive_name, archive_ext) - print_message("create archive {} with {}", archive, files_to_archive) - with open_archive(archive) as arch: - for f in files_to_archive: - archive_add(arch, f) - - -def make_deps_archive(target=None, name=None, full=False): - archive_name = name or "deps_{}_{}_{}.tar.xz".format( - TRAVIS_OS_NAME, PLATFORM, target) - print_message("Create archive {}.", archive_name) - files_to_archive = [INSTALL_DIR] - files_to_archive += HOME.glob('BUILD_*/LOGS') - if PLATFORM == 'native_mixed': - files_to_archive += [HOME/'BUILD_native_static'/'INSTALL'] - if PLATFORM.startswith('android'): - files_to_archive.append(HOME/'BUILD_neutral'/'INSTALL') - if PLATFORM == 'android': - for arch in ('arm', 'arm64', 'x86', 'x86_64'): - base_dir = HOME/"BUILD_android_{}".format(arch) - files_to_archive.append(base_dir/'INSTALL') - if (base_dir/'meson_cross_file.txt').exists(): - files_to_archive.append(base_dir/'meson_cross_file.txt') - files_to_archive += HOME.glob('BUILD_*/android-ndk*') - files_to_archive += HOME.glob('BUILD_*/android-sdk*') - if (BASE_DIR/'meson_cross_file.txt').exists(): - files_to_archive.append(BASE_DIR/'meson_cross_file.txt') - - manifest_file = BASE_DIR/'manifest.txt' - write_manifest(manifest_file, archive_name, target, PLATFORM) - files_to_archive.append(manifest_file) - - relative_path = HOME - if full: - files_to_archive += ARCHIVE_DIR.glob(".*_ok") - files_to_archive += BASE_DIR.glob('*/.*_ok') - files_to_archive += (HOME/"BUILD_native_dyn").glob('*/.*_ok') - files_to_archive += (HOME/"BUILD_native_static").glob('*/.*_ok') - files_to_archive += HOME.glob('BUILD_android*/**/.*_ok') - files_to_archive += SOURCE_DIR.glob('*/.*_ok') - if PLATFORM.startswith('armhf'): - files_to_archive += (SOURCE_DIR/'armhf').glob('*') - toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*') - for subdir in toolchains_subdirs: - if not subdir.match('tools'): - files_to_archive.append(subdir) - - with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar: - for name in set(files_to_archive): - print('.{}'.format(name), flush=True) - tar.add(str(name), arcname=str(name.relative_to(relative_path))) - -def make_flatpak_cache_archive(): - archive_name = "base_deps_{os}_{platform}.tar.xz".format( - os=TRAVIS_OS_NAME, - platform=PLATFORM) - cache = BASE_DIR/'.flatpak-builder' - print_message("make flatpak cache archive") - nb_files = 0 - def pseudo_filter(tarinfo): - nonlocal nb_files - nb_files += 1 - if (nb_files % 1000) == 0: - print('.', flush=True) - return tarinfo - with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar: - tar.add(str(cache), arcname=str(cache.relative_to(BASE_DIR)), filter=pseudo_filter) - - -def update_flathub_git(): - GIT_REPO_DIR = GIT_EXPORT_DIR/"org.kiwix.desktop" - env = dict(os.environ) - env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = "KiwixBot" - env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr" - def call(command, cwd=None): - cwd = cwd or GIT_REPO_DIR - print_message("call {}", command) - subprocess.check_call(command, env=env, cwd=str(cwd)) - command = ['git', 'clone', FLATPAK_HTTP_GIT_REMOTE] - call(command, cwd=GIT_EXPORT_DIR) - shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), str(GIT_REPO_DIR)) - patch_dir = KBUILD_SOURCE_DIR/'kiwixbuild'/'patches' - for dep in ('libaria2', 'mustache', 'pugixml', 'xapian', 'zstd'): - for f in patch_dir.glob('{}_*.patch'.format(dep)): - shutil.copy(str(f), str(GIT_REPO_DIR/'patches')) - command = ['git', 'add', '-A', '.'] - call(command) - command = ['git', 'commit', '-m', - 'Update to version {}'.format(main_project_versions['kiwix-desktop'])] - try: - call(command) - except subprocess.CalledProcessError: - # This may fail if there is nothing to commit (a rebuild of the CI for exemple) - pass - command = ['git', 'config', 'remote.origin.pushurl', FLATPAK_GIT_REMOTE] - call(command) - - - -for p in (NIGHTLY_KIWIX_ARCHIVES_DIR, - NIGHTLY_ZIM_ARCHIVES_DIR, - RELEASE_KIWIX_ARCHIVES_DIR, - RELEASE_ZIM_ARCHIVES_DIR, - DIST_KIWIX_ARCHIVES_DIR, - DIST_ZIM_ARCHIVES_DIR, - BINTRAY_ARCHIVES_DIR, - BASE_EXPORT_DIR, - GIT_EXPORT_DIR): - try: - p.mkdir(parents=True) - except FileExistsError: - pass - -make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None - -def download_base_archive(base_name): - url = 'http://tmp.kiwix.org/ci/{}'.format(base_name) - file_path = str(HOME/base_name) - batch_size = 1024*1024*8 - with urlopen(url) as resource, open(file_path, 'wb') as file: - while True: - batch = resource.read(batch_size) - if not batch: - break - print(".", end="", flush=True) - file.write(batch) - return file_path - -def export_dist(target): - if target in ('kiwix-lib', 'kiwix-tools', 'kiwix-desktop'): - out_dir = DIST_KIWIX_ARCHIVES_DIR - else: - out_dir = DIST_ZIM_ARCHIVES_DIR - - if target in main_project_versions: - try: - (out_dir/target).mkdir(parents=True) - except FileExistsError: - pass - - full_target_name = "{}-{}".format(target, main_project_versions[target]) - if target != 'kiwix-desktop': - in_file = BASE_DIR/full_target_name/'meson-dist'/'{}.tar.xz'.format( - full_target_name) - else: - in_file = BASE_DIR/full_target_name/'{}.tar.gz'.format(full_target_name) - if in_file.exists(): - print_message("Copying {} to {}", in_file, out_dir/target) - shutil.copy(str(in_file), str(out_dir/target)) - else: - print_message("No {} to copy.", in_file) - - -if PLATFORM != 'flatpak': - # The first thing we need to do is to (potentially) download already compiled base dependencies. - base_dep_archive_name = "base_deps_{os}_{platform}_{version}_{release}.tar.xz".format( - os=TRAVIS_OS_NAME, - platform=PLATFORM, - version=base_deps_meta_version, - release='release' if make_release else 'debug') - - print_message("Getting archive {}", base_dep_archive_name) - try: - local_filename = download_base_archive(base_dep_archive_name) - with tarfile.open(local_filename) as f: - f.extractall(str(HOME)) - except URLError: - print_message("Cannot get archive. Build dependencies") - if PLATFORM == 'android': - for arch in ('arm', 'arm64', 'x86', 'x86_64'): - archive_name = "base_deps_{os}_android_{arch}_{version}_{release}.tar.xz".format( - os=TRAVIS_OS_NAME, - arch=arch, - version=base_deps_meta_version, - release='release' if make_release else 'debug') - print_message("Getting archive {}", archive_name) - try: - local_filename = download_base_archive(archive_name) - with tarfile.open(local_filename) as f: - f.extractall(str(HOME)) - except URLError: - pass - else: - run_kiwix_build('alldependencies', platform=PLATFORM) - make_deps_archive(name=base_dep_archive_name, full=True) -else: - base_dep_archive_name = "base_deps_{os}_{platform}.tar.xz".format( - os=TRAVIS_OS_NAME, - platform=PLATFORM) - - print_message("Getting archive {}", base_dep_archive_name) - try: - local_filename = download_base_archive(base_dep_archive_name) - BASE_DIR.mkdir(parents=True) - with tarfile.open(local_filename) as f: - f.extractall(str(BASE_DIR)) - except URLError: - print_message("Cannot get archive. Move on") - - -# A basic compilation to be sure everything is working (for a PR) -if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release: - if PLATFORM.startswith('android'): - TARGETS = ('kiwix-lib',) - elif PLATFORM.startswith('iOS'): - TARGETS = ('kiwix-lib',) - elif PLATFORM.startswith('native_'): - if TRAVIS_OS_NAME == "osx": - TARGETS = ('kiwix-lib', 'zim-tools', 'zimwriterfs') - elif PLATFORM == 'native_dyn' and KIWIX_DESKTOP_ONLY: - TARGETS = ('kiwix-desktop', ) - elif PLATFORM == 'native_mixed': - TARGETS = ('libzim', ) - else: - TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs') - elif PLATFORM == 'flatpak': - TARGETS = ('kiwix-desktop', ) - else: - TARGETS = ('kiwix-tools', ) - - for target in TARGETS: - run_kiwix_build(target, - platform=PLATFORM) - - if PLATFORM == 'native_mixed': - make_archive('libzim', 'linux-x86_64') - elif PLATFORM == 'native_static': - for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'): - make_archive(target, 'linux-x86_64') - elif PLATFORM == 'win32_static': - make_archive('kiwix-tools', 'win-i686') - elif PLATFORM == 'armhf_static': - make_archive('kiwix-tools', 'linux-armhf') - elif PLATFORM == 'i586_static': - make_archive('kiwix-tools', 'linux-i586') - elif PLATFORM == 'flatpak': - make_flatpak_cache_archive() - - sys.exit(0) - - -TARGETS = tuple() -if PLATFORM == 'android': - TARGETS = ('kiwix-lib-app',) -elif PLATFORM.startswith('android_'): - TARGETS = ('libzim', 'kiwix-lib') -elif PLATFORM.startswith('iOS'): - TARGETS = ('libzim', 'kiwix-lib') -elif PLATFORM.startswith('native_'): - if TRAVIS_OS_NAME == "osx": - TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib') - else: - if PLATFORM == 'native_dyn' and KIWIX_DESKTOP_ONLY: - TARGETS = ('kiwix-desktop', ) - elif PLATFORM == 'native_mixed': - TARGETS = ('libzim', ) - else: - TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools') -elif PLATFORM == 'flatpak': - TARGETS = ('kiwix-desktop', ) -else: - TARGETS = ('libzim', 'zim-tools', 'kiwix-lib', 'kiwix-tools') - -for target in TARGETS: - if environ['TRAVIS_EVENT_TYPE'] == 'cron' and PLATFORM not in ('android', 'flatpak'): - run_kiwix_build(target, - platform=PLATFORM, - build_deps_only=True) - make_deps_archive(target=target) - - if target == 'kiwix-lib-app': - extra_postfix = release_versions.get('kiwix-lib') - if extra_postfix: - environ['KIWIXLIB_BUILDVERSION'] = extra_postfix - - run_kiwix_build(target, - platform=PLATFORM, - make_release=make_release) - print_message("target is {}", target) - if target == 'kiwix-desktop': - create_desktop_image() - if make_release and PLATFORM == 'native_dyn' and release_versions.get(target) == 0: - run_kiwix_build(target, - platform=PLATFORM, - make_release=True, - make_dist=True) - export_dist(target) - -# We have build everything. Now create archives for public deployement. -if PLATFORM == 'native_mixed': - make_archive('libzim', 'linux-x86_64') -elif PLATFORM == 'native_static': - for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'): - make_archive(target, 'linux-x86_64') -elif PLATFORM == 'win32_static': - make_archive('kiwix-tools', 'win-i686') -elif PLATFORM == 'armhf_static': - make_archive('kiwix-tools', 'linux-armhf') -elif PLATFORM == 'i586_static': - make_archive('kiwix-tools', 'linux-i586') -elif ( make_release - and PLATFORM == 'flatpak' - and release_versions.get('kiwix-desktop') is not None): - update_flathub_git() -elif PLATFORM == 'android' and 'kiwix-lib-app' in TARGETS: - if make_release and release_versions.get('kiwix-lib') is not None: - postfix = main_project_versions['kiwix-lib'] - extra_postfix = release_versions.get('kiwix-lib') - if extra_postfix: - postfix = "{}-{}".format(postfix, extra_postfix) - - basename = "kiwixlib-{}".format(postfix) - - output_release_dir = HOME/'BUILD_android'/'kiwix-lib-app'/'kiwixLibAndroid'/'build' - shutil.copy(str(output_release_dir/'outputs'/'aar'/'kiwixLibAndroid-release.aar'), - str(BINTRAY_ARCHIVES_DIR/(basename+'.aar'))) - shutil.copy(str(output_release_dir/'pom.xml'), - str(BINTRAY_ARCHIVES_DIR/(basename+'.pom'))) - - - json_filename = '{}_bintray_info.json'.format(basename) - data = { - 'version': postfix, - 'files': [basename+ext for ext in ('.aar', '.pom')] - } - with open(str(BINTRAY_ARCHIVES_DIR/json_filename), 'w') as f: - json.dump(data, f) - diff --git a/travis/deploy.sh b/travis/deploy.sh deleted file mode 100755 index fcb56cd..0000000 --- a/travis/deploy.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash - -set -e - -EXPORT_DIR=${HOME}/EXPORT -NIGHTLY_KIWIX_ARCHIVES_DIR=${EXPORT_DIR}/NIGHTLY_KIWIX_ARCHIVES/${NIGHTLY_DATE} -NIGHTLY_ZIM_ARCHIVES_DIR=${EXPORT_DIR}/NIGHTLY_ZIM_ARCHIVES/${NIGHTLY_DATE} -RELEASE_KIWIX_ARCHIVES_DIR=${EXPORT_DIR}/RELEASE_KIWIX_ARCHIVES -RELEASE_ZIM_ARCHIVES_DIR=${EXPORT_DIR}/RELEASE_ZIM_ARCHIVES -DIST_KIWIX_ARCHIVES_DIR=${EXPORT_DIR}/DIST_KIWIX_ARCHIVES -DIST_ZIM_ARCHIVES_DIR=${EXPORT_DIR}/DIST_ZIM_ARCHIVES -BINTRAY_ARCHIVES_DIR=${EXPORT_DIR}/BINTRAY_ARCHIVES - -if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]] -then - echo "Publishing nightlies" - ls ${NIGHTLY_KIWIX_ARCHIVES_DIR} - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${NIGHTLY_KIWIX_ARCHIVES_DIR} \ - ci@download.kiwix.org:/data/download/nightly - ls ${NIGHTLY_ZIM_ARCHIVES_DIR} - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${NIGHTLY_ZIM_ARCHIVES_DIR} \ - ci@download.openzim.org:/data/openzim/nightly - -elif [[ "x$TRAVIS_TAG" != "x" ]] -then - RELEASE_ARCHIVES=$(find $RELEASE_KIWIX_ARCHIVES_DIR -type f) - if [[ "x$RELEASE_ARCHIVES" != "x" ]] - then - echo "Publishing kiwix archives $RELEASE_ARCHIVES" - for archive in $RELEASE_ARCHIVES - do - subdir=$(basename $(dirname $archive)) - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${archive} \ - ci@download.kiwix.org:/data/download/release/${subdir} - done - fi - - RELEASE_ARCHIVES=$(find $RELEASE_ZIM_ARCHIVES_DIR -type f) - if [[ "x$RELEASE_ARCHIVES" != "x" ]] - then - echo "Publishing zim archives $RELEASE_ARCHIVES" - for archive in $RELEASE_ARCHIVES - do - subdir=$(basename $(dirname $archive)) - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${archive} \ - ci@download.openzim.org:/data/openzim/release/${subdir} - done - fi - - DIST_KIWIX_ARCHIVES=$(find $DIST_KIWIX_ARCHIVES_DIR -type f) - if [[ "x$DIST_KIWIX_ARCHIVES" != "x" ]] - then - echo "Publishing dist kiwix archives $DIST_KIWIX_ARCHIVES" - for archive in $DIST_KIWIX_ARCHIVES - do - subdir=$(basename $(dirname $archive)) - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${archive} \ - ci@download.kiwix.org:/data/download/release/${subdir} - done - fi - - DIST_ZIM_ARCHIVES=$(find $DIST_ZIM_ARCHIVES_DIR -type f) - if [[ "x$DIST_ZIM_ARCHIVES" != "x" ]] - then - echo "Publishing dist zim archives $DIST_ZIM_ARCHIVES" - for archive in $DIST_ZIM_ARCHIVES - do - subdir=$(basename $(dirname $archive)) - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${archive} \ - ci@download.openzim.org:/data/openzim/release/${subdir} - done - fi - - (cd ${EXPORT_DIR}/GIT - GIT_REPOS=$(ls -l | awk '/^d/ { print $9 }') - if [[ "x$GIT_REPOS" != "x" ]] - then - echo "Pushing repositories $GIT_REPOS" - for repo in $GIT_REPOS - do - (cd $repo; - GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -i $SSH_KEY" git push - ) - done - fi - ) - - ls $BINTRAY_ARCHIVES_DIR - BINTRAY_ARCHIVES=$(find $BINTRAY_ARCHIVES_DIR -name "*_bintray_info.json" -type f) - if [[ "x$BINTRAY_ARCHIVES" != "x" ]] - then - echo "Publish bintray $BINTRAY_ARCHIVES" - for archive_info in $BINTRAY_ARCHIVES - do - ${TRAVIS_BUILD_DIR}/scripts/upload_kiwix_lib_android_to_bintray.py $archive_info - done - fi - -fi - diff --git a/travis/f30_builder.dockerfile b/travis/f30_builder.dockerfile deleted file mode 100644 index 99d68fe..0000000 --- a/travis/f30_builder.dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM fedora:30 - -ENV LANG C.UTF-8 - -RUN dnf install -y --nodocs \ -# Base build tools - make automake libtool cmake git-core subversion pkg-config gcc-c++ \ - wget unzip ninja-build ccache which patch \ -# Cross win32 compiler - mingw32-gcc-c++ mingw32-bzip2-static mingw32-win-iconv-static \ - mingw32-winpthreads-static mingw32-zlib-static mingw32-xz-libs-static \ - mingw32-libmicrohttpd \ -# Other tools (to remove) -# vim less grep - && dnf remove -y "*-doc" \ - && dnf autoremove -y \ - && dnf clean all - -# Create user -RUN useradd --create-home ci_builder -USER ci_builder -WORKDIR /home/ci_builder -ENV PATH="/home/ci_builder/.local/bin:${PATH}" - -ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build -ENV TRAVIS_OS_NAME linux_f30 - -CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py diff --git a/travis/focal_builder.dockerfile b/travis/focal_builder.dockerfile deleted file mode 100644 index dba937a..0000000 --- a/travis/focal_builder.dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu:focal - -ENV LANG C.UTF-8 - -RUN apt update -q && \ - apt install -q -y --no-install-recommends \ -# Base build tools - build-essential automake libtool cmake ccache pkg-config autopoint patch \ - python3-pip python3-setuptools python3-wheel git subversion wget unzip \ - ninja-build \ -# Packaged dependencies - libbz2-dev libmagic-dev uuid-dev zlib1g-dev \ - libmicrohttpd-dev aria2 \ -# Qt packages - libqt5gui5 qtbase5-dev qtwebengine5-dev libqt5svg5-dev qt5-image-formats-plugins qt5-default \ -# To create the appimage of kiwix-desktop - libfuse2 fuse patchelf \ -# Flatpak tools - elfutils flatpak flatpak-builder \ -# Cross win32 compiler - g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \ -# Other tools (to remove) -# vim less grep \ - && \ - apt-get clean -y && \ - rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* - -# Create user -RUN useradd --create-home ci_builder -USER ci_builder -WORKDIR /home/ci_builder -ENV PATH="/home/ci_builder/.local/bin:${PATH}" - -ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build -ENV TRAVIS_OS_NAME linux_focal - -CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py diff --git a/travis/install_extra_deps.sh b/travis/install_extra_deps.sh deleted file mode 100755 index 5777030..0000000 --- a/travis/install_extra_deps.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [[ "$TRAVIS_OS_NAME" == "osx" ]] -then - pip3 install pillow - pip3 install . - - wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-mac.zip - unzip ninja-mac.zip ninja -else - wget https://bootstrap.pypa.io/get-pip.py - python3.5 get-pip.py --user - python3.5 -m pip install --user --upgrade pip wheel - python3.5 -m pip install --user pillow - python3.5 -m pip install --user . - python3.5 -m pip show -f kiwix-build - - wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip - unzip ninja-linux.zip ninja -fi - -mkdir -p $HOME/bin -cp ninja $HOME/bin - diff --git a/travis/travisci_builder_id_key.enc b/travis/travisci_builder_id_key.enc deleted file mode 100644 index 7d09269..0000000 Binary files a/travis/travisci_builder_id_key.enc and /dev/null differ diff --git a/travis/travisci_builder_id_key.pub b/travis/travisci_builder_id_key.pub deleted file mode 100644 index 12e88a2..0000000 --- a/travis/travisci_builder_id_key.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCWxcsEawR4TrXmGgMXXOhuYjXtO1V/BpEQ2tuf2tTiloZ0n1cPtxeLp3ic7F91p/e+58U1QlpLkxs9Ia0REXvr2ppd3FuIRjADs6g9iqK4jgAx/KKVjy2vhtxFtDKP5xCkbqSRjDcYnSZwrllEPbsOEIgTvRBaOyyqBNbctjSN6DhB+hY1bsPNJntDL0/XLqSYFXBnxd+unRLiItRJX1C42tpTwWyr9OvpjuPCjWVo/6WqohU8xjW29xPDpmAnWCX02JD//hRamJ+zfDPxf5AGTVJhk7of4Jjj4pTunhTXRLGhA3UBPApbI//NA2nAOc8feOchu1VfO6n5JuVbnXDzUlMFVqSNJ6ihYBTEczN+T/xbdc3Uyzi3JZX7IW2lH1YpxJoGEzJ67HjNPHdKgQzqVGlKl00qUrL1aMJ+1rOm8xDvPYVjHidORPUn0x+2fvxJ0VVr4crVwmTa23ft79MS2sLbw5DbPkURVUkEH88nSwcbF9fbuCPE4RL15NS7ZNwDqRMyQIxnAe7Sf+UWz45HxTrdvgjrsk0PCYX+KCgnXEexXWDDlbXWuiJa1h2VhKb2zM3AUmtRMK0pFTi/xCrFFwDIYYIBOMxMb0nYmruv5Feh/cez1FNYq9+UKrnBAV7DAZCLibM2/u46OAgbbtZS70zaX1JvcX44yUgjpClNcQ== TravisCIBuilder diff --git a/travis/upload_all_log.sh b/travis/upload_all_log.sh deleted file mode 100755 index 057f509..0000000 --- a/travis/upload_all_log.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -e - -cd $HOME - -tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM} - -scp -vrp -i ${SSH_KEY} \ - fail_log_${PLATFORM}.tar.gz \ - ci@tmp.kiwix.org:/data/tmp/ci diff --git a/travis/upload_base_deps.sh b/travis/upload_base_deps.sh deleted file mode 100755 index 6c5f232..0000000 --- a/travis/upload_base_deps.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e - -BASE_EXPORT_DIR=$HOME/EXPORT/BASE -ls $BASE_EXPORT_DIR - -BASE_ARCHIVES=$(find $BASE_EXPORT_DIR -type f) -echo $BASE_ARCHIVES -if [[ "x$BASE_ARCHIVES" != "x" ]] -then - scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - $BASE_ARCHIVES \ - ci@tmp.kiwix.org:/data/tmp/ci -fi diff --git a/travis/xenial_builder.dockerfile b/travis/xenial_builder.dockerfile deleted file mode 100644 index e0b7bcb..0000000 --- a/travis/xenial_builder.dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu:xenial - -ENV LANG C.UTF-8 - -RUN apt update -q && \ - dpkg --add-architecture i386 && \ - apt install -q -y --no-install-recommends \ -# Base build tools - build-essential automake libtool cmake ccache pkg-config autopoint patch \ - python3-pip python3-setuptools python3-wheel git subversion wget unzip \ - ninja-build \ -# Python (2) is needed to install android-ndk - python \ -# Packaged dependencies - libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \ - libmicrohttpd-dev \ -# Cross compile i586 - libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \ -# Other tools (to remove) -# vim less grep \ - && \ - apt-get clean -y && \ - rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* && \ - pip3 install --upgrade pip && \ - pip3 install meson==0.52.1 pytest gcovr - -# Create user -RUN useradd --create-home ci_builder -USER ci_builder -WORKDIR /home/ci_builder -ENV PATH="/home/ci_builder/.local/bin:${PATH}" - -ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build -ENV GRADLE_USER_HOME /home/ci_builder -ENV TRAVIS_OS_NAME linux_xenial - -CMD pip3 install --user --no-deps ./kiwix-build && kiwix-build/travis/compile_all.py