diff --git a/.github/workflows/build_apps.yml b/.github/workflows/build_apps.yml deleted file mode 100644 index 5030abb..0000000 --- a/.github/workflows/build_apps.yml +++ /dev/null @@ -1,51 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: CI - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: macos-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - - # Checks-out the kiwix/apple repository - - uses: actions/checkout@v4 - with: - repository: kiwix/apple - ref: main - path: apple - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4 - with: - repository: kiwix/kiwix-apple-custom - ref: main - path: custom - - - run: ls -la - - run: ls -la custom/ - - run: ls -la apple/ - - # copy build file to the main folder and run from there - - run: cp custom/build_project.zsh . - - run: chmod +x build_project.zsh - - name: Build Project - env: - DWDS_HTTP_BASIC_ACCESS_AUTHENTICATION: ${{ secrets.DWDS_HTTP_BASIC_ACCESS_AUTHENTICATION }} - run: zsh ./build_project.zsh - diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 9258f93..f846f93 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -26,4 +26,4 @@ jobs: run: | cd custom - python .github/workflows/tag_validator.py ${{ steps.vars.outputs.tag }} + python src/tag_validator.py ${{ steps.vars.outputs.tag }} | python src/generate_and_download.py diff --git a/src/brand.py b/src/brand.py new file mode 100644 index 0000000..e424219 --- /dev/null +++ b/src/brand.py @@ -0,0 +1,22 @@ +from pathlib import Path +import sys + +INFO_JSON = 'info.json' + +class Brand: + + def __init__(self, name): + if Path(name).is_dir() == False: + self._exit_with_error(f"The directory of the brand: '{name}' does not exist") + self.info_file = Path(name)/INFO_JSON + if self.info_file.exists() == False: + self._exit_with_error(f"There is no {INFO_JSON} file for brand {name}") + self.name = name + + @staticmethod + def all_info_files(): + return list(Path().rglob(INFO_JSON)) + + def _exit_with_error(self, msg): + print(f"Error: {msg}") + sys.exit(1) diff --git a/src/custom_apps.py b/src/custom_apps.py new file mode 100644 index 0000000..3b9b2bc --- /dev/null +++ b/src/custom_apps.py @@ -0,0 +1,68 @@ +from pathlib import Path +from info_parser import InfoParser +from brand import Brand +import subprocess +import yaml + +INFO_JSON = 'info.json' + +class CustomApps: + + def __init__(self, brands=["all"], build_version=None): + self.build_version = build_version + if brands == ["all"]: + self.info_files = Brand.all_info_files() + else: + self.info_files = [] + for brand_name in brands: + brand = Brand(brand_name) + self.info_files.append(brand.info_file) + + def create_custom_project_file(self, path): + """Create the project file based on the main repo project.yml + It will contain the targets we need for each custom app, and their build settings, + pointing to their individual info.plist files + + Args: + path (Path): the output file path where the project yaml will be saved + """ + dict = {"include": ["project.yml"]} + targets = {} + for info in self.info_files: + parser = InfoParser(info, build_version=self.build_version) + targets = targets | parser.as_project_yml() + + dict["targets"] = targets + with open(path, "w") as file: + yaml.dump(dict, file) + + def create_plists(self, custom_plist): + """Generate the plist files for each brand + + Args: + custom_plist (Path): the path to the original plist file we are basing this of, + it should be a copy from the Kiwix target + """ + for info in self.info_files: + parser = InfoParser(info, build_version=self.build_version) + parser.create_plist(based_on_plist_file=custom_plist) + + def download_zim_files(self): + """Download all the zim files that were declared in the info.json files + """ + for cmd in self._curl_download_commands(): + subprocess.call(cmd) + + # private + def _curl_download_commands(self): + """Yield all the curl commands we need to download each zim file from all info.json files + + Yields: + array: commands that can be feeded into subprocess.call() + """ + for info in self.info_files: + parser = InfoParser(info, build_version=self.build_version) + url = parser.zimurl() + file_path = parser.zim_file_path() + auth = parser.download_auth() + yield ["curl", "-L", url, "-u", auth, "-o", file_path] diff --git a/src/generate_and_download.py b/src/generate_and_download.py new file mode 100644 index 0000000..468e9d1 --- /dev/null +++ b/src/generate_and_download.py @@ -0,0 +1,44 @@ +"""Generate the custom app plist files, and a custom_project.yml. +Based on the arguments passed in: +where the subfolder name will become the "brand name" of the custom app. +""" + +from custom_apps import CustomApps +from pathlib import Path +import argparse + +def main(): + parser = argparse.ArgumentParser( + description="Builder of custom apps, based on the passed in (optional) brand name and (optional) build version") + parser.add_argument( + "brand_name", + nargs='?', + default='all', + help="The brand name to be built, if not provided will fall back to all apps", + type=str + ) + + parser.add_argument( + "build_version", + nargs='?', + default=None, + help="The optional build version to use, if not provided will fall back to the build_version defined in the info.json value", + type=int + ) + args = parser.parse_args() + brand = args.brand_name + build_version = args.build_version + + custom_apps = CustomApps(brands=[brand], build_version=build_version) + # create the plist files + custom_apps.create_plists(custom_plist=Path("Custom.plist")) + + # download the zim files + custom_apps.download_zim_files() + + # finally create the project file, containing all brands as targets + custom_apps.create_custom_project_file(path=Path("custom_project.yml")) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/info_parser.py b/src/info_parser.py new file mode 100644 index 0000000..7205593 --- /dev/null +++ b/src/info_parser.py @@ -0,0 +1,156 @@ +from urllib.parse import urlparse +import json +from pathlib import Path +import os +import re +import shutil +import plistlib + +JSON_KEY_ZIM_URL = "zim_url" +JSON_KEY_AUTH = "zim_auth" +JSON_KEY_APP_NAME = "app_name" +JSON_KEY_ENFORCED_LANGUAGE = "enforced_lang" +CUSTOM_ZIM_FILE_KEY = "CUSTOM_ZIM_FILE" +JSON_TO_PLIST_MAPPING = { + "app_store_id": "APP_STORE_ID", + "about_app_url": "CUSTOM_ABOUT_WEBSITE", + "about_text": "CUSTOM_ABOUT_TEXT", + "settings_default_external_link_to": "SETTINGS_DEFAULT_EXTERNAL_LINK_TO", + "settings_show_search_snippet": "SETTINGS_SHOW_SEARCH_SNIPPET", + "settings_show_external_link_option": "SETTINGS_SHOW_EXTERNAL_LINK_OPTION" +} + + +class InfoParser: + + def __init__(self, json_path, build_version=None): + """Parse a specific info.json file for a brand + + Args: + json_path (Path): of the branded info.json file + build_number (int, optional): If defined it will be used instead of the info.json[build_version]. Defaults to None. + """ + self.brand_name = self._brandname_from(json_path) + self.build_version = build_version + content = json_path.read_text() + self.data = json.loads(content) + assert (JSON_KEY_ZIM_URL in self.data) + self.zim_file_name = self._filename_from( + self.data[JSON_KEY_ZIM_URL]) + + def create_plist(self, based_on_plist_file): + with based_on_plist_file.open(mode="rb") as file: + plist = plistlib.load(file) + for keyValues in self._plist_key_values(): + for key in keyValues: + plist[key] = keyValues[key] + plist[CUSTOM_ZIM_FILE_KEY] = self.zim_file_name + out_path = self._info_plist_path() + out_path.parent.mkdir(parents=True, exist_ok=True) + with out_path.open(mode="wb") as out_file: + plistlib.dump(plist, out_file) + + def as_project_yml(self): + dict = { + "templates": ["ApplicationTemplate"], + "settings": {"base": { + "MARKETING_VERSION": self._app_version(), + "PRODUCT_BUNDLE_IDENTIFIER": f"org.kiwix.custom.{self.brand_name}", + "INFOPLIST_FILE": f"custom/{self._info_plist_path()}", + "INFOPLIST_KEY_CFBundleDisplayName": self._app_name(), + "INFOPLIST_KEY_UILaunchStoryboardName": "SplashScreen.storyboard", + "DEVELOPMENT_LANGUAGE": self._dev_language() + # without specifying DEVELOPMENT_LANGUAGE, + # the default value of it: English will be added to the list of + # selectable languages in iOS Settings, + # even if the en.lproj is excluded from the sources. + # If DEVELOPMENT_LANGUAGE is not added, enforcing a single language is not effective, + # therefore it's better to set it to the enforced language value if there's such. + } + }, + "sources": [ + {"path": f"custom/{self.brand_name}"}, + {"path": "custom/SplashScreen.storyboard", + "destinationFilters": ["iOS"] + }, + {"path": "Support", + "excludes": [ + "*.xcassets", + "Info.plist" + ] + self._excluded_languages() + }, + ] + } + return {self.brand_name: dict} + + def zimurl(self): + return self.data[JSON_KEY_ZIM_URL] + + def zim_file_path(self): + url = Path(self.zimurl()) + return Path()/self.brand_name/url.name + + def download_auth(self): + auth_key = self.data[JSON_KEY_AUTH] + return os.getenv(auth_key) + + def _info_plist_path(self): + return Path()/self.brand_name/f"{self.brand_name}.plist" + + def _plist_key_values(self): + for json_key in JSON_TO_PLIST_MAPPING: + if json_key in self.data: + plistKey = JSON_TO_PLIST_MAPPING[json_key] + value = self.data[json_key] + yield {plistKey: value} + + def _app_version(self): + build_version = self.build_version or self.data["build_version"] + return f"{self._app_version_from(self.zim_file_name)}.{build_version}" + + def _app_name(self): + return self.data[JSON_KEY_APP_NAME] + + def _dev_language(self): + enforced = self._enforced_language() + if enforced == None: + return "en" + else: + return enforced + + def _enforced_language(self): + if JSON_KEY_ENFORCED_LANGUAGE in self.data: + return self.data[JSON_KEY_ENFORCED_LANGUAGE] + else: + return None + + def _brandname_from(self, filepath): + return filepath.parent.name.lower() + + def _filename_from(self, url): + return Path(urlparse(url).path).stem + + def _app_version_from(self, file_name): + p = re.compile('(?P\d{4})-(?P\d{1,2})') + m = p.search(file_name) + year = int(m.group('year')) + month = int(m.group('month')) + assert (year > 2000) + assert (month > 0) + assert (month <= 12) + # downgrade the version by 1000 for testing the release + year -= 1000 + return ".".join([str(year), str(month)]) + + def _excluded_languages(self): + enforced = self._enforced_language() + if enforced == None: + return ["**/qqq.lproj"] + else: + # Copy the enforced lang to the custom folder + for lang_file in Path().parent.rglob(f'{enforced}.lproj'): + lang_file.copy + shutil.copytree( + lang_file, Path().parent/"custom"/self.brand_name, dirs_exist_ok=True) + # exclude all other languages under Support/*.lproj + return ["**/*.lproj"] diff --git a/.github/workflows/tag_validator.py b/src/tag_validator.py similarity index 59% rename from .github/workflows/tag_validator.py rename to src/tag_validator.py index 0701e33..c5fae37 100644 --- a/.github/workflows/tag_validator.py +++ b/src/tag_validator.py @@ -2,11 +2,11 @@ import argparse import re -from pathlib import Path import sys +from brand import Brand -def is_valid(tag): +def _is_valid(tag): # Regex verify the tag format pattern = re.compile( r'^(?P\w+)_(?P\d+)(?:_(?P\w+))?$') @@ -14,21 +14,16 @@ def is_valid(tag): if match: groups = match.groupdict() - brand = groups.get('brand_folder') + brand_name = groups.get('brand_folder') build_nr = int(groups.get('build_nr')) - if Path(brand).is_dir(): - print(f"valid tag found: {tag} (brand: { - brand}, build number: {build_nr})") - return True - else: - exist_with_error(f"The directory of the tag: '{ - brand}' doesn't exist") + brand = Brand(brand_name) + print(f"{brand.name} {build_nr}") else: - exist_with_error(f"Invalid tag: {tag}") + _exit_with_error(f"Invalid tag: {tag}") return False -def exist_with_error(msg): +def _exit_with_error(msg): print(f"Error: {msg}") sys.exit(1) @@ -42,7 +37,7 @@ def main(): type=str ) args = parser.parse_args() - return is_valid(args.tag) + return _is_valid(args.tag) if __name__ == "__main__": diff --git a/tests/Support/Info.plist b/tests/Support/Info.plist new file mode 100644 index 0000000..b0f8377 --- /dev/null +++ b/tests/Support/Info.plist @@ -0,0 +1,67 @@ + + + + + APP_STORE_ID + $(APP_STORE_ID) + BGTaskSchedulerPermittedIdentifiers + + org.kiwix.library_refresh + + CFBundleDocumentTypes + + + CFBundleTypeName + OpenZIM Content File + LSHandlerRank + Owner + LSItemContentTypes + + org.openzim.zim + + + + CFBundleURLTypes + + + CFBundleTypeRole + Viewer + CFBundleURLSchemes + + kiwix + + + + ITSAppUsesNonExemptEncryption + + UIBackgroundModes + + fetch + + UIFileSharingEnabled + + UTExportedTypeDeclarations + + + UTTypeConformsTo + + public.data + public.content + + UTTypeDescription + OpenZIM Content File + UTTypeIconFiles + + UTTypeIdentifier + org.openzim.zim + UTTypeTagSpecification + + public.filename-extension + + zim + + + + + + diff --git a/tests/custom_apps_test.py b/tests/custom_apps_test.py new file mode 100644 index 0000000..5829345 --- /dev/null +++ b/tests/custom_apps_test.py @@ -0,0 +1,24 @@ +import unittest +from src.custom_apps import CustomApps +from pathlib import Path + + +class CustomAppsTest(unittest.TestCase): + + def setUp(self): + self.custom = CustomApps() + + def test_custom_plist(self): + self.custom.create_plists( + custom_plist=Path()/"tests"/"Support"/"Info.plist") + + def test_custom_project_creation(self): + self.custom.create_custom_project_file( + path=Path()/"custom_project_test.yml") + + def x_test_downloads(self): + self.custom.download_zim_files() + + def x_test_download_commands(self): + for cmd in self.custom._curl_download_commands(): + print(cmd) diff --git a/tests/info_parser_test.py b/tests/info_parser_test.py new file mode 100644 index 0000000..94330ca --- /dev/null +++ b/tests/info_parser_test.py @@ -0,0 +1,89 @@ +import unittest +from src.info_parser import InfoParser +from pathlib import Path +import yaml +import os + + +class InfoParserTest(unittest.TestCase): + + def setUp(self): + self.parser = InfoParser(Path()/"tests"/"test.json") + + def test_json_to_project_yml(self): + project = self.parser.as_project_yml() + print("custom_project.yml targets:") + print(yaml.dump(project)) + + def test_info_plist_path(self): + custom_info = self.parser._info_plist_path() + self.assertEqual(custom_info, Path()/"tests"/"tests.plist") + + def test_file_name_from_url(self): + url = "https://www.dwds.de/kiwix/f/dwds_de_dictionary_nopic_2023-11-20.zim" + file_name = self.parser._filename_from(url) + self.assertEqual(file_name, "dwds_de_dictionary_nopic_2023-11-20") + + url = "https://www.dwds.de/kiwix/f/dwds_de_dictionary_nopic_2023-11.zim" + file_name = self.parser._filename_from(url) + self.assertEqual(file_name, "dwds_de_dictionary_nopic_2023-11") + + def test_brand_name_from_file_path(self): + filepath = Path().home()/"some"/"dev"/"path"/"project"/"dwds"/"info.json" + brand_name = self.parser._brandname_from(filepath) + self.assertEqual(brand_name, "dwds") + + def test_version_from_filename(self): + version = self.parser._app_version_from( + "dwds_de_dictionary_nopic_2023-11-20") + self.assertEqual(version, "1023.11") + + version = self.parser._app_version_from( + "dwds_de_dictionary_nopic_2023-09-20") + self.assertEqual(version, "1023.9") + + version = self.parser._app_version_from( + "dwds_de_dictionary_nopic_2023-01") + self.assertEqual(version, "1023.1") + + version = self.parser._app_version_from( + "dwds_de_dictionary_nopic_2023-12") + self.assertEqual(version, "1023.12") + + def test_app_name(self): + app_name = self.parser._app_name() + self.assertEqual(app_name, "DWDS") + + def test_enforced_language(self): + enforced_language = self.parser._enforced_language() + self.assertEqual(enforced_language, "de") + + def test_excluded_languages(self): + excluded = self.parser._excluded_languages() + self.assertIn("**/*.lproj", excluded) + + def test_app_version(self): + self.assertEqual(self.parser._app_version(), "1023.12.3") + + def test_app_version_using_a_tag(self): + parser = InfoParser(Path()/"tests"/"test.json", build_version=15) + self.assertEqual(parser._app_version(), "1023.12.15") + + parser = InfoParser(Path()/"tests"/"test.json", build_version=33) + self.assertEqual(parser._app_version(), "1023.12.33") + + def test_as_plist(self): + self.parser.create_plist( + based_on_plist_file=Path()/"tests"/"Support"/"Info.plist") + + def test_zimurl(self): + self.assertEqual(self.parser.zimurl( + ), "https://www.dwds.de/kiwix/f/dwds_de_dictionary_nopic_2023-12-15.zim") + + def test_zimfile_path(self): + self.assertEqual(self.parser.zim_file_path(), + Path()/"tests"/"dwds_de_dictionary_nopic_2023-12-15.zim") + + def test_auth_value(self): + self.assertEqual(self.parser.download_auth(), os.getenv( + "DWDS_HTTP_BASIC_ACCESS_AUTHENTICATION")) diff --git a/tests/test.json b/tests/test.json new file mode 100644 index 0000000..95fe19c --- /dev/null +++ b/tests/test.json @@ -0,0 +1,13 @@ +{ + "about_app_url": "https://www.dwds.de", + "about_text": "Für Schreibende, Lernende, Lehrende und Sprachinteressierte: Das Digitale Wörterbuch der deutschen Sprache (DWDS) ist das große Bedeutungswörterbuch des Deutschen der Gegenwart. Es bietet umfassende und wissenschaftlich verlässliche lexikalische Informationen, kostenlos und werbefrei.", + "app_name": "DWDS", + "app_store_id": "id6473090365", + "enforced_lang": "de", + "settings_default_external_link_to": "alwaysLoad", + "settings_show_search_snippet": false, + "settings_show_external_link_option": false, + "zim_auth": "DWDS_HTTP_BASIC_ACCESS_AUTHENTICATION", + "zim_url": "https://www.dwds.de/kiwix/f/dwds_de_dictionary_nopic_2023-12-15.zim", + "build_version": 3 +}