CD per brand

.github/workflows/build_apps.yml

@@ -1,51 +0,0 @@
-# This is a basic workflow to help you get started with Actions
-
-name: CI
-
-# Controls when the workflow will run
-on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
-  # This workflow contains a single job called "build"
-  build:
-    # The type of runner that the job will run on
-    runs-on: macos-latest
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-
-      # Checks-out the kiwix/apple repository
-      - uses: actions/checkout@v4
-        with:
-          repository: kiwix/apple
-          ref: main
-          path: apple
-    
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v4
-        with:
-          repository: kiwix/kiwix-apple-custom
-          ref: main
-          path: custom
-
-      - run: ls -la
-      - run: ls -la custom/
-      - run: ls -la apple/
-
-      # copy build file to the main folder and run from there
-      - run: cp custom/build_project.zsh .
-      - run: chmod +x build_project.zsh
-      - name: Build Project
-        env:
-          DWDS_HTTP_BASIC_ACCESS_AUTHENTICATION: ${{ secrets.DWDS_HTTP_BASIC_ACCESS_AUTHENTICATION }}
-        run: zsh ./build_project.zsh
-      

.github/workflows/cd.yml

@@ -26,4 +26,4 @@ jobs:
         run:
           |
           cd custom
-          python .github/workflows/tag_validator.py ${{ steps.vars.outputs.tag }}
+          python src/tag_validator.py ${{ steps.vars.outputs.tag }} | python src/generate_and_download.py

src/brand.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+import sys
+
+INFO_JSON = 'info.json'
+
+class Brand:
+    
+    def __init__(self, name):
+        if Path(name).is_dir() == False:
+            self._exit_with_error(f"The directory of the brand: '{name}' does not exist")
+        self.info_file = Path(name)/INFO_JSON
+        if self.info_file.exists() == False:
+            self._exit_with_error(f"There is no {INFO_JSON} file for brand {name}")
+        self.name = name
+        
+    @staticmethod
+    def all_info_files():
+        return list(Path().rglob(INFO_JSON))
+        
+    def _exit_with_error(self, msg):
+        print(f"Error: {msg}")
+        sys.exit(1)

src/custom_apps.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+from info_parser import InfoParser
+from brand import Brand
+import subprocess
+import yaml
+
+INFO_JSON = 'info.json'
+
+class CustomApps:
+
+    def __init__(self, brands=["all"], build_version=None):
+        self.build_version = build_version
+        if brands == ["all"]:
+            self.info_files = Brand.all_info_files()
+        else:
+            self.info_files = []
+            for brand_name in brands:
+                brand = Brand(brand_name)
+                self.info_files.append(brand.info_file)
+
+    def create_custom_project_file(self, path):
+        """Create the project file based on the main repo project.yml
+        It will contain the targets we need for each custom app, and their build settings,
+        pointing to their individual info.plist files
+
+        Args:
+            path (Path): the output file path where the project yaml will be saved
+        """
+        dict = {"include": ["project.yml"]}
+        targets = {}
+        for info in self.info_files:
+            parser = InfoParser(info, build_version=self.build_version)
+            targets = targets | parser.as_project_yml()
+
+        dict["targets"] = targets
+        with open(path, "w") as file:
+            yaml.dump(dict, file)
+
+    def create_plists(self, custom_plist):
+        """Generate the plist files for each brand
+
+        Args:
+            custom_plist (Path): the path to the original plist file we are basing this of,
+            it should be a copy from the Kiwix target
+        """
+        for info in self.info_files:
+            parser = InfoParser(info, build_version=self.build_version)
+            parser.create_plist(based_on_plist_file=custom_plist)
+
+    def download_zim_files(self):
+        """Download all the zim files that were declared in the info.json files
+        """
+        for cmd in self._curl_download_commands():
+            subprocess.call(cmd)
+
+    # private    
+    def _curl_download_commands(self):
+        """Yield all the curl commands we need to download each zim file from all info.json files
+
+        Yields:
+            array: commands that can be feeded into subprocess.call()
+        """
+        for info in self.info_files:
+            parser = InfoParser(info, build_version=self.build_version)
+            url = parser.zimurl()
+            file_path = parser.zim_file_path()
+            auth = parser.download_auth()
+            yield ["curl", "-L", url, "-u", auth, "-o", file_path]

src/generate_and_download.py

@@ -0,0 +1,44 @@
+"""Generate the custom app plist files, and a custom_project.yml. 
+Based on the arguments passed in:  
+where the subfolder name will become the "brand name" of the custom app.
+"""
+
+from custom_apps import CustomApps
+from pathlib import Path
+import argparse
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Builder of custom apps, based on the passed in (optional) brand name and (optional) build version")
+    parser.add_argument(
+        "brand_name",
+        nargs='?',
+        default='all',
+        help="The brand name to be built, if not provided will fall back to all apps",
+        type=str
+    )
+
+    parser.add_argument(
+        "build_version",
+        nargs='?',
+        default=None,
+        help="The optional build version to use, if not provided will fall back to the build_version defined in the info.json value",
+        type=int
+    )
+    args = parser.parse_args()
+    brand = args.brand_name
+    build_version = args.build_version
+
+    custom_apps = CustomApps(brands=[brand], build_version=build_version)
+    # create the plist files
+    custom_apps.create_plists(custom_plist=Path("Custom.plist"))
+
+    # download the zim files
+    custom_apps.download_zim_files()
+
+    # finally create the project file, containing all brands as targets
+    custom_apps.create_custom_project_file(path=Path("custom_project.yml"))
+
+
+if __name__ == "__main__":
+    main()    

src/info_parser.py

@@ -0,0 +1,156 @@
+from urllib.parse import urlparse
+import json
+from pathlib import Path
+import os
+import re
+import shutil
+import plistlib
+
+JSON_KEY_ZIM_URL = "zim_url"
+JSON_KEY_AUTH = "zim_auth"
+JSON_KEY_APP_NAME = "app_name"
+JSON_KEY_ENFORCED_LANGUAGE = "enforced_lang"
+CUSTOM_ZIM_FILE_KEY = "CUSTOM_ZIM_FILE"
+JSON_TO_PLIST_MAPPING = {
+    "app_store_id": "APP_STORE_ID",
+    "about_app_url": "CUSTOM_ABOUT_WEBSITE",
+    "about_text": "CUSTOM_ABOUT_TEXT",
+    "settings_default_external_link_to": "SETTINGS_DEFAULT_EXTERNAL_LINK_TO",
+    "settings_show_search_snippet": "SETTINGS_SHOW_SEARCH_SNIPPET",
+    "settings_show_external_link_option": "SETTINGS_SHOW_EXTERNAL_LINK_OPTION"
+}
+
+
+class InfoParser:
+
+    def __init__(self, json_path, build_version=None):
+        """Parse a specific info.json file for a brand
+
+        Args:
+            json_path (Path): of the branded info.json file
+            build_number (int, optional): If defined it will be used instead of the info.json[build_version]. Defaults to None.
+        """
+        self.brand_name = self._brandname_from(json_path)
+        self.build_version = build_version
+        content = json_path.read_text()
+        self.data = json.loads(content)
+        assert (JSON_KEY_ZIM_URL in self.data)
+        self.zim_file_name = self._filename_from(
+            self.data[JSON_KEY_ZIM_URL])
+
+    def create_plist(self, based_on_plist_file):
+        with based_on_plist_file.open(mode="rb") as file:
+            plist = plistlib.load(file)
+            for keyValues in self._plist_key_values():
+                for key in keyValues:
+                    plist[key] = keyValues[key]
+            plist[CUSTOM_ZIM_FILE_KEY] = self.zim_file_name
+            out_path = self._info_plist_path()
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+            with out_path.open(mode="wb") as out_file:
+                plistlib.dump(plist, out_file)
+
+    def as_project_yml(self):
+        dict = {
+            "templates": ["ApplicationTemplate"],
+            "settings": {"base": {
+                "MARKETING_VERSION": self._app_version(),
+                "PRODUCT_BUNDLE_IDENTIFIER": f"org.kiwix.custom.{self.brand_name}",
+                "INFOPLIST_FILE": f"custom/{self._info_plist_path()}",
+                "INFOPLIST_KEY_CFBundleDisplayName": self._app_name(),
+                "INFOPLIST_KEY_UILaunchStoryboardName": "SplashScreen.storyboard",
+                "DEVELOPMENT_LANGUAGE": self._dev_language()
+                # without specifying DEVELOPMENT_LANGUAGE,
+                # the default value of it: English will be added to the list of
+                # selectable languages in iOS Settings,
+                # even if the en.lproj is excluded from the sources.
+                # If DEVELOPMENT_LANGUAGE is not added, enforcing a single language is not effective,
+                # therefore it's better to set it to the enforced language value if there's such.
+            }
+            },
+            "sources": [
+                {"path": f"custom/{self.brand_name}"},
+                {"path": "custom/SplashScreen.storyboard",
+                 "destinationFilters": ["iOS"]
+                 },
+                {"path": "Support",
+                 "excludes": [
+                     "*.xcassets",
+                     "Info.plist"
+                 ] + self._excluded_languages()
+                 },
+            ]
+        }
+        return {self.brand_name: dict}
+
+    def zimurl(self):
+        return self.data[JSON_KEY_ZIM_URL]
+
+    def zim_file_path(self):
+        url = Path(self.zimurl())
+        return Path()/self.brand_name/url.name
+
+    def download_auth(self):
+        auth_key = self.data[JSON_KEY_AUTH]
+        return os.getenv(auth_key)
+
+    def _info_plist_path(self):
+        return Path()/self.brand_name/f"{self.brand_name}.plist"
+
+    def _plist_key_values(self):
+        for json_key in JSON_TO_PLIST_MAPPING:
+            if json_key in self.data:
+                plistKey = JSON_TO_PLIST_MAPPING[json_key]
+                value = self.data[json_key]
+                yield {plistKey: value}
+
+    def _app_version(self):
+        build_version = self.build_version or self.data["build_version"]
+        return f"{self._app_version_from(self.zim_file_name)}.{build_version}"
+
+    def _app_name(self):
+        return self.data[JSON_KEY_APP_NAME]
+
+    def _dev_language(self):
+        enforced = self._enforced_language()
+        if enforced == None:
+            return "en"
+        else:
+            return enforced
+
+    def _enforced_language(self):
+        if JSON_KEY_ENFORCED_LANGUAGE in self.data:
+            return self.data[JSON_KEY_ENFORCED_LANGUAGE]
+        else:
+            return None
+
+    def _brandname_from(self, filepath):
+        return filepath.parent.name.lower()
+
+    def _filename_from(self, url):
+        return Path(urlparse(url).path).stem
+
+    def _app_version_from(self, file_name):
+        p = re.compile('(?P<year>\d{4})-(?P<month>\d{1,2})')
+        m = p.search(file_name)
+        year = int(m.group('year'))
+        month = int(m.group('month'))
+        assert (year > 2000)
+        assert (month > 0)
+        assert (month <= 12)
+        # downgrade the version by 1000 for testing the release
+        year -= 1000
+        return ".".join([str(year), str(month)])
+
+    def _excluded_languages(self):
+        enforced = self._enforced_language()
+        if enforced == None:
+            return ["**/qqq.lproj"]
+        else:
+            # Copy the enforced lang to the custom folder
+            for lang_file in Path().parent.rglob(f'{enforced}.lproj'):
+                lang_file.copy
+                shutil.copytree(
+                    lang_file, Path().parent/"custom"/self.brand_name, dirs_exist_ok=True)
+            # exclude all other languages under Support/*.lproj
+            return ["**/*.lproj"]

src/tag_validator.py

@@ -2,11 +2,11 @@
 
 import argparse
 import re
-from pathlib import Path
 import sys
+from brand import Brand
 
 
-def is_valid(tag):
+def _is_valid(tag):
     # Regex verify the tag format
     pattern = re.compile(
         r'^(?P<brand_folder>\w+)_(?P<build_nr>\d+)(?:_(?P<extra_tag>\w+))?$')
@@ -14,21 +14,16 @@ def is_valid(tag):
 
     if match:
         groups = match.groupdict()
-        brand = groups.get('brand_folder')
+        brand_name = groups.get('brand_folder')
         build_nr = int(groups.get('build_nr'))
-        if Path(brand).is_dir():
+        brand = Brand(brand_name)
-            print(f"valid tag found: {tag} (brand: {
+        print(f"{brand.name} {build_nr}")
-                  brand}, build number: {build_nr})")
-            return True
-        else:
-            exist_with_error(f"The directory of the tag: '{
-                             brand}' doesn't exist")
     else:
-        exist_with_error(f"Invalid tag: {tag}")
+        _exit_with_error(f"Invalid tag: {tag}")
         return False
 
 
-def exist_with_error(msg):
+def _exit_with_error(msg):
     print(f"Error: {msg}")
     sys.exit(1)
 
@@ -42,7 +37,7 @@ def main():
         type=str
     )
     args = parser.parse_args()
-    return is_valid(args.tag)
+    return _is_valid(args.tag)
 
 
 if __name__ == "__main__":

tests/Support/Info.plist

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>APP_STORE_ID</key>
+	<string>$(APP_STORE_ID)</string>
+	<key>BGTaskSchedulerPermittedIdentifiers</key>
+	<array>
+		<string>org.kiwix.library_refresh</string>
+	</array>
+	<key>CFBundleDocumentTypes</key>
+	<array>
+		<dict>
+			<key>CFBundleTypeName</key>
+			<string>OpenZIM Content File</string>
+			<key>LSHandlerRank</key>
+			<string>Owner</string>
+			<key>LSItemContentTypes</key>
+			<array>
+				<string>org.openzim.zim</string>
+			</array>
+		</dict>
+	</array>
+	<key>CFBundleURLTypes</key>
+	<array>
+		<dict>
+			<key>CFBundleTypeRole</key>
+			<string>Viewer</string>
+			<key>CFBundleURLSchemes</key>
+			<array>
+				<string>kiwix</string>
+			</array>
+		</dict>
+	</array>
+	<key>ITSAppUsesNonExemptEncryption</key>
+	<false/>
+	<key>UIBackgroundModes</key>
+	<array>
+		<string>fetch</string>
+	</array>
+	<key>UIFileSharingEnabled</key>
+	<true/>
+	<key>UTExportedTypeDeclarations</key>
+	<array>
+		<dict>
+			<key>UTTypeConformsTo</key>
+			<array>
+				<string>public.data</string>
+				<string>public.content</string>
+			</array>
+			<key>UTTypeDescription</key>
+			<string>OpenZIM Content File</string>
+			<key>UTTypeIconFiles</key>
+			<array/>
+			<key>UTTypeIdentifier</key>
+			<string>org.openzim.zim</string>
+			<key>UTTypeTagSpecification</key>
+			<dict>
+				<key>public.filename-extension</key>
+				<array>
+					<string>zim</string>
+				</array>
+			</dict>
+		</dict>
+	</array>
+</dict>
+</plist>

tests/custom_apps_test.py

@@ -0,0 +1,24 @@
+import unittest
+from src.custom_apps import CustomApps
+from pathlib import Path
+
+
+class CustomAppsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.custom = CustomApps()
+
+    def test_custom_plist(self):
+        self.custom.create_plists(
+            custom_plist=Path()/"tests"/"Support"/"Info.plist")
+
+    def test_custom_project_creation(self):
+        self.custom.create_custom_project_file(
+            path=Path()/"custom_project_test.yml")
+
+    def x_test_downloads(self):
+        self.custom.download_zim_files()
+
+    def x_test_download_commands(self):
+        for cmd in self.custom._curl_download_commands():
+            print(cmd)

tests/info_parser_test.py

@@ -0,0 +1,89 @@
+import unittest
+from src.info_parser import InfoParser
+from pathlib import Path
+import yaml
+import os
+
+
+class InfoParserTest(unittest.TestCase):
+
+    def setUp(self):
+        self.parser = InfoParser(Path()/"tests"/"test.json")
+
+    def test_json_to_project_yml(self):
+        project = self.parser.as_project_yml()
+        print("custom_project.yml targets:")
+        print(yaml.dump(project))
+
+    def test_info_plist_path(self):
+        custom_info = self.parser._info_plist_path()
+        self.assertEqual(custom_info, Path()/"tests"/"tests.plist")
+
+    def test_file_name_from_url(self):
+        url = "https://www.dwds.de/kiwix/f/dwds_de_dictionary_nopic_2023-11-20.zim"
+        file_name = self.parser._filename_from(url)
+        self.assertEqual(file_name, "dwds_de_dictionary_nopic_2023-11-20")
+
+        url = "https://www.dwds.de/kiwix/f/dwds_de_dictionary_nopic_2023-11.zim"
+        file_name = self.parser._filename_from(url)
+        self.assertEqual(file_name, "dwds_de_dictionary_nopic_2023-11")
+
+    def test_brand_name_from_file_path(self):
+        filepath = Path().home()/"some"/"dev"/"path"/"project"/"dwds"/"info.json"
+        brand_name = self.parser._brandname_from(filepath)
+        self.assertEqual(brand_name, "dwds")
+
+    def test_version_from_filename(self):
+        version = self.parser._app_version_from(
+            "dwds_de_dictionary_nopic_2023-11-20")
+        self.assertEqual(version, "1023.11")
+
+        version = self.parser._app_version_from(
+            "dwds_de_dictionary_nopic_2023-09-20")
+        self.assertEqual(version, "1023.9")
+
+        version = self.parser._app_version_from(
+            "dwds_de_dictionary_nopic_2023-01")
+        self.assertEqual(version, "1023.1")
+
+        version = self.parser._app_version_from(
+            "dwds_de_dictionary_nopic_2023-12")
+        self.assertEqual(version, "1023.12")
+
+    def test_app_name(self):
+        app_name = self.parser._app_name()
+        self.assertEqual(app_name, "DWDS")
+
+    def test_enforced_language(self):
+        enforced_language = self.parser._enforced_language()
+        self.assertEqual(enforced_language, "de")
+
+    def test_excluded_languages(self):
+        excluded = self.parser._excluded_languages()
+        self.assertIn("**/*.lproj", excluded)
+
+    def test_app_version(self):
+        self.assertEqual(self.parser._app_version(), "1023.12.3")
+
+    def test_app_version_using_a_tag(self):
+        parser = InfoParser(Path()/"tests"/"test.json", build_version=15)
+        self.assertEqual(parser._app_version(), "1023.12.15")
+
+        parser = InfoParser(Path()/"tests"/"test.json", build_version=33)
+        self.assertEqual(parser._app_version(), "1023.12.33")
+
+    def test_as_plist(self):
+        self.parser.create_plist(
+            based_on_plist_file=Path()/"tests"/"Support"/"Info.plist")
+
+    def test_zimurl(self):
+        self.assertEqual(self.parser.zimurl(
+        ), "https://www.dwds.de/kiwix/f/dwds_de_dictionary_nopic_2023-12-15.zim")
+
+    def test_zimfile_path(self):
+        self.assertEqual(self.parser.zim_file_path(),
+                         Path()/"tests"/"dwds_de_dictionary_nopic_2023-12-15.zim")
+
+    def test_auth_value(self):
+        self.assertEqual(self.parser.download_auth(), os.getenv(
+            "DWDS_HTTP_BASIC_ACCESS_AUTHENTICATION"))

tests/test.json

@@ -0,0 +1,13 @@
+{
+    "about_app_url": "https://www.dwds.de",
+    "about_text": "Für Schreibende, Lernende, Lehrende und Sprachinteressierte: Das Digitale Wörterbuch der deutschen Sprache (DWDS) ist das große Bedeutungswörterbuch des Deutschen der Gegenwart. Es bietet umfassende und wissenschaftlich verlässliche lexikalische Informationen, kostenlos und werbefrei.",
+    "app_name": "DWDS",
+    "app_store_id": "id6473090365",
+    "enforced_lang": "de",
+    "settings_default_external_link_to": "alwaysLoad",
+    "settings_show_search_snippet": false,
+    "settings_show_external_link_option": false,
+    "zim_auth": "DWDS_HTTP_BASIC_ACCESS_AUTHENTICATION",
+    "zim_url": "https://www.dwds.de/kiwix/f/dwds_de_dictionary_nopic_2023-12-15.zim",
+    "build_version": 3
+}