From 6a8e254ed17989c4d0bc671d09c3f169de61f5eb Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 17 Nov 2023 16:04:49 +0000 Subject: [PATCH 1/2] Introduce Continuous Deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automatically building and deploying in Github Actions for Nightlies and Releases. Triggered by the following: - every day at 01:32am (nightly mode) - manualy (nightly mode) - on release publication (release mode) This workflow makes extensive use of secrets with no additional safe-guard, given: - `schedule` (nightly) runs only off `main` branch. - `workflow_dispatch` (manual) can run on any in-repo branch (but uses the workflow from `main`) - Release publication requires push access to repo. There are thus two *modes*: Release and Nightly (also used on manual dispatch). The mode sets the `VERSION` either to the YYYY-MM-DD date for nightly or the tag-name for the release. It has four *targets*: `macOS dmg`, `macOS app-store`, `iOS ipa` and `iOS app-store` - **macOS dmg**: universal notarized macOS App in a dmg uploaded to `Kiwix-$VERSION.dmg` - **macOS app-store**: universal notarized macOS App uploaded to the App Store. - **iOS ipa**: iOS App uploaded to `Kiwix-$VERSION.ipa` - **iOS app-store**: iOS App uploaded to the App Store Code Signing is *automatic* (xcode decides which one to use based on availability). We use Apple Distribution one for the app-store targets. IPA uses Apple Development and dmg uses Developer ID. ⚠️ This allows updates CI workflow to make use of the shared xcbuild action --- .github/actions/install-cert/action.yml | 31 ++++ .github/actions/xcbuild/action.yml | 133 +++++++++++++++ .github/dmg-bg.png | Bin 0 -> 2338 bytes .github/dmg-settings.py | 46 ++++++ .github/retry-if-retcode.py | 75 +++++++++ .github/upload_file.py | 110 +++++++++++++ .github/workflows/cd.yml | 209 ++++++++++++++++++++++++ .github/workflows/ci.yml | 105 +++++------- 8 files changed, 648 insertions(+), 61 deletions(-) create mode 100644 .github/actions/install-cert/action.yml create mode 100644 .github/actions/xcbuild/action.yml create mode 100644 .github/dmg-bg.png create mode 100644 .github/dmg-settings.py create mode 100644 .github/retry-if-retcode.py create mode 100644 .github/upload_file.py create mode 100644 .github/workflows/cd.yml diff --git a/.github/actions/install-cert/action.yml b/.github/actions/install-cert/action.yml new file mode 100644 index 00000000..fea88550 --- /dev/null +++ b/.github/actions/install-cert/action.yml @@ -0,0 +1,31 @@ +name: Install Certificate in Keychain +description: Install a single cert in existing keychain + +inputs: + KEYCHAIN: + required: true + KEYCHAIN_PASSWORD: + required: true + SIGNING_CERTIFICATE: + required: true + SIGNING_CERTIFICATE_P12_PASSWORD: + required: true + +runs: + using: composite + steps: + - name: Install certificate + shell: bash + env: + KEYCHAIN: ${{ inputs.KEYCHAIN }} + KEYCHAIN_PASSWORD: ${{ inputs.KEYCHAIN_PASSWORD }} + CERTIFICATE_PATH: /tmp/cert.p12 + SIGNING_CERTIFICATE: ${{ inputs.SIGNING_CERTIFICATE }} + SIGNING_CERTIFICATE_P12_PASSWORD: ${{ inputs.SIGNING_CERTIFICATE_P12_PASSWORD }} + run: | + security unlock-keychain -p $KEYCHAIN_PASSWORD $KEYCHAIN + echo "${SIGNING_CERTIFICATE}" | base64 --decode -o $CERTIFICATE_PATH + security import $CERTIFICATE_PATH -k $KEYCHAIN -P "${SIGNING_CERTIFICATE_P12_PASSWORD}" -A -T /usr/bin/codesign -T /usr/bin/security -T /usr/bin/productbuild + rm $CERTIFICATE_PATH + security find-identity -v $KEYCHAIN + security set-key-partition-list -S apple-tool:,apple: -s -k $KEYCHAIN_PASSWORD $KEYCHAIN diff --git a/.github/actions/xcbuild/action.yml b/.github/actions/xcbuild/action.yml new file mode 100644 index 00000000..cafc8d63 --- /dev/null +++ b/.github/actions/xcbuild/action.yml @@ -0,0 +1,133 @@ +name: Build with XCode +description: Run xcodebuild for Kiwix + +inputs: + action: + required: true + version: + required: true + xc-destination: + required: true + upload-to: + required: true + libkiwix-version: + required: true + APPLE_DEVELOPMENT_SIGNING_CERTIFICATE: + required: true + APPLE_DEVELOPMENT_SIGNING_P12_PASSWORD: + required: true + DEPLOYMENT_SIGNING_CERTIFICATE: + required: false + DEPLOYMENT_SIGNING_CERTIFICATE_P12_PASSWORD: + required: false + KEYCHAIN: + required: false + default: /Users/runner/build.keychain-db + KEYCHAIN_PASSWORD: + required: false + default: mysecretpassword + KEYCHAIN_PROFILE: + required: false + default: build-profile + XC_WORKSPACE: + required: false + default: Kiwix.xcodeproj/project.xcworkspace/ + XC_SCHEME: + required: false + default: Kiwix + XC_CONFIG: + required: false + default: Release + EXTRA_XCODEBUILD: + required: false + default: "" + +runs: + using: composite + steps: + + # not necessary on github runner but serves as documentation for local setup + - name: Update Apple Intermediate Certificate + shell: bash + run: | + curl -L -o ~/Downloads/AppleWWDRCAG3.cer https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer + sudo security import ~/Downloads/AppleWWDRCAG3.cer \ + -k /Library/Keychains/System.keychain \ + -T /usr/bin/codesign \ + -T /usr/bin/security \ + -T /usr/bin/productbuild || true + + - name: Set Xcode version (15.0.1) + shell: bash + # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode + run: sudo xcode-select -s /Applications/Xcode_15.0.1.app + + - name: Create Keychain + shell: bash + env: + KEYCHAIN: ${{ inputs.KEYCHAIN }} + KEYCHAIN_PASSWORD: ${{ inputs.KEYCHAIN_PASSWORD }} + KEYCHAIN_PROFILE: ${{ inputs.KEYCHAIN_PROFILE }} + CERTIFICATE_PATH: /tmp/cert.p12 + APPLE_DEVELOPER_CERTIFICATE_PATH: /tmp/dev-cert.p12 + SIGNING_CERTIFICATE: ${{ inputs.SIGNING_CERTIFICATE }} + SIGNING_CERTIFICATE_P12_PASSWORD: ${{ inputs.SIGNING_CERTIFICATE_P12_PASSWORD }} + APPLE_DEVELOPER_ID_SIGNING_CERTIFICATE: ${{ inputs.APPLE_DEVELOPER_ID_SIGNING_CERTIFICATE }} + APPLE_DEVELOPER_ID_SIGNING_P12_PASSWORD: ${{ inputs.APPLE_DEVELOPER_ID_SIGNING_P12_PASSWORD }} + run: | + security create-keychain -p $KEYCHAIN_PASSWORD $KEYCHAIN + security default-keychain -s $KEYCHAIN + security set-keychain-settings $KEYCHAIN + security unlock-keychain -p $KEYCHAIN_PASSWORD $KEYCHAIN + + - name: Add Apple Development certificate to Keychain + uses: ./.github/actions/install-cert + with: + SIGNING_CERTIFICATE: ${{ inputs.APPLE_DEVELOPMENT_SIGNING_CERTIFICATE }} + SIGNING_CERTIFICATE_P12_PASSWORD: ${{ inputs.APPLE_DEVELOPMENT_SIGNING_P12_PASSWORD }} + KEYCHAIN: ${{ inputs.KEYCHAIN }} + KEYCHAIN_PASSWORD: ${{ inputs.KEYCHAIN_PASSWORD }} + + - name: Add Distribution certificate to Keychain + if: ${{ inputs.DEPLOYMENT_SIGNING_CERTIFICATE }} + uses: ./.github/actions/install-cert + with: + SIGNING_CERTIFICATE: ${{ inputs.DEPLOYMENT_SIGNING_CERTIFICATE }} + SIGNING_CERTIFICATE_P12_PASSWORD: ${{ inputs.DEPLOYMENT_SIGNING_CERTIFICATE_P12_PASSWORD }} + KEYCHAIN: ${{ inputs.KEYCHAIN }} + KEYCHAIN_PASSWORD: ${{ inputs.KEYCHAIN_PASSWORD }} + + - name: Download CoreKiwix.xcframework + env: + XCF_URL: https://download.kiwix.org/release/libkiwix/libkiwix_xcframework-${{ inputs.libkiwix-version }}.tar.gz + shell: bash + run: curl -L -o - $XCF_URL | tar -x --strip-components 2 + + - name: Prepare Xcode + shell: bash + run: xcrun xcodebuild -checkFirstLaunchStatus || xcrun xcodebuild -runFirstLaunch + + - name: Dump build settings + env: + XC_WORKSPACE: ${{ inputs.XC_WORKSPACE }} + XC_SCHEME: ${{ inputs.XC_SCHEME }} + shell: bash + run: xcrun xcodebuild -workspace $XC_WORKSPACE -scheme $XC_SCHEME -showBuildSettings + # build is launched up to twice as it's common the build fails, looking for CoreKiwix module + + - name: Install retry command + shell: bash + run: brew install kadwanev/brew/retry + + - name: Build with Xcode + env: + FRAMEWORK_SEARCH_PATHS: ${{ env.PWD }} + ACTION: ${{ inputs.action }} + VERSION: ${{ inputs.version }} + XC_WORKSPACE: ${{ inputs.XC_WORKSPACE }} + XC_SCHEME: ${{ inputs.XC_SCHEME }} + XC_CONFIG: ${{ inputs.XC_CONFIG }} + XC_DESTINATION: ${{ inputs.xc-destination }} + EXTRA_XCODEBUILD: ${{ inputs.EXTRA_XCODEBUILD }} + shell: bash + run: retry -t 2 -- xcrun xcodebuild ${EXTRA_XCODEBUILD} -workspace $XC_WORKSPACE -scheme $XC_SCHEME -destination "$XC_DESTINATION" -configuration $XC_CONFIG -onlyUsePackageVersionsFromResolvedFile -allowProvisioningUpdates -verbose -archivePath $PWD/Kiwix-$VERSION.xcarchive ${ACTION} diff --git a/.github/dmg-bg.png b/.github/dmg-bg.png new file mode 100644 index 0000000000000000000000000000000000000000..8523ba63fa054216ed74d4685f5698324d57e93c GIT binary patch literal 2338 zcmeAS@N?(olHy`uVBq!ia0y~yV4A|fz&L?}2`DoEO8NsJC0XJcQ4*Y=R#Ki=l*-_k zlAn~S;F+74o*I;zm{M7IGS!BGf#Zm$i(^Q|tv7f5GlI(*4t!L3A1Rt5+?8Ff*;Ie% z+SJVImyXYFB+TWp-O+jbXyb<0A8h=sDytp?hD8C+bZi%{-`QnUN;N#o(fB&u1zj&uXWrPYetBEfyZN4lT5|g(?%SIAZ|9JQG$GL>cq!Sv9Y4a=QiY|G8Db_=wk6 z;5~1??SI?3KTm1Dxf{KAx?PRV%y}kx4Hbt!9sT|E9qT^h-OFt6?6Q7yPtZT6d!5`? z8NLp4o&F^eGu@txe{9@#_5U4j`QAFKJWH91Q)>7n=HGpG;X!uJ)m_{6-cP%}<@y(0#JA_gw4O{F<>X`uXES zobIO`_B6JgF6C`mUq9jXt*G|*votNE*dFmNDgC+jEc^O>@@4N=oZtJU^Yz7Ee!Gyf=7pey#cyxzW2SC8lHzlsYJnktwzSDs$B4H)J(7c#R6ACdMrKiM}|db{pV z=`{V?RDG)s^9jF7qYp3Jw#4G=b#vxp0#U!;S2#21s?VI)v#vHW^TV6(@l({->FfQ; zzO^~~Gf+M%gJVJI(d6aY>%Dpo>;7)?&DSXij{esA^;~`PO!EzEgg>?~zIfu}3WE%d z181Z&7!N+Pb^JHiRhHdf?yhkj(B-$DovUY`y-mODd{oe!qVqDq;QY4oTg}A}n#mkM ze;Qql+p72CZwhDd+n=lYt;$bG0~KtEx$f}cR9}?bnNLgZ$n|qQ;x(yR6}x=C_~*ka zKqIq#wi@@ppX?sqe`_HyY)yWLSQ`g-aW#MZura0T(g9#F_}X6e>k0j(Zx@fc!MCk4H?oZv#zRyzJIaU+cs7_Lk1fQa`$~;?JViT!H)rfj9ZK z%k%8|CHCWR{P~mLh+8Fx4kZi-4t8va%-+5*pTG472iKcowg0` zIfKz?J!@W zdwXBqM){R`IsJ2V>~|!6VffgV8t`xLgx6>EU#u-YZ?d=c+95NAdmGk#{P(o|%(c7! z|IXW)K5N?X^FY@ek)EY+kAK~fa((;PiCI@$-_E`dj4NIdg?krV(%Sh{>g&YL&9Aqw zTeh+_Hz9M=Qsr7V#!EYHgsnZj@5=9#LvJ;;8)llD>`$pOx|p*-ZnSjAtkH&X=}vYv jOJT*-p!Rh;12e;c@SCz?`g`SpjUEP1S3j3^P6 int: + attempts = 0 + while True: + ps = subprocess.run(command, check=False) + attempts += 1 + + # either suceeded or returned an unexpected exit-code, returning. + if ps.returncode == 0 or ps.returncode != retcode: + return ps.returncode + + if attempts >= max_attempts: + print(f"Reached {max_attempts=}") + return ps.returncode + + print( + f"Received retcode={ps.returncode} on attempt #{attempts}. " + f"Retrying in {sleep_seconds}s." + ) + if sleep_seconds: + time.sleep(sleep_seconds) + + +def main(): + parser = argparse.ArgumentParser( + prog="retry-if-retcode", epilog=r"/!\ Append your command after those args!" + ) + + parser.add_argument( + "--retcode", + required=True, + help="Return code to retry when received", + type=int, + ) + + parser.add_argument( + "--attempts", + required=False, + help="Max number of attempts", + type=int, + default=10, + ) + + parser.add_argument( + "--sleep", + required=False, + help="Nb. of seconds to sleep in-between retries", + type=int, + default=1, + ) + + args, command = parser.parse_known_args() + if not command: + print("You must supply a command to run") + return 1 + + return run_command( + max_attempts=args.attempts, + retcode=args.retcode, + sleep_seconds=args.sleep, + command=command, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/upload_file.py b/.github/upload_file.py new file mode 100644 index 00000000..f50f424d --- /dev/null +++ b/.github/upload_file.py @@ -0,0 +1,110 @@ +import argparse +import os +import pathlib +import subprocess +import sys +import urllib.parse + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="scp-upload", + description="Upload files to Kiwix server", + ) + + parser.add_argument( + "--src", required=True, help="filepath to be uploaded", dest="src_path" + ) + + parser.add_argument( + "--dest", + required=True, + help="destination as user@host[:port]/folder/", + dest="dest", + ) + + parser.add_argument( + "--ssh-key", + required=False, + help="filepath to the private key to use for upload", + default=os.getenv("SSH_KEY", ""), + dest="ssh_key", + ) + + args = parser.parse_args() + + ssh_path = ( + pathlib.Path(args.ssh_key or os.getenv("SSH_KEY", "")).expanduser().resolve() + ) + src_path = pathlib.Path(args.src_path).expanduser().resolve() + dest = urllib.parse.urlparse(f"ssh://{args.dest}") + dest_path = pathlib.Path(dest.path) + + if not src_path.exists() or not ssh_path.is_file(): + print(f"Source file “{src_path}” missing") + return 1 + + if not ssh_path.exists() or not ssh_path.is_file(): + print(f"SSH Key “{ssh_path}” missing") + return 1 + + if not dest_path or dest_path == pathlib.Path("") or dest_path == pathlib.Path("/"): + print(f"Must upload in a subfoler, not “{dest_path}”") + return 1 + + return upload( + src_path=src_path, host=dest.netloc, dest_path=dest_path, ssh_path=ssh_path + ) + + +def upload( + src_path: pathlib.Path, host: str, dest_path: pathlib.Path, ssh_path: pathlib.Path +) -> int: + if ":" in host: + host, port = host.split(":", 1) + else: + port = "22" + + # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode + # as the latter would exit on any mkdir error while it is most likely + # the first parts of the destination is already present and thus can't be created + sftp_commands = "\n".join( + [ + f"mkdir {part}" + for part in list(reversed(dest_path.parents)) + [str(dest_path)] + ] + ) + command = [ + "sftp", + "-i", + str(ssh_path), + "-P", + port, + "-o", + "StrictHostKeyChecking=no", + host, + ] + print(f"Creating dest path: {dest_path}") + subprocess.run(command, input=sftp_commands, text=True, check=True) + + command = [ + "scp", + "-c", + "aes128-ctr", + "-rp", + "-P", + port, + "-i", + str(ssh_path), + "-o", + "StrictHostKeyChecking=no", + str(src_path), + f"{host}:{dest_path}/", + ] + print(f"Sending archive with command {' '.join(command)}") + subprocess.run(command, check=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 00000000..2b219efc --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,209 @@ +name: CD + +on: + schedule: + - cron: '32 1 * * *' + workflow_dispatch: + release: + types: [published] + +env: + LIBKIWIX_VERSION: "13.0.0" + KEYCHAIN: /Users/runner/build.keychain-db + KEYCHAIN_PASSWORD: mysecretpassword + KEYCHAIN_PROFILE: build-profile + SSH_KEY: /tmp/id_rsa + APPLE_STORE_AUTH_KEY_PATH: /tmp/authkey.p8 + +jobs: + build_and_deploy: + strategy: + fail-fast: false + matrix: + destination: + - platform: macOS + uploadto: dmg + - platform: macOS + uploadto: app-store + - platform: iOS + uploadto: ipa + xcode_extra: -sdk iphoneos + - platform: iOS + uploadto: app-store + xcode_extra: -sdk iphoneos + runs-on: macos-13 + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Decide whether building nightly or release + env: + PLATFORM: ${{ matrix.destination.platform }} + UPLOAD_TO: ${{ matrix.destination.uploadto }} + EXTRA_XCODEBUILD: ${{ matrix.destination.xcode_extra }} + APPLE_STORE_AUTH_KEY_PATH: ${{ env.APPLE_STORE_AUTH_KEY_PATH }} + APPLE_STORE_AUTH_KEY_ID: ${{ secrets.APPLE_STORE_AUTH_KEY_ID }} + APPLE_STORE_AUTH_KEY_ISSUER_ID: ${{ secrets.APPLE_STORE_AUTH_KEY_ISSUER_ID }} + shell: python + run: | + import datetime + import os + if os.getenv("GITHUB_EVENT_NAME", "") == "release": + is_release = True + version = os.getenv("GITHUB_REF_NAME") + upload_folder = f"release/{version}" + else: + is_release = False + version = str(datetime.date.today()) + upload_folder = f"nightly/{version}" + + export_method = "developer-id" if os.getenv("UPLOAD_TO") == "dmg" else "app-store" + + extra_xcode = os.getenv("EXTRA_XCODEBUILD", "") + if os.getenv("PLATFORM") == "iOS": + extra_xcode += f" -authenticationKeyPath {os.getenv('APPLE_STORE_AUTH_KEY_PATH')}" + extra_xcode += f" -authenticationKeyID {os.getenv('APPLE_STORE_AUTH_KEY_ID')}" + extra_xcode += f" -authenticationKeyIssuerID {os.getenv('APPLE_STORE_AUTH_KEY_ISSUER_ID')}" + + with open(os.getenv("GITHUB_ENV"), "a") as fh: + fh.write(f"VERSION={version}\n") + fh.write(f"ISRELEASE={'yes' if is_release else ''}\n") + fh.write(f"EXPORT_METHOD={export_method}\n") + fh.write(f"UPLOAD_FOLDER={upload_folder}\n") + fh.write(f"EXTRA_XCODEBUILD={extra_xcode}\n") + + - name: Prepare use of Developper ID Certificate + if: ${{ matrix.destination.uploadto == 'dmg' }} + shell: bash + env: + APPLE_DEVELOPER_ID_SIGNING_CERTIFICATE: ${{ secrets.APPLE_DEVELOPER_ID_SIGNING_CERTIFICATE }} + APPLE_DEVELOPER_ID_SIGNING_P12_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_SIGNING_P12_PASSWORD }} + APPLE_DEVELOPER_ID_SIGNING_IDENTITY: ${{ secrets.APPLE_DEVELOPER_ID_SIGNING_IDENTITY }} + run: | + echo "SIGNING_CERTIFICATE=${APPLE_DEVELOPER_ID_SIGNING_CERTIFICATE}" >> "$GITHUB_ENV" + echo "SIGNING_CERTIFICATE_P12_PASSWORD=${APPLE_DEVELOPER_ID_SIGNING_P12_PASSWORD}" >> "$GITHUB_ENV" + echo "SIGNING_IDENTITY=${APPLE_DEVELOPER_ID_SIGNING_IDENTITY}" >> "$GITHUB_ENV" + + - name: Prepare use of Apple Development Certificate + if: ${{ matrix.destination.uploadto == 'ipa' }} + shell: bash + env: + APPLE_DEVELOPMENT_SIGNING_CERTIFICATE: ${{ secrets.APPLE_DEVELOPMENT_SIGNING_CERTIFICATE }} + APPLE_DEVELOPMENT_SIGNING_P12_PASSWORD: ${{ secrets.APPLE_DEVELOPMENT_SIGNING_P12_PASSWORD }} + APPLE_DEVELOPMENT_SIGNING_IDENTITY: ${{ secrets.APPLE_DEVELOPMENT_SIGNING_IDENTITY }} + run: | + echo "SIGNING_CERTIFICATE=${APPLE_DEVELOPMENT_SIGNING_CERTIFICATE}" >> "$GITHUB_ENV" + echo "SIGNING_CERTIFICATE_P12_PASSWORD=${APPLE_DEVELOPMENT_SIGNING_P12_PASSWORD}" >> "$GITHUB_ENV" + echo "SIGNING_IDENTITY=${APPLE_DEVELOPMENT_SIGNING_IDENTITY}" >> "$GITHUB_ENV" + + - name: Prepare use of Apple Distribution Certificate + if: ${{ matrix.destination.uploadto == 'app-store' }} + shell: bash + env: + APPLE_DISTRIBUTION_SIGNING_CERTIFICATE: ${{ secrets.APPLE_DISTRIBUTION_SIGNING_CERTIFICATE }} + APPLE_DISTRIBUTION_SIGNING_P12_PASSWORD: ${{ secrets.APPLE_DISTRIBUTION_SIGNING_P12_PASSWORD }} + APPLE_DEVELOPMENT_SIGNING_IDENTITY: ${{ secrets.APPLE_DEVELOPMENT_SIGNING_IDENTITY }} + run: | + echo "SIGNING_CERTIFICATE=${APPLE_DISTRIBUTION_SIGNING_CERTIFICATE}" >> "$GITHUB_ENV" + echo "SIGNING_CERTIFICATE_P12_PASSWORD=${APPLE_DISTRIBUTION_SIGNING_P12_PASSWORD}" >> "$GITHUB_ENV" + echo "SIGNING_IDENTITY=${APPLE_DEVELOPMENT_SIGNING_IDENTITY}" >> "$GITHUB_ENV" + + - name: Add Apple Store Key + env: + APPLE_STORE_AUTH_KEY_PATH: ${{ env.APPLE_STORE_AUTH_KEY_PATH }} + APPLE_STORE_AUTH_KEY: ${{ secrets.APPLE_STORE_AUTH_KEY }} + shell: bash + run: echo "${APPLE_STORE_AUTH_KEY}" | base64 --decode -o $APPLE_STORE_AUTH_KEY_PATH + + - name: Build xcarchive + uses: ./.github/actions/xcbuild + with: + action: archive + xc-destination: generic/platform=${{ matrix.destination.platform }} + upload-to: ${{ matrix.destination.uploadto }} + libkiwix-version: ${{ env.LIBKIWIX_VERSION }} + version: ${{ env.VERSION }} + APPLE_DEVELOPMENT_SIGNING_CERTIFICATE: ${{ secrets.APPLE_DEVELOPMENT_SIGNING_CERTIFICATE }} + APPLE_DEVELOPMENT_SIGNING_P12_PASSWORD: ${{ secrets.APPLE_DEVELOPMENT_SIGNING_P12_PASSWORD }} + DEPLOYMENT_SIGNING_CERTIFICATE: ${{ env.SIGNING_CERTIFICATE }} + DEPLOYMENT_SIGNING_CERTIFICATE_P12_PASSWORD: ${{ env.SIGNING_CERTIFICATE_P12_PASSWORD }} + KEYCHAIN: ${{ env.KEYCHAIN }} + KEYCHAIN_PASSWORD: ${{ env.KEYCHAIN_PASSWORD }} + KEYCHAIN_PROFILE: ${{ env.KEYCHAIN_PROFILE }} + EXTRA_XCODEBUILD: ${{ env.EXTRA_XCODEBUILD }} + + - name: Add altool credentials to Keychain + shell: bash + env: + APPLE_SIGNING_ALTOOL_USERNAME: ${{ secrets.APPLE_SIGNING_ALTOOL_USERNAME }} + APPLE_SIGNING_ALTOOL_PASSWORD: ${{ secrets.APPLE_SIGNING_ALTOOL_PASSWORD }} + APPLE_SIGNING_TEAM: ${{ secrets.APPLE_SIGNING_TEAM }} + run: | + security find-identity -v $KEYCHAIN + security unlock-keychain -p $KEYCHAIN_PASSWORD $KEYCHAIN + xcrun notarytool store-credentials \ + --apple-id "${APPLE_SIGNING_ALTOOL_USERNAME}" \ + --password "${APPLE_SIGNING_ALTOOL_PASSWORD}" \ + --team-id "${APPLE_SIGNING_TEAM}" \ + --validate \ + --keychain $KEYCHAIN \ + $KEYCHAIN_PROFILE + + - name: Prepare export for ${{ env.EXPORT_METHOD }} + if: ${{ matrix.destination.uploadto != 'ipa' }} + run: | + plutil -create xml1 ./export.plist + plutil -insert destination -string upload ./export.plist + plutil -insert method -string $EXPORT_METHOD ./export.plist + + - name: Prepare export for IPA + if: ${{ matrix.destination.uploadto == 'ipa' }} + run: | + plutil -create xml1 ./export.plist + plutil -insert method -string ad-hoc ./export.plist + plutil -insert provisioningProfiles -dictionary ./export.plist + plutil -replace provisioningProfiles -json '{ "self.Kiwix" : "iOS Team Provisioning Profile" }' ./export.plist + + - name: Upload Archive to Apple (App Store or Notarization) + env: + APPLE_STORE_AUTH_KEY_PATH: ${{ env.APPLE_STORE_AUTH_KEY_PATH }} + APPLE_STORE_AUTH_KEY_ID: ${{ secrets.APPLE_STORE_AUTH_KEY_ID }} + APPLE_STORE_AUTH_KEY_ISSUER_ID: ${{ secrets.APPLE_STORE_AUTH_KEY_ISSUER_ID }} + run: python .github/retry-if-retcode.py --sleep 60 --attempts 5 --retcode 70 xcrun xcodebuild -exportArchive -archivePath $PWD/Kiwix-$VERSION.xcarchive -exportPath $PWD/export/ -exportOptionsPlist export.plist -authenticationKeyPath $APPLE_STORE_AUTH_KEY_PATH -allowProvisioningUpdates -authenticationKeyID $APPLE_STORE_AUTH_KEY_ID -authenticationKeyIssuerID $APPLE_STORE_AUTH_KEY_ISSUER_ID + + - name: Export notarized App from archive + if: ${{ matrix.destination.uploadto == 'dmg' }} + env: + APPLE_STORE_AUTH_KEY_PATH: ${{ env.APPLE_STORE_AUTH_KEY_PATH }} + APPLE_STORE_AUTH_KEY_ID: ${{ secrets.APPLE_STORE_AUTH_KEY_ID }} + APPLE_STORE_AUTH_KEY_ISSUER_ID: ${{ secrets.APPLE_STORE_AUTH_KEY_ISSUER_ID }} + run: python .github/retry-if-retcode.py --sleep 60 --attempts 20 --retcode 65 xcrun xcodebuild -exportNotarizedApp -archivePath $PWD/Kiwix-$VERSION.xcarchive -exportPath $PWD/export/ -authenticationKeyPath $APPLE_STORE_AUTH_KEY_PATH -allowProvisioningUpdates -authenticationKeyID $APPLE_STORE_AUTH_KEY_ID -authenticationKeyIssuerID $APPLE_STORE_AUTH_KEY_ISSUER_ID + + - name: Create DMG + if: ${{ matrix.destination.uploadto == 'dmg' }} + run: | + pip install dmgbuild + dmgbuild -s .github/dmg-settings.py -Dapp=$PWD/export/Kiwix.app -Dbg=.github/dmg-bg.png "Kiwix-$VERSION" $PWD/Kiwix-$VERSION.dmg + + - name: Notarize DMG + if: ${{ matrix.destination.uploadto == 'dmg' }} + run: | + xcrun notarytool submit --keychain $KEYCHAIN --keychain-profile $KEYCHAIN_PROFILE --wait $PWD/Kiwix-$VERSION.dmg + xcrun stapler staple $PWD/Kiwix-$VERSION.dmg + + - name: Add SSH_KEY to filesystem + shell: bash + env: + PRIVATE_KEY: ${{ secrets.SSH_KEY }} + run: | + echo "${PRIVATE_KEY}" > $SSH_KEY + chmod 600 $SSH_KEY + + - name: Upload DMG + if: ${{ matrix.destination.uploadto == 'dmg' }} + run: python .github/upload_file.py --src ${PWD}/Kiwix-${VERSION}.dmg --dest ci@master.download.kiwix.org:30022/data/download/${UPLOAD_FOLDER} --ssh-key ${SSH_KEY} + + - name: Upload IPA + if: ${{ matrix.destination.uploadto == 'ipa' }} + run: | + mv ${PWD}/export/Kiwix.ipa ${PWD}/export/Kiwix-${VERSION}.ipa + python .github/upload_file.py --src ${PWD}/export/Kiwix-${VERSION}.ipa --dest ci@master.download.kiwix.org:30022/data/download/${UPLOAD_FOLDER} --ssh-key ${SSH_KEY} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae6b1fa3..c61a3bb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,74 +8,57 @@ on: env: LIBKIWIX_VERSION: "13.0.0" + APPLE_STORE_AUTH_KEY_PATH: /tmp/authkey.p8 jobs: build: + runs-on: macos-13 strategy: fail-fast: false matrix: destination: - platform: macOS - name: Any Mac - platform: iOS - name: Any iOS Device - runs-on: macos-13 - env: - XC_WORKSPACE: Kiwix.xcodeproj/project.xcworkspace/ - XC_SCHEME: Kiwix - XC_CONFIG: Release - XC_DESTINATION: platform=${{ matrix.destination.platform }},name=${{ matrix.destination.name }} - CERTIFICATE: /tmp/apple-development.p12 - SIGNING_IDENTITY: ${{ secrets.APPLE_DEVELOPMENT_SIGNING_IDENTITY }} - KEYCHAIN: /Users/runner/build.keychain-db - KEYCHAIN_PASSWORD: mysecretpassword - KEYCHAIN_PROFILE: build-profile + xcode_extra: -sdk iphoneos steps: - - name: install Apple certificate - shell: bash - run: | - echo "${{ secrets.APPLE_DEVELOPMENT_SIGNING_CERTIFICATE }}" | base64 --decode -o $CERTIFICATE - security create-keychain -p $KEYCHAIN_PASSWORD $KEYCHAIN - security default-keychain -s $KEYCHAIN - security set-keychain-settings $KEYCHAIN - security unlock-keychain -p $KEYCHAIN_PASSWORD $KEYCHAIN - security import $CERTIFICATE -k $KEYCHAIN -P "${{ secrets.APPLE_DEVELOPMENT_SIGNING_P12_PASSWORD }}" -A -T /usr/bin/codesign -T /usr/bin/security -T /usr/bin/productbuild - rm $CERTIFICATE - security set-key-partition-list -S apple-tool:,apple: -s -k $KEYCHAIN_PASSWORD $KEYCHAIN - security find-identity -v $KEYCHAIN - xcrun notarytool store-credentials \ - --apple-id "${{ secrets.APPLE_SIGNING_ALTOOL_USERNAME }}" \ - --password "${{ secrets.APPLE_SIGNING_ALTOOL_PASSWORD }}" \ - --team-id "${{ secrets.APPLE_SIGNING_TEAM }}" \ - --validate \ - --keychain $KEYCHAIN \ - $KEYCHAIN_PROFILE - # not necessary on github runner but serves as documentation for local setup - - name: Update Apple Intermediate Certificate - run: | - curl -L -o ~/Downloads/AppleWWDRCAG3.cer https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer - sudo security import ~/Downloads/AppleWWDRCAG3.cer \ - -k /Library/Keychains/System.keychain \ - -T /usr/bin/codesign \ - -T /usr/bin/security \ - -T /usr/bin/productbuild || true - - name: Set Xcode version (15.0.1) - # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode - run: sudo xcode-select -s /Applications/Xcode_15.0.1.app - - name: Checkout code - uses: actions/checkout@v3 - - name: Download CoreKiwix.xcframework - env: - XCF_URL: https://download.kiwix.org/release/libkiwix/libkiwix_xcframework-${{ env.LIBKIWIX_VERSION }}.tar.gz - run: curl -L -o - $XCF_URL | tar -x --strip-components 2 - - name: Prepare Xcode - run: xcrun xcodebuild -checkFirstLaunchStatus || xcrun xcodebuild -runFirstLaunch - - name: Dump build settings - run: xcrun xcodebuild -workspace $XC_WORKSPACE -scheme $XC_SCHEME -showBuildSettings - # build is launched up to twice as it's common the build fails, looking for CoreKiwix module - - name: Install retry command - run: brew install kadwanev/brew/retry - - name: Build for ${{ matrix.destination.platform }}/${{ matrix.destination.name }} - env: - FRAMEWORK_SEARCH_PATHS: /Users/runner/work/apple/apple/ - run: retry -t 2 -- xcrun xcodebuild -workspace $XC_WORKSPACE -scheme $XC_SCHEME -destination "$XC_DESTINATION" -configuration $XC_CONFIG -onlyUsePackageVersionsFromResolvedFile -derivedDataPath $PWD/build -allowProvisioningUpdates -verbose build + - name: Checkout code + uses: actions/checkout@v3 + + - name: Add Apple Store Key + if: ${{ matrix.destination.platform == 'iOS' }} + env: + APPLE_STORE_AUTH_KEY_PATH: ${{ env.APPLE_STORE_AUTH_KEY_PATH }} + APPLE_STORE_AUTH_KEY: ${{ secrets.APPLE_STORE_AUTH_KEY }} + shell: bash + run: echo "${APPLE_STORE_AUTH_KEY}" | base64 --decode -o $APPLE_STORE_AUTH_KEY_PATH + + - name: Extend EXTRA_XCODEBUILD + if: ${{ matrix.destination.platform == 'iOS' }} + env: + EXTRA_XCODEBUILD: ${{ matrix.destination.xcode_extra }} + APPLE_STORE_AUTH_KEY_PATH: ${{ env.APPLE_STORE_AUTH_KEY_PATH }} + APPLE_STORE_AUTH_KEY_ID: ${{ secrets.APPLE_STORE_AUTH_KEY_ID }} + APPLE_STORE_AUTH_KEY_ISSUER_ID: ${{ secrets.APPLE_STORE_AUTH_KEY_ISSUER_ID }} + shell: python + run: | + import os + extra_xcode = os.getenv("EXTRA_XCODEBUILD", "") + extra_xcode += f" -authenticationKeyPath {os.getenv('APPLE_STORE_AUTH_KEY_PATH')}" + extra_xcode += f" -authenticationKeyID {os.getenv('APPLE_STORE_AUTH_KEY_ID')}" + extra_xcode += f" -authenticationKeyIssuerID {os.getenv('APPLE_STORE_AUTH_KEY_ISSUER_ID')}" + + with open(os.getenv("GITHUB_ENV"), "a") as fh: + fh.write(f"EXTRA_XCODEBUILD={extra_xcode}\n") + + - name: Build + uses: ./.github/actions/xcbuild + with: + action: build + xc-destination: generic/platform=${{ matrix.destination.platform }} + upload-to: dev + libkiwix-version: ${{ env.LIBKIWIX_VERSION }} + version: CI + APPLE_DEVELOPMENT_SIGNING_CERTIFICATE: ${{ secrets.APPLE_DEVELOPMENT_SIGNING_CERTIFICATE }} + APPLE_DEVELOPMENT_SIGNING_P12_PASSWORD: ${{ secrets.APPLE_DEVELOPMENT_SIGNING_P12_PASSWORD }} + EXTRA_XCODEBUILD: ${{ env.EXTRA_XCODEBUILD }} + From 5cf9877233210d6763b57ffc248c0c7607596f7a Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Wed, 22 Nov 2023 15:46:54 +0000 Subject: [PATCH 2/2] External PR access to CI Instead of accepting `pull_request` event which doesn't passes secrets, we are now using `pull_request_target` event which does. To prevent leaking secrets, we are using an initial job that sets the `environment` dynamically based on the event. If it's an external PR, then it deploys to `external` environment. This `external` environment has been created, includes required secrets and mandates approval by automactic, kelson or rgaudin Because `pull_request_target` is meant to run on `main` code and not PR's one, we also change the checkout ref. With this setup, CI runs unattended for pushes on main or PRs from our own repo but requires validation for external ones Note: this is based on April 2023 instructions at https://iterative.ai/blog/testing-external-contributions-using-github-actions-secrets --- .github/workflows/ci.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c61a3bb4..e43f967e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI on: - pull_request: + pull_request_target: push: branches: - main @@ -11,7 +11,14 @@ env: APPLE_STORE_AUTH_KEY_PATH: /tmp/authkey.p8 jobs: + authorize: + # sets environment based on origin of PR: internal (non-existent) for own-repo or external (requires reviewer to run) for external repos + environment: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'external' || 'internal' }} + runs-on: ubuntu-22.04 + steps: + - run: true build: + needs: authorize runs-on: macos-13 strategy: fail-fast: false @@ -23,6 +30,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + with: + # /!\ important: this checks out code from the HEAD of the PR instead of the main branch (for pull_request_target) + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Add Apple Store Key if: ${{ matrix.destination.platform == 'iOS' }}