From f16e89b8dd4147e075ec73ada07e96615aab7d26 Mon Sep 17 00:00:00 2001 From: Jaifroid Date: Sat, 2 Apr 2022 13:50:24 +0100 Subject: [PATCH] Enable cloud build of portable Electron app Former-commit-id: 1c5e898a947e22570cd316f57752572b766d7508 [formerly adc83096414fde0ae7ea6b98e7c1969da334f2dd] [formerly b7c83d0b68641e73d5420f5cff0ad099c11d257a] [formerly ed05c6d523b53343f8ca11ce7a393119516b0d31 [formerly bcf276bb09b80748b9aa7549abb461a6794562ec [formerly 3222ffef425df55b11b2e65dbb8ed52e2ed8afed]]] Former-commit-id: 8dd2062df66de78c92caf8367226d9d4f424b712 [formerly 701fbbc2ffd196622b5990444f6e9ffe871ebf77 [formerly 631300705c1d2cdb26791965eebd88f8d7b1ab84]] Former-commit-id: 1c572683b4f3083d374222e49312e4272b972f94 [formerly 391e9b68962a335f891f2ac0a86ca30e85679167] Former-commit-id: d0839384c27c4b92589f45250748945e956a2254 --- .github/workflows/build-electron.yml | 9 +- scripts/Build-Electron.ps1 | 12 +- scripts/Create-DraftRelease.ps1 | 44 ++++--- scripts/Publish-ElectronPackages.ps1 | 175 ++++++++++++++++++++------- 4 files changed, 170 insertions(+), 70 deletions(-) diff --git a/.github/workflows/build-electron.yml b/.github/workflows/build-electron.yml index ee9e9147..b011b9d8 100644 --- a/.github/workflows/build-electron.yml +++ b/.github/workflows/build-electron.yml @@ -59,11 +59,16 @@ jobs: $INPUT_TARGET = $Env:INPUT_TARGET $CRON_LAUNCHED = $Env:CRON_LAUNCHED ./scripts/Rewrite-AppVersion.ps1 + - name: build portable Electron app + run: | + $INPUT_VERSION_E = $Env:INPUT_VERSION -replace '^([0-9.]+)E?(.*)', '$1-E$2' + ./scripts/Create-DraftRelease -buildonly -tag_name $INPUT_VERSION_E -portableonly -wingetprompt N - name: publish run: | npm run publish $SSH_KEY = $Env:SSH_KEY echo "$SSH_KEY" > .\scripts\ssh_key + $GITHUB_TOKEN = $Env:GITHUB_TOKEN $INPUT_VERSION = $Env:INPUT_VERSION $INPUT_TARGET = $Env:INPUT_TARGET $CRON_LAUNCHED = $Env:CRON_LAUNCHED @@ -88,11 +93,13 @@ jobs: $INPUT_TARGET = $Env:INPUT_TARGET $CRON_LAUNCHED = $Env:CRON_LAUNCHED ./scripts/Rewrite-AppVersion.ps1 + - name: build NWJS app + run: ./scripts/Build-NWJS.ps1 -only32bit - name: publish run: | - ./scripts/Build-NWJS.ps1 -only32bit $SSH_KEY = $Env:SSH_KEY echo "$SSH_KEY" > .\scripts\ssh_key + $GITHUB_TOKEN = $Env:GITHUB_TOKEN $INPUT_VERSION = $Env:INPUT_VERSION $INPUT_TARGET = $Env:INPUT_TARGET $CRON_LAUNCHED = $Env:CRON_LAUNCHED diff --git a/scripts/Build-Electron.ps1 b/scripts/Build-Electron.ps1 index 2d6475c6..c6e58aea 100644 --- a/scripts/Build-Electron.ps1 +++ b/scripts/Build-Electron.ps1 @@ -1,10 +1,8 @@ # This script is intended to be run by Create-DraftRelease, and must be dot-sourced (run with `. ./Build-Electron.ps1` or `. /path/to/Build-Electron.ps1`) # because it modifies variables needed in Create-DraftRelease $base_dir = "$PSScriptRoot/../bld/electron/" -if ($text_tag -match 'WikiMed') { - $electronbuild = "local" -} -if (($electronbuild -eq "")) { +"`nSelected base_tag: $base_tag" +if ($electronbuild -eq "") { "" $electronbuild_check = Read-Host "Do you want to build Electron packages on GitHub instead of locally? [Y/N]" $electronbuild_check = -Not ( $electronbuild_check -imatch 'n' ) @@ -56,7 +54,7 @@ if ($alt_tag -imatch 'WikiMed|Wikivoyage') { $WinInstaller = $base_dir + "$alt_tag.by.Kiwix.Setup.$numeric_tag-E.exe" } if ($electronbuild -eq "local") { - if (-Not (Test-Path $WinInstaller -PathType Leaf)) { + if (-Not (Test-Path $WinInstaller -PathType Leaf) -and (-not $portableonly)) { "No package found: building $WinInstaller..." if (-Not $dryrun) { npm run dist-win @@ -67,7 +65,7 @@ if ($electronbuild -eq "local") { } } } else { - "Package found." + "Package found or only building portable package." } } # Portable app isn't built in the cloud, so we always run this @@ -117,7 +115,7 @@ if (-Not ($old_windows_support -or (Test-Path $comp_electron_archive -PathType L "Compressing: $AddAppPackage, $compressed_assets_dir to $comp_electron_archive" if (-Not $dryrun) { "$AddAppPackage", "$compressed_assets_dir" | Compress-Archive -DestinationPath $comp_electron_archive -Force } } -if ($electronbuild -eq "local") { +if ($electronbuild -eq "local" -and (-not $portableonly)) { # Package Electron app for Linux "`nChecking for Electron packages for Linux..." $LinuxBasePackage = $base_dir + "Kiwix JS $alt_tag-$numeric_tag-E" diff --git a/scripts/Create-DraftRelease.ps1 b/scripts/Create-DraftRelease.ps1 index 3f82f9af..8a17694e 100644 --- a/scripts/Create-DraftRelease.ps1 +++ b/scripts/Create-DraftRelease.ps1 @@ -7,8 +7,9 @@ param ( [switch]$draftonly = $false, [switch]$buildonly = $false, [string]$electronbuild = "", # 'local' or 'cloud' + [switch]$portableonly = $false, # If set, only the portable electron build will be built. Implies local electron build. [switch]$updatewinget = $false, - [string]$respondtowingetprompt = "" # Provide an override response (Y/N) to the winget prompt at the end of the script - for automation + [string]$wingetprompt = "" # Provide an override response (Y/N) to the winget prompt at the end of the script - for automation ) # DEV: To build Electron packages for all platforms and NWJS for XP and Vista in a single release, use, e.g., "v1.3.0E+N" (Electron + NWJS) # DEV: To build UWP + Electron in a single release (for WikiMed or Wikivoyage), use "v1.3.0+E" (plus Electron) @@ -20,8 +21,11 @@ param ( # Provide parameters $release_uri = 'https://api.github.com/repos/kiwix/kiwix-js-windows/releases' -$github_token = Get-Content -Raw "$PSScriptRoot/github_token" - +if ($GITHUB_TOKEN) { + $github_token = $GITHUB_TOKEN +} else { + $github_token = Get-Content -Raw "$PSScriptRoot/github_token" +} $init_params = Get-Content -Raw "$PSScriptRoot\..\www\js\init.js" $serviceworker = Select-String 'appVersion' "$PSScriptRoot\..\service-worker.js" -List @@ -159,29 +163,26 @@ if ($json_object -imatch '"name":\s"([\w]+-[^"]+)') { "Package name: $package_name" # Determine type of Electron build if any -$electronbuild = "" -if ($text_tag -match 'WikiMed') { - $electronbuild = "local" -} if (($flavour -match '_E') -or $plus_electron) { - if (($electronbuild -eq "")) { + if ($portableonly) { $electronbuild = 'local' } + if ($electronbuild -eq "") { "" $electronbuild_check = Read-Host "Do you want to build Electron packages on GitHub instead of locally? [Y/N]" $electronbuild_check = -Not ( $electronbuild_check -imatch 'n' ) if ($electronbuild_check) { "`nSelecting cloud build..." $electronbuild = 'cloud' - if (-Not ($release_tag_name -cmatch '-E$')) { - $original_release_tag_name = $release_tag_name - $release_tag_name = $release_tag_name -creplace '-?E$|-WikiMed|-Wikivoyage', '' - $release_tag_name = $release_tag_name + '-E' - "Changing release tag name to $release_tag_name" - } } else { "`nSelecting local build..." $electronbuild = 'local' } } + if (-Not ($release_tag_name -cmatch '-E$') -and ($electronbuild -eq 'cloud')) { + $original_release_tag_name = $release_tag_name + $release_tag_name = $release_tag_name -creplace '-?E$|-WikiMed|-Wikivoyage', '' + $release_tag_name = $release_tag_name + '-E' + "Changing release tag name to $release_tag_name" + } } # Create the Draft Release text @@ -214,7 +215,7 @@ $release_params = @{ # Post to the release server if (-Not ($dryrun -or $buildonly -or $updatewinget)) { $release = Invoke-RestMethod @release_params -} elseif (-Not $updatewinget) { +} elseif (-Not ($updatewinget -or $buildonly -or $updatewinget)) { "[DRYRUN] Release Body:`n$release_body" } @@ -283,7 +284,10 @@ if ($dryrun -or $buildonly -or $release.assets_url -imatch '^https:') { $AppImageArchives = @() if ($flavour -eq '_E') { "Building Electron packages..." + $base_tag_origin = $base_tag + $base_tag = $base_tag -replace '^([0-9.]+).*', '$1-E' . $PSScriptRoot/Build-Electron.ps1 # Note that we are dot-sourcing this, so that variables will be available in this scope + $base_tag = $base_tag_origin } elseif ($flavour -eq '_N') { # Package NWJS app if necessary $base_dir = "$PSScriptRoot/../bld/nwjs" @@ -442,7 +446,7 @@ if ($dryrun -or $buildonly -or $release.assets_url -imatch '^https:') { if ($plus_electron) { "Building add-on: Electron packages..." $base_tag_origin = $base_tag - $base_tag = $base_tag -replace '^([\d.]+)', '$1-E' + $base_tag = $base_tag -replace '^([0-9.]+).*', '$1-E' . $PSScriptRoot/Build-Electron.ps1 $base_tag = $base_tag_origin } @@ -471,7 +475,7 @@ if ($dryrun -or $buildonly -or $release.assets_url -imatch '^https:') { } if ($plus_electron) { $upload_assets += $AppImageArchives - if ($electronbuild -eq 'local') { $upload_assets += $WinInstaller } + if ($electronbuild -eq 'local' -and (-not $portableonly)) { $upload_assets += $WinInstaller } if ($old_windows_support) { $upload_assets += $nwjs_archives } else { @@ -555,13 +559,13 @@ if ($dryrun -or $buildonly -or $release.assets_url -imatch '^https:') { } # Now update winget manifest if we are not building NWJS or Electron if ($flavour -eq '' -or $flavour -eq '_E') { - if ($respondtowingetprompt) { - $wingetcreate_check = $respondtowingetprompt + if ($wingetprompt) { + $wingetcreate_check = $wingetprompt } else { $wingetcreate_check = Read-Host "Would you like to update the WinGet repository with these new builds?`nWARNING: be sure you have published the draft release (if in doubt answer N)! [Y/N]" } $wingetcreate_check = $wingetcreate_check -imatch 'y' - if ($original_release_tag_name) { + if ($original_release_tag_name -and (-not $wingetprompt)) { $check = Read-Host "Did you change the Release Tag Name? (Y/N)" if ($check -imatch 'N') { "You must change the Tag name!" diff --git a/scripts/Publish-ElectronPackages.ps1 b/scripts/Publish-ElectronPackages.ps1 index a052cd36..749fa51a 100644 --- a/scripts/Publish-ElectronPackages.ps1 +++ b/scripts/Publish-ElectronPackages.ps1 @@ -1,9 +1,15 @@ -# Publish Kiwix Electron packages to Kiwix download server +# Publish Kiwix Electron packages to a GitHub draft release and/or to Kiwix download server [CmdletBinding()] param ( - [string]$test = "" + [string]$test = "", + [switch]$dryrun = $false, + [switch]$githubonly = $false, + [string]$tag = "" ) - +if ($tag) { + # If user overrode the INPUT_VERSION, use it + $INPUT_VERSION = $tag +} $target = "/data/download/release/kiwix-js-electron" $keyfile = "$PSScriptRoot\ssh_key" $keyfile = $keyfile -ireplace '[\\/]', '/' @@ -19,8 +25,6 @@ if ($CRON_LAUNCHED) { $target = "/data/download/nightly/$current_date" } -"`nUploading packages to https://download.kiwix.org$target/ ...`n" -& "C:\Program Files\Git\usr\bin\ssh.exe" @('-o', 'StrictHostKeyChecking=no', '-i', "$keyfile", 'ci@download.kiwix.org', "mkdir -p $target") if ((Get-Content ./package.json) -match 'nwVersion') { $Packages = $(ls bld/NWJS/*.*) } else { @@ -29,45 +33,132 @@ if ((Get-Content ./package.json) -match 'nwVersion') { if ($test) { $Packages = @($test) } -$Packages | % { - $file = $_ - if ($file -match '\.(exe|zip|msix)$') { - $directory = $file -replace '^(.+[\\/])[^\\/]+$', '$1' - $filename = $file -replace '^.+[\\/]([^\\/]+)$', '$1' - # Convert all spaces and hyphens to underscore - $filename = $filename -replace '[\s-]', '_' - $filename = $filename -creplace '_N([_.])', '_NWJS$1' - # Swap architecture and release number, and remove redundant -win - $filename = $filename -replace '(windows(?:_XP)?)(.+)_win(_ia32[^.]*)', '$1$3$2' - # Convert filename to lowercase - $filename = $filename.ToLower() - # Convert back appname to hyphens - $filename = $filename -replace 'kiwix_js_(electron|windows)', 'kiwix-js-$1' - # Fix Windows Setup version so that it is clear it is a Windows executable - $filename = $filename -replace 'electron_setup', 'electron_win_setup' - # Change underscore to hyphen in win type and remove redundant E - $filename = ($filename -creplace '_xp([_.])', '-xp$1') -creplace '_e([_.])', '$1' - # Move nwjs - $filename = $filename -replace '-windows(.*)_nwjs', '-nwjs_win$1' - # Change ia32 to i386 - $filename = $filename -replace 'ia32', 'i386' - if ($CRON_LAUNCHED) { - # Remove the version number - $filename = $filename -replace '_[0-9.]+([-_.])', '$1' + +if (-not $CRON_LAUNCHED) { + "`nChecking for a draft publishing target on GitHub..." + if (-not $GITHUB_TOKEN) { + $GITHUB_TOKEN = Get-Content -Raw "$PSScriptRoot/github_token" + } + $draft_release_params = @{ + Uri = "https://api.github.com/repos/kiwix/kiwix-js-windows/releases" + Method = 'GET' + Headers = @{ + 'Authorization' = "token $GITHUB_TOKEN" + 'Accept' = 'application/vnd.github.v3+json' } - # Put back together - $renamed_file = "$directory$filename" - if ($test) { - "`n$file was renamed to $renamed_file" - } else { - # Rename the file - if ($file -ne $renamed_file) { - mv $file $renamed_file + ContentType = "application/json" + } + $releases = Invoke-RestMethod @draft_release_params + $release_found = $false + $release = $null + $releases| Where-Object { $release_found -eq $False } | % { + $release = $_ + if (($release.draft -eq $true) -and ($release.tag_name -eq $INPUT_VERSION) ) { + $release_found = $true + } + } + if ($release_found) { + if ($dryrun) { + $release_json = $release | ConvertTo-Json + "[DRYRUN:] Draft release found: `n$release_json" + } + $upload_uri = $release.upload_url -ireplace '\{[^{}]+}', '' + "`nUploading assets to: $upload_uri..." + ForEach($asset in $packages) { + if (-Not $asset) { Continue } + if (-Not ($file -match '\.(exe|zip|msix)$')) { Continue } + # Replace backslash with forward slash + $asset_name = $asset -replace '^.*[\\/]([^\\/]+)$', '$1' + # Replace spaces with hyphens + $asset_name = $asset_name -replace '\s', '-'; + # Establish upload params + $upload_params = @{ + Uri = $upload_uri + "?name=$asset_name" + Method = 'POST' + Headers = @{ + 'Authorization' = "token $GITHUB_TOKEN" + 'Accept' = 'application/vnd.github.v3+json' + } + # Body = [System.IO.File]::ReadAllBytes($upload_file) + InFile = $asset + ContentType = 'application/octet-stream' + } + "`n*** Uploading $asset..." + # Upload asset to the release server + # $upload = [System.IO.File]::ReadAllBytes($upload_file) | Invoke-RestMethod @upload_params + if (-Not $dryrun) { + # Disable progress because it causes high CPU usage on large files, and slows down upload + $ProgressPreference = 'SilentlyContinue' + $upload = Invoke-RestMethod @upload_params + } + if ($dryrun -or $upload.name -eq ($asset_name -replace '\s', '.')) { + if (-Not $dryrun) { + "Upload successfully posted as " + $upload.url + "Full details:" + echo $upload + } else { + echo "DRYRUN with these upload parameters:`n" + @upload_params + } + } else { + "`nI'm sorry, this upload appears to have failed! Please upload manually or try again..." + if ($upload) { + "`nThe server returned:" + echo $upload + } else { + "The server did not respond." + } + } + } + } else { + "No draft release matching the tag $INPUT_VERSION was found." + } + +} + +if (-not $githubonly) { + "`nUploading packages to https://download.kiwix.org$target/ ...`n" + & "C:\Program Files\Git\usr\bin\ssh.exe" @('-o', 'StrictHostKeyChecking=no', '-i', "$keyfile", 'ci@download.kiwix.org', "mkdir -p $target") + + $Packages | % { + $file = $_ + if ($file -match '\.(exe|zip|msix)$') { + $directory = $file -replace '^(.+[\\/])[^\\/]+$', '$1' + $filename = $file -replace '^.+[\\/]([^\\/]+)$', '$1' + # Convert all spaces and hyphens to underscore + $filename = $filename -replace '[\s-]', '_' + $filename = $filename -creplace '_N([_.])', '_NWJS$1' + # Swap architecture and release number, and remove redundant -win + $filename = $filename -replace '(windows(?:_XP)?)(.+)_win(_ia32[^.]*)', '$1$3$2' + # Convert filename to lowercase + $filename = $filename.ToLower() + # Convert back appname to hyphens + $filename = $filename -replace 'kiwix_js_(electron|windows)', 'kiwix-js-$1' + # Fix Windows Setup version so that it is clear it is a Windows executable + $filename = $filename -replace 'electron_setup', 'electron_win_setup' + # Change underscore to hyphen in win type and remove redundant E + $filename = ($filename -creplace '_xp([_.])', '-xp$1') -creplace '_e([_.])', '$1' + # Move nwjs + $filename = $filename -replace '-windows(.*)_nwjs', '-nwjs_win$1' + # Change ia32 to i386 + $filename = $filename -replace 'ia32', 'i386' + if ($CRON_LAUNCHED) { + # Remove the version number + $filename = $filename -replace '_[0-9.]+([-_.])', '$1' + } + # Put back together + $renamed_file = "$directory$filename" + if ($test) { + "`n$file was renamed to $renamed_file" + } else { + # Rename the file + if ($file -ne $renamed_file) { + mv $file $renamed_file + } + # Replace absolute path with relative, and normalize to forward slashes + $renamed_file = $renamed_file -replace '^.*?([\\/]bld)', '.$1' -replace '[\\/]', '/' + "Copying $renamed_file to $target..." + & "C:\Program Files\Git\usr\bin\scp.exe" @('-o', 'StrictHostKeyChecking=no', '-i', "$keyfile", "$renamed_file", "ci@download.kiwix.org:$target") } - # Replace absolute path with relative, and normalize to forward slashes - $renamed_file = $renamed_file -replace '^.*?([\\/]bld)', '.$1' -replace '[\\/]', '/' - "Copying $renamed_file to $target..." - & "C:\Program Files\Git\usr\bin\scp.exe" @('-o', 'StrictHostKeyChecking=no', '-i', "$keyfile", "$renamed_file", "ci@download.kiwix.org:$target") } } }