Enable cloud build of portable Electron app

Former-commit-id: 1c5e898a947e22570cd316f57752572b766d7508 [formerly adc83096414fde0ae7ea6b98e7c1969da334f2dd] [formerly b7c83d0b68641e73d5420f5cff0ad099c11d257a] [formerly ed05c6d523b53343f8ca11ce7a393119516b0d31 [formerly bcf276bb09b80748b9aa7549abb461a6794562ec [formerly 3222ffef425df55b11b2e65dbb8ed52e2ed8afed]]]
Former-commit-id: 8dd2062df66de78c92caf8367226d9d4f424b712 [formerly 701fbbc2ffd196622b5990444f6e9ffe871ebf77 [formerly 631300705c1d2cdb26791965eebd88f8d7b1ab84]]
Former-commit-id: 1c572683b4f3083d374222e49312e4272b972f94 [formerly 391e9b68962a335f891f2ac0a86ca30e85679167]
Former-commit-id: d0839384c27c4b92589f45250748945e956a2254

.github/workflows/build-electron.yml

@@ -59,11 +59,16 @@ jobs:
         $INPUT_TARGET = $Env:INPUT_TARGET
         $CRON_LAUNCHED = $Env:CRON_LAUNCHED
         ./scripts/Rewrite-AppVersion.ps1
+    - name: build portable Electron app
+      run: |
+        $INPUT_VERSION_E = $Env:INPUT_VERSION -replace '^([0-9.]+)E?(.*)', '$1-E$2'
+        ./scripts/Create-DraftRelease -buildonly -tag_name $INPUT_VERSION_E -portableonly -wingetprompt N
     - name: publish
       run: |
         npm run publish
         $SSH_KEY = $Env:SSH_KEY
         echo "$SSH_KEY" > .\scripts\ssh_key
+        $GITHUB_TOKEN = $Env:GITHUB_TOKEN
         $INPUT_VERSION = $Env:INPUT_VERSION
         $INPUT_TARGET = $Env:INPUT_TARGET
         $CRON_LAUNCHED = $Env:CRON_LAUNCHED
@@ -88,11 +93,13 @@ jobs:
         $INPUT_TARGET = $Env:INPUT_TARGET
         $CRON_LAUNCHED = $Env:CRON_LAUNCHED
         ./scripts/Rewrite-AppVersion.ps1
+    - name: build NWJS app
+      run: ./scripts/Build-NWJS.ps1 -only32bit
     - name: publish
       run: |
-        ./scripts/Build-NWJS.ps1 -only32bit
         $SSH_KEY = $Env:SSH_KEY
         echo "$SSH_KEY" > .\scripts\ssh_key
+        $GITHUB_TOKEN = $Env:GITHUB_TOKEN
         $INPUT_VERSION = $Env:INPUT_VERSION
         $INPUT_TARGET = $Env:INPUT_TARGET
         $CRON_LAUNCHED = $Env:CRON_LAUNCHED

scripts/Build-Electron.ps1

@@ -1,10 +1,8 @@
 # This script is intended to be run by Create-DraftRelease, and must be dot-sourced (run with `. ./Build-Electron.ps1` or `. /path/to/Build-Electron.ps1`)
 # because it modifies variables needed in Create-DraftRelease
 $base_dir = "$PSScriptRoot/../bld/electron/"
-if ($text_tag -match 'WikiMed') {
+"`nSelected base_tag: $base_tag"
-  $electronbuild = "local"
+if ($electronbuild -eq "") {
-}
-if (($electronbuild -eq "")) {
   ""
   $electronbuild_check = Read-Host "Do you want to build Electron packages on GitHub instead of locally? [Y/N]"
   $electronbuild_check = -Not ( $electronbuild_check -imatch 'n' )
@@ -56,7 +54,7 @@ if ($alt_tag -imatch 'WikiMed|Wikivoyage') {
   $WinInstaller = $base_dir + "$alt_tag.by.Kiwix.Setup.$numeric_tag-E.exe"
 }
 if ($electronbuild -eq "local") {
-  if (-Not (Test-Path $WinInstaller -PathType Leaf)) {
+  if (-Not (Test-Path $WinInstaller -PathType Leaf) -and (-not $portableonly)) {
     "No package found: building $WinInstaller..."
     if (-Not $dryrun) {
       npm run dist-win
@@ -67,7 +65,7 @@ if ($electronbuild -eq "local") {
       }
     }
   } else {
-    "Package found."
+    "Package found or only building portable package."
   }
 }
 # Portable app isn't built in the cloud, so we always run this
@@ -117,7 +115,7 @@ if (-Not ($old_windows_support -or (Test-Path $comp_electron_archive -PathType L
   "Compressing: $AddAppPackage, $compressed_assets_dir to $comp_electron_archive"
   if (-Not $dryrun) { "$AddAppPackage", "$compressed_assets_dir" | Compress-Archive -DestinationPath $comp_electron_archive -Force }
 }
-if ($electronbuild -eq "local") {
+if ($electronbuild -eq "local" -and (-not $portableonly)) {
   # Package Electron app for Linux
   "`nChecking for Electron packages for Linux..."
   $LinuxBasePackage = $base_dir + "Kiwix JS $alt_tag-$numeric_tag-E"

scripts/Create-DraftRelease.ps1

@@ -7,8 +7,9 @@ param (
     [switch]$draftonly = $false,
     [switch]$buildonly = $false,
     [string]$electronbuild = "", # 'local' or 'cloud'
+    [switch]$portableonly = $false, # If set, only the portable electron build will be built. Implies local electron build.
     [switch]$updatewinget = $false,
-    [string]$respondtowingetprompt = "" # Provide an override response (Y/N) to the winget prompt at the end of the script - for automation
+    [string]$wingetprompt = "" # Provide an override response (Y/N) to the winget prompt at the end of the script - for automation
 )
 # DEV: To build Electron packages for all platforms and NWJS for XP and Vista in a single release, use, e.g., "v1.3.0E+N" (Electron + NWJS)
 # DEV: To build UWP + Electron in a single release (for WikiMed or Wikivoyage), use "v1.3.0+E" (plus Electron)
@@ -20,8 +21,11 @@ param (
 
 # Provide parameters
 $release_uri = 'https://api.github.com/repos/kiwix/kiwix-js-windows/releases'
-$github_token = Get-Content -Raw "$PSScriptRoot/github_token"
+if ($GITHUB_TOKEN) {
-
+  $github_token = $GITHUB_TOKEN
+} else {
+  $github_token = Get-Content -Raw "$PSScriptRoot/github_token"
+}
 $init_params = Get-Content -Raw "$PSScriptRoot\..\www\js\init.js"
 $serviceworker = Select-String 'appVersion' "$PSScriptRoot\..\service-worker.js" -List
 
@@ -159,29 +163,26 @@ if ($json_object -imatch '"name":\s"([\w]+-[^"]+)') {
 "Package name: $package_name"
 
 # Determine type of Electron build if any
-$electronbuild = ""
-if ($text_tag -match 'WikiMed') {
-  $electronbuild = "local"
-}
 if (($flavour -match '_E') -or $plus_electron) {
-  if (($electronbuild -eq "")) {
+  if ($portableonly) { $electronbuild = 'local' }
+  if ($electronbuild -eq "") {
     ""
     $electronbuild_check = Read-Host "Do you want to build Electron packages on GitHub instead of locally? [Y/N]"
     $electronbuild_check = -Not ( $electronbuild_check -imatch 'n' )
     if ($electronbuild_check) {
       "`nSelecting cloud build..."
       $electronbuild = 'cloud'
-      if (-Not ($release_tag_name -cmatch '-E$')) {
-        $original_release_tag_name = $release_tag_name
-        $release_tag_name = $release_tag_name -creplace '-?E$|-WikiMed|-Wikivoyage', ''
-        $release_tag_name = $release_tag_name + '-E' 
-        "Changing release tag name to $release_tag_name"
-      }
     } else {
       "`nSelecting local build..."
       $electronbuild = 'local'
     }
   }
+  if (-Not ($release_tag_name -cmatch '-E$') -and ($electronbuild -eq 'cloud')) {
+    $original_release_tag_name = $release_tag_name
+    $release_tag_name = $release_tag_name -creplace '-?E$|-WikiMed|-Wikivoyage', ''
+    $release_tag_name = $release_tag_name + '-E' 
+    "Changing release tag name to $release_tag_name"
+  }
 }
 
 # Create the Draft Release text
@@ -214,7 +215,7 @@ $release_params = @{
 # Post to the release server
 if (-Not ($dryrun -or $buildonly -or $updatewinget)) { 
   $release = Invoke-RestMethod @release_params 
-} elseif (-Not $updatewinget) {
+} elseif (-Not ($updatewinget -or $buildonly -or $updatewinget)) {
   "[DRYRUN] Release Body:`n$release_body"
 }
 
@@ -283,7 +284,10 @@ if ($dryrun -or $buildonly -or $release.assets_url -imatch '^https:') {
   $AppImageArchives = @()
   if ($flavour -eq '_E') {
     "Building Electron packages..."
+    $base_tag_origin = $base_tag
+    $base_tag = $base_tag -replace '^([0-9.]+).*', '$1-E'
     . $PSScriptRoot/Build-Electron.ps1 # Note that we are dot-sourcing this, so that variables will be available in this scope
+    $base_tag = $base_tag_origin
   } elseif ($flavour -eq '_N') {
     # Package NWJS app if necessary
     $base_dir = "$PSScriptRoot/../bld/nwjs"
@@ -442,7 +446,7 @@ if ($dryrun -or $buildonly -or $release.assets_url -imatch '^https:') {
   if ($plus_electron) {
     "Building add-on: Electron packages..."
     $base_tag_origin = $base_tag
-    $base_tag = $base_tag -replace '^([\d.]+)', '$1-E'
+    $base_tag = $base_tag -replace '^([0-9.]+).*', '$1-E'
     . $PSScriptRoot/Build-Electron.ps1
     $base_tag = $base_tag_origin
   } 
@@ -471,7 +475,7 @@ if ($dryrun -or $buildonly -or $release.assets_url -imatch '^https:') {
   }
   if ($plus_electron) {
     $upload_assets += $AppImageArchives
-    if ($electronbuild -eq 'local') { $upload_assets += $WinInstaller }
+    if ($electronbuild -eq 'local' -and (-not $portableonly)) { $upload_assets += $WinInstaller }
     if ($old_windows_support) {
       $upload_assets += $nwjs_archives
     } else {
@@ -555,13 +559,13 @@ if ($dryrun -or $buildonly -or $release.assets_url -imatch '^https:') {
   }
   # Now update winget manifest if we are not building NWJS or Electron
   if ($flavour -eq '' -or $flavour -eq '_E') {
-    if ($respondtowingetprompt) {
+    if ($wingetprompt) {
-      $wingetcreate_check = $respondtowingetprompt
+      $wingetcreate_check = $wingetprompt
     } else {
       $wingetcreate_check = Read-Host "Would you like to update the WinGet repository with these new builds?`nWARNING: be sure you have published the draft release (if in doubt answer N)! [Y/N]"
     }
     $wingetcreate_check = $wingetcreate_check -imatch 'y'
-    if ($original_release_tag_name) {
+    if ($original_release_tag_name -and (-not $wingetprompt)) {
       $check = Read-Host "Did you change the Release Tag Name? (Y/N)"
       if ($check -imatch 'N') {
         "You must change the Tag name!"

scripts/Publish-ElectronPackages.ps1

@@ -1,9 +1,15 @@
-# Publish Kiwix Electron packages to Kiwix download server
+# Publish Kiwix Electron packages to a GitHub draft release and/or to Kiwix download server
 [CmdletBinding()]
 param (
-    [string]$test = ""
+    [string]$test = "",
+    [switch]$dryrun = $false,
+    [switch]$githubonly = $false,
+    [string]$tag = ""
 )
-
+if ($tag) {
+    # If user overrode the INPUT_VERSION, use it
+    $INPUT_VERSION = $tag
+}
 $target = "/data/download/release/kiwix-js-electron"
 $keyfile = "$PSScriptRoot\ssh_key"
 $keyfile = $keyfile -ireplace '[\\/]', '/'
@@ -19,8 +25,6 @@ if ($CRON_LAUNCHED) {
     $target = "/data/download/nightly/$current_date"
 }
 
-"`nUploading packages to https://download.kiwix.org$target/ ...`n"
-& "C:\Program Files\Git\usr\bin\ssh.exe" @('-o', 'StrictHostKeyChecking=no', '-i', "$keyfile", 'ci@download.kiwix.org', "mkdir -p $target")
 if ((Get-Content ./package.json) -match 'nwVersion') {
     $Packages = $(ls bld/NWJS/*.*)
 } else {
@@ -29,45 +33,132 @@ if ((Get-Content ./package.json) -match 'nwVersion') {
 if ($test) {
     $Packages = @($test)
 }
-$Packages | % {
+
-    $file = $_
+if (-not $CRON_LAUNCHED) {
-    if ($file -match '\.(exe|zip|msix)$') {
+    "`nChecking for a draft publishing target on GitHub..."
-        $directory = $file -replace '^(.+[\\/])[^\\/]+$', '$1'
+    if (-not $GITHUB_TOKEN) {
-        $filename = $file -replace '^.+[\\/]([^\\/]+)$', '$1'
+        $GITHUB_TOKEN = Get-Content -Raw "$PSScriptRoot/github_token"
-        # Convert all spaces and hyphens to underscore
+    }
-        $filename = $filename -replace '[\s-]', '_'
+    $draft_release_params = @{
-        $filename = $filename -creplace '_N([_.])', '_NWJS$1'
+        Uri = "https://api.github.com/repos/kiwix/kiwix-js-windows/releases"
-        # Swap architecture and release number, and remove redundant -win
+        Method = 'GET'
-        $filename = $filename -replace '(windows(?:_XP)?)(.+)_win(_ia32[^.]*)', '$1$3$2'
+        Headers = @{
-        # Convert filename to lowercase
+        'Authorization' = "token $GITHUB_TOKEN"
-        $filename = $filename.ToLower()
+        'Accept' = 'application/vnd.github.v3+json'
-        # Convert back appname to hyphens
-        $filename = $filename -replace 'kiwix_js_(electron|windows)', 'kiwix-js-$1'
-        # Fix Windows Setup version so that it is clear it is a Windows executable
-        $filename = $filename -replace 'electron_setup', 'electron_win_setup'
-        # Change underscore to hyphen in win type and remove redundant E
-        $filename = ($filename -creplace '_xp([_.])', '-xp$1') -creplace '_e([_.])', '$1'
-        # Move nwjs
-        $filename = $filename -replace '-windows(.*)_nwjs', '-nwjs_win$1'
-        # Change ia32 to i386
-        $filename = $filename -replace 'ia32', 'i386'
-        if ($CRON_LAUNCHED) {
-            # Remove the version number
-            $filename = $filename -replace '_[0-9.]+([-_.])', '$1'
         }
-        # Put back together
+        ContentType = "application/json"
-        $renamed_file = "$directory$filename"
+    }
-        if ($test) {
+    $releases = Invoke-RestMethod @draft_release_params
-            "`n$file was renamed to $renamed_file"
+    $release_found = $false
-        } else {
+    $release = $null
-            # Rename the file
+    $releases| Where-Object { $release_found -eq $False } | % {
-            if ($file -ne $renamed_file) {
+        $release = $_
-                mv $file $renamed_file
+        if (($release.draft -eq $true) -and ($release.tag_name -eq $INPUT_VERSION) ) {
+            $release_found = $true
+        }
+    }
+    if ($release_found) {
+        if ($dryrun) {
+            $release_json = $release | ConvertTo-Json
+            "[DRYRUN:] Draft release found: `n$release_json"
+        }
+        $upload_uri = $release.upload_url -ireplace '\{[^{}]+}', '' 
+        "`nUploading assets to: $upload_uri..."
+        ForEach($asset in $packages) {
+            if (-Not $asset) { Continue }
+            if (-Not ($file -match '\.(exe|zip|msix)$')) { Continue }
+            # Replace backslash with forward slash
+            $asset_name = $asset -replace '^.*[\\/]([^\\/]+)$', '$1'
+            # Replace spaces with hyphens
+            $asset_name = $asset_name -replace '\s', '-';
+            # Establish upload params
+            $upload_params = @{
+                Uri = $upload_uri + "?name=$asset_name"
+                Method = 'POST'
+                Headers = @{
+                    'Authorization' = "token $GITHUB_TOKEN"
+                    'Accept' = 'application/vnd.github.v3+json'
+                }
+                # Body = [System.IO.File]::ReadAllBytes($upload_file)
+                InFile = $asset
+                ContentType = 'application/octet-stream'
+            }
+            "`n*** Uploading $asset..."
+            # Upload asset to the release server
+            # $upload = [System.IO.File]::ReadAllBytes($upload_file) | Invoke-RestMethod @upload_params
+            if (-Not $dryrun) {
+                # Disable progress because it causes high CPU usage on large files, and slows down upload
+                $ProgressPreference = 'SilentlyContinue'
+                $upload = Invoke-RestMethod @upload_params
+            }
+            if ($dryrun -or $upload.name -eq ($asset_name -replace '\s', '.')) {
+                if (-Not $dryrun) {
+                    "Upload successfully posted as " + $upload.url
+                    "Full details:"
+                    echo $upload
+                } else {
+                    echo "DRYRUN with these upload parameters:`n" + @upload_params 
+                }
+            } else {
+                "`nI'm sorry, this upload appears to have failed! Please upload manually or try again..."
+                if ($upload) {
+                    "`nThe server returned:"
+                    echo $upload
+                } else {
+                    "The server did not respond."
+                }
+            }
+        }
+    } else {
+        "No draft release matching the tag $INPUT_VERSION was found."
+    }
+
+}  
+
+if (-not $githubonly) {
+    "`nUploading packages to https://download.kiwix.org$target/ ...`n"
+    & "C:\Program Files\Git\usr\bin\ssh.exe" @('-o', 'StrictHostKeyChecking=no', '-i', "$keyfile", 'ci@download.kiwix.org', "mkdir -p $target")
+
+    $Packages | % {
+        $file = $_
+        if ($file -match '\.(exe|zip|msix)$') {
+            $directory = $file -replace '^(.+[\\/])[^\\/]+$', '$1'
+            $filename = $file -replace '^.+[\\/]([^\\/]+)$', '$1'
+            # Convert all spaces and hyphens to underscore
+            $filename = $filename -replace '[\s-]', '_'
+            $filename = $filename -creplace '_N([_.])', '_NWJS$1'
+            # Swap architecture and release number, and remove redundant -win
+            $filename = $filename -replace '(windows(?:_XP)?)(.+)_win(_ia32[^.]*)', '$1$3$2'
+            # Convert filename to lowercase
+            $filename = $filename.ToLower()
+            # Convert back appname to hyphens
+            $filename = $filename -replace 'kiwix_js_(electron|windows)', 'kiwix-js-$1'
+            # Fix Windows Setup version so that it is clear it is a Windows executable
+            $filename = $filename -replace 'electron_setup', 'electron_win_setup'
+            # Change underscore to hyphen in win type and remove redundant E
+            $filename = ($filename -creplace '_xp([_.])', '-xp$1') -creplace '_e([_.])', '$1'
+            # Move nwjs
+            $filename = $filename -replace '-windows(.*)_nwjs', '-nwjs_win$1'
+            # Change ia32 to i386
+            $filename = $filename -replace 'ia32', 'i386'
+            if ($CRON_LAUNCHED) {
+                # Remove the version number
+                $filename = $filename -replace '_[0-9.]+([-_.])', '$1'
+            }
+            # Put back together
+            $renamed_file = "$directory$filename"
+            if ($test) {
+                "`n$file was renamed to $renamed_file"
+            } else {
+                # Rename the file
+                if ($file -ne $renamed_file) {
+                    mv $file $renamed_file
+                }
+                # Replace absolute path with relative, and normalize to forward slashes
+                $renamed_file = $renamed_file -replace '^.*?([\\/]bld)', '.$1' -replace '[\\/]', '/'
+                "Copying $renamed_file to $target..."
+                & "C:\Program Files\Git\usr\bin\scp.exe" @('-o', 'StrictHostKeyChecking=no', '-i', "$keyfile", "$renamed_file", "ci@download.kiwix.org:$target")
             }
-            # Replace absolute path with relative, and normalize to forward slashes
-            $renamed_file = $renamed_file -replace '^.*?([\\/]bld)', '.$1' -replace '[\\/]', '/'
-            "Copying $renamed_file to $target..."
-            & "C:\Program Files\Git\usr\bin\scp.exe" @('-o', 'StrictHostKeyChecking=no', '-i', "$keyfile", "$renamed_file", "ci@download.kiwix.org:$target")
         }
     }
 }