Build & Package signed Linux builds (#527)

.github/workflows/build_and_deploy.yaml

@@ -19,6 +19,11 @@ on:
         required: true
         type: boolean
         default: true
+      linux:
+        description: Whether to build Linux
+        required: true
+        type: boolean
+        default: true
       deploy:
         description: Whether to deploy artifacts
         required: true
@@ -54,6 +59,17 @@ jobs:
       base-url: https://packages.element.io/${{ inputs.mode == 'release' && 'desktop' || 'nightly' }}
       version: ${{ needs.prepare.outputs.macos-version }}
 
+  linux:
+    if: github.event_name != 'workflow_dispatch' || inputs.linux
+    needs: prepare
+    name: Linux
+    uses: ./.github/workflows/build_linux.yaml
+    secrets: inherit
+    with:
+      sqlcipher: system
+      deploy-mode: true
+      version: ${{ needs.prepare.outputs.linux-version }}
+
   deploy:
     needs:
       - macos
@@ -68,8 +84,20 @@ jobs:
           name: packages.element.io
           path: packages.element.io
 
+      - name: Deploy debian repo
+        if: github.event_name != 'workflow_dispatch' || inputs.linux
+        run: |
+          mv packages.element.io/debian .
+          aws s3 cp --recursive debian/ s3://$R2_BUCKET/debian --endpoint-url $R2_URL --region auto
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_TOKEN }}
+          R2_URL: ${{ secrets.CF_R2_S3_API }}
+
       - name: Deploy artifacts
-        run: aws s3 cp --recursive packages.element.io/ s3://$R2_BUCKET/$DEPLOYMENT_DIR --endpoint-url $R2_URL --region auto
+        if: github.event_name != 'workflow_dispatch' || inputs.macos
+        run: |
+          aws s3 cp --recursive packages.element.io/ s3://$R2_BUCKET/$DEPLOYMENT_DIR --endpoint-url $R2_URL --region auto
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_TOKEN }}

.github/workflows/build_linux.yaml

@@ -3,14 +3,34 @@
 # the correct cache scoping, and additional care must be taken to not run untrusted actions on the develop branch.
 on:
     workflow_call:
+        secrets:
+            GPG_PRIVATE_KEY:
+                required: false
+            GPG_PASSPHRASE:
+                required: false
+            CF_R2_ACCESS_KEY_ID:
+                required: false
+            CF_R2_TOKEN:
+                required: false
+            CF_R2_S3_API:
+                required: false
         inputs:
+            version:
+                type: string
+                required: false
+                description: "Version string to override the one in package.json, used for non-release builds"
             sqlcipher:
                 type: string
                 required: true
                 description: "How to link sqlcipher, one of 'system' | 'static'"
+            deploy-mode:
+                type: string
+                required: false
+                description: "Whether to arrange artifacts in the arrangement needed for deployment, skipping unrelated ones, this uses reprepro and requires 'packages.element.io' environment"
 jobs:
     build:
         runs-on: ubuntu-latest
+        environment: ${{ inputs.deploy-mode && 'packages.element.io' || '' }}
         steps:
             - uses: actions/checkout@v3
 
@@ -50,15 +70,59 @@ jobs:
               env:
                   SQLCIPHER_STATIC: ${{ inputs.sqlcipher == 'static' && '1' || '' }}
 
-            - name: Build App
+            - name: '[Nightly] Resolve version'
-              run: "yarn build --publish never -l"
+              id: nightly
+              if: inputs.version != ''
+              run: |
+                  echo "config-args=--nightly '${{ inputs.version }}'" >> $GITHUB_OUTPUT
 
-            - name: Install .deb
+            - name: Generate debian control file
-              run: "sudo apt install ./dist/*.deb"
+              run: |
+                  cp element.io/${{ inputs.version && 'nightly' || 'release' }}/control.template debcontrol
+                  INPUT_VERSION="${{ inputs.version }}"
+                  VERSION=${INPUT_VERSION:-$(cat package.json | jq -r .version)}
+                  echo "Version: $VERSION" >> debcontrol
+
+            - name: Build App
+              run: |
+                  scripts/generate-builder-config.ts ${{ steps.nightly.outputs.config-args }} --deb-custom-control=debcontrol
+                  yarn build --publish never -l --config electron-builder.json
+
+            - name: Load GPG key
+              if: inputs.deploy-mode
+              uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 # v5
+              with:
+                  gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+                  passphrase: ${{ secrets.GPG_PASSPHRASE }}
+                  fingerprint: 75741890063E5E9A46135D01C2850B265AC085BD
+
+            - name: Prepare artifacts for deployment (reprepro)
+              if: inputs.deploy-mode
+              run: |
+                  # Clear out the template packages.element.io directory, it has a dedicated deploy workflow
+                  rm -R packages.element.io/*
+                  
+                  # Install reprepro
+                  sudo apt-get install -y reprepro
+                  
+                  # Fetch reprepro database
+                  aws s3 cp --recursive s3://$R2_BUCKET debian/db/ --endpoint-url $R2_URL --region auto
+                  
+                  grep Codename debian/conf/distributions | sed -n 's/Codename: //p' | while read -r target ; do
+                      reprepro -b debian includedeb "$target" ./dist/*.deb
+                  done
+                  
+                  # Store reprepro database
+                  aws s3 cp --recursive debian/db/ s3://$R2_BUCKET --endpoint-url $R2_URL --region auto
+              env:
+                  R2_BUCKET: packages-element-io-db
+                  AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+                  AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_TOKEN }}
+                  R2_URL: ${{ secrets.CF_R2_S3_API }}
 
             - name: Upload Artifacts
               uses: actions/upload-artifact@v3
               with:
-                  name: linux-sqlcipher-${{ inputs.sqlcipher }}
+                  name: ${{ inputs.deploy-mode && 'packages.element.io' || format('linux-sqlcipher-{0}', inputs.sqlcipher) }}
-                  path: dist
+                  path: ${{ inputs.deploy-mode && 'packages.element.io' || 'dist' }}
                   retention-days: 1

.github/workflows/build_prepare.yaml

@@ -28,13 +28,17 @@ on:
             macos-version:
                 description: "The version string the next macOS Nightly should use, only output for calculate-nightly-versions"
                 value: ${{ jobs.prepare.outputs.macos-version }}
+            linux-version:
+                description: "The version string the next Linux Nightly should use, only output for calculate-nightly-versions"
+                value: ${{ jobs.prepare.outputs.linux-version }}
 jobs:
     prepare:
         name: Prepare
         environment: ${{ inputs.calculate-nightly-versions && 'packages.element.io' || '' }}
         runs-on: ubuntu-latest
         outputs:
-            macos-version: ${{ steps.macos.outputs.version }}
+            macos-version: ${{ steps.versions.outputs.macos }}
+            linux-version: ${{ steps.versions.outputs.linux }}
         steps:
             - uses: actions/checkout@v3
 
@@ -64,13 +68,18 @@ jobs:
                       electronVersion
                       hakDependencies.json
 
-            - name: Calculate macOS Nightly version
+            - name: Calculate Nightly versions
-              id: macos
+              id: versions
               if: inputs.calculate-nightly-versions
               run: |
-                  LATEST=$(aws s3 cp s3://$R2_BUCKET/nightly/update/macos/releases.json - --endpoint-url $R2_URL --region auto | jq -r .currentRelease)
+                  MACOS=$(aws s3 cp s3://$R2_BUCKET/nightly/update/macos/releases.json - --endpoint-url $R2_URL --region auto | jq -r .currentRelease)
-                  echo "version=$(scripts/generate-nightly-version.ts --latest $LATEST)" >> $GITHUB_OUTPUT
+                  echo "macos=$(scripts/generate-nightly-version.ts --latest $MACOS)" >> $GITHUB_OUTPUT
+
+                  LINUX=$(aws s3 cp s3://$R2_BUCKET/debian/dists/default/main/binary-amd64/Packages - --endpoint-url $R2_URL --region auto | grep "Package: element-nightly" -A 50 | grep Version -m1 | sed -n 's/Version: //p')
+                  echo "linux=$(scripts/generate-nightly-version.ts --latest $LINUX)" >> $GITHUB_OUTPUT
               env:
                   AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
                   AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_TOKEN }}
+                  # XXX: UPDATE THIS BEFORE WHEN GOING LIVE
+                  R2_BUCKET: 'packages-element-io-test'
                   R2_URL: ${{ secrets.CF_R2_S3_API }}

debian/conf/distributions

@@ -0,0 +1,52 @@
+Origin: riot.im
+Codename: default
+Architectures: amd64 i386 source
+Components: main
+SignWith: D7B0B66941D01538
+Tracking: minimal
+
+Origin: riot.im
+Suite: oldoldstable
+Codename: jessie
+Architectures: amd64 i386 source
+Components: main
+SignWith: D7B0B66941D01538
+Tracking: minimal
+
+Origin: riot.im
+Suite: oldstable
+Codename: stretch
+Architectures: amd64 i386 source
+Components: main
+SignWith: D7B0B66941D01538
+Tracking: minimal
+
+Origin: riot.im
+Suite: testing
+Codename: bullseye
+Architectures: amd64 i386 source
+Components: main
+SignWith: D7B0B66941D01538
+Tracking: minimal
+
+Origin: riot.im
+Suite: unstable
+Codename: sid
+Architectures: amd64 i386 source
+Components: main
+SignWith: D7B0B66941D01538
+Tracking: minimal
+
+Origin: riot.im
+Codename: xenial
+Architectures: amd64 i386 source
+Components: main
+SignWith: D7B0B66941D01538
+Tracking: minimal
+
+Origin: riot.im
+Codename: bionic
+Architectures: amd64 i386 source
+Components: main
+SignWith: D7B0B66941D01538
+Tracking: minimal

debian/conf/options

@@ -0,0 +1,5 @@
+# don't delete old debs by default
+keepunreferencedfiles
+
+# output dir
+outdir +b/../packages.element.io/debian

package.json

@@ -35,7 +35,6 @@
     "docker:build:native": "scripts/in-docker.sh yarn run hak",
     "docker:build": "scripts/in-docker.sh yarn run build",
     "docker:install": "scripts/in-docker.sh yarn install",
-    "debrepo": "scripts/mkrepo.sh",
     "clean": "rimraf webapp.asar dist packages deploys lib",
     "hak": "ts-node scripts/hak/index.ts",
     "test": "jest"

scripts/generate-nightly-version.ts

@@ -18,19 +18,19 @@ const argv = parseArgs<{
 
 function parseVersion(version: string): [Date, number] {
     const year = parseInt(version.slice(0, 4), 10);
-    const month = parseInt(version.slice(4, 2), 10);
+    const month = parseInt(version.slice(4, 6), 10);
-    const day = parseInt(version.slice(6, 2), 10);
+    const day = parseInt(version.slice(6, 8), 10);
-    const num = parseInt(version.slice(8, 2), 10);
+    const num = parseInt(version.slice(8, 10), 10);
     return [new Date(year, month - 1, day), num];
 }
 
 const [latestDate, latestNum] = argv.latest ? parseVersion(argv.latest) : [];
 
 const now = new Date();
-const month = (now.getMonth() + 1).toString().padStart(2, '0');
+const month = (now.getMonth() + 1).toString().padStart(2, "0");
-const date = now.getDate().toString().padStart(2, '0');
+const date = now.getDate().toString().padStart(2, "0");
 let buildNum = 1;
-if (latestDate && new Date(latestDate).getDate().toString().padStart(2, '0') === date) {
+if (latestDate && new Date(latestDate).getDate().toString().padStart(2, "0") === date) {
     buildNum = latestNum! + 1;
 }
 
@@ -38,4 +38,4 @@ if (buildNum > 99) {
     throw new Error("Maximum number of Nightlies exceeded on this day.");
 }
 
-console.log(now.getFullYear() + month + date + buildNum.toString().padStart(2, '0') + buildNum);
+console.log(now.getFullYear() + month + date + buildNum.toString().padStart(2, "0"));

scripts/mkrepo.sh

@@ -1,32 +0,0 @@
-#!/bin/sh
-
-# Take the deb and bundle it into a apt repository
-if [[ $# -lt 1 ]]
-then
-    echo "Usage $0 <config dir>"
-    exit
-fi
-
-confdir=$1
-
-set -ex
-
-ver=`jq -r .version package.json`
-distdir=$PWD/dist
-confdir=$PWD/$confdir
-
-repodir=`mktemp -d -t repo`
-mkdir $repodir/conf
-cp $confdir/conf_distributions $repodir/conf/distributions
-
-pushd $repodir
-for i in `cat conf/distributions | grep Codename | cut -d ' ' -f 2`
-do
-    reprepro includedeb $i $distdir/element-desktop_${ver}_amd64.deb
-done
-
-tar cvzf $distdir/element-desktop_repo_$ver.tar.gz .
-
-popd
-
-rm -r $repodir