Merge pull request #29 from vector-im/dbkr/fetch_develop

Support fetching the latest develop build

package.json

@@ -46,7 +46,6 @@
     "eslint-config-google": "^0.7.1",
     "eslint-plugin-babel": "^4.1.2",
     "find-npm-prefix": "^1.0.2",
-    "follow-redirects": "^1.9.0",
     "fs-extra": "^8.1.0",
     "glob": "^7.1.6",
     "matrix-js-sdk": "^2.4.6-rc.1",

scripts/fetch-package.js

@@ -4,10 +4,10 @@ const process = require('process');
 const path = require('path');
 const fs = require('fs');
 const fsPromises = require('fs').promises;
-const { https } = require('follow-redirects');
 const childProcess = require('child_process');
 const tar = require('tar');
 const asar = require('asar');
+const needle = require('needle');
 
 const riotDesktopPackageJson = require('../package.json');
 
@@ -15,30 +15,84 @@ const PUB_KEY_URL = "https://packages.riot.im/riot-release-key.asc";
 const PACKAGE_URL_PREFIX = "https://github.com/vector-im/riot-web/releases/download/";
 const ASAR_PATH = 'webapp.asar';
 
-async function downloadToFile(url, filename) {
+async function getLatestDevelopUrl(bkToken) {
-    console.log("Downloading " + url + "...");
+    const buildsResult = await needle('get',
-    const outStream = await fs.createWriteStream(filename);
+        "https://api.buildkite.com/v2/organizations/matrix-dot-org/pipelines/riot-web/builds",
-
+        {
-    return new Promise((resolve, reject) => {
+            branch: 'develop',
-        https.get(url, (resp) => {
+            state: 'passed',
-            if (resp.statusCode / 100 !== 2) {
+            per_page: 1,
-                reject("Download failed: " + resp.statusCode);
+        },
-                return;
+        {
+            headers: {
+                authorization: "Bearer " + bkToken,
+            },
+        },
+    );
+    const latestBuild = buildsResult.body[0];
+    console.log("Latest build is " + latestBuild.number);
+    let artifactUrl;
+    for (const job of latestBuild.jobs) {
+        // Strip any colon-form emoji from the build name
+        if (job.name && job.name.replace(/:\w*:\s*/, '') === 'Package') {
+            artifactUrl = job.artifacts_url;
+            break;
+        }
+    }
+    if (artifactUrl === undefined) {
+        throw new Error("Couldn't find artifact URL - has the name of the package job changed?");
     }
 
-            resp.on('data', (chunk) => {
+    const artifactsResult = await needle('get', artifactUrl, {},
-                outStream.write(chunk);
+        {
-            });
+            headers: {
-            resp.on('end', (chunk) => {
+                authorization: "Bearer " + bkToken,
-                outStream.end();
+            },
-                resolve();
+        },
-            });
+    );
-        });
+    let dlUrl;
-    }).catch(async (e) => {
+    let dlFilename;
-        outStream.end();
+    for (const artifact of artifactsResult.body) {
+        if (artifact.filename && /^riot-.*\.tar.gz$/.test(artifact.filename)) {
+            dlUrl = artifact.download_url;
+            dlFilename = artifact.filename;
+            break;
+        }
+    }
+    if (dlUrl === undefined) {
+        throw new Error("Couldn't find artifact download URL - has the artifact filename changed?");
+    }
+    console.log("Fetching artifact URL...");
+    const dlResult = await needle('get', dlUrl, {},
+        {
+            headers: {
+                authorization: "Bearer " + bkToken,
+            },
+            // This URL will give us a Location header, but will also give us
+            // a JSON object with the direct URL. We'll take the URL and pass it
+            // back, then we can easily support specifying a URL directly.
+            follow_max: 0,
+        },
+    );
+    return [dlFilename, dlResult.body.url];
+}
+
+async function downloadToFile(url, filename) {
+    console.log("Downloading " + url + "...");
+
+    try {
+        await needle('get', url, null,
+            {
+                follow_max: 5,
+                output: filename,
+            },
+        );
+    } catch (e) {
+        try {
             await fsPromises.unlink(filename);
+        } catch (_) {}
         throw e;
-    });
+    }
 }
 
 async function verifyFile(filename) {
@@ -60,6 +114,8 @@ async function main() {
     let deployDir = 'deploys';
     let cfgDir;
     let targetVersion;
+    let filename;
+    let url;
 
     while (process.argv.length > 2) {
         switch (process.argv[2]) {
@@ -90,6 +146,21 @@ async function main() {
 
     if (targetVersion === undefined) {
         targetVersion = 'v' + riotDesktopPackageJson.version;
+    } else if (targetVersion === 'develop') {
+        const buildKiteApiKey = process.env.BUILDKITE_API_KEY;
+        if (buildKiteApiKey === undefined) {
+            console.log("Set BUILDKITE_API_KEY to fetch latest develop version");
+            console.log(
+                "Sorry - Buildkite's API requires authentication to access builds, " +
+                "even if those builds are accessible on the web with no auth.",
+            );
+            process.exit(1);
+        }
+        [filename, url] = await getLatestDevelopUrl(buildKiteApiKey);
+        verify = false; // develop builds aren't signed
+    } else {
+        filename = 'riot-' + targetVersion + '.tar.gz';
+        url = PACKAGE_URL_PREFIX + targetVersion + '/' + filename;
     }
 
     const haveGpg = await new Promise((resolve) => {
@@ -113,14 +184,7 @@ async function main() {
                 }
                 resolve(!error);
             });
-            https.get(PUB_KEY_URL, (resp) => {
+            needle.get(PUB_KEY_URL).pipe(gpgProc.stdin);
-                resp.on('data', (chunk) => {
-                    gpgProc.stdin.write(chunk);
-                });
-                resp.on('end', (chunk) => {
-                    gpgProc.stdin.end();
-                });
-            });
         });
         return 0;
     }
@@ -138,7 +202,7 @@ async function main() {
     }
 
     let haveDeploy = false;
-    const expectedDeployDir = path.join(deployDir, 'riot-' + targetVersion);
+    const expectedDeployDir = path.join(deployDir, path.basename(filename).replace(/\.tar\.gz/, ''));
     try {
         await fs.opendir(expectedDeployDir);
         console.log(expectedDeployDir + "already exists");
@@ -147,9 +211,7 @@ async function main() {
     }
 
     if (!haveDeploy) {
-        const filename = 'riot-' + targetVersion + '.tar.gz';
         const outPath = path.join(pkgDir, filename);
-        const url = PACKAGE_URL_PREFIX + targetVersion + '/' + filename;
         try {
             await fsPromises.stat(outPath);
             console.log("Already have " + filename + ": not redownloading");

yarn.lock

@@ -1065,7 +1065,7 @@ debug@3.1.0:
   dependencies:
     ms "2.0.0"
 
-debug@^3.0.0, debug@^3.1.0, debug@^3.2.6:
+debug@^3.1.0, debug@^3.2.6:
   version "3.2.6"
   resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b"
   integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==
@@ -1713,13 +1713,6 @@ flush-write-stream@^1.0.0:
     inherits "^2.0.3"
     readable-stream "^2.3.6"
 
-follow-redirects@^1.9.0:
-  version "1.9.0"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.9.0.tgz#8d5bcdc65b7108fe1508649c79c12d732dcedb4f"
-  integrity sha512-CRcPzsSIbXyVDl0QI01muNDu69S8trU4jArW9LpOt2WtC6LyUJetcIrmfHsRBx7/Jb6GHJUiuqyYxPooFfNt6A==
-  dependencies:
-    debug "^3.0.0"
-
 forever-agent@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"