Support universal builds

Hopefully adequately documented in the README

README.md

@@ -58,15 +58,19 @@ run Element locally, skip to the next section.
 If you'd like to build the native modules (for searching in encrypted rooms and
 secure storage), do this first. This will take 10 minutes or so, and will
 require a number of native tools to be installed, depending on your OS (eg.
-rust, tcl, make/nmake). If you don't need these features, you can skip this
+rust, tcl, make/nmake).
-step.
+
+You'll also to need to make sure you've built the native modules for the same
+architecture as your package, so for anything more advanced than just building
+the modules and app for the host architecture see 'Other Architectures'.
+
+If you don't need these features, you can skip this step.
+
+To just build these for your native architecture:
 ```
 yarn run build:native
 ```
 
-On Windows, this will automatically determine the architecture to build for based
-on the environment. Make sure that you have all the [tools required to perform the native modules build](docs/windows-requirements.md)
-
 Now you can build the package:
 
 ```
@@ -78,15 +82,6 @@ This will do a couple of things:
  * Run electron-builder to build a package. The package built will match the operating system
    you're running the build process on.
 
-If you're on Windows, you can choose to build specifically for 32 or 64 bit:
-```
-yarn run build32
-```
-or
-```
-yarn run build64
-```
-
 This build step will not build any native modules.
 
 You can also build using docker, which will always produce the linux package:
@@ -112,6 +107,70 @@ yarn add electron
 yarn start
 ```
 
+Other Architectures
+===================
+Building the native modules will build for the host architecture (and only the
+host architecture) by default. On Windows, this will automatically determine
+the architecture to build for based on the environment. Make sure that you have
+all the [tools required to perform the native modules build](docs/windows-requirements.md)
+
+
+On macOS, you can build universal native modules too:
+```
+yarn run build:native:universal
+```
+
+...or you can build for a specific architecture:
+```
+yarn run build:native --target x86_64-apple-darwin
+```
+or
+```
+yarn run build:native --target aarch64-apple-darwin
+```
+
+You'll then need to create a built bundle with the same architecture.
+To bundle a universal build for macOS, run:
+
+```
+yarn run builduniversal
+```
+
+If you're on Windows, you can choose to build specifically for 32 or 64 bit:
+```
+yarn run build32
+```
+or
+```
+yarn run build64
+```
+
+Note that the native module build system keeps the different architectures
+separate, so you can keep native modules for several architectures at the same
+time and switch which are active using a `yarn run hak copy` command, passing
+the appropriate architectures. This will error if you haven't yet built those
+architectures. eg:
+
+```
+yarn run build:native --target x86_64-apple-darwin
+# We've now built & linked into place native modules for Intel
+yarn run build:native --target aarch64-apple-darwin
+# We've now built Apple Silicon modules too, and linked them into place as the active ones
+
+yarn run hak copy --target x86_64-apple-darwin
+# We've now switched back to our Intel modules
+yarn run hak copy --target x86_64-apple-darwin --target aarch64-apple-darwin
+# Now our native modules are universal x86_64+aarch64 binaries
+```
+
+The current set of native modules are stored in `.hak/hakModules`,
+so you can use this to check what architecture is currently in place, eg:
+
+```
+$ lipo -info .hak/hakModules/keytar/build/Release/keytar.node 
+Architectures in the fat file: .hak/hakModules/keytar/build/Release/keytar.node are: x86_64 arm64 
+```
+
 Config
 ======
 If you'd like the packaged Element to have a configuration file, you can create a

hak/matrix-seshat/build.js

@@ -32,7 +32,7 @@ module.exports = async function(hakEnv, moduleInfo) {
 
 async function buildOpenSslWin(hakEnv, moduleInfo) {
     const version = moduleInfo.cfg.dependencies.openssl;
-    const openSslDir = path.join(moduleInfo.moduleDotHakDir, `openssl-${version}`);
+    const openSslDir = path.join(moduleInfo.moduleTargetDotHakDir, `openssl-${version}`);
 
     const openSslArch = hakEnv.getTargetArch() === 'x64' ? 'VC-WIN64A' : 'VC-WIN32';
 
@@ -134,7 +134,7 @@ async function buildOpenSslWin(hakEnv, moduleInfo) {
 
 async function buildSqlCipherWin(hakEnv, moduleInfo) {
     const version = moduleInfo.cfg.dependencies.sqlcipher;
-    const sqlCipherDir = path.join(moduleInfo.moduleDotHakDir, `sqlcipher-${version}`);
+    const sqlCipherDir = path.join(moduleInfo.moduleTargetDotHakDir, `sqlcipher-${version}`);
     const buildDir = path.join(sqlCipherDir, 'bld');
 
     await mkdirp(buildDir);
@@ -171,7 +171,7 @@ async function buildSqlCipherWin(hakEnv, moduleInfo) {
 
 async function buildSqlCipherUnix(hakEnv, moduleInfo) {
     const version = moduleInfo.cfg.dependencies.sqlcipher;
-    const sqlCipherDir = path.join(moduleInfo.moduleDotHakDir, `sqlcipher-${version}`);
+    const sqlCipherDir = path.join(moduleInfo.moduleTargetDotHakDir, `sqlcipher-${version}`);
 
     const args = [
         '--prefix=' + moduleInfo.depPrefix + '',

hak/matrix-seshat/fetchDeps.js

@@ -62,9 +62,10 @@ async function getSqlCipher(hakEnv, moduleInfo) {
         await bob;
     }
 
+    // Extract the tarball to per-target directories, then we avoid cross-contaiminating archs
     await tar.x({
         file: sqlCipherTarball,
-        cwd: moduleInfo.moduleDotHakDir,
+        cwd: moduleInfo.moduleTargetDotHakDir,
     });
 
     if (hakEnv.isWin()) {
@@ -124,6 +125,6 @@ async function getOpenSsl(hakEnv, moduleInfo) {
     console.log("extracting " + openSslTarball + " in " + moduleInfo.moduleDotHakDir);
     await tar.x({
         file: openSslTarball,
-        cwd: moduleInfo.moduleDotHakDir,
+        cwd: moduleInfo.moduleTargetDotHakDir,
     });
 }

package.json

@@ -23,8 +23,10 @@
     "lint:js": "eslint src/ scripts/ hak/",
     "lint:types": "tsc --noEmit",
     "build:native": "yarn run hak",
+    "build:native:universal": "yarn run hak --target x86_64-apple-darwin fetchandbuild && yarn run hak --target aarch64-apple-darwin fetchandbuild && yarn run hak --target x86_64-apple-darwin --target aarch64-apple-darwin copyandlink",
     "build32": "yarn run build:ts && yarn run build:res && electron-builder --ia32",
     "build64": "yarn run build:ts && yarn run build:res && electron-builder --x64",
+    "builduniversal": "yarn run build:ts && yarn run build:res && electron-builder --universal",
     "build": "yarn run build:ts && yarn run build:res && electron-builder",
     "build:ts": "tsc",
     "build:res": "node scripts/copy-res.js",

scripts/hak/copy.js

@@ -16,6 +16,7 @@ limitations under the License.
 
 const path = require('path');
 const fsProm = require('fs').promises;
+const childProcess = require('child_process');
 
 const rimraf = require('rimraf');
 const glob = require('glob');
@@ -40,10 +41,9 @@ async function copy(hakEnv, moduleInfo) {
     }
 
     if (moduleInfo.cfg.copy) {
-        console.log(
+        // If there are multiple moduleBuildDirs, singular moduleBuildDir
-            "Copying files from " +
+        // is the same as moduleBuildDirs[0], so we're just listing the contents
-            moduleInfo.moduleBuildDir + " to " + moduleInfo.moduleOutDir,
+        // of the first one.
-        );
         const files = await new Promise(async (resolve, reject) => {
             glob(moduleInfo.cfg.copy, {
                 nosort: true,
@@ -53,13 +53,46 @@ async function copy(hakEnv, moduleInfo) {
                 err ? reject(err) : resolve(files);
             });
         });
-        for (const f of files) {
-            console.log("\t" + f);
-            const src = path.join(moduleInfo.moduleBuildDir, f);
-            const dst = path.join(moduleInfo.moduleOutDir, f);
 
-            await mkdirp(path.dirname(dst));
+        if (moduleInfo.moduleBuildDirs.length > 1) {
-            await fsProm.copyFile(src, dst);
+            if (!hakEnv.isMac()) {
+                console.error(
+                    "You asked me to copy multiple targets but I've only been taught " +
+                    "how to do that on a mac.",
+                );
+                throw new Error("Can't copy multiple targets on this platform");
+            }
+
+            for (const f of files) {
+                const components = moduleInfo.moduleBuildDirs.map(dir => path.join(dir, f));
+                const dst = path.join(moduleInfo.moduleOutDir, f);
+
+                await mkdirp(path.dirname(dst));
+                await new Promise((resolve, reject) => {
+                    childProcess.execFile('lipo',
+                        ['-create', '-output', dst, ...components], (err) => {
+                            if (err) {
+                                reject(err);
+                            } else {
+                                resolve();
+                            }
+                        },
+                    );
+                });
+            }
+        } else {
+            console.log(
+                "Copying files from " +
+                moduleInfo.moduleBuildDir + " to " + moduleInfo.moduleOutDir,
+            );
+            for (const f of files) {
+                console.log("\t" + f);
+                const src = path.join(moduleInfo.moduleBuildDir, f);
+                const dst = path.join(moduleInfo.moduleOutDir, f);
+
+                await mkdirp(path.dirname(dst));
+                await fsProm.copyFile(src, dst);
+            }
         }
     }
 }

scripts/hak/index.js

@@ -35,6 +35,13 @@ const MODULECOMMANDS = [
     'clean',
 ];
 
+// Shortcuts for multiple commands at once (useful for building universal binaries
+// because you can run the fetch/fetchDeps/build for each arch and then copy/link once)
+const METACOMMANDS = {
+    'fetchandbuild': ['check', 'fetch', 'fetchDeps', 'build'],
+    'copyandlink': ['copy', 'link'],
+};
+
 // Scripts valid in a hak.json 'scripts' section
 const HAKSCRIPTS = [
     'check',
@@ -53,19 +60,24 @@ async function main() {
         process.exit(1);
     }
 
+    const targetIds = [];
     // Apply `--target <target>` option if specified
-    const targetIndex = process.argv.indexOf('--target');
+    // Can be specified multiple times for the copy command to bundle
-    let targetId;
+    // multiple archs into a single universal output module)
-    if (targetIndex >= 0) {
+    while (true) { // eslint-disable-line no-constant-condition
+        const targetIndex = process.argv.indexOf('--target');
+        if (targetIndex === -1) break;
+
         if ((targetIndex + 1) >= process.argv.length) {
             console.error("--target option specified without a target");
             process.exit(1);
         }
         // Extract target ID and remove from args
-        targetId = process.argv.splice(targetIndex, 2)[1];
+        targetIds.push(process.argv.splice(targetIndex, 2)[1]);
     }
 
-    const hakEnv = new HakEnv(prefix, packageJson, targetId);
+    const hakEnvs = targetIds.map(tid => new HakEnv(prefix, packageJson, tid));
+    const hakEnv = hakEnvs[0];
 
     const deps = {};
 
@@ -87,10 +99,12 @@ async function main() {
             cfg: hakJson,
             moduleHakDir: path.join(prefix, 'hak', dep),
             moduleDotHakDir: path.join(hakEnv.dotHakDir, dep),
-            moduleBuildDir: path.join(hakEnv.dotHakDir, dep, 'build'),
+            moduleTargetDotHakDir: path.join(hakEnv.dotHakDir, dep, hakEnv.getTargetId()),
+            moduleBuildDir: path.join(hakEnv.dotHakDir, dep, hakEnv.getTargetId(), 'build'),
+            moduleBuildDirs: hakEnvs.map(h => path.join(h.dotHakDir, dep, h.getTargetId(), 'build')),
             moduleOutDir: path.join(hakEnv.dotHakDir, 'hakModules', dep),
-            nodeModuleBinDir: path.join(hakEnv.dotHakDir, dep, 'build', 'node_modules', '.bin'),
+            nodeModuleBinDir: path.join(hakEnv.dotHakDir, dep, hakEnv.getTargetId(), 'build', 'node_modules', '.bin'),
-            depPrefix: path.join(hakEnv.dotHakDir, dep, 'opt'),
+            depPrefix: path.join(hakEnv.dotHakDir, dep, hakEnv.getTargetId(), 'opt'),
             scripts: {},
         };
 
@@ -104,10 +118,18 @@ async function main() {
     let cmds;
     if (process.argv.length < 3) {
         cmds = ['check', 'fetch', 'fetchDeps', 'build', 'copy', 'link'];
+    } else if (METACOMMANDS[process.argv[2]]) {
+        cmds = METACOMMANDS[process.argv[2]];
     } else {
         cmds = [process.argv[2]];
     }
 
+    if (hakEnvs.length > 1 && cmds.some(c => !['copy', 'link'].includes(c))) {
+        // We allow link here too for convenience because it's completely arch independent
+        console.error("Multiple targets only supported with the copy command");
+        return;
+    }
+
     let modules = process.argv.slice(3);
     if (modules.length === 0) modules = Object.keys(deps);