Working seshat building on Windows

hak/matrix-seshat/build.js

@@ -17,18 +17,158 @@ limitations under the License.
 const path = require('path');
 const child_process = require('child_process');
 
+const mkdirp = require('mkdirp');
+const fsExtra = require('fs-extra');
+
 module.exports = async function(hakEnv, moduleInfo) {
-    if (hakEnv.isWin()) await buildOpenSsl(hakEnv, moduleInfo);
+    if (hakEnv.isWin()) {
-    await buildSqlCipher(hakEnv, moduleInfo);
+        await buildOpenSslWin(hakEnv, moduleInfo);
+        await buildSqlCipherWin(hakEnv, moduleInfo);
+    } else {
+        await buildSqlCipherUnix(hakEnv, moduleInfo);
+    }
     await buildMatrixSeshat(hakEnv, moduleInfo);
 }
 
-async function buildOpenSsl(hakEnv, moduleInfo) {
+async function buildOpenSslWin(hakEnv, moduleInfo) {
-    const openSslDir = path.join(moduleInfo.moduleHakDir, 'openssl-1.1.1d');
+    const openSslDir = path.join(moduleInfo.moduleDotHakDir, 'openssl-1.1.1d');
+
+    const openSslArch = hakEnv.arch === 'x64' ? 'VC-WIN64A' : 'VC-WIN32';
+
+    console.log("Building openssl in " + openSslDir);
+    await new Promise((resolve, reject) => {
+        const proc = child_process.spawn(
+            'perl',
+            [
+                'Configure',
+                '--prefix=' + moduleInfo.depPrefix,
+                 // sqlcipher only uses about a tiny part of openssl. We link statically
+                 // so will only pull in the symbols we use, but we may as well turn off
+                 // as much as possible to save on build time.
+                'no-afalgeng',
+                'no-capieng',
+                'no-cms',
+                'no-ct',
+                'no-deprecated',
+                'no-dgram',
+                'no-dso',
+                'no-ec',
+                'no-ec2m',
+                'no-gost',
+                'no-nextprotoneg',
+                'no-ocsp',
+                'no-sock',
+                'no-srp',
+                'no-srtp',
+                'no-tests',
+                'no-ssl',
+                'no-tls',
+                'no-dtls',
+                'no-shared',
+                'no-aria',
+                'no-camellia',
+                'no-cast',
+                'no-chacha',
+                'no-cmac',
+                'no-des',
+                'no-dh',
+                'no-dsa',
+                'no-ecdh',
+                'no-ecdsa',
+                'no-idea',
+                'no-md4',
+                'no-mdc2',
+                'no-ocb',
+                'no-poly1305',
+                'no-rc2',
+                'no-rc4',
+                'no-rmd160',
+                'no-scrypt',
+                'no-seed',
+                'no-siphash',
+                'no-sm2',
+                'no-sm3',
+                'no-sm4',
+                'no-whirlpool',
+                openSslArch,
+            ],
+            {
+                cwd: openSslDir,
+                stdio: 'inherit',
+            },
+        );
+        proc.on('exit', (code) => {
+            code ? reject(code) : resolve();
+        });
+    });
+
+    await new Promise((resolve, reject) => {
+        const proc = child_process.spawn(
+            'nmake',
+            ['build_libs'],
+            {
+                cwd: openSslDir,
+                stdio: 'inherit',
+            },
+        );
+        proc.on('exit', (code) => {
+            code ? reject(code) : resolve();
+        });
+    });
+
+    await new Promise((resolve, reject) => {
+        const proc = child_process.spawn(
+            'nmake',
+            ['install_dev'],
+            {
+                cwd: openSslDir,
+                stdio: 'inherit',
+            },
+        );
+        proc.on('exit', (code) => {
+            code ? reject(code) : resolve();
+        });
+    });
 }
 
-async function buildSqlCipher(hakEnv, moduleInfo) {
+async function buildSqlCipherWin(hakEnv, moduleInfo) {
-    const sqlCipherDir = path.join(moduleInfo.moduleHakDir, 'sqlcipher-4.3.0');
+    const sqlCipherDir = path.join(moduleInfo.moduleDotHakDir, 'sqlcipher-4.3.0');
+    const buildDir = path.join(sqlCipherDir, 'bld');
+
+    await mkdirp(buildDir);
+
+    await new Promise((resolve, reject) => {
+        const proc = child_process.spawn(
+            'nmake',
+            ['/f', path.join('..', 'Makefile.msc'), 'libsqlite3.lib', 'TOP=..'],
+            {
+                cwd: buildDir,
+                stdio: 'inherit',
+	        env: Object.assign({}, process.env, {
+                    CCOPTS: "-DSQLITE_HAS_CODEC -I" + path.join(moduleInfo.depPrefix, 'include'),
+                    LTLIBPATHS: "/LIBPATH:" + path.join(moduleInfo.depPrefix, 'lib'),
+                    LTLIBS: "libcrypto.lib",
+                }),
+            },
+        );
+        proc.on('exit', (code) => {
+            code ? reject(code) : resolve();
+        });
+    });
+
+    await fsExtra.copy(
+        path.join(buildDir, 'libsqlite3.lib'),
+        path.join(moduleInfo.depPrefix, 'lib', 'sqlcipher.lib'),
+    );
+
+    await fsExtra.copy(
+        path.join(buildDir, 'sqlite3.h'),
+        path.join(moduleInfo.depPrefix, 'include', 'sqlcipher.h'),
+    );
+}
+
+async function buildSqlCipherUnix(hakEnv, moduleInfo) {
+    const sqlCipherDir = path.join(moduleInfo.moduleDotHakDir, 'sqlcipher-4.3.0');
 
     const args = [
         '--prefix=' + moduleInfo.depPrefix + '',
@@ -88,17 +228,24 @@ async function buildSqlCipher(hakEnv, moduleInfo) {
 }
 
 async function buildMatrixSeshat(hakEnv, moduleInfo) {
-    await new Promise((resolve) => {
+    const env = Object.assign({
+        SQLCIPHER_STATIC: 1,
+        SQLCIPHER_LIB_DIR: path.join(moduleInfo.depPrefix, 'lib'),
+        SQLCIPHER_INCLUDE_DIR: path.join(moduleInfo.depPrefix, 'include'),
+    }, hakEnv.makeGypEnv());
+
+    if (hakEnv.isWin()) {
+        env.RUSTFLAGS = '-Ctarget-feature=+crt-static -Clink-args=libcrypto.lib';
+    }
+
+    console.log("Running neon with env", env);
+    await new Promise((resolve, reject) => {
         const proc = child_process.spawn(
-            path.join(moduleInfo.nodeModuleBinDir, 'neon'),
+            path.join(moduleInfo.nodeModuleBinDir, 'neon' + (hakEnv.isWin() ? '.cmd' : '')),
             ['build', '--release'],
             {
                 cwd: moduleInfo.moduleBuildDir,
-                env: Object.assign({
+                env,
-                    SQLCIPHER_STATIC: 1,
-                    SQLCIPHER_LIB_DIR: path.join(moduleInfo.depPrefix, 'lib'),
-                    SQLCIPHER_INCLUDE_DIR: path.join(moduleInfo.depPrefix, 'include'),
-                }, hakEnv.makeGypEnv()),
                 stdio: 'inherit',
             },
         );

hak/matrix-seshat/fetchDeps.js

@@ -15,7 +15,9 @@ limitations under the License.
 */
 
 const path = require('path');
+const child_process = require('child_process');
 
+const fs = require('fs');
 const fsProm = require('fs').promises;
 const needle = require('needle');
 const tar = require('tar');
@@ -29,7 +31,7 @@ module.exports = async function(hakEnv, moduleInfo) {
 }
 
 async function getSqlCipher(hakEnv, moduleInfo) {
-    const sqlCipherDir = path.join(moduleInfo.moduleHakDir, 'sqlcipher-4.3.0');
+    const sqlCipherDir = path.join(moduleInfo.moduleDotHakDir, 'sqlcipher-4.3.0');
 
     let haveSqlcipher;
     try {
@@ -41,7 +43,7 @@ async function getSqlCipher(hakEnv, moduleInfo) {
 
     if (haveSqlcipher) return;
 
-    const sqlCipherTarball = path.join(moduleInfo.moduleHakDir, 'sqlcipher-4.3.0.tar.gz');
+    const sqlCipherTarball = path.join(moduleInfo.moduleDotHakDir, 'sqlcipher-4.3.0.tar.gz');
     let haveSqlcipherTar;
     try {
         await fsProm.stat(sqlCipherTarball);
@@ -59,12 +61,36 @@ async function getSqlCipher(hakEnv, moduleInfo) {
 
     await tar.x({
         file: sqlCipherTarball,
-        cwd: moduleInfo.moduleHakDir,
+        cwd: moduleInfo.moduleDotHakDir,
     });
+
+    if (hakEnv.isWin()) {
+        // On Windows, we need to patch the makefile because it forces TEMP_STORE to
+        // default to files (1) but the README specifically says you '*must*' set it
+        // set it to 2 (default to memory).
+        const patchFile = path.join(moduleInfo.moduleHakDir, 'sqlcipher-4.3.0-win.patch');
+
+        await new Promise((resolve, reject) => {
+	    const readStream = fs.createReadStream(patchFile);
+
+            const proc = child_process.spawn(
+                'patch',
+                ['-p1'],
+                {
+                    cwd: sqlCipherDir,
+                    stdio: ['pipe', 'inherit', 'inherit'],
+                },
+            );
+            proc.on('exit', (code) => {
+                code ? reject(code) : resolve();
+            });
+	    readStream.pipe(proc.stdin);
+        });
+    }
 }
 
 async function getOpenSsl(hakEnv, moduleInfo) {
-    const openSslDir = path.join(moduleInfo.moduleHakDir, 'openssl-1.1.1d');
+    const openSslDir = path.join(moduleInfo.moduleDotHakDir, 'openssl-1.1.1d');
 
     let haveOpenSsl;
     try {
@@ -76,7 +102,7 @@ async function getOpenSsl(hakEnv, moduleInfo) {
 
     if (haveOpenSsl) return;
 
-    const openSslTarball = path.join(moduleInfo.depDir, 'openssl-1.1.1d.tar.gz');
+    const openSslTarball = path.join(moduleInfo.moduleDotHakDir, 'openssl-1.1.1d.tar.gz');
     let haveOpenSslTar;
     try {
         await fsProm.stat(openSslTarball);
@@ -91,8 +117,9 @@ async function getOpenSsl(hakEnv, moduleInfo) {
         });
     }
 
+    console.log("extracting " + openSslTarball + " in " +  moduleInfo.moduleDotHakDir);
     await tar.x({
         file: openSslTarball,
-        cwd: moduleInfo.depDir,
+        cwd: moduleInfo.moduleDotHakDir,
     });
 }

package.json

@@ -43,6 +43,7 @@
     "eslint-plugin-babel": "^4.1.2",
     "find-npm-prefix": "^1.0.2",
     "follow-redirects": "^1.9.0",
+    "fs-extra": "^8.1.0",
     "glob": "^7.1.6",
     "matrix-js-sdk": "^2.4.6-rc.1",
     "mkdirp": "^1.0.3",

riot.im/config.json

@@ -1,5 +1,4 @@
 {
-    "update_base_url": "https://packages.riot.im/desktop/update/",
     "default_server_name": "matrix.org",
     "brand": "Riot",
     "integrations_ui_url": "https://scalar.vector.im/",
@@ -31,7 +30,8 @@
         }
     },
     "features": {
-        "feature_lazyloading": "enable"
+        "feature_lazyloading": "enable",
+        "feature_event_indexing": "labs"
     },
     "enable_presence_by_hs_url": {
         "https://matrix.org": false,

scripts/hak/clean.js

@@ -20,7 +20,7 @@ const rimraf = require('rimraf');
 
 async function clean(hakEnv, moduleInfo) {
     await new Promise((resolve, reject) => {
-        rimraf(moduleInfo.moduleHakDir, (err) => {
+        rimraf(moduleInfo.moduleDotHakDir, (err) => {
             if (err) {
                 reject(err);
             } else {

scripts/hak/fetch.js

@@ -62,10 +62,10 @@ async function fetch(hakEnv, moduleInfo) {
 
     const tarballUrl = versions[orderedVersions[0]]['dist.tarball'];
 
-    await mkdirp(moduleInfo.moduleHakDir);
+    await mkdirp(moduleInfo.moduleDotHakDir);
 
     const parsedUrl = url.parse(tarballUrl);
-    const tarballFile = path.join(moduleInfo.moduleHakDir, path.basename(parsedUrl.path));
+    const tarballFile = path.join(moduleInfo.moduleDotHakDir, path.basename(parsedUrl.path));
 
     let haveTarball;
     try {
@@ -89,9 +89,10 @@ async function fetch(hakEnv, moduleInfo) {
         strip: 1,
     });
 
+    console.log("Running yarn install in " + moduleInfo.moduleBuildDir);
     await new Promise((resolve, reject) => {
         const proc = child_process.spawn(
-            'yarn',
+            hakEnv.isWin() ? 'yarn.cmd' : 'yarn',
             ['install', '--ignore-scripts'],
             {
                 stdio: 'inherit',

scripts/hak/fetchDeps.js

@@ -14,7 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+const mkdirp = require('mkdirp');
+
 async function fetchDeps(hakEnv, moduleInfo) {
+    await mkdirp(moduleInfo.moduleDotHakDir);
     if (moduleInfo.scripts.fetchDeps) {
         await moduleInfo.scripts.fetchDeps(hakEnv, moduleInfo);
     }

scripts/hak/index.js

@@ -71,7 +71,8 @@ async function main() {
             name: dep,
             version: hakDepsCfg[dep],
             cfg: hakJson,
-            moduleHakDir: path.join(hakEnv.dotHakDir, dep),
+            moduleHakDir: path.join(prefix, 'hak', dep),
+            moduleDotHakDir: path.join(hakEnv.dotHakDir, dep),
             moduleBuildDir: path.join(hakEnv.dotHakDir, dep, 'build'),
             moduleOutDir: path.join(hakEnv.dotHakDir, dep, 'out'),
             nodeModuleBinDir: path.join(hakEnv.dotHakDir, dep, 'build', 'node_modules', '.bin'),

scripts/hak/link.js

@@ -32,12 +32,22 @@ async function link(hakEnv, moduleInfo) {
     } catch (e) {
         await fsProm.writeFile(
             yarnrc, 
-            '--link-folder ' + path.join(hakEnv.dotHakDir, 'links') + os.EOL,
+            // XXX: 1. This must be absolute, as yarn will resolve link directories
+            // relative to the closest project root, which means when we run it
+            // in the dependency project, it will put the link directory in its
+            // own project folder rather than the main project.
+            // 2. We put a colon on the end of the key, which is not what yarn's
+            // stringifier does and not really the format of the file, but it happens
+            // to work and also work around the bug where the parser breaks on values
+            // with a colon in them (which absolute windows paths do).
+            '--link-folder: ' + path.join(hakEnv.dotHakDir, 'links') + os.EOL,
         );
     }
 
+    const yarnCmd = 'yarn' + (hakEnv.isWin() ? '.cmd' : '');
+
     await new Promise((resolve, reject) => {
-        const proc = child_process.spawn('yarn', ['link'], {
+        const proc = child_process.spawn(yarnCmd, ['link'], {
             cwd: moduleInfo.moduleOutDir,
             stdio: 'inherit',
         });
@@ -47,7 +57,7 @@ async function link(hakEnv, moduleInfo) {
     });
 
     await new Promise((resolve, reject) => {
-        const proc = child_process.spawn('yarn', ['link', moduleInfo.name], {
+        const proc = child_process.spawn(yarnCmd, ['link', moduleInfo.name], {
             cwd: hakEnv.projectRoot,
             stdio: 'inherit',
         });