diff --git a/hak/matrix-seshat/build.js b/hak/matrix-seshat/build.js index 070af0c..f15474a 100644 --- a/hak/matrix-seshat/build.js +++ b/hak/matrix-seshat/build.js @@ -17,18 +17,158 @@ limitations under the License. const path = require('path'); const child_process = require('child_process'); +const mkdirp = require('mkdirp'); +const fsExtra = require('fs-extra'); + module.exports = async function(hakEnv, moduleInfo) { - if (hakEnv.isWin()) await buildOpenSsl(hakEnv, moduleInfo); - await buildSqlCipher(hakEnv, moduleInfo); + if (hakEnv.isWin()) { + await buildOpenSslWin(hakEnv, moduleInfo); + await buildSqlCipherWin(hakEnv, moduleInfo); + } else { + await buildSqlCipherUnix(hakEnv, moduleInfo); + } await buildMatrixSeshat(hakEnv, moduleInfo); } -async function buildOpenSsl(hakEnv, moduleInfo) { - const openSslDir = path.join(moduleInfo.moduleHakDir, 'openssl-1.1.1d'); +async function buildOpenSslWin(hakEnv, moduleInfo) { + const openSslDir = path.join(moduleInfo.moduleDotHakDir, 'openssl-1.1.1d'); + + const openSslArch = hakEnv.arch === 'x64' ? 'VC-WIN64A' : 'VC-WIN32'; + + console.log("Building openssl in " + openSslDir); + await new Promise((resolve, reject) => { + const proc = child_process.spawn( + 'perl', + [ + 'Configure', + '--prefix=' + moduleInfo.depPrefix, + // sqlcipher only uses about a tiny part of openssl. We link statically + // so will only pull in the symbols we use, but we may as well turn off + // as much as possible to save on build time. + 'no-afalgeng', + 'no-capieng', + 'no-cms', + 'no-ct', + 'no-deprecated', + 'no-dgram', + 'no-dso', + 'no-ec', + 'no-ec2m', + 'no-gost', + 'no-nextprotoneg', + 'no-ocsp', + 'no-sock', + 'no-srp', + 'no-srtp', + 'no-tests', + 'no-ssl', + 'no-tls', + 'no-dtls', + 'no-shared', + 'no-aria', + 'no-camellia', + 'no-cast', + 'no-chacha', + 'no-cmac', + 'no-des', + 'no-dh', + 'no-dsa', + 'no-ecdh', + 'no-ecdsa', + 'no-idea', + 'no-md4', + 'no-mdc2', + 'no-ocb', + 'no-poly1305', + 'no-rc2', + 'no-rc4', + 'no-rmd160', + 'no-scrypt', + 'no-seed', + 'no-siphash', + 'no-sm2', + 'no-sm3', + 'no-sm4', + 'no-whirlpool', + openSslArch, + ], + { + cwd: openSslDir, + stdio: 'inherit', + }, + ); + proc.on('exit', (code) => { + code ? reject(code) : resolve(); + }); + }); + + await new Promise((resolve, reject) => { + const proc = child_process.spawn( + 'nmake', + ['build_libs'], + { + cwd: openSslDir, + stdio: 'inherit', + }, + ); + proc.on('exit', (code) => { + code ? reject(code) : resolve(); + }); + }); + + await new Promise((resolve, reject) => { + const proc = child_process.spawn( + 'nmake', + ['install_dev'], + { + cwd: openSslDir, + stdio: 'inherit', + }, + ); + proc.on('exit', (code) => { + code ? reject(code) : resolve(); + }); + }); } -async function buildSqlCipher(hakEnv, moduleInfo) { - const sqlCipherDir = path.join(moduleInfo.moduleHakDir, 'sqlcipher-4.3.0'); +async function buildSqlCipherWin(hakEnv, moduleInfo) { + const sqlCipherDir = path.join(moduleInfo.moduleDotHakDir, 'sqlcipher-4.3.0'); + const buildDir = path.join(sqlCipherDir, 'bld'); + + await mkdirp(buildDir); + + await new Promise((resolve, reject) => { + const proc = child_process.spawn( + 'nmake', + ['/f', path.join('..', 'Makefile.msc'), 'libsqlite3.lib', 'TOP=..'], + { + cwd: buildDir, + stdio: 'inherit', + env: Object.assign({}, process.env, { + CCOPTS: "-DSQLITE_HAS_CODEC -I" + path.join(moduleInfo.depPrefix, 'include'), + LTLIBPATHS: "/LIBPATH:" + path.join(moduleInfo.depPrefix, 'lib'), + LTLIBS: "libcrypto.lib", + }), + }, + ); + proc.on('exit', (code) => { + code ? reject(code) : resolve(); + }); + }); + + await fsExtra.copy( + path.join(buildDir, 'libsqlite3.lib'), + path.join(moduleInfo.depPrefix, 'lib', 'sqlcipher.lib'), + ); + + await fsExtra.copy( + path.join(buildDir, 'sqlite3.h'), + path.join(moduleInfo.depPrefix, 'include', 'sqlcipher.h'), + ); +} + +async function buildSqlCipherUnix(hakEnv, moduleInfo) { + const sqlCipherDir = path.join(moduleInfo.moduleDotHakDir, 'sqlcipher-4.3.0'); const args = [ '--prefix=' + moduleInfo.depPrefix + '', @@ -88,17 +228,24 @@ async function buildSqlCipher(hakEnv, moduleInfo) { } async function buildMatrixSeshat(hakEnv, moduleInfo) { - await new Promise((resolve) => { + const env = Object.assign({ + SQLCIPHER_STATIC: 1, + SQLCIPHER_LIB_DIR: path.join(moduleInfo.depPrefix, 'lib'), + SQLCIPHER_INCLUDE_DIR: path.join(moduleInfo.depPrefix, 'include'), + }, hakEnv.makeGypEnv()); + + if (hakEnv.isWin()) { + env.RUSTFLAGS = '-Ctarget-feature=+crt-static -Clink-args=libcrypto.lib'; + } + + console.log("Running neon with env", env); + await new Promise((resolve, reject) => { const proc = child_process.spawn( - path.join(moduleInfo.nodeModuleBinDir, 'neon'), + path.join(moduleInfo.nodeModuleBinDir, 'neon' + (hakEnv.isWin() ? '.cmd' : '')), ['build', '--release'], { cwd: moduleInfo.moduleBuildDir, - env: Object.assign({ - SQLCIPHER_STATIC: 1, - SQLCIPHER_LIB_DIR: path.join(moduleInfo.depPrefix, 'lib'), - SQLCIPHER_INCLUDE_DIR: path.join(moduleInfo.depPrefix, 'include'), - }, hakEnv.makeGypEnv()), + env, stdio: 'inherit', }, ); diff --git a/hak/matrix-seshat/fetchDeps.js b/hak/matrix-seshat/fetchDeps.js index 504fd94..713bfb4 100644 --- a/hak/matrix-seshat/fetchDeps.js +++ b/hak/matrix-seshat/fetchDeps.js @@ -15,7 +15,9 @@ limitations under the License. */ const path = require('path'); +const child_process = require('child_process'); +const fs = require('fs'); const fsProm = require('fs').promises; const needle = require('needle'); const tar = require('tar'); @@ -29,7 +31,7 @@ module.exports = async function(hakEnv, moduleInfo) { } async function getSqlCipher(hakEnv, moduleInfo) { - const sqlCipherDir = path.join(moduleInfo.moduleHakDir, 'sqlcipher-4.3.0'); + const sqlCipherDir = path.join(moduleInfo.moduleDotHakDir, 'sqlcipher-4.3.0'); let haveSqlcipher; try { @@ -41,7 +43,7 @@ async function getSqlCipher(hakEnv, moduleInfo) { if (haveSqlcipher) return; - const sqlCipherTarball = path.join(moduleInfo.moduleHakDir, 'sqlcipher-4.3.0.tar.gz'); + const sqlCipherTarball = path.join(moduleInfo.moduleDotHakDir, 'sqlcipher-4.3.0.tar.gz'); let haveSqlcipherTar; try { await fsProm.stat(sqlCipherTarball); @@ -59,12 +61,36 @@ async function getSqlCipher(hakEnv, moduleInfo) { await tar.x({ file: sqlCipherTarball, - cwd: moduleInfo.moduleHakDir, + cwd: moduleInfo.moduleDotHakDir, }); + + if (hakEnv.isWin()) { + // On Windows, we need to patch the makefile because it forces TEMP_STORE to + // default to files (1) but the README specifically says you '*must*' set it + // set it to 2 (default to memory). + const patchFile = path.join(moduleInfo.moduleHakDir, 'sqlcipher-4.3.0-win.patch'); + + await new Promise((resolve, reject) => { + const readStream = fs.createReadStream(patchFile); + + const proc = child_process.spawn( + 'patch', + ['-p1'], + { + cwd: sqlCipherDir, + stdio: ['pipe', 'inherit', 'inherit'], + }, + ); + proc.on('exit', (code) => { + code ? reject(code) : resolve(); + }); + readStream.pipe(proc.stdin); + }); + } } async function getOpenSsl(hakEnv, moduleInfo) { - const openSslDir = path.join(moduleInfo.moduleHakDir, 'openssl-1.1.1d'); + const openSslDir = path.join(moduleInfo.moduleDotHakDir, 'openssl-1.1.1d'); let haveOpenSsl; try { @@ -76,7 +102,7 @@ async function getOpenSsl(hakEnv, moduleInfo) { if (haveOpenSsl) return; - const openSslTarball = path.join(moduleInfo.depDir, 'openssl-1.1.1d.tar.gz'); + const openSslTarball = path.join(moduleInfo.moduleDotHakDir, 'openssl-1.1.1d.tar.gz'); let haveOpenSslTar; try { await fsProm.stat(openSslTarball); @@ -91,8 +117,9 @@ async function getOpenSsl(hakEnv, moduleInfo) { }); } + console.log("extracting " + openSslTarball + " in " + moduleInfo.moduleDotHakDir); await tar.x({ file: openSslTarball, - cwd: moduleInfo.depDir, + cwd: moduleInfo.moduleDotHakDir, }); } diff --git a/package.json b/package.json index c8772df..69dcc61 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "eslint-plugin-babel": "^4.1.2", "find-npm-prefix": "^1.0.2", "follow-redirects": "^1.9.0", + "fs-extra": "^8.1.0", "glob": "^7.1.6", "matrix-js-sdk": "^2.4.6-rc.1", "mkdirp": "^1.0.3", diff --git a/riot.im/config.json b/riot.im/config.json index 9a98dae..3aeb363 100644 --- a/riot.im/config.json +++ b/riot.im/config.json @@ -1,5 +1,4 @@ { - "update_base_url": "https://packages.riot.im/desktop/update/", "default_server_name": "matrix.org", "brand": "Riot", "integrations_ui_url": "https://scalar.vector.im/", @@ -31,7 +30,8 @@ } }, "features": { - "feature_lazyloading": "enable" + "feature_lazyloading": "enable", + "feature_event_indexing": "labs" }, "enable_presence_by_hs_url": { "https://matrix.org": false, diff --git a/scripts/hak/clean.js b/scripts/hak/clean.js index 20152f4..9a15a21 100644 --- a/scripts/hak/clean.js +++ b/scripts/hak/clean.js @@ -20,7 +20,7 @@ const rimraf = require('rimraf'); async function clean(hakEnv, moduleInfo) { await new Promise((resolve, reject) => { - rimraf(moduleInfo.moduleHakDir, (err) => { + rimraf(moduleInfo.moduleDotHakDir, (err) => { if (err) { reject(err); } else { diff --git a/scripts/hak/fetch.js b/scripts/hak/fetch.js index 703916f..4587ce7 100644 --- a/scripts/hak/fetch.js +++ b/scripts/hak/fetch.js @@ -62,10 +62,10 @@ async function fetch(hakEnv, moduleInfo) { const tarballUrl = versions[orderedVersions[0]]['dist.tarball']; - await mkdirp(moduleInfo.moduleHakDir); + await mkdirp(moduleInfo.moduleDotHakDir); const parsedUrl = url.parse(tarballUrl); - const tarballFile = path.join(moduleInfo.moduleHakDir, path.basename(parsedUrl.path)); + const tarballFile = path.join(moduleInfo.moduleDotHakDir, path.basename(parsedUrl.path)); let haveTarball; try { @@ -89,9 +89,10 @@ async function fetch(hakEnv, moduleInfo) { strip: 1, }); + console.log("Running yarn install in " + moduleInfo.moduleBuildDir); await new Promise((resolve, reject) => { const proc = child_process.spawn( - 'yarn', + hakEnv.isWin() ? 'yarn.cmd' : 'yarn', ['install', '--ignore-scripts'], { stdio: 'inherit', diff --git a/scripts/hak/fetchDeps.js b/scripts/hak/fetchDeps.js index 1e623d3..361e97b 100644 --- a/scripts/hak/fetchDeps.js +++ b/scripts/hak/fetchDeps.js @@ -14,7 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ +const mkdirp = require('mkdirp'); + async function fetchDeps(hakEnv, moduleInfo) { + await mkdirp(moduleInfo.moduleDotHakDir); if (moduleInfo.scripts.fetchDeps) { await moduleInfo.scripts.fetchDeps(hakEnv, moduleInfo); } diff --git a/scripts/hak/index.js b/scripts/hak/index.js index 3480dbb..143a9ce 100644 --- a/scripts/hak/index.js +++ b/scripts/hak/index.js @@ -71,7 +71,8 @@ async function main() { name: dep, version: hakDepsCfg[dep], cfg: hakJson, - moduleHakDir: path.join(hakEnv.dotHakDir, dep), + moduleHakDir: path.join(prefix, 'hak', dep), + moduleDotHakDir: path.join(hakEnv.dotHakDir, dep), moduleBuildDir: path.join(hakEnv.dotHakDir, dep, 'build'), moduleOutDir: path.join(hakEnv.dotHakDir, dep, 'out'), nodeModuleBinDir: path.join(hakEnv.dotHakDir, dep, 'build', 'node_modules', '.bin'), diff --git a/scripts/hak/link.js b/scripts/hak/link.js index 9b5f2fb..d3f696a 100644 --- a/scripts/hak/link.js +++ b/scripts/hak/link.js @@ -32,12 +32,22 @@ async function link(hakEnv, moduleInfo) { } catch (e) { await fsProm.writeFile( yarnrc, - '--link-folder ' + path.join(hakEnv.dotHakDir, 'links') + os.EOL, + // XXX: 1. This must be absolute, as yarn will resolve link directories + // relative to the closest project root, which means when we run it + // in the dependency project, it will put the link directory in its + // own project folder rather than the main project. + // 2. We put a colon on the end of the key, which is not what yarn's + // stringifier does and not really the format of the file, but it happens + // to work and also work around the bug where the parser breaks on values + // with a colon in them (which absolute windows paths do). + '--link-folder: ' + path.join(hakEnv.dotHakDir, 'links') + os.EOL, ); } + const yarnCmd = 'yarn' + (hakEnv.isWin() ? '.cmd' : ''); + await new Promise((resolve, reject) => { - const proc = child_process.spawn('yarn', ['link'], { + const proc = child_process.spawn(yarnCmd, ['link'], { cwd: moduleInfo.moduleOutDir, stdio: 'inherit', }); @@ -47,7 +57,7 @@ async function link(hakEnv, moduleInfo) { }); await new Promise((resolve, reject) => { - const proc = child_process.spawn('yarn', ['link', moduleInfo.name], { + const proc = child_process.spawn(yarnCmd, ['link', moduleInfo.name], { cwd: hakEnv.projectRoot, stdio: 'inherit', });