Move i18n scripts in its own module

package.json

@@ -11,11 +11,8 @@
   },
   "license": "Apache-2.0",
   "files": [],
-  "bin": {
-    "matrix-gen-i18n": "scripts/gen-i18n.js"
-  },
   "scripts": {
-    "i18n": "matrix-gen-i18n",
+    "i18n": "gen-18n",
     "mkdirs": "mkdirp packages deploys",
     "fetch": "yarn run mkdirs && node scripts/fetch-package.js",
     "asar-webapp": "asar p webapp webapp.asar",
@@ -53,6 +50,7 @@
     "find-npm-prefix": "^1.0.2",
     "fs-extra": "^8.1.0",
     "glob": "^7.1.6",
+    "matrix-web-i18n": "github:matrix-org/matrix-web-i18n",
     "mkdirp": "^1.0.3",
     "needle": "^2.5.0",
     "node-pre-gyp": "^0.15.0",

scripts/gen-i18n.js

@@ -1,305 +0,0 @@
-#!/usr/bin/env node
-
-/*
-Copyright 2017 New Vector Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * Regenerates the translations en_EN file by walking the source tree and
- * parsing each file with the appropriate parser. Emits a JSON file with the
- * translatable strings mapped to themselves in the order they appeared
- * in the files and grouped by the file they appeared in.
- *
- * Usage: node scripts/gen-i18n.js
- */
-const fs = require('fs');
-const path = require('path');
-
-const walk = require('walk');
-
-const parser = require("@babel/parser");
-const traverse = require("@babel/traverse");
-
-const TRANSLATIONS_FUNCS = ['_t', '_td'];
-
-const INPUT_TRANSLATIONS_FILE = 'src/i18n/strings/en_EN.json';
-const OUTPUT_FILE = 'src/i18n/strings/en_EN.json';
-
-// NB. The sync version of walk is broken for single files so we walk
-// all of res rather than just res/home.html.
-// https://git.daplie.com/Daplie/node-walk/merge_requests/1 fixes it,
-// or if we get bored waiting for it to be merged, we could switch
-// to a project that's actively maintained.
-const SEARCH_PATHS = ['src', 'res'];
-
-function getObjectValue(obj, key) {
-    for (const prop of obj.properties) {
-        if (prop.key.type === 'Identifier' && prop.key.name === key) {
-            return prop.value;
-        }
-    }
-    return null;
-}
-
-function getTKey(arg) {
-    if (arg.type === 'Literal' || arg.type === "StringLiteral") {
-        return arg.value;
-    } else if (arg.type === 'BinaryExpression' && arg.operator === '+') {
-        return getTKey(arg.left) + getTKey(arg.right);
-    } else if (arg.type === 'TemplateLiteral') {
-        return arg.quasis.map((q) => {
-            return q.value.raw;
-        }).join('');
-    }
-    return null;
-}
-
-function getFormatStrings(str) {
-    // Match anything that starts with %
-    // We could make a regex that matched the full placeholder, but this
-    // would just not match invalid placeholders and so wouldn't help us
-    // detect the invalid ones.
-    // Also note that for simplicity, this just matches a % character and then
-    // anything up to the next % character (or a single %, or end of string).
-    const formatStringRe = /%([^%]+|%|$)/g;
-    const formatStrings = new Set();
-
-    let match;
-    while ( (match = formatStringRe.exec(str)) !== null ) {
-        const placeholder = match[1]; // Minus the leading '%'
-        if (placeholder === '%') continue; // Literal % is %%
-
-        const placeholderMatch = placeholder.match(/^\((.*?)\)(.)/);
-        if (placeholderMatch === null) {
-            throw new Error("Invalid format specifier: '"+match[0]+"'");
-        }
-        if (placeholderMatch.length < 3) {
-            throw new Error("Malformed format specifier");
-        }
-        const placeholderName = placeholderMatch[1];
-        const placeholderFormat = placeholderMatch[2];
-
-        if (placeholderFormat !== 's') {
-            throw new Error(`'${placeholderFormat}' used as format character: you probably meant 's'`);
-        }
-
-        formatStrings.add(placeholderName);
-    }
-
-    return formatStrings;
-}
-
-function getTranslationsJs(file) {
-    const contents = fs.readFileSync(file, { encoding: 'utf8' });
-
-    const trs = new Set();
-
-    try {
-        const plugins = [
-            // https://babeljs.io/docs/en/babel-parser#plugins
-            "classProperties",
-            "objectRestSpread",
-            "throwExpressions",
-            "exportDefaultFrom",
-            "decorators-legacy",
-        ];
-
-        if (file.endsWith(".js") || file.endsWith(".jsx")) {
-            // all JS is assumed to be flow or react
-            plugins.push("flow", "jsx");
-        } else if (file.endsWith(".ts")) {
-            // TS can't use JSX unless it's a TSX file (otherwise angle casts fail)
-            plugins.push("typescript");
-        } else if (file.endsWith(".tsx")) {
-            // When the file is a TSX file though, enable JSX parsing
-            plugins.push("typescript", "jsx");
-        }
-
-        const babelParsed = parser.parse(contents, {
-            allowImportExportEverywhere: true,
-            errorRecovery: true,
-            sourceFilename: file,
-            tokens: true,
-            plugins,
-        });
-        traverse.default(babelParsed, {
-            enter: (p) => {
-                const node = p.node;
-                if (p.isCallExpression() && node.callee && TRANSLATIONS_FUNCS.includes(node.callee.name)) {
-                    const tKey = getTKey(node.arguments[0]);
-
-                    // This happens whenever we call _t with non-literals (ie. whenever we've
-                    // had to use a _td to compensate) so is expected.
-                    if (tKey === null) return;
-
-                    // check the format string against the args
-                    // We only check _t: _td has no args
-                    if (node.callee.name === '_t') {
-                        try {
-                            const placeholders = getFormatStrings(tKey);
-                            for (const placeholder of placeholders) {
-                                if (node.arguments.length < 2) {
-                                    throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`);
-                                }
-                                const value = getObjectValue(node.arguments[1], placeholder);
-                                if (value === null) {
-                                    throw new Error(`No value found for placeholder '${placeholder}'`);
-                                }
-                            }
-
-                            // Validate tag replacements
-                            if (node.arguments.length > 2) {
-                                const tagMap = node.arguments[2];
-                                for (const prop of tagMap.properties || []) {
-                                    if (prop.key.type === 'Literal') {
-                                        const tag = prop.key.value;
-                                        // RegExp same as in src/languageHandler.js
-                                        // eslint-disable-next-line
-                                        const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`);
-                                        if (!tKey.match(regexp)) {
-                                            throw new Error(`No match for ${regexp} in ${tKey}`);
-                                        }
-                                    }
-                                }
-                            }
-                        } catch (e) {
-                            console.log();
-                            console.error(`ERROR: ${file}:${node.loc.start.line} ${tKey}`);
-                            console.error(e);
-                            process.exit(1);
-                        }
-                    }
-
-                    let isPlural = false;
-                    if (node.arguments.length > 1 && node.arguments[1].type === 'ObjectExpression') {
-                        const countVal = getObjectValue(node.arguments[1], 'count');
-                        if (countVal) {
-                            isPlural = true;
-                        }
-                    }
-
-                    if (isPlural) {
-                        trs.add(tKey + "|other");
-                        const plurals = enPlurals[tKey];
-                        if (plurals) {
-                            for (const pluralType of Object.keys(plurals)) {
-                                trs.add(tKey + "|" + pluralType);
-                            }
-                        }
-                    } else {
-                        trs.add(tKey);
-                    }
-                }
-            },
-        });
-    } catch (e) {
-        console.error(e);
-        process.exit(1);
-    }
-
-    return trs;
-}
-
-function getTranslationsOther(file) {
-    const contents = fs.readFileSync(file, { encoding: 'utf8' });
-
-    const trs = new Set();
-
-    // Taken from element-web src/components/structures/HomePage.js
-    const translationsRegex = /_t\(['"]([\s\S]*?)['"]\)/mg;
-    let matches;
-    while (matches = translationsRegex.exec(contents)) {
-        trs.add(matches[1]);
-    }
-    return trs;
-}
-
-// gather en_EN plural strings from the input translations file:
-// the en_EN strings are all in the source with the exception of
-// pluralised strings, which we need to pull in from elsewhere.
-const inputTranslationsRaw = JSON.parse(fs.readFileSync(INPUT_TRANSLATIONS_FILE, { encoding: 'utf8' }));
-const enPlurals = {};
-
-for (const key of Object.keys(inputTranslationsRaw)) {
-    const parts = key.split("|");
-    if (parts.length > 1) {
-        const plurals = enPlurals[parts[0]] || {};
-        plurals[parts[1]] = inputTranslationsRaw[key];
-        enPlurals[parts[0]] = plurals;
-    }
-}
-
-const translatables = new Set();
-
-const walkOpts = {
-    listeners: {
-        names: function(root, nodeNamesArray) {
-            // Sort the names case insensitively and alphabetically to
-            // maintain some sense of order between the different strings.
-            nodeNamesArray.sort((a, b) => {
-                a = a.toLowerCase();
-                b = b.toLowerCase();
-                if (a > b) return 1;
-                if (a < b) return -1;
-                return 0;
-            });
-        },
-        file: function(root, fileStats, next) {
-            const fullPath = path.join(root, fileStats.name);
-
-            let trs;
-            if (fileStats.name.endsWith('.js') || fileStats.name.endsWith('.ts') || fileStats.name.endsWith('.tsx')) {
-                trs = getTranslationsJs(fullPath);
-            } else if (fileStats.name.endsWith('.html')) {
-                trs = getTranslationsOther(fullPath);
-            } else {
-                return;
-            }
-            console.log(`${fullPath} (${trs.size} strings)`);
-            for (const tr of trs.values()) {
-                // Convert DOS line endings to unix
-                translatables.add(tr.replace(/\r\n/g, "\n"));
-            }
-        },
-    },
-};
-
-for (const path of SEARCH_PATHS) {
-    if (fs.existsSync(path)) {
-        walk.walkSync(path, walkOpts);
-    }
-}
-
-const trObj = {};
-for (const tr of translatables) {
-    if (tr.includes("|")) {
-        if (inputTranslationsRaw[tr]) {
-            trObj[tr] = inputTranslationsRaw[tr];
-        } else {
-            trObj[tr] = tr.split("|")[0];
-        }
-    } else {
-        trObj[tr] = tr;
-    }
-}
-
-fs.writeFileSync(
-    OUTPUT_FILE,
-    JSON.stringify(trObj, translatables.values(), 4) + "\n",
-);
-
-console.log();
-console.log(`Wrote ${translatables.size} strings to ${OUTPUT_FILE}`);
-

yarn.lock

@@ -26,6 +26,13 @@
   dependencies:
     "@babel/highlight" "^7.10.4"
 
+"@babel/code-frame@^7.12.13":
+  version "7.12.13"
+  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.13.tgz#dcfc826beef65e75c50e21d3837d7d95798dd658"
+  integrity sha512-HV1Cm0Q3ZrpCR93tkWOYiuYIgLxZXZFVG2VgK+MBWjUqZTundupbfx2aXarXuw5Ko5aMcjtJgbSs4vUGBS5v6g==
+  dependencies:
+    "@babel/highlight" "^7.12.13"
+
 "@babel/generator@^7.10.5":
   version "7.10.5"
   resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.10.5.tgz#1b903554bc8c583ee8d25f1e8969732e6b829a69"
@@ -35,6 +42,15 @@
     jsesc "^2.5.1"
     source-map "^0.5.0"
 
+"@babel/generator@^7.13.16":
+  version "7.13.16"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.13.16.tgz#0befc287031a201d84cdfc173b46b320ae472d14"
+  integrity sha512-grBBR75UnKOcUWMp8WoDxNsWCFl//XCK6HWTrBQKTr5SV9f5g0pNOjdyzi/DTBv12S9GnYPInIXQBTky7OXEMg==
+  dependencies:
+    "@babel/types" "^7.13.16"
+    jsesc "^2.5.1"
+    source-map "^0.5.0"
+
 "@babel/helper-function-name@^7.10.4":
   version "7.10.4"
   resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.10.4.tgz#d2d3b20c59ad8c47112fa7d2a94bc09d5ef82f1a"
@@ -44,6 +60,15 @@
     "@babel/template" "^7.10.4"
     "@babel/types" "^7.10.4"
 
+"@babel/helper-function-name@^7.12.13":
+  version "7.12.13"
+  resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.12.13.tgz#93ad656db3c3c2232559fd7b2c3dbdcbe0eb377a"
+  integrity sha512-TZvmPn0UOqmvi5G4vvw0qZTpVptGkB1GL61R6lKvrSdIxGm5Pky7Q3fpKiIkQCAtRCBUwB0PaThlx9vebCDSwA==
+  dependencies:
+    "@babel/helper-get-function-arity" "^7.12.13"
+    "@babel/template" "^7.12.13"
+    "@babel/types" "^7.12.13"
+
 "@babel/helper-get-function-arity@^7.10.4":
   version "7.10.4"
   resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.4.tgz#98c1cbea0e2332f33f9a4661b8ce1505b2c19ba2"
@@ -51,6 +76,13 @@
   dependencies:
     "@babel/types" "^7.10.4"
 
+"@babel/helper-get-function-arity@^7.12.13":
+  version "7.12.13"
+  resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.13.tgz#bc63451d403a3b3082b97e1d8b3fe5bd4091e583"
+  integrity sha512-DjEVzQNz5LICkzN0REdpD5prGoidvbdYk1BVgRUOINaWJP2t6avB27X1guXK1kXNrX0WMfsrm1A/ZBthYuIMQg==
+  dependencies:
+    "@babel/types" "^7.12.13"
+
 "@babel/helper-split-export-declaration@^7.10.4":
   version "7.10.4"
   resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.10.4.tgz#2c70576eaa3b5609b24cb99db2888cc3fc4251d1"
@@ -58,11 +90,23 @@
   dependencies:
     "@babel/types" "^7.10.4"
 
+"@babel/helper-split-export-declaration@^7.12.13":
+  version "7.12.13"
+  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.13.tgz#e9430be00baf3e88b0e13e6f9d4eaf2136372b05"
+  integrity sha512-tCJDltF83htUtXx5NLcaDqRmknv652ZWCHyoTETf1CXYJdPC7nohZohjUgieXhv0hTJdRf2FjDueFehdNucpzg==
+  dependencies:
+    "@babel/types" "^7.12.13"
+
 "@babel/helper-validator-identifier@^7.10.4":
   version "7.10.4"
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz#a78c7a7251e01f616512d31b10adcf52ada5e0d2"
   integrity sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw==
 
+"@babel/helper-validator-identifier@^7.12.11":
+  version "7.12.11"
+  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz#c9a1f021917dcb5ccf0d4e453e399022981fc9ed"
+  integrity sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==
+
 "@babel/highlight@^7.0.0":
   version "7.5.0"
   resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.5.0.tgz#56d11312bd9248fa619591d02472be6e8cb32540"
@@ -81,11 +125,25 @@
     chalk "^2.0.0"
     js-tokens "^4.0.0"
 
+"@babel/highlight@^7.12.13":
+  version "7.13.10"
+  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.13.10.tgz#a8b2a66148f5b27d666b15d81774347a731d52d1"
+  integrity sha512-5aPpe5XQPzflQrFwL1/QoeHkP2MsA4JCntcXHRhEsdsfPVkvPi2w7Qix4iV7t5S/oC9OodGrggd8aco1g3SZFg==
+  dependencies:
+    "@babel/helper-validator-identifier" "^7.12.11"
+    chalk "^2.0.0"
+    js-tokens "^4.0.0"
+
 "@babel/parser@^7.10.4", "@babel/parser@^7.10.5", "@babel/parser@^7.7.0":
   version "7.10.5"
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.5.tgz#e7c6bf5a7deff957cec9f04b551e2762909d826b"
   integrity sha512-wfryxy4bE1UivvQKSQDU4/X6dr+i8bctjUjj8Zyt3DQy7NtPizJXT8M52nqpNKL+nq2PW8lxk4ZqLj0fD4B4hQ==
 
+"@babel/parser@^7.12.13", "@babel/parser@^7.13.16":
+  version "7.13.16"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.16.tgz#0f18179b0448e6939b1f3f5c4c355a3a9bcdfd37"
+  integrity sha512-6bAg36mCwuqLO0hbR+z7PHuqWiCeP7Dzg73OpQwsAB1Eb8HnGEz5xYBzCfbu+YjoaJsJs+qheDxVAuqbt3ILEw==
+
 "@babel/runtime@^7.7.2":
   version "7.11.2"
   resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.11.2.tgz#f549c13c754cc40b87644b9fa9f09a6a95fe0736"
@@ -102,6 +160,29 @@
     "@babel/parser" "^7.10.4"
     "@babel/types" "^7.10.4"
 
+"@babel/template@^7.12.13":
+  version "7.12.13"
+  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.12.13.tgz#530265be8a2589dbb37523844c5bcb55947fb327"
+  integrity sha512-/7xxiGA57xMo/P2GVvdEumr8ONhFOhfgq2ihK3h1e6THqzTAkHbkXgB0xI9yeTfIUoH3+oAeHhqm/I43OTbbjA==
+  dependencies:
+    "@babel/code-frame" "^7.12.13"
+    "@babel/parser" "^7.12.13"
+    "@babel/types" "^7.12.13"
+
+"@babel/traverse@^7.13.17":
+  version "7.13.17"
+  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.13.17.tgz#c85415e0c7d50ac053d758baec98b28b2ecfeea3"
+  integrity sha512-BMnZn0R+X6ayqm3C3To7o1j7Q020gWdqdyP50KEoVqaCO2c/Im7sYZSmVgvefp8TTMQ+9CtwuBp0Z1CZ8V3Pvg==
+  dependencies:
+    "@babel/code-frame" "^7.12.13"
+    "@babel/generator" "^7.13.16"
+    "@babel/helper-function-name" "^7.12.13"
+    "@babel/helper-split-export-declaration" "^7.12.13"
+    "@babel/parser" "^7.13.16"
+    "@babel/types" "^7.13.17"
+    debug "^4.1.0"
+    globals "^11.1.0"
+
 "@babel/traverse@^7.7.0":
   version "7.10.5"
   resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.10.5.tgz#77ce464f5b258be265af618d8fddf0536f20b564"
@@ -126,6 +207,14 @@
     lodash "^4.17.19"
     to-fast-properties "^2.0.0"
 
+"@babel/types@^7.12.13", "@babel/types@^7.13.16", "@babel/types@^7.13.17":
+  version "7.13.17"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.13.17.tgz#48010a115c9fba7588b4437dd68c9469012b38b4"
+  integrity sha512-RawydLgxbOPDlTLJNtoIypwdmAy//uQIzlKt2+iBiJaRlVuI6QLUxVAyWGNfOzp8Yu4L4lLIacoCyTNtpb4wiA==
+  dependencies:
+    "@babel/helper-validator-identifier" "^7.12.11"
+    to-fast-properties "^2.0.0"
+
 "@develar/schema-utils@~2.6.5":
   version "2.6.5"
   resolved "https://registry.yarnpkg.com/@develar/schema-utils/-/schema-utils-2.6.5.tgz#3ece22c5838402419a6e0425f85742b961d9b6c6"
@@ -2634,6 +2723,11 @@ flush-write-stream@^1.0.0:
     inherits "^2.0.3"
     readable-stream "^2.3.6"
 
+foreachasync@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/foreachasync/-/foreachasync-3.0.0.tgz#5502987dc8714be3392097f32e0071c9dee07cf6"
+  integrity sha1-VQKYfchxS+M5IJfzLgBxyd7gfPY=
+
 forever-agent@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
@@ -4003,6 +4097,14 @@ make-fetch-happen@^5.0.0:
     socks-proxy-agent "^4.0.0"
     ssri "^6.0.0"
 
+"matrix-web-i18n@github:matrix-org/matrix-web-i18n":
+  version "1.0.1"
+  resolved "https://codeload.github.com/matrix-org/matrix-web-i18n/tar.gz/6df80d0197711e9f9733e9e3e2ef1d38a2ab8949"
+  dependencies:
+    "@babel/parser" "^7.13.16"
+    "@babel/traverse" "^7.13.17"
+    walk "^2.3.14"
+
 meant@^1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/meant/-/meant-1.0.3.tgz#67769af9de1d158773e928ae82c456114903554c"
@@ -6440,6 +6542,13 @@ version-range@^1.0.0:
   dependencies:
     version-compare "^1.0.0"
 
+walk@^2.3.14:
+  version "2.3.14"
+  resolved "https://registry.yarnpkg.com/walk/-/walk-2.3.14.tgz#60ec8631cfd23276ae1e7363ce11d626452e1ef3"
+  integrity sha512-5skcWAUmySj6hkBdH6B6+3ddMjVQYH5Qy9QGbPmN8kVmLteXk+yVXg+yfk1nbX30EYakahLrr8iPcCxJQSCBeg==
+  dependencies:
+    foreachasync "^3.0.0"
+
 wcwidth@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/wcwidth/-/wcwidth-1.0.1.tgz#f0b0dcf915bc5ff1528afadb2c0e17b532da2fe8"