diff --git a/package.json b/package.json index d651e48..5245a18 100644 --- a/package.json +++ b/package.json @@ -11,11 +11,8 @@ }, "license": "Apache-2.0", "files": [], - "bin": { - "matrix-gen-i18n": "scripts/gen-i18n.js" - }, "scripts": { - "i18n": "matrix-gen-i18n", + "i18n": "gen-18n", "mkdirs": "mkdirp packages deploys", "fetch": "yarn run mkdirs && node scripts/fetch-package.js", "asar-webapp": "asar p webapp webapp.asar", @@ -53,6 +50,7 @@ "find-npm-prefix": "^1.0.2", "fs-extra": "^8.1.0", "glob": "^7.1.6", + "matrix-web-i18n": "github:matrix-org/matrix-web-i18n", "mkdirp": "^1.0.3", "needle": "^2.5.0", "node-pre-gyp": "^0.15.0", diff --git a/scripts/gen-i18n.js b/scripts/gen-i18n.js deleted file mode 100644 index 0863071..0000000 --- a/scripts/gen-i18n.js +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env node - -/* -Copyright 2017 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/** - * Regenerates the translations en_EN file by walking the source tree and - * parsing each file with the appropriate parser. Emits a JSON file with the - * translatable strings mapped to themselves in the order they appeared - * in the files and grouped by the file they appeared in. - * - * Usage: node scripts/gen-i18n.js - */ -const fs = require('fs'); -const path = require('path'); - -const walk = require('walk'); - -const parser = require("@babel/parser"); -const traverse = require("@babel/traverse"); - -const TRANSLATIONS_FUNCS = ['_t', '_td']; - -const INPUT_TRANSLATIONS_FILE = 'src/i18n/strings/en_EN.json'; -const OUTPUT_FILE = 'src/i18n/strings/en_EN.json'; - -// NB. The sync version of walk is broken for single files so we walk -// all of res rather than just res/home.html. -// https://git.daplie.com/Daplie/node-walk/merge_requests/1 fixes it, -// or if we get bored waiting for it to be merged, we could switch -// to a project that's actively maintained. -const SEARCH_PATHS = ['src', 'res']; - -function getObjectValue(obj, key) { - for (const prop of obj.properties) { - if (prop.key.type === 'Identifier' && prop.key.name === key) { - return prop.value; - } - } - return null; -} - -function getTKey(arg) { - if (arg.type === 'Literal' || arg.type === "StringLiteral") { - return arg.value; - } else if (arg.type === 'BinaryExpression' && arg.operator === '+') { - return getTKey(arg.left) + getTKey(arg.right); - } else if (arg.type === 'TemplateLiteral') { - return arg.quasis.map((q) => { - return q.value.raw; - }).join(''); - } - return null; -} - -function getFormatStrings(str) { - // Match anything that starts with % - // We could make a regex that matched the full placeholder, but this - // would just not match invalid placeholders and so wouldn't help us - // detect the invalid ones. - // Also note that for simplicity, this just matches a % character and then - // anything up to the next % character (or a single %, or end of string). - const formatStringRe = /%([^%]+|%|$)/g; - const formatStrings = new Set(); - - let match; - while ( (match = formatStringRe.exec(str)) !== null ) { - const placeholder = match[1]; // Minus the leading '%' - if (placeholder === '%') continue; // Literal % is %% - - const placeholderMatch = placeholder.match(/^\((.*?)\)(.)/); - if (placeholderMatch === null) { - throw new Error("Invalid format specifier: '"+match[0]+"'"); - } - if (placeholderMatch.length < 3) { - throw new Error("Malformed format specifier"); - } - const placeholderName = placeholderMatch[1]; - const placeholderFormat = placeholderMatch[2]; - - if (placeholderFormat !== 's') { - throw new Error(`'${placeholderFormat}' used as format character: you probably meant 's'`); - } - - formatStrings.add(placeholderName); - } - - return formatStrings; -} - -function getTranslationsJs(file) { - const contents = fs.readFileSync(file, { encoding: 'utf8' }); - - const trs = new Set(); - - try { - const plugins = [ - // https://babeljs.io/docs/en/babel-parser#plugins - "classProperties", - "objectRestSpread", - "throwExpressions", - "exportDefaultFrom", - "decorators-legacy", - ]; - - if (file.endsWith(".js") || file.endsWith(".jsx")) { - // all JS is assumed to be flow or react - plugins.push("flow", "jsx"); - } else if (file.endsWith(".ts")) { - // TS can't use JSX unless it's a TSX file (otherwise angle casts fail) - plugins.push("typescript"); - } else if (file.endsWith(".tsx")) { - // When the file is a TSX file though, enable JSX parsing - plugins.push("typescript", "jsx"); - } - - const babelParsed = parser.parse(contents, { - allowImportExportEverywhere: true, - errorRecovery: true, - sourceFilename: file, - tokens: true, - plugins, - }); - traverse.default(babelParsed, { - enter: (p) => { - const node = p.node; - if (p.isCallExpression() && node.callee && TRANSLATIONS_FUNCS.includes(node.callee.name)) { - const tKey = getTKey(node.arguments[0]); - - // This happens whenever we call _t with non-literals (ie. whenever we've - // had to use a _td to compensate) so is expected. - if (tKey === null) return; - - // check the format string against the args - // We only check _t: _td has no args - if (node.callee.name === '_t') { - try { - const placeholders = getFormatStrings(tKey); - for (const placeholder of placeholders) { - if (node.arguments.length < 2) { - throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`); - } - const value = getObjectValue(node.arguments[1], placeholder); - if (value === null) { - throw new Error(`No value found for placeholder '${placeholder}'`); - } - } - - // Validate tag replacements - if (node.arguments.length > 2) { - const tagMap = node.arguments[2]; - for (const prop of tagMap.properties || []) { - if (prop.key.type === 'Literal') { - const tag = prop.key.value; - // RegExp same as in src/languageHandler.js - // eslint-disable-next-line - const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`); - if (!tKey.match(regexp)) { - throw new Error(`No match for ${regexp} in ${tKey}`); - } - } - } - } - } catch (e) { - console.log(); - console.error(`ERROR: ${file}:${node.loc.start.line} ${tKey}`); - console.error(e); - process.exit(1); - } - } - - let isPlural = false; - if (node.arguments.length > 1 && node.arguments[1].type === 'ObjectExpression') { - const countVal = getObjectValue(node.arguments[1], 'count'); - if (countVal) { - isPlural = true; - } - } - - if (isPlural) { - trs.add(tKey + "|other"); - const plurals = enPlurals[tKey]; - if (plurals) { - for (const pluralType of Object.keys(plurals)) { - trs.add(tKey + "|" + pluralType); - } - } - } else { - trs.add(tKey); - } - } - }, - }); - } catch (e) { - console.error(e); - process.exit(1); - } - - return trs; -} - -function getTranslationsOther(file) { - const contents = fs.readFileSync(file, { encoding: 'utf8' }); - - const trs = new Set(); - - // Taken from element-web src/components/structures/HomePage.js - const translationsRegex = /_t\(['"]([\s\S]*?)['"]\)/mg; - let matches; - while (matches = translationsRegex.exec(contents)) { - trs.add(matches[1]); - } - return trs; -} - -// gather en_EN plural strings from the input translations file: -// the en_EN strings are all in the source with the exception of -// pluralised strings, which we need to pull in from elsewhere. -const inputTranslationsRaw = JSON.parse(fs.readFileSync(INPUT_TRANSLATIONS_FILE, { encoding: 'utf8' })); -const enPlurals = {}; - -for (const key of Object.keys(inputTranslationsRaw)) { - const parts = key.split("|"); - if (parts.length > 1) { - const plurals = enPlurals[parts[0]] || {}; - plurals[parts[1]] = inputTranslationsRaw[key]; - enPlurals[parts[0]] = plurals; - } -} - -const translatables = new Set(); - -const walkOpts = { - listeners: { - names: function(root, nodeNamesArray) { - // Sort the names case insensitively and alphabetically to - // maintain some sense of order between the different strings. - nodeNamesArray.sort((a, b) => { - a = a.toLowerCase(); - b = b.toLowerCase(); - if (a > b) return 1; - if (a < b) return -1; - return 0; - }); - }, - file: function(root, fileStats, next) { - const fullPath = path.join(root, fileStats.name); - - let trs; - if (fileStats.name.endsWith('.js') || fileStats.name.endsWith('.ts') || fileStats.name.endsWith('.tsx')) { - trs = getTranslationsJs(fullPath); - } else if (fileStats.name.endsWith('.html')) { - trs = getTranslationsOther(fullPath); - } else { - return; - } - console.log(`${fullPath} (${trs.size} strings)`); - for (const tr of trs.values()) { - // Convert DOS line endings to unix - translatables.add(tr.replace(/\r\n/g, "\n")); - } - }, - }, -}; - -for (const path of SEARCH_PATHS) { - if (fs.existsSync(path)) { - walk.walkSync(path, walkOpts); - } -} - -const trObj = {}; -for (const tr of translatables) { - if (tr.includes("|")) { - if (inputTranslationsRaw[tr]) { - trObj[tr] = inputTranslationsRaw[tr]; - } else { - trObj[tr] = tr.split("|")[0]; - } - } else { - trObj[tr] = tr; - } -} - -fs.writeFileSync( - OUTPUT_FILE, - JSON.stringify(trObj, translatables.values(), 4) + "\n", -); - -console.log(); -console.log(`Wrote ${translatables.size} strings to ${OUTPUT_FILE}`); - diff --git a/yarn.lock b/yarn.lock index d8b4e5b..7fb03ee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -26,6 +26,13 @@ dependencies: "@babel/highlight" "^7.10.4" +"@babel/code-frame@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.13.tgz#dcfc826beef65e75c50e21d3837d7d95798dd658" + integrity sha512-HV1Cm0Q3ZrpCR93tkWOYiuYIgLxZXZFVG2VgK+MBWjUqZTundupbfx2aXarXuw5Ko5aMcjtJgbSs4vUGBS5v6g== + dependencies: + "@babel/highlight" "^7.12.13" + "@babel/generator@^7.10.5": version "7.10.5" resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.10.5.tgz#1b903554bc8c583ee8d25f1e8969732e6b829a69" @@ -35,6 +42,15 @@ jsesc "^2.5.1" source-map "^0.5.0" +"@babel/generator@^7.13.16": + version "7.13.16" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.13.16.tgz#0befc287031a201d84cdfc173b46b320ae472d14" + integrity sha512-grBBR75UnKOcUWMp8WoDxNsWCFl//XCK6HWTrBQKTr5SV9f5g0pNOjdyzi/DTBv12S9GnYPInIXQBTky7OXEMg== + dependencies: + "@babel/types" "^7.13.16" + jsesc "^2.5.1" + source-map "^0.5.0" + "@babel/helper-function-name@^7.10.4": version "7.10.4" resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.10.4.tgz#d2d3b20c59ad8c47112fa7d2a94bc09d5ef82f1a" @@ -44,6 +60,15 @@ "@babel/template" "^7.10.4" "@babel/types" "^7.10.4" +"@babel/helper-function-name@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.12.13.tgz#93ad656db3c3c2232559fd7b2c3dbdcbe0eb377a" + integrity sha512-TZvmPn0UOqmvi5G4vvw0qZTpVptGkB1GL61R6lKvrSdIxGm5Pky7Q3fpKiIkQCAtRCBUwB0PaThlx9vebCDSwA== + dependencies: + "@babel/helper-get-function-arity" "^7.12.13" + "@babel/template" "^7.12.13" + "@babel/types" "^7.12.13" + "@babel/helper-get-function-arity@^7.10.4": version "7.10.4" resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.4.tgz#98c1cbea0e2332f33f9a4661b8ce1505b2c19ba2" @@ -51,6 +76,13 @@ dependencies: "@babel/types" "^7.10.4" +"@babel/helper-get-function-arity@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.13.tgz#bc63451d403a3b3082b97e1d8b3fe5bd4091e583" + integrity sha512-DjEVzQNz5LICkzN0REdpD5prGoidvbdYk1BVgRUOINaWJP2t6avB27X1guXK1kXNrX0WMfsrm1A/ZBthYuIMQg== + dependencies: + "@babel/types" "^7.12.13" + "@babel/helper-split-export-declaration@^7.10.4": version "7.10.4" resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.10.4.tgz#2c70576eaa3b5609b24cb99db2888cc3fc4251d1" @@ -58,11 +90,23 @@ dependencies: "@babel/types" "^7.10.4" +"@babel/helper-split-export-declaration@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.13.tgz#e9430be00baf3e88b0e13e6f9d4eaf2136372b05" + integrity sha512-tCJDltF83htUtXx5NLcaDqRmknv652ZWCHyoTETf1CXYJdPC7nohZohjUgieXhv0hTJdRf2FjDueFehdNucpzg== + dependencies: + "@babel/types" "^7.12.13" + "@babel/helper-validator-identifier@^7.10.4": version "7.10.4" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz#a78c7a7251e01f616512d31b10adcf52ada5e0d2" integrity sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw== +"@babel/helper-validator-identifier@^7.12.11": + version "7.12.11" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz#c9a1f021917dcb5ccf0d4e453e399022981fc9ed" + integrity sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw== + "@babel/highlight@^7.0.0": version "7.5.0" resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.5.0.tgz#56d11312bd9248fa619591d02472be6e8cb32540" @@ -81,11 +125,25 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/highlight@^7.12.13": + version "7.13.10" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.13.10.tgz#a8b2a66148f5b27d666b15d81774347a731d52d1" + integrity sha512-5aPpe5XQPzflQrFwL1/QoeHkP2MsA4JCntcXHRhEsdsfPVkvPi2w7Qix4iV7t5S/oC9OodGrggd8aco1g3SZFg== + dependencies: + "@babel/helper-validator-identifier" "^7.12.11" + chalk "^2.0.0" + js-tokens "^4.0.0" + "@babel/parser@^7.10.4", "@babel/parser@^7.10.5", "@babel/parser@^7.7.0": version "7.10.5" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.5.tgz#e7c6bf5a7deff957cec9f04b551e2762909d826b" integrity sha512-wfryxy4bE1UivvQKSQDU4/X6dr+i8bctjUjj8Zyt3DQy7NtPizJXT8M52nqpNKL+nq2PW8lxk4ZqLj0fD4B4hQ== +"@babel/parser@^7.12.13", "@babel/parser@^7.13.16": + version "7.13.16" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.16.tgz#0f18179b0448e6939b1f3f5c4c355a3a9bcdfd37" + integrity sha512-6bAg36mCwuqLO0hbR+z7PHuqWiCeP7Dzg73OpQwsAB1Eb8HnGEz5xYBzCfbu+YjoaJsJs+qheDxVAuqbt3ILEw== + "@babel/runtime@^7.7.2": version "7.11.2" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.11.2.tgz#f549c13c754cc40b87644b9fa9f09a6a95fe0736" @@ -102,6 +160,29 @@ "@babel/parser" "^7.10.4" "@babel/types" "^7.10.4" +"@babel/template@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.12.13.tgz#530265be8a2589dbb37523844c5bcb55947fb327" + integrity sha512-/7xxiGA57xMo/P2GVvdEumr8ONhFOhfgq2ihK3h1e6THqzTAkHbkXgB0xI9yeTfIUoH3+oAeHhqm/I43OTbbjA== + dependencies: + "@babel/code-frame" "^7.12.13" + "@babel/parser" "^7.12.13" + "@babel/types" "^7.12.13" + +"@babel/traverse@^7.13.17": + version "7.13.17" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.13.17.tgz#c85415e0c7d50ac053d758baec98b28b2ecfeea3" + integrity sha512-BMnZn0R+X6ayqm3C3To7o1j7Q020gWdqdyP50KEoVqaCO2c/Im7sYZSmVgvefp8TTMQ+9CtwuBp0Z1CZ8V3Pvg== + dependencies: + "@babel/code-frame" "^7.12.13" + "@babel/generator" "^7.13.16" + "@babel/helper-function-name" "^7.12.13" + "@babel/helper-split-export-declaration" "^7.12.13" + "@babel/parser" "^7.13.16" + "@babel/types" "^7.13.17" + debug "^4.1.0" + globals "^11.1.0" + "@babel/traverse@^7.7.0": version "7.10.5" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.10.5.tgz#77ce464f5b258be265af618d8fddf0536f20b564" @@ -126,6 +207,14 @@ lodash "^4.17.19" to-fast-properties "^2.0.0" +"@babel/types@^7.12.13", "@babel/types@^7.13.16", "@babel/types@^7.13.17": + version "7.13.17" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.13.17.tgz#48010a115c9fba7588b4437dd68c9469012b38b4" + integrity sha512-RawydLgxbOPDlTLJNtoIypwdmAy//uQIzlKt2+iBiJaRlVuI6QLUxVAyWGNfOzp8Yu4L4lLIacoCyTNtpb4wiA== + dependencies: + "@babel/helper-validator-identifier" "^7.12.11" + to-fast-properties "^2.0.0" + "@develar/schema-utils@~2.6.5": version "2.6.5" resolved "https://registry.yarnpkg.com/@develar/schema-utils/-/schema-utils-2.6.5.tgz#3ece22c5838402419a6e0425f85742b961d9b6c6" @@ -2634,6 +2723,11 @@ flush-write-stream@^1.0.0: inherits "^2.0.3" readable-stream "^2.3.6" +foreachasync@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/foreachasync/-/foreachasync-3.0.0.tgz#5502987dc8714be3392097f32e0071c9dee07cf6" + integrity sha1-VQKYfchxS+M5IJfzLgBxyd7gfPY= + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -4003,6 +4097,14 @@ make-fetch-happen@^5.0.0: socks-proxy-agent "^4.0.0" ssri "^6.0.0" +"matrix-web-i18n@github:matrix-org/matrix-web-i18n": + version "1.0.1" + resolved "https://codeload.github.com/matrix-org/matrix-web-i18n/tar.gz/6df80d0197711e9f9733e9e3e2ef1d38a2ab8949" + dependencies: + "@babel/parser" "^7.13.16" + "@babel/traverse" "^7.13.17" + walk "^2.3.14" + meant@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/meant/-/meant-1.0.3.tgz#67769af9de1d158773e928ae82c456114903554c" @@ -6440,6 +6542,13 @@ version-range@^1.0.0: dependencies: version-compare "^1.0.0" +walk@^2.3.14: + version "2.3.14" + resolved "https://registry.yarnpkg.com/walk/-/walk-2.3.14.tgz#60ec8631cfd23276ae1e7363ce11d626452e1ef3" + integrity sha512-5skcWAUmySj6hkBdH6B6+3ddMjVQYH5Qy9QGbPmN8kVmLteXk+yVXg+yfk1nbX30EYakahLrr8iPcCxJQSCBeg== + dependencies: + foreachasync "^3.0.0" + wcwidth@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/wcwidth/-/wcwidth-1.0.1.tgz#f0b0dcf915bc5ff1528afadb2c0e17b532da2fe8"