Merge pull request #10 from matrix-org/t3chguy/localazy-2

package.json

@@ -13,8 +13,8 @@
     ],
     "bin": {
         "matrix-gen-i18n": "scripts/gen-i18n.js",
-        "matrix-prune-i18n": "scripts/prune-i18n.js",
-        "matrix-compare-i18n-files": "scripts/compare-file.js"
+        "matrix-compare-i18n-files": "scripts/compare-file.js",
+        "matrix-i18n-rekey": "scripts/rekey.js"
     },
     "scripts": {
         "build:ts": "tsc",
@@ -37,11 +37,13 @@
         "@babel/parser": "^7.18.5",
         "@babel/traverse": "^7.18.5",
         "lodash": "^4.17.21",
+        "minimist": "^1.2.8",
         "walk": "^2.3.15"
     },
     "devDependencies": {
         "@types/babel__traverse": "^7.17.1",
         "@types/lodash": "^4.14.197",
+        "@types/minimist": "^1.2.2",
         "@types/node": "^18.0.0",
         "@types/walk": "^2.3.1",
         "typescript": "^4.7.4"

scripts/common.ts

@@ -0,0 +1,67 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import fs from "fs";
+import { isBinaryExpression, isStringLiteral, isTemplateLiteral, Node } from "@babel/types";
+
+export const NESTING_KEY = process.env["NESTING_KEY"] || "|";
+export const INPUT_FILE = process.env["INPUT_FILE"] || 'src/i18n/strings/en_EN.json';
+export const OUTPUT_FILE = process.env["OUTPUT_FILE"] || 'src/i18n/strings/en_EN.json';
+
+export type Translation = string | {
+    one?: string;
+    other: string;
+};
+
+export interface Translations {
+    [key: string]: Translation | Translations;
+}
+
+export function getPath(key: string): string[] {
+    return key.split(NESTING_KEY);
+}
+
+export function getKeys(translations: Translations | Translation, path = ""): string[] {
+    // base case
+    if (typeof translations === "string" || "other" in translations) {
+        return [path];
+    }
+
+    if (path) path += NESTING_KEY;
+    return Object.keys(translations).flatMap(key => getKeys(translations[key], path + key));
+}
+
+export function getTranslations(file = INPUT_FILE): Readonly<Translations> {
+     return JSON.parse(fs.readFileSync(file, { encoding: 'utf8' }));
+}
+
+export function putTranslations(translations: Translations, file = OUTPUT_FILE): void {
+    fs.writeFileSync(
+        file,
+        JSON.stringify(translations, null, 4) + "\n"
+    );
+}
+
+export function getTKey(arg: Node): string | null {
+    if (isStringLiteral(arg)) {
+        return arg.value;
+    } else if (isBinaryExpression(arg) && arg.operator === '+') {
+        return getTKey(arg.left)! + getTKey(arg.right)!;
+    } else if (isTemplateLiteral(arg)) {
+        return arg.quasis.map(q => q.value.raw).join('');
+    }
+    return null;
+}

scripts/find-usage.ts

@@ -0,0 +1,183 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Finds code usages of a specific i18n key or count of usages per key if no key specified
+ */
+
+import * as path from "path";
+import * as fs from "fs";
+import { WalkOptions, walkSync } from "walk";
+import * as parser from "@babel/parser";
+import traverse from "@babel/traverse";
+import {
+    isIdentifier,
+    isCallExpression,
+    isNewExpression,
+} from "@babel/types";
+import { ParserPlugin } from "@babel/parser";
+import _ from "lodash";
+import { getTKey } from "./common";
+
+// Find the package.json for the project we're running against
+const projectPackageJsonPath = path.join(process.cwd(), 'package.json');
+const projectPackageJson = require(projectPackageJsonPath);
+
+const TRANSLATIONS_FUNCS = ['_t', '_td', '_tDom']
+    // Add some addition translation functions to look out that are specified
+    // per project in package.json under the
+    // "matrix_i18n_extra_translation_funcs" key
+    .concat(projectPackageJson.matrix_i18n_extra_translation_funcs || []);
+
+// NB. The sync version of walk is broken for single files,
+// so we walk all of res rather than just res/home.html.
+// https://git.daplie.com/Daplie/node-walk/merge_requests/1 fixes it,
+// or if we get bored waiting for it to be merged, we could switch
+// to a project that's actively maintained.
+const SEARCH_PATHS = ['src', 'res'];
+
+function getTranslationsJs(file: string): Map<string, string[]> {
+    const contents = fs.readFileSync(file, { encoding: 'utf8' });
+
+    const trs = new Map<string, string[]>();
+
+    try {
+        const plugins: ParserPlugin[] = [
+            // https://babeljs.io/docs/en/babel-parser#plugins
+            "classProperties",
+            "objectRestSpread",
+            "throwExpressions",
+            "exportDefaultFrom",
+            "decorators-legacy",
+        ];
+
+        if (file.endsWith(".js") || file.endsWith(".jsx")) {
+            // All JS is assumed to be React
+            plugins.push("jsx");
+        } else if (file.endsWith(".ts")) {
+            // TS can't use JSX unless it's a TSX file (otherwise angle casts fail)
+            plugins.push("typescript");
+        } else if (file.endsWith(".tsx")) {
+            // When the file is a TSX file though, enable JSX parsing
+            plugins.push("typescript", "jsx");
+        }
+
+        const babelParsed = parser.parse(contents, {
+            allowImportExportEverywhere: true,
+            errorRecovery: true,
+            sourceFilename: file,
+            tokens: true,
+            plugins,
+        });
+        traverse(babelParsed, {
+            enter: (p) => {
+                if (
+                    (isNewExpression(p.node) || isCallExpression(p.node)) &&
+                    isIdentifier(p.node.callee) &&
+                    TRANSLATIONS_FUNCS.includes(p.node.callee.name)
+                ) {
+                    const tKey = getTKey(p.node.arguments[0]);
+
+                    // This happens whenever we call _t with non-literals (ie. whenever we've
+                    // had to use a _td to compensate) so is expected.
+                    if (tKey === null) return;
+
+                    if (trs.has(tKey)) {
+                        trs.get(tKey)!.push(file);
+                    } else {
+                        trs.set(tKey, [file]);
+                    }
+                }
+            },
+        });
+    } catch (e) {
+        console.error(e);
+        process.exit(1);
+    }
+
+    return trs;
+}
+
+function getTranslationsOther(file: string): Map<string, string[]> {
+    const contents = fs.readFileSync(file, { encoding: 'utf8' });
+
+    const trs = new Map<string, string[]>();
+
+    // Taken from element-web src/components/structures/HomePage.js
+    const translationsRegex = /_t\(['"]([\s\S]*?)['"]\)/mg;
+    let matches: RegExpExecArray | null;
+    while (matches = translationsRegex.exec(contents)) {
+        if (trs.has(matches[1])) {
+            trs.get(matches[1])!.push(file);
+        } else {
+            trs.set(matches[1], [file]);
+        }
+    }
+    return trs;
+}
+
+const keyUsages = new Map<string, string[]>();
+
+const walkOpts: WalkOptions = {
+    listeners: {
+        names: function(root, nodeNamesArray) {
+            // Sort the names case insensitively and alphabetically to
+            // maintain some sense of order between the different strings.
+            nodeNamesArray.sort((a, b) => {
+                a = a.toLowerCase();
+                b = b.toLowerCase();
+                if (a > b) return 1;
+                if (a < b) return -1;
+                return 0;
+            });
+        },
+        file: function(root, fileStats, next) {
+            const fullPath = path.join(root, fileStats.name);
+
+            let trs: Map<string, string[]>;
+            if (fileStats.name.endsWith('.js') || fileStats.name.endsWith('.ts') || fileStats.name.endsWith('.tsx')) {
+                trs = getTranslationsJs(fullPath);
+            } else if (fileStats.name.endsWith('.html')) {
+                trs = getTranslationsOther(fullPath);
+            } else {
+                return;
+            }
+
+            for (const key of trs.keys()) {
+                if (keyUsages.has(key)) {
+                    keyUsages.get(key)!.push(...trs.get(key)!);
+                } else {
+                    keyUsages.set(key, trs.get(key)!);
+                }
+            }
+        },
+    }
+};
+
+for (const path of SEARCH_PATHS) {
+    if (fs.existsSync(path)) {
+        walkSync(path, walkOpts);
+    }
+}
+
+const key = process.argv[2];
+
+if (key) {
+    console.log(`Consumers of "${key}":`, keyUsages.get(key));
+} else {
+    const sorted = _.sortBy([...keyUsages.keys()], k => -keyUsages.get(k)!.length);
+    console.table(Object.fromEntries(sorted.map(key => [key.substring(0, 120), keyUsages.get(key)!.length])));
+}