Sync missing translation keys in app/locales/*.json files to a Google Sheet

.gitignore

@@ -6,6 +6,8 @@ yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
 
+*-credentials.json
+
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 

ops/lost-in-translation.js

@@ -0,0 +1,193 @@
+/* eslint-disable no-await-in-loop, global-require */
+const { parse } = require('path');
+const { readdirSync } = require('fs');
+const { spawnSync } = require('child_process');
+const { GoogleSpreadsheet } = require('google-spreadsheet');
+
+/**
+ * This script finds all the english translation keys across
+ * all commits (including branches and pull requests) for each
+ * locale.json in app/locales. Afterwards it looks at each locale
+ * separately to see if it's missing any one the english keys.
+ *
+ * Important: You need to run it from the root directory.
+ *
+ * Usage:
+ *   node ops/lost-in-translation.js
+ */
+
+/**
+ * Calls git cli with arguments and returns stdout.
+ */
+function git(...args) {
+  const { stdout } = spawnSync('git', args, { encoding: 'utf-8' });
+  return stdout;
+}
+
+/**
+ * Returns the commit hashes for a file path across all local branches.
+ */
+function findCommitHashesForFile(filePath) {
+  const output = git('log', '--pretty=format:"%h"', '--all', '--', filePath);
+  const hashes = output.split('\n').map(line => line.replace(/"/g, ''));
+  return hashes;
+}
+
+/**
+ * Returns the JSON representation of the contents of a file at certain commit hash.
+ */
+function retrieveJSONForFileAtCommitHash(filePath, commitHash) {
+  const contentsAtCommitHash = git('show', `${commitHash}:${filePath}`);
+  try {
+    return JSON.parse(contentsAtCommitHash);
+  } catch (error) {
+    // Sometimes the file is not in a proper JSON format. Simply return nothing in that case.
+    return {};
+  }
+}
+
+/**
+ * Normalizes translation key like we do in app/server.ts.
+ */
+function normalizeTranslationKey(translationKey) {
+  return translationKey.replace(/[\s\n\t]+/g, ' ').trim();
+}
+
+/**
+ * Find all the locales (ie. en-IN, no, se) in the provided directory.
+ */
+function retrieveAllLocales(directoryPath) {
+  const filenames = readdirSync(directoryPath);
+  const locales = filenames.map(filename => parse(filename).name);
+  return locales;
+}
+
+/**
+ * Add rows to a Google Spreadsheet that are not already added.
+ */
+async function addUniqueRowsToGoogleSheet(sheet, rowsToAdd) {
+  const alreadyAddedRows = await sheet.getRows({ limit: 1000 });
+  const uniqueRowsToAdd = rowsToAdd.filter(
+    rowToAdd =>
+      !alreadyAddedRows.find(
+        alreadyAddedRow => alreadyAddedRow.key === rowToAdd.key
+      )
+  );
+  await sheet.addRows(uniqueRowsToAdd);
+  return uniqueRowsToAdd;
+}
+
+/**
+ * Retrieve all the sheets in a Google Spreadsheet document.
+ */
+async function retrieveSheetsInDocument(doc) {
+  const sheets = [];
+  for (let sheetIndex = 0; sheetIndex < doc.sheetCount; sheetIndex += 1) {
+    const sheet = await doc.sheetsByIndex[sheetIndex];
+    sheets.push(sheet);
+  }
+  return sheets;
+}
+
+/**
+ * Step 1: Find all the (english) translation keys across all branches and PRs.
+ *
+ * If a Dutch developer has made a feature in a branch, we expect that him/her added a key
+ * to one or many locale files (usually the english locale (en.json) or to their own locale
+ * file (nl.json), or both).
+ *
+ * But they probably haven't added translations to all the other locales, because they don't
+ * speak the other languages. And this is the problem, because now we need find people to help
+ * translate the added keys to all the other locales.
+ *
+ * So what we do here is simply to find *all* the english translation keys across the entire
+ * project. That means looking at all the english translations keys in all the locale files
+ * since the start of the project across all branches and PRs. We throw them into a set that
+ * we use in the next step.
+ */
+const allLocales = retrieveAllLocales('app/locales/');
+const allEnglishTranslationKeys = new Set([]);
+for (const locale of allLocales) {
+  const filePath = `app/locales/${locale}.json`;
+  for (const commitHash of findCommitHashesForFile(filePath)) {
+    const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash);
+    for (const translationKey of Object.keys(translation)) {
+      allEnglishTranslationKeys.add(normalizeTranslationKey(translationKey));
+    }
+  }
+}
+
+/**
+ * Step 2: For each locale file check if there are any missing english translation keys
+ *         across all branches and PRs and all commits/changes.
+ *
+ * If there's a missing english translation key, we know that we're most likely missing
+ * a translation for that locale. So what we need to do is to translate it, or get help
+ * to translate it.
+ *
+ * We throw it into a dictionary where the key is the locale and the value is a set of
+ * missing translations from english to that locale.
+ */
+const translationKeysByLocale = {};
+for (const locale of allLocales) {
+  const filePath = `app/locales/${locale}.json`;
+  for (const commitHash of findCommitHashesForFile(filePath)) {
+    const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash);
+    const translationKeys = Object.keys(translation).map(key =>
+      normalizeTranslationKey(key)
+    );
+    if (translationKeysByLocale[locale] === undefined) {
+      translationKeysByLocale[locale] = new Set([]);
+    }
+    translationKeysByLocale[locale] = new Set([
+      ...translationKeysByLocale[locale],
+      ...translationKeys
+    ]);
+  }
+}
+
+/**
+ * Step 3: Add rows of missing translations to Google Spreadsheet.
+ *
+ * https://docs.google.com/spreadsheets/d/1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw
+ */
+(async () => {
+  const doc = new GoogleSpreadsheet(
+    '1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw'
+  );
+  await doc.useServiceAccountAuth(
+    require('./coronastatus-translation-486cef09736e-credentials.json')
+  );
+  await doc.loadInfo();
+  const sheets = await retrieveSheetsInDocument(doc);
+
+  for (const locale of allLocales) {
+    // Create sheet for this locale if it doesn't already exist.
+    let matchingSheet = sheets.find(sheet => sheet.title === locale);
+    if (!matchingSheet) {
+      matchingSheet = await doc.addSheet({
+        title: locale,
+        headerValues: ['key', 'translation']
+      });
+    }
+
+    // Find rows that don't already exist by looking at the key column.
+    const rows = [];
+    for (const englishTranslationKey of allEnglishTranslationKeys) {
+      if (!translationKeysByLocale[locale].has(englishTranslationKey)) {
+        const row = { key: englishTranslationKey, translation: '' };
+        rows.push(row);
+      }
+    }
+
+    // Add and print out how many rows we added.
+    const addedRows = await addUniqueRowsToGoogleSheet(matchingSheet, rows);
+    console.log(
+      `Added ${addedRows.length} of ${rows.length} missing translations to the ${locale} sheet.`
+    );
+
+    // Avoid getting rate limited by Google's API (max writes per 100 seconds).
+    console.log('Waiting before processing the next sheet.');
+    await new Promise(resolve => setTimeout(resolve, 20 * 1000));
+  }
+})();

package.json

@@ -24,6 +24,7 @@
     "ejs": "^3.0.1",
     "express": "^4.17.1",
     "express-rate-limit": "^5.1.1",
+    "google-spreadsheet": "^3.0.10",
     "i18n": "^0.8.6",
     "node-fetch": "^2.6.0",
     "pg": "^7.18.2",