[6.x] Implement a build tool for locale files integration (#19826)

scripts/i18n_integrate.js

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+require('../src/setup_node_env');
+require('../src/dev/run_i18n_integrate');

src/dev/i18n/__fixtures__/integrate_locale_files/fr.json

@@ -0,0 +1,63 @@
+{
+  "formats": {
+    "number": {
+      "currency": {
+        "style": "currency"
+      },
+      "percent": {
+        "style": "percent"
+      }
+    },
+    "date": {
+      "short": {
+        "month": "numeric",
+        "day": "numeric",
+        "year": "2-digit"
+      },
+      "medium": {
+        "month": "short",
+        "day": "numeric",
+        "year": "numeric"
+      },
+      "long": {
+        "month": "long",
+        "day": "numeric",
+        "year": "numeric"
+      },
+      "full": {
+        "weekday": "long",
+        "month": "long",
+        "day": "numeric",
+        "year": "numeric"
+      }
+    },
+    "time": {
+      "short": {
+        "hour": "numeric",
+        "minute": "numeric"
+      },
+      "medium": {
+        "hour": "numeric",
+        "minute": "numeric",
+        "second": "numeric"
+      },
+      "long": {
+        "hour": "numeric",
+        "minute": "numeric",
+        "second": "numeric",
+        "timeZoneName": "short"
+      },
+      "full": {
+        "hour": "numeric",
+        "minute": "numeric",
+        "second": "numeric",
+        "timeZoneName": "short"
+      }
+    }
+  },
+  "messages": {
+    "plugin-1.message-id-1": "Translated text 1",
+    "plugin-1.message-id-2": "Translated text 2",
+    "plugin-2.message-id": "Translated text"
+  }
+}

src/dev/i18n/__snapshots__/integrate_locale_files.test.js.snap

@@ -0,0 +1,163 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`dev/i18n/integrate_locale_files integrateLocaleFiles splits locale file by plugins and writes them into the right folders 1`] = `
+Array [
+  "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_1/translations/fr.json",
+  "{
+  \\"formats\\": {
+    \\"number\\": {
+      \\"currency\\": {
+        \\"style\\": \\"currency\\"
+      },
+      \\"percent\\": {
+        \\"style\\": \\"percent\\"
+      }
+    },
+    \\"date\\": {
+      \\"short\\": {
+        \\"month\\": \\"numeric\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"2-digit\\"
+      },
+      \\"medium\\": {
+        \\"month\\": \\"short\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      },
+      \\"long\\": {
+        \\"month\\": \\"long\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      },
+      \\"full\\": {
+        \\"weekday\\": \\"long\\",
+        \\"month\\": \\"long\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      }
+    },
+    \\"time\\": {
+      \\"short\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\"
+      },
+      \\"medium\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\"
+      },
+      \\"long\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\",
+        \\"timeZoneName\\": \\"short\\"
+      },
+      \\"full\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\",
+        \\"timeZoneName\\": \\"short\\"
+      }
+    }
+  },
+  \\"messages\\": {
+    \\"plugin-1.message-id-1\\": \\"Translated text 1\\",
+    \\"plugin-1.message-id-2\\": \\"Translated text 2\\"
+  }
+}",
+]
+`;
+
+exports[`dev/i18n/integrate_locale_files integrateLocaleFiles splits locale file by plugins and writes them into the right folders 2`] = `
+Array [
+  "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_2/translations/fr.json",
+  "{
+  \\"formats\\": {
+    \\"number\\": {
+      \\"currency\\": {
+        \\"style\\": \\"currency\\"
+      },
+      \\"percent\\": {
+        \\"style\\": \\"percent\\"
+      }
+    },
+    \\"date\\": {
+      \\"short\\": {
+        \\"month\\": \\"numeric\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"2-digit\\"
+      },
+      \\"medium\\": {
+        \\"month\\": \\"short\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      },
+      \\"long\\": {
+        \\"month\\": \\"long\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      },
+      \\"full\\": {
+        \\"weekday\\": \\"long\\",
+        \\"month\\": \\"long\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      }
+    },
+    \\"time\\": {
+      \\"short\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\"
+      },
+      \\"medium\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\"
+      },
+      \\"long\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\",
+        \\"timeZoneName\\": \\"short\\"
+      },
+      \\"full\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\",
+        \\"timeZoneName\\": \\"short\\"
+      }
+    }
+  },
+  \\"messages\\": {
+    \\"plugin-2.message-id\\": \\"Translated text\\"
+  }
+}",
+]
+`;
+
+exports[`dev/i18n/integrate_locale_files integrateLocaleFiles splits locale file by plugins and writes them into the right folders 3`] = `
+Array [
+  "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_1/translations",
+  "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_2/translations",
+]
+`;
+
+exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 1`] = `
+"
+Missing translations:
+plugin-1.message-id-2"
+`;
+
+exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 2`] = `
+"
+Unused translations:
+plugin-1.message-id-3"
+`;
+
+exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 3`] = `
+"
+Unused translations:
+plugin-2.message
+Missing translations:
+plugin-2.message-id"
+`;

src/dev/i18n/__snapshots__/utils.test.js.snap

@@ -9,6 +9,8 @@ exports[`i18n utils should create verbose parser error message 1`] = `
 "
 `;
 
+exports[`i18n utils should normalizePath 1`] = `"src/dev/i18n"`;
+
 exports[`i18n utils should not escape linebreaks 1`] = `
 "Text
  with

src/dev/i18n/extract_default_translations.js

@@ -26,6 +26,7 @@ import {
   extractHandlebarsMessages,
 } from './extractors';
 import { globAsync, readFileAsync, normalizePath } from './utils';
+
 import { createFailError, isFailError } from '../run';
 
 function addMessageToMap(targetMap, key, value, reporter) {
@@ -147,3 +148,13 @@ export async function extractMessagesFromPathToMap(inputPath, targetMap, config,
     })
   );
 }
+
+export async function getDefaultMessagesMap(inputPaths, config, reporter) {
+  const defaultMessagesMap = new Map();
+
+  for (const inputPath of filterPaths(inputPaths, config.paths)) {
+    await extractMessagesFromPathToMap(inputPath, defaultMessagesMap, config, reporter);
+  }
+
+  return defaultMessagesMap;
+}

src/dev/i18n/integrate_locale_files.js

@@ -0,0 +1,112 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import path from 'path';
+
+import {
+  difference,
+  readFileAsync,
+  writeFileAsync,
+  accessAsync,
+  makeDirAsync,
+  normalizePath,
+  ErrorReporter,
+} from './utils';
+import { paths, exclude } from '../../../.i18nrc.json';
+import { getDefaultMessagesMap } from './extract_default_translations';
+import { createFailError } from '../run';
+import { serializeToJson } from './serializers/json';
+
+export function verifyMessages(localizedMessagesMap, defaultMessagesMap) {
+  let errorMessage = '';
+
+  const defaultMessagesIds = [...defaultMessagesMap.keys()];
+  const localizedMessagesIds = [...localizedMessagesMap.keys()];
+
+  const unusedTranslations = difference(localizedMessagesIds, defaultMessagesIds);
+  if (unusedTranslations.length > 0) {
+    errorMessage += `\nUnused translations:\n${unusedTranslations.join(', ')}`;
+  }
+
+  const missingTranslations = difference(defaultMessagesIds, localizedMessagesIds);
+  if (missingTranslations.length > 0) {
+    errorMessage += `\nMissing translations:\n${missingTranslations.join(', ')}`;
+  }
+
+  if (errorMessage) {
+    throw createFailError(errorMessage);
+  }
+}
+
+function groupMessagesByNamespace(localizedMessagesMap) {
+  const localizedMessagesByNamespace = new Map();
+  const knownNamespaces = Object.keys(paths);
+
+  for (const [messageId, messageValue] of localizedMessagesMap) {
+    const namespace = knownNamespaces.find(key => messageId.startsWith(`${key}.`));
+
+    if (!namespace) {
+      throw createFailError(`Unknown namespace in id ${messageId}.`);
+    }
+
+    if (!localizedMessagesByNamespace.has(namespace)) {
+      localizedMessagesByNamespace.set(namespace, []);
+    }
+
+    localizedMessagesByNamespace
+      .get(namespace)
+      .push([messageId, { message: messageValue.text || messageValue }]);
+  }
+
+  return localizedMessagesByNamespace;
+}
+
+async function writeMessages(localizedMessagesByNamespace, fileName, formats, log) {
+  for (const [namespace, messages] of localizedMessagesByNamespace) {
+    const destPath = path.resolve(paths[namespace], 'translations');
+
+    try {
+      await accessAsync(destPath);
+    } catch (_) {
+      await makeDirAsync(destPath);
+    }
+
+    const writePath = path.resolve(destPath, fileName);
+    await writeFileAsync(writePath, serializeToJson(messages, formats));
+    log.success(`Translations have been integrated to ${normalizePath(writePath)}`);
+  }
+}
+
+export async function integrateLocaleFiles(filePath, log) {
+  const reporter = new ErrorReporter();
+  const defaultMessagesMap = await getDefaultMessagesMap(['.'], { paths, exclude }, reporter);
+  const localizedMessages = JSON.parse((await readFileAsync(filePath)).toString());
+
+  if (!localizedMessages.formats) {
+    throw createFailError(`Locale file should contain formats object.`);
+  }
+
+  const localizedMessagesMap = new Map(Object.entries(localizedMessages.messages));
+  verifyMessages(localizedMessagesMap, defaultMessagesMap);
+
+  // use basename of filePath to write the same locale name as the source file has
+  const fileName = path.basename(filePath);
+  const localizedMessagesByNamespace = groupMessagesByNamespace(localizedMessagesMap);
+  await writeMessages(localizedMessagesByNamespace, fileName, localizedMessages.formats, log);
+}