diff --git a/scripts/i18n_integrate.js b/scripts/i18n_integrate.js new file mode 100644 index 0000000000000..9fbdf424682b0 --- /dev/null +++ b/scripts/i18n_integrate.js @@ -0,0 +1,21 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +require('../src/setup_node_env'); +require('../src/dev/run_i18n_integrate'); diff --git a/src/dev/i18n/__fixtures__/integrate_locale_files/fr.json b/src/dev/i18n/__fixtures__/integrate_locale_files/fr.json new file mode 100644 index 0000000000000..72312c447961e --- /dev/null +++ b/src/dev/i18n/__fixtures__/integrate_locale_files/fr.json @@ -0,0 +1,63 @@ +{ + "formats": { + "number": { + "currency": { + "style": "currency" + }, + "percent": { + "style": "percent" + } + }, + "date": { + "short": { + "month": "numeric", + "day": "numeric", + "year": "2-digit" + }, + "medium": { + "month": "short", + "day": "numeric", + "year": "numeric" + }, + "long": { + "month": "long", + "day": "numeric", + "year": "numeric" + }, + "full": { + "weekday": "long", + "month": "long", + "day": "numeric", + "year": "numeric" + } + }, + "time": { + "short": { + "hour": "numeric", + "minute": "numeric" + }, + "medium": { + "hour": "numeric", + "minute": "numeric", + "second": "numeric" + }, + "long": { + "hour": "numeric", + "minute": "numeric", + "second": "numeric", + "timeZoneName": "short" + }, + "full": { + "hour": "numeric", + "minute": "numeric", + "second": "numeric", + "timeZoneName": "short" + } + } + }, + "messages": { + "plugin-1.message-id-1": "Translated text 1", + "plugin-1.message-id-2": "Translated text 2", + "plugin-2.message-id": "Translated text" + } +} diff --git a/src/dev/i18n/__snapshots__/integrate_locale_files.test.js.snap b/src/dev/i18n/__snapshots__/integrate_locale_files.test.js.snap new file mode 100644 index 0000000000000..f455387afe1e4 --- /dev/null +++ b/src/dev/i18n/__snapshots__/integrate_locale_files.test.js.snap @@ -0,0 +1,163 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`dev/i18n/integrate_locale_files integrateLocaleFiles splits locale file by plugins and writes them into the right folders 1`] = ` +Array [ + "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_1/translations/fr.json", + "{ + \\"formats\\": { + \\"number\\": { + \\"currency\\": { + \\"style\\": \\"currency\\" + }, + \\"percent\\": { + \\"style\\": \\"percent\\" + } + }, + \\"date\\": { + \\"short\\": { + \\"month\\": \\"numeric\\", + \\"day\\": \\"numeric\\", + \\"year\\": \\"2-digit\\" + }, + \\"medium\\": { + \\"month\\": \\"short\\", + \\"day\\": \\"numeric\\", + \\"year\\": \\"numeric\\" + }, + \\"long\\": { + \\"month\\": \\"long\\", + \\"day\\": \\"numeric\\", + \\"year\\": \\"numeric\\" + }, + \\"full\\": { + \\"weekday\\": \\"long\\", + \\"month\\": \\"long\\", + \\"day\\": \\"numeric\\", + \\"year\\": \\"numeric\\" + } + }, + \\"time\\": { + \\"short\\": { + \\"hour\\": \\"numeric\\", + \\"minute\\": \\"numeric\\" + }, + \\"medium\\": { + \\"hour\\": \\"numeric\\", + \\"minute\\": \\"numeric\\", + \\"second\\": \\"numeric\\" + }, + \\"long\\": { + \\"hour\\": \\"numeric\\", + \\"minute\\": \\"numeric\\", + \\"second\\": \\"numeric\\", + \\"timeZoneName\\": \\"short\\" + }, + \\"full\\": { + \\"hour\\": \\"numeric\\", + \\"minute\\": \\"numeric\\", + \\"second\\": \\"numeric\\", + \\"timeZoneName\\": \\"short\\" + } + } + }, + \\"messages\\": { + \\"plugin-1.message-id-1\\": \\"Translated text 1\\", + \\"plugin-1.message-id-2\\": \\"Translated text 2\\" + } +}", +] +`; + +exports[`dev/i18n/integrate_locale_files integrateLocaleFiles splits locale file by plugins and writes them into the right folders 2`] = ` +Array [ + "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_2/translations/fr.json", + "{ + \\"formats\\": { + \\"number\\": { + \\"currency\\": { + \\"style\\": \\"currency\\" + }, + \\"percent\\": { + \\"style\\": \\"percent\\" + } + }, + \\"date\\": { + \\"short\\": { + \\"month\\": \\"numeric\\", + \\"day\\": \\"numeric\\", + \\"year\\": \\"2-digit\\" + }, + \\"medium\\": { + \\"month\\": \\"short\\", + \\"day\\": \\"numeric\\", + \\"year\\": \\"numeric\\" + }, + \\"long\\": { + \\"month\\": \\"long\\", + \\"day\\": \\"numeric\\", + \\"year\\": \\"numeric\\" + }, + \\"full\\": { + \\"weekday\\": \\"long\\", + \\"month\\": \\"long\\", + \\"day\\": \\"numeric\\", + \\"year\\": \\"numeric\\" + } + }, + \\"time\\": { + \\"short\\": { + \\"hour\\": \\"numeric\\", + \\"minute\\": \\"numeric\\" + }, + \\"medium\\": { + \\"hour\\": \\"numeric\\", + \\"minute\\": \\"numeric\\", + \\"second\\": \\"numeric\\" + }, + \\"long\\": { + \\"hour\\": \\"numeric\\", + \\"minute\\": \\"numeric\\", + \\"second\\": \\"numeric\\", + \\"timeZoneName\\": \\"short\\" + }, + \\"full\\": { + \\"hour\\": \\"numeric\\", + \\"minute\\": \\"numeric\\", + \\"second\\": \\"numeric\\", + \\"timeZoneName\\": \\"short\\" + } + } + }, + \\"messages\\": { + \\"plugin-2.message-id\\": \\"Translated text\\" + } +}", +] +`; + +exports[`dev/i18n/integrate_locale_files integrateLocaleFiles splits locale file by plugins and writes them into the right folders 3`] = ` +Array [ + "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_1/translations", + "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_2/translations", +] +`; + +exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 1`] = ` +" +Missing translations: +plugin-1.message-id-2" +`; + +exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 2`] = ` +" +Unused translations: +plugin-1.message-id-3" +`; + +exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 3`] = ` +" +Unused translations: +plugin-2.message +Missing translations: +plugin-2.message-id" +`; diff --git a/src/dev/i18n/__snapshots__/utils.test.js.snap b/src/dev/i18n/__snapshots__/utils.test.js.snap index 7118c1e772fd8..9b7a66d8353f2 100644 --- a/src/dev/i18n/__snapshots__/utils.test.js.snap +++ b/src/dev/i18n/__snapshots__/utils.test.js.snap @@ -9,6 +9,8 @@ exports[`i18n utils should create verbose parser error message 1`] = ` " `; +exports[`i18n utils should normalizePath 1`] = `"src/dev/i18n"`; + exports[`i18n utils should not escape linebreaks 1`] = ` "Text with diff --git a/src/dev/i18n/extract_default_translations.js b/src/dev/i18n/extract_default_translations.js index d3bb156ea5b48..b0e3d60a6d7c5 100644 --- a/src/dev/i18n/extract_default_translations.js +++ b/src/dev/i18n/extract_default_translations.js @@ -26,6 +26,7 @@ import { extractHandlebarsMessages, } from './extractors'; import { globAsync, readFileAsync, normalizePath } from './utils'; + import { createFailError, isFailError } from '../run'; function addMessageToMap(targetMap, key, value, reporter) { @@ -147,3 +148,13 @@ export async function extractMessagesFromPathToMap(inputPath, targetMap, config, }) ); } + +export async function getDefaultMessagesMap(inputPaths, config, reporter) { + const defaultMessagesMap = new Map(); + + for (const inputPath of filterPaths(inputPaths, config.paths)) { + await extractMessagesFromPathToMap(inputPath, defaultMessagesMap, config, reporter); + } + + return defaultMessagesMap; +} diff --git a/src/dev/i18n/integrate_locale_files.js b/src/dev/i18n/integrate_locale_files.js new file mode 100644 index 0000000000000..fa06f6399f460 --- /dev/null +++ b/src/dev/i18n/integrate_locale_files.js @@ -0,0 +1,112 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import path from 'path'; + +import { + difference, + readFileAsync, + writeFileAsync, + accessAsync, + makeDirAsync, + normalizePath, + ErrorReporter, +} from './utils'; +import { paths, exclude } from '../../../.i18nrc.json'; +import { getDefaultMessagesMap } from './extract_default_translations'; +import { createFailError } from '../run'; +import { serializeToJson } from './serializers/json'; + +export function verifyMessages(localizedMessagesMap, defaultMessagesMap) { + let errorMessage = ''; + + const defaultMessagesIds = [...defaultMessagesMap.keys()]; + const localizedMessagesIds = [...localizedMessagesMap.keys()]; + + const unusedTranslations = difference(localizedMessagesIds, defaultMessagesIds); + if (unusedTranslations.length > 0) { + errorMessage += `\nUnused translations:\n${unusedTranslations.join(', ')}`; + } + + const missingTranslations = difference(defaultMessagesIds, localizedMessagesIds); + if (missingTranslations.length > 0) { + errorMessage += `\nMissing translations:\n${missingTranslations.join(', ')}`; + } + + if (errorMessage) { + throw createFailError(errorMessage); + } +} + +function groupMessagesByNamespace(localizedMessagesMap) { + const localizedMessagesByNamespace = new Map(); + const knownNamespaces = Object.keys(paths); + + for (const [messageId, messageValue] of localizedMessagesMap) { + const namespace = knownNamespaces.find(key => messageId.startsWith(`${key}.`)); + + if (!namespace) { + throw createFailError(`Unknown namespace in id ${messageId}.`); + } + + if (!localizedMessagesByNamespace.has(namespace)) { + localizedMessagesByNamespace.set(namespace, []); + } + + localizedMessagesByNamespace + .get(namespace) + .push([messageId, { message: messageValue.text || messageValue }]); + } + + return localizedMessagesByNamespace; +} + +async function writeMessages(localizedMessagesByNamespace, fileName, formats, log) { + for (const [namespace, messages] of localizedMessagesByNamespace) { + const destPath = path.resolve(paths[namespace], 'translations'); + + try { + await accessAsync(destPath); + } catch (_) { + await makeDirAsync(destPath); + } + + const writePath = path.resolve(destPath, fileName); + await writeFileAsync(writePath, serializeToJson(messages, formats)); + log.success(`Translations have been integrated to ${normalizePath(writePath)}`); + } +} + +export async function integrateLocaleFiles(filePath, log) { + const reporter = new ErrorReporter(); + const defaultMessagesMap = await getDefaultMessagesMap(['.'], { paths, exclude }, reporter); + const localizedMessages = JSON.parse((await readFileAsync(filePath)).toString()); + + if (!localizedMessages.formats) { + throw createFailError(`Locale file should contain formats object.`); + } + + const localizedMessagesMap = new Map(Object.entries(localizedMessages.messages)); + verifyMessages(localizedMessagesMap, defaultMessagesMap); + + // use basename of filePath to write the same locale name as the source file has + const fileName = path.basename(filePath); + const localizedMessagesByNamespace = groupMessagesByNamespace(localizedMessagesMap); + await writeMessages(localizedMessagesByNamespace, fileName, localizedMessages.formats, log); +} diff --git a/src/dev/i18n/integrate_locale_files.test.js b/src/dev/i18n/integrate_locale_files.test.js new file mode 100644 index 0000000000000..04f7b607192c0 --- /dev/null +++ b/src/dev/i18n/integrate_locale_files.test.js @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import path from 'path'; + +import { verifyMessages, integrateLocaleFiles } from './integrate_locale_files'; +import { normalizePath } from './utils'; + +const localePath = path.resolve(__dirname, '__fixtures__', 'integrate_locale_files', 'fr.json'); + +const mockDefaultMessagesMap = new Map([ + ['plugin-1.message-id-1', 'Message text 1'], + ['plugin-1.message-id-2', 'Message text 2'], + ['plugin-2.message-id', 'Message text'], +]); + +jest.mock('./extract_default_translations.js', () => ({ + getDefaultMessagesMap: () => mockDefaultMessagesMap, +})); + +jest.mock('../../../.i18nrc.json', () => ({ + paths: { + 'plugin-1': 'src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_1', + 'plugin-2': 'src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_2', + }, + exclude: [], +})); + +const utils = require('./utils'); +utils.writeFileAsync = jest.fn(); +utils.makeDirAsync = jest.fn(); + +describe('dev/i18n/integrate_locale_files', () => { + describe('verifyMessages', () => { + test('validates localized messages', () => { + const localizedMessagesMap = new Map([ + ['plugin-1.message-id-1', 'Translated text 1'], + ['plugin-1.message-id-2', 'Translated text 2'], + ['plugin-2.message-id', 'Translated text'], + ]); + + expect(() => verifyMessages(localizedMessagesMap, mockDefaultMessagesMap)).not.toThrow(); + }); + + test('throws an error for unused id and missing id', () => { + const localizedMessagesMapWithMissingMessage = new Map([ + ['plugin-1.message-id-1', 'Translated text 1'], + ['plugin-2.message-id', 'Translated text'], + ]); + + const localizedMessagesMapWithUnusedMessage = new Map([ + ['plugin-1.message-id-1', 'Translated text 1'], + ['plugin-1.message-id-2', 'Translated text 2'], + ['plugin-1.message-id-3', 'Translated text 3'], + ['plugin-2.message-id', 'Translated text'], + ]); + + const localizedMessagesMapWithIdTypo = new Map([ + ['plugin-1.message-id-1', 'Message text 1'], + ['plugin-1.message-id-2', 'Message text 2'], + ['plugin-2.message', 'Message text'], + ]); + + expect(() => + verifyMessages(localizedMessagesMapWithMissingMessage, mockDefaultMessagesMap) + ).toThrowErrorMatchingSnapshot(); + expect(() => + verifyMessages(localizedMessagesMapWithUnusedMessage, mockDefaultMessagesMap) + ).toThrowErrorMatchingSnapshot(); + expect(() => + verifyMessages(localizedMessagesMapWithIdTypo, mockDefaultMessagesMap) + ).toThrowErrorMatchingSnapshot(); + }); + }); + + describe('integrateLocaleFiles', () => { + test('splits locale file by plugins and writes them into the right folders', async () => { + const success = jest.fn(); + await integrateLocaleFiles(localePath, { success }); + + const [[path1, json1], [path2, json2]] = utils.writeFileAsync.mock.calls; + const [[dirPath1], [dirPath2]] = utils.makeDirAsync.mock.calls; + + expect([normalizePath(path1), json1]).toMatchSnapshot(); + expect([normalizePath(path2), json2]).toMatchSnapshot(); + expect([normalizePath(dirPath1), normalizePath(dirPath2)]).toMatchSnapshot(); + }); + }); +}); diff --git a/src/dev/i18n/serializers/json.js b/src/dev/i18n/serializers/json.js index 4dfb86da54f08..edf51e66ac651 100644 --- a/src/dev/i18n/serializers/json.js +++ b/src/dev/i18n/serializers/json.js @@ -19,10 +19,10 @@ import { i18n } from '@kbn/i18n'; -export function serializeToJson(defaultMessages) { - const resultJsonObject = { formats: i18n.formats, messages: {} }; +export function serializeToJson(messages, formats = i18n.formats) { + const resultJsonObject = { formats, messages: {} }; - for (const [mapKey, mapValue] of defaultMessages) { + for (const [mapKey, mapValue] of messages) { if (mapValue.description) { resultJsonObject.messages[mapKey] = { text: mapValue.message, comment: mapValue.description }; } else { diff --git a/src/dev/i18n/serializers/json.test.js b/src/dev/i18n/serializers/json.test.js index a1dd3e0e9cd6a..b910b3b9fbcc3 100644 --- a/src/dev/i18n/serializers/json.test.js +++ b/src/dev/i18n/serializers/json.test.js @@ -21,7 +21,7 @@ import { serializeToJson } from './json'; describe('dev/i18n/serializers/json', () => { test('should serialize default messages to JSON', () => { - const messages = new Map([ + const messages = [ ['plugin1.message.id-1', { message: 'Message text 1 ' }], [ 'plugin2.message.id-2', @@ -30,7 +30,7 @@ describe('dev/i18n/serializers/json', () => { description: 'Message description', }, ], - ]); + ]; expect(serializeToJson(messages)).toMatchSnapshot(); }); diff --git a/src/dev/i18n/serializers/json5.js b/src/dev/i18n/serializers/json5.js index df3872b24c356..1af7c56d40676 100644 --- a/src/dev/i18n/serializers/json5.js +++ b/src/dev/i18n/serializers/json5.js @@ -22,15 +22,15 @@ import { i18n } from '@kbn/i18n'; const ESCAPE_SINGLE_QUOTE_REGEX = /\\([\s\S])|(')/g; -export function serializeToJson5(defaultMessages) { +export function serializeToJson5(messages, formats = i18n.formats) { // .slice(0, -4): remove closing curly braces from json to append messages let jsonBuffer = Buffer.from( - JSON5.stringify({ formats: i18n.formats, messages: {} }, { quote: `'`, space: 2 }) + JSON5.stringify({ formats, messages: {} }, { quote: `'`, space: 2 }) .slice(0, -4) .concat('\n') ); - for (const [mapKey, mapValue] of defaultMessages) { + for (const [mapKey, mapValue] of messages) { const formattedMessage = mapValue.message.replace(ESCAPE_SINGLE_QUOTE_REGEX, '\\$1$2'); const formattedDescription = mapValue.description ? mapValue.description.replace(ESCAPE_SINGLE_QUOTE_REGEX, '\\$1$2') diff --git a/src/dev/i18n/serializers/json5.test.js b/src/dev/i18n/serializers/json5.test.js index 0488294537e84..6c5ece278989d 100644 --- a/src/dev/i18n/serializers/json5.test.js +++ b/src/dev/i18n/serializers/json5.test.js @@ -21,7 +21,7 @@ import { serializeToJson5 } from './json5'; describe('dev/i18n/serializers/json5', () => { test('should serialize default messages to JSON5', () => { - const messages = new Map([ + const messages = [ [ 'plugin1.message.id-1', { @@ -35,7 +35,7 @@ describe('dev/i18n/serializers/json5', () => { description: 'Message description', }, ], - ]); + ]; expect(serializeToJson5(messages).toString()).toMatchSnapshot(); }); diff --git a/src/dev/i18n/utils.js b/src/dev/i18n/utils.js index 4286cc0e23929..922a774d50934 100644 --- a/src/dev/i18n/utils.js +++ b/src/dev/i18n/utils.js @@ -31,10 +31,10 @@ import { import fs from 'fs'; import glob from 'glob'; import { promisify } from 'util'; -import chalk from 'chalk'; -import parser from 'intl-messageformat-parser'; import normalize from 'normalize-path'; import path from 'path'; +import chalk from 'chalk'; +import parser from 'intl-messageformat-parser'; import { createFailError } from '../run'; @@ -46,6 +46,8 @@ const HTML_KEY_PREFIX = 'html_'; export const readFileAsync = promisify(fs.readFile); export const writeFileAsync = promisify(fs.writeFile); +export const makeDirAsync = promisify(fs.mkdir); +export const accessAsync = promisify(fs.access); export const globAsync = promisify(glob); export function normalizePath(inputPath) { diff --git a/src/dev/i18n/utils.test.js b/src/dev/i18n/utils.test.js index a6df4fa1d346d..fe70b45c0270a 100644 --- a/src/dev/i18n/utils.test.js +++ b/src/dev/i18n/utils.test.js @@ -27,6 +27,7 @@ import { formatJSString, checkValuesProperty, createParserErrorMessage, + normalizePath, extractMessageValueFromNode, } from './utils'; @@ -107,6 +108,10 @@ describe('i18n utils', () => { } }); + test('should normalizePath', () => { + expect(normalizePath(__dirname)).toMatchSnapshot(); + }); + test('should validate conformity of "values" and "defaultMessage"', () => { const valuesKeys = ['url', 'username', 'password']; const defaultMessage = 'Test message with {username}, {password} and [markdown link]({url}).'; diff --git a/src/dev/run_i18n_integrate.js b/src/dev/run_i18n_integrate.js new file mode 100644 index 0000000000000..e81d37d5c1bc6 --- /dev/null +++ b/src/dev/run_i18n_integrate.js @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import chalk from 'chalk'; + +import { createFailError, run } from './run'; +import { integrateLocaleFiles } from './i18n/integrate_locale_files'; + +run(async ({ flags: { path }, log }) => { + if (!path || typeof path === 'boolean') { + throw createFailError(`${chalk.white.bgRed(' I18N ERROR ')} --path option isn't provided.`); + } + + if (Array.isArray(path)) { + throw createFailError( + `${chalk.white.bgRed(' I18N ERROR ')} --path should be specified only once` + ); + } + + await integrateLocaleFiles(path, log); +});