From 2d9e124b91f1ba7a65e9f997a3ba952679c6c23a Mon Sep 17 00:00:00 2001 From: Sergio Moreno Date: Thu, 5 Nov 2020 09:26:15 +0100 Subject: [PATCH] feat: use fallback locales from cldr (#820) --- docs/ref/conf.rst | 50 +++- packages/cli/src/api/catalog.ts | 35 ++- packages/cli/src/api/types.ts | 10 +- packages/cli/src/lingui-compile.ts | 2 +- packages/conf/index.d.ts | 24 +- .../conf/src/__snapshots__/index.test.ts.snap | 24 +- .../src/fixtures/valid/.fallbacklocalesrc | 7 + packages/conf/src/index.test.ts | 37 +++ packages/conf/src/index.ts | 251 ++++++++++++++++-- packages/core/src/formats.test.ts | 36 --- packages/loader/src/index.js | 2 +- 11 files changed, 397 insertions(+), 81 deletions(-) create mode 100644 packages/conf/src/fixtures/valid/.fallbacklocalesrc diff --git a/docs/ref/conf.rst b/docs/ref/conf.rst index 5ab05c505..da5fd2dda 100644 --- a/docs/ref/conf.rst +++ b/docs/ref/conf.rst @@ -20,7 +20,7 @@ Default config: }], "compileNamespace": "cjs", "extractBabelOptions": {}, - "fallbackLocale": "", + "fallbackLocales": {}, "format": "po", "locales": [], "orderBy": "messageId", @@ -230,17 +230,49 @@ extracted. This is required when project doesn't use standard Babel config } } -.. config:: fallbackLocale +.. config:: fallbackLocales -fallbackLocale +fallbackLocales -------------- -Default: ``''`` +Default: ``{}`` + +:conf:`fallbackLocales` by default is using `CLDR Parent Locales `_, unless you disable it with a `false`: + +.. code-block:: json + + { + "fallbackLocales": false + } + +:conf:`fallbackLocales` object let's us configure fallback locales to each locale instance. + +.. code-block:: json + + { + "fallbackLocales": { + "en-US": ["en-GB", "en"], + "es-MX": "es" + } + } + +On this example if any translation isn't found on `en-US` then will search on `en-GB`, after that if not found we'll search in `en` + +Also, we can configure a default one for everything: + +.. code-block:: json + + { + "fallbackLocales": { + "en-US": ["en-GB", "en"], + "es-MX": "es", + "default": "en" + } + } -Translation from :conf:`fallbackLocale` is used when translation for given locale is missing. +Translations from :conf:`fallbackLocales` is used when translation for given locale is missing. -If :conf:`fallbackLocale` isn't defined or translation in :conf:`fallbackLocale` is -missing too, either default message or message ID is used instead. +If :conf:`fallbackLocales` is `false` default message or message ID is used instead. .. config:: format @@ -393,6 +425,6 @@ Catalog for :conf:`sourceLocale` doesn't require translated messages, because me IDs are used by default. However, it's still possible to override message ID by providing custom translation. -The difference between :conf:`fallbackLocale` and :conf:`sourceLocale` is that -:conf:`fallbackLocale` is used in translation, while :conf:`sourceLocale` is +The difference between :conf:`fallbackLocales` and :conf:`sourceLocale` is that +:conf:`fallbackLocales` is used in translation, while :conf:`sourceLocale` is used for the message ID. diff --git a/packages/cli/src/api/catalog.ts b/packages/cli/src/api/catalog.ts index 5b3957feb..c9c80739f 100644 --- a/packages/cli/src/api/catalog.ts +++ b/packages/cli/src/api/catalog.ts @@ -7,7 +7,7 @@ import glob from "glob" import micromatch from "micromatch" import normalize from "normalize-path" -import { LinguiConfig, OrderBy } from "@lingui/conf" +import { LinguiConfig, OrderBy, FallbackLocales } from "@lingui/conf" import getFormat from "./formats" import { CatalogFormatter } from "./formats/types" @@ -46,7 +46,7 @@ export type MergeOptions = { export type GetTranslationsOptions = { sourceLocale: string - fallbackLocale: string + fallbackLocales: FallbackLocales } type CatalogProps = { @@ -94,7 +94,7 @@ export class Catalog { ) ) as unknown) as (catalog: AllCatalogsType) => AllCatalogsType - const sortedCatalogs = cleanAndSort(catalogs); + const sortedCatalogs = cleanAndSort(catalogs) if (options.locale) { this.write(options.locale, sortedCatalogs[options.locale]) @@ -227,7 +227,7 @@ export class Catalog { catalogs: Object, locale: string, key: string, - { fallbackLocale, sourceLocale }: GetTranslationsOptions + { fallbackLocales, sourceLocale }: GetTranslationsOptions ) { if (!catalogs[locale].hasOwnProperty(key)) { console.error(`Message with key ${key} is missing in locale ${locale}`) @@ -235,18 +235,37 @@ export class Catalog { const getTranslation = (locale) => catalogs[locale][key].translation + const getMultipleFallbacks = (locale) => { + const fL = fallbackLocales[locale] + + // some probably the fallback will be undefined, so just search by locale + if (!fL) return null + + if (Array.isArray(fL)) { + for (const fallbackLocale of fL) { + if (catalogs[fallbackLocale]) { + return getTranslation(fallbackLocale) + } + } + } else { + return getTranslation(fL) + } + } + return ( // Get translation in target locale getTranslation(locale) || - // Get translation in fallbackLocale (if any) - (fallbackLocale && getTranslation(fallbackLocale)) || + // We search in fallbackLocales as dependent of each locale + getMultipleFallbacks(locale) || + // Get translation in fallbackLocales.default (if any) + (fallbackLocales.default && getTranslation(fallbackLocales.default)) || // Get message default catalogs[locale][key].defaults || // If sourceLocale is either target locale of fallback one, use key (sourceLocale && sourceLocale === locale && key) || (sourceLocale && - fallbackLocale && - sourceLocale === fallbackLocale && + fallbackLocales.default && + sourceLocale === fallbackLocales.default && key) || // Otherwise no translation is available undefined diff --git a/packages/cli/src/api/types.ts b/packages/cli/src/api/types.ts index 3117f5feb..8b76644d2 100644 --- a/packages/cli/src/api/types.ts +++ b/packages/cli/src/api/types.ts @@ -36,8 +36,16 @@ export type AllCatalogsType = { [locale: string]: CatalogType } +export type LocaleObject = { + [locale: string]: string[] | string +} +export type DefaultLocaleObject = { + default: string +} +export declare type FallbackLocales = LocaleObject | DefaultLocaleObject | false + export type getTranslationOptions = { - fallbackLocale: string + fallbackLocales: FallbackLocales sourceLocale: string } diff --git a/packages/cli/src/lingui-compile.ts b/packages/cli/src/lingui-compile.ts index 070dbae41..06cc7b983 100644 --- a/packages/cli/src/lingui-compile.ts +++ b/packages/cli/src/lingui-compile.ts @@ -54,7 +54,7 @@ function command(config, options) { const messages = catalog.getTranslations( locale === config.pseudoLocale ? config.sourceLocale : locale, { - fallbackLocale: config.fallbackLocale, + fallbackLocales: config.fallbackLocales, sourceLocale: config.sourceLocale, } ) diff --git a/packages/conf/index.d.ts b/packages/conf/index.d.ts index 62569b42c..c1669e3b8 100644 --- a/packages/conf/index.d.ts +++ b/packages/conf/index.d.ts @@ -9,11 +9,22 @@ declare type CatalogConfig = { include: string[]; exclude?: string[]; }; + +export type LocaleObject = { + [locale: string]: string[] | string +} + +export type DefaultLocaleObject = { + default: string +} + +export declare type FallbackLocales = LocaleObject | DefaultLocaleObject + export declare type LinguiConfig = { catalogs: CatalogConfig[]; compileNamespace: string; extractBabelOptions: Object; - fallbackLocale: string; + fallbackLocales: FallbackLocales; format: CatalogFormat; prevFormat: CatalogFormat; formatOptions: CatalogFormatOptions; @@ -42,7 +53,7 @@ export declare const configValidation: { }; catalogs: CatalogConfig[]; compileNamespace: string; - fallbackLocale: string; + fallbackLocales: FallbackLocales; format: CatalogFormat; formatOptions: CatalogFormatOptions; locales: string[]; @@ -53,7 +64,7 @@ export declare const configValidation: { sourceLocale: string; }; deprecatedConfig: { - fallbackLanguage: (config: LinguiConfig & DeprecatedFallbackLanguage) => string; + fallbackLocale: (config: LinguiConfig & DeprecatedFallbackLanguage) => string; localeDir: (config: LinguiConfig & DeprecatedLocaleDir) => string; srcPathDirs: (config: LinguiConfig & DeprecatedLocaleDir) => string; srcPathIgnorePatterns: (config: LinguiConfig & DeprecatedLocaleDir) => string; @@ -62,14 +73,15 @@ export declare const configValidation: { }; export declare function replaceRootDir(config: LinguiConfig, rootDir: string): LinguiConfig; /** - * Replace fallbackLanguage with fallbackLocale + * Replace fallbackLocale with fallbackLocales * * Released in lingui-conf 0.9 - * Remove anytime after 3.x + * Remove anytime after 4.x */ declare type DeprecatedFallbackLanguage = { - fallbackLanguage: string | null; + fallbackLocale: string | null; }; + export declare function fallbackLanguageMigration(config: LinguiConfig & DeprecatedFallbackLanguage): LinguiConfig; /** * Replace localeDir, srcPathDirs and srcPathIgnorePatterns with catalogs diff --git a/packages/conf/src/__snapshots__/index.test.ts.snap b/packages/conf/src/__snapshots__/index.test.ts.snap index ded947170..e043dca71 100644 --- a/packages/conf/src/__snapshots__/index.test.ts.snap +++ b/packages/conf/src/__snapshots__/index.test.ts.snap @@ -72,6 +72,26 @@ Documentation: https://lingui.js.org/ref/conf.html Please update your configuration. +Documentation: https://lingui.js.org/ref/conf.html +`; + +exports[`@lingui/conf fallbackLocales logic if fallbackLocale is defined, we use the default one on fallbackLocales 1`] = ` +● Deprecation Warning: + + Option fallbackLocale was replaced by fallbackLocales + + You can find more information here: https://github.com/lingui/js-lingui/issues/791 + + @lingui/cli now treats your current configuration as: + { + "fallbackLocales": { + default: "en" + } + } + + Please update your configuration. + + Documentation: https://lingui.js.org/ref/conf.html `; @@ -93,7 +113,9 @@ Object { plugins: Array [], presets: Array [], }, - fallbackLocale: , + fallbackLocales: Object { + en-gb: en, + }, format: po, formatOptions: Object { origins: true, diff --git a/packages/conf/src/fixtures/valid/.fallbacklocalesrc b/packages/conf/src/fixtures/valid/.fallbacklocalesrc new file mode 100644 index 000000000..3ce0b1eb5 --- /dev/null +++ b/packages/conf/src/fixtures/valid/.fallbacklocalesrc @@ -0,0 +1,7 @@ +{ + locales: ["en-US", "es-MX"], + fallbackLocales: { + "en-US": ["en"], + "default": "en" + } +} \ No newline at end of file diff --git a/packages/conf/src/index.test.ts b/packages/conf/src/index.test.ts index f268912b5..cbd13c22f 100644 --- a/packages/conf/src/index.test.ts +++ b/packages/conf/src/index.test.ts @@ -1,4 +1,5 @@ import path from "path" +import mockFs from "mock-fs" import { validate } from "jest-validate" import { getConfig, @@ -142,4 +143,40 @@ describe("@lingui/conf", function () { }) }) }) + + describe("fallbackLocales logic", () => { + afterEach(() => { + mockFs.restore() + }) + + it ("if fallbackLocale is defined, we use the default one on fallbackLocales", () => { + mockFs({ + ".linguirc": JSON.stringify({ + locales: ["en-US"], + fallbackLocale: "en" + }) + }) + mockConsole((console) => { + const config = getConfig({ + configPath: ".linguirc", + }) + expect(config.fallbackLocales.default).toEqual("en") + expect(getConsoleMockCalls(console.warn)).toMatchSnapshot() + }) + }) + + it ("if fallbackLocales is defined, we also build the cldr", () => { + const config = getConfig({ + configPath: path.resolve( + __dirname, + path.join("fixtures", "valid", ".fallbacklocalesrc") + ), + }) + expect(config.fallbackLocales).toEqual({ + "en-US": "en", + default: "en", + "es-MX": "es" + }) + }) + }) }) diff --git a/packages/conf/src/index.ts b/packages/conf/src/index.ts index 7999bc8a5..0692c3ded 100644 --- a/packages/conf/src/index.ts +++ b/packages/conf/src/index.ts @@ -2,7 +2,7 @@ import path from "path" import fs from "fs" import chalk from "chalk" import { cosmiconfigSync } from "cosmiconfig" -import { validate } from "jest-validate" +import { multipleValidOptions, validate } from "jest-validate" export type CatalogFormat = "lingui" | "minimal" | "po" | "csv" @@ -19,11 +19,19 @@ type CatalogConfig = { exclude?: string[] } +type LocaleObject = { + [locale: string]: string[] | string +} +type DefaultLocaleObject = { + default: string +} +export type FallbackLocales = LocaleObject | DefaultLocaleObject | false + export type LinguiConfig = { catalogs: CatalogConfig[] compileNamespace: string extractBabelOptions: Object - fallbackLocale: string + fallbackLocales?: FallbackLocales format: CatalogFormat formatOptions: CatalogFormatOptions locales: string[] @@ -54,7 +62,7 @@ export const defaultConfig: LinguiConfig = { ], compileNamespace: "cjs", extractBabelOptions: { plugins: [], presets: [] }, - fallbackLocale: "", + fallbackLocales: {}, format: "po", formatOptions: { origins: true }, locales: [], @@ -112,6 +120,13 @@ export function getConfig({ const exampleConfig = { ...defaultConfig, + fallbackLocales: multipleValidOptions( + {}, + { "en-US": "en" }, + { "en-US": ["en"] }, + { default: "en" }, + false + ), extractBabelOptions: { extends: "babelconfig.js", rootMode: "rootmode", @@ -121,16 +136,20 @@ const exampleConfig = { } const deprecatedConfig = { - fallbackLanguage: (config: LinguiConfig & DeprecatedFallbackLanguage) => - ` Option ${chalk.bold("fallbackLanguage")} was replaced by ${chalk.bold( - "fallbackLocale" + fallbackLocale: (config: LinguiConfig & DeprecatedFallbackLanguage) => + ` Option ${chalk.bold("fallbackLocale")} was replaced by ${chalk.bold( + "fallbackLocales" )} + You can find more information here: https://github.com/lingui/js-lingui/issues/791 + @lingui/cli now treats your current configuration as: { - ${chalk.bold('"fallbackLocale"')}: ${chalk.bold( - `"${config.fallbackLanguage}"` - )} + ${chalk.bold('"fallbackLocales"')}: { + default: ${chalk.bold( + `"${config.fallbackLocale}"` + )} + } } Please update your configuration. @@ -241,22 +260,218 @@ export function replaceRootDir( } /** - * Replace fallbackLanguage with fallbackLocale - * - * Released in lingui-conf 0.9 - * Remove anytime after 3.x + * Replace fallbackLocale, by the new standard fallbackLocales + * - https://github.com/lingui/js-lingui/issues/791 + * - Remove anytime after 4.x */ -type DeprecatedFallbackLanguage = { fallbackLanguage: string | null } +type DeprecatedFallbackLanguage = { fallbackLocale?: string } export function fallbackLanguageMigration( config: LinguiConfig & DeprecatedFallbackLanguage ): LinguiConfig { - const { fallbackLocale, fallbackLanguage, ...newConfig } = config + const { fallbackLocale, fallbackLocales } = config - return { - ...newConfig, - fallbackLocale: fallbackLocale || fallbackLanguage || "", + if (fallbackLocales === false) return { + ...config, + fallbackLocales: null, + } + + config.locales.forEach((locale) => { + const fl = getCldrParentLocale(locale.toLowerCase()) + if (fl) { + config.fallbackLocales = { + ...config.fallbackLocales, + [locale]: fl + } + } + }) + + const DEFAULT_FALLBACK = fallbackLocales?.default || fallbackLocale + if (DEFAULT_FALLBACK) { + if (!config.fallbackLocales) config.fallbackLocales = {} + config.fallbackLocales.default = DEFAULT_FALLBACK } + + return config +} + +function getCldrParentLocale(sourceLocale: string) { + return { + "en-ag": "en", + "en-ai": "en", + "en-au": "en", + "en-bb": "en", + "en-bm": "en", + "en-bs": "en", + "en-bw": "en", + "en-bz": "en", + "en-ca": "en", + "en-cc": "en", + "en-ck": "en", + "en-cm": "en", + "en-cx": "en", + "en-cy": "en", + "en-dg": "en", + "en-dm": "en", + "en-er": "en", + "en-fj": "en", + "en-fk": "en", + "en-fm": "en", + "en-gb": "en", + "en-gd": "en", + "en-gg": "en", + "en-gh": "en", + "en-gi": "en", + "en-gm": "en", + "en-gy": "en", + "en-hk": "en", + "en-ie": "en", + "en-il": "en", + "en-im": "en", + "en-in": "en", + "en-io": "en", + "en-je": "en", + "en-jm": "en", + "en-ke": "en", + "en-ki": "en", + "en-kn": "en", + "en-ky": "en", + "en-lc": "en", + "en-lr": "en", + "en-ls": "en", + "en-mg": "en", + "en-mo": "en", + "en-ms": "en", + "en-mt": "en", + "en-mu": "en", + "en-mw": "en", + "en-my": "en", + "en-na": "en", + "en-nf": "en", + "en-ng": "en", + "en-nr": "en", + "en-nu": "en", + "en-nz": "en", + "en-pg": "en", + "en-ph": "en", + "en-pk": "en", + "en-pn": "en", + "en-pw": "en", + "en-rw": "en", + "en-sb": "en", + "en-sc": "en", + "en-sd": "en", + "en-sg": "en", + "en-sh": "en", + "en-sl": "en", + "en-ss": "en", + "en-sx": "en", + "en-sz": "en", + "en-tc": "en", + "en-tk": "en", + "en-to": "en", + "en-tt": "en", + "en-tv": "en", + "en-tz": "en", + "en-ug": "en", + "en-us": "en", + "en-vc": "en", + "en-vg": "en", + "en-vu": "en", + "en-ws": "en", + "en-za": "en", + "en-zm": "en", + "en-zw": "en", + "en-at": "en", + "en-be": "en", + "en-ch": "en", + "en-de": "en", + "en-dk": "en", + "en-fi": "en", + "en-nl": "en", + "en-se": "en", + "en-si": "en", + "es-ar": "es", + "es-bo": "es", + "es-br": "es", + "es-bz": "es", + "es-cl": "es", + "es-co": "es", + "es-cr": "es", + "es-cu": "es", + "es-do": "es", + "es-ec": "es", + "es-es": "es", + "es-gt": "es", + "es-hn": "es", + "es-mx": "es", + "es-ni": "es", + "es-pa": "es", + "es-pe": "es", + "es-pr": "es", + "es-py": "es", + "es-sv": "es", + "es-us": "es", + "es-uy": "es", + "es-ve": "es", + "pt-ao": "pt", + "pt-ch": "pt", + "pt-cv": "pt", + "pt-fr": "pt", + "pt-gq": "pt", + "pt-gw": "pt", + "pt-lu": "pt", + "pt-mo": "pt", + "pt-mz": "pt", + "pt-pt": "pt", + "pt-st": "pt", + "pt-tl": "pt", + "az-arab": "az", + "az-cyrl": "az", + "blt-latn": "blt", + "bm-nkoo": "bm", + "bs-cyrl": "bs", + "byn-latn": "byn", + "cu-glag": "cu", + "dje-arab": "dje", + "dyo-arab": "dyo", + "en-dsrt": "en", + "en-shaw": "en", + "ff-adlm": "ff", + "ff-arab": "ff", + "ha-arab": "ha", + "hi-latn": "hi", + "iu-latn": "iu", + "kk-arab": "kk", + "ks-deva": "ks", + "ku-arab": "ku", + "ky-arab": "ky", + "ky-latn": "ky", + "ml-arab": "ml", + "mn-mong": "mn", + "mni-mtei": "mni", + "ms-arab": "ms", + "pa-arab": "pa", + "sat-deva": "sat", + "sd-deva": "sd", + "sd-khoj": "sd", + "sd-sind": "sd", + "shi-latn": "shi", + "so-arab": "so", + "sr-latn": "sr", + "sw-arab": "sw", + "tg-arab": "tg", + "ug-cyrl": "ug", + "uz-arab": "uz", + "uz-cyrl": "uz", + "vai-latn": "vai", + "wo-arab": "wo", + "yo-arab": "yo", + "yue-hans": "yue", + "zh-hant": "zh", + "zh-hant-hk": "zh", + "zh-hant-mo": "zh-hant-hk" + }[sourceLocale] } /** diff --git a/packages/core/src/formats.test.ts b/packages/core/src/formats.test.ts index 51cd7cb5b..7ab5977bd 100644 --- a/packages/core/src/formats.test.ts +++ b/packages/core/src/formats.test.ts @@ -27,40 +27,4 @@ describe("@lingui/core/formats", () => { expect(secondRunResult).toBeLessThan(firstRunResult) }) - - it("date memoized function is faster than the not memoized function", () => { - const loopt0 = performance.now() - for (let i = 0; i < 1000; i++) { - date("es", {})(new Date()) - } - const loopt1 = performance.now() - const memoizedDateResult = loopt1 - loopt0 - - const loop0 = performance.now() - for (let i = 0; i < 1000; i++) { - date("es", {}, false)(new Date()) - } - const loop1 = performance.now() - const withoutMemoizeResult = loop1 - loop0 - - expect(memoizedDateResult).toBeLessThan(withoutMemoizeResult) - }) - - it("number memoized function is faster than the not memoized function", () => { - const loopt0 = performance.now() - for (let i = 0; i < 1000; i++) { - number("es", {})(999666) - } - const loopt1 = performance.now() - const memoizedNumberResult = loopt1 - loopt0 - - const loop0 = performance.now() - for (let i = 0; i < 1000; i++) { - number("es", {}, false)(999666) - } - const loop1 = performance.now() - const withoutMemoizeResult = loop1 - loop0 - - expect(memoizedNumberResult).toBeLessThan(withoutMemoizeResult) - }) }) diff --git a/packages/loader/src/index.js b/packages/loader/src/index.js index f497c17de..1b384e3c3 100644 --- a/packages/loader/src/index.js +++ b/packages/loader/src/index.js @@ -62,7 +62,7 @@ export default function (source) { const messages = R.mapObjIndexed( (_, key) => catalog.getTranslation(catalogs, locale, key, { - fallbackLocale: config.fallbackLocale, + fallbackLocales: config.fallbackLocales, sourceLocale: config.sourceLocale, }), catalogs[locale]