From dcb41346cf54d19e185003f4c9a7577dce9f0cd8 Mon Sep 17 00:00:00 2001 From: Amr Nashawaty Date: Wed, 12 Jun 2024 22:23:22 +0300 Subject: [PATCH] feat: atlas push pull scripts: FC-55 (#422) --- .gitignore | 8 +- Makefile | 12 ++ README.md | 41 +++++ i18n_scripts/requirements.txt | 3 + i18n_scripts/translation.py | 309 ++++++++++++++++++++++++++++++++++ 5 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 Makefile create mode 100644 i18n_scripts/requirements.txt create mode 100644 i18n_scripts/translation.py diff --git a/.gitignore b/.gitignore index 8a80e27f8..274b62ec6 100644 --- a/.gitignore +++ b/.gitignore @@ -115,4 +115,10 @@ vendor/ venv/ Podfile.lock config_settings.yaml -default_config/ \ No newline at end of file +default_config/ + +# Translations ignored files +.venv/ +I18N/ +*.lproj/ +!en.lproj/ diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..5f97f7c59 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +clean_translations_temp_directory: + rm -rf I18N/ + +translation_requirements: + pip3 install -r i18n_scripts/requirements.txt + +pull_translations: clean_translations_temp_directory + atlas pull $(ATLAS_OPTIONS) translations/openedx-app-ios/I18N:I18N + python3 i18n_scripts/translation.py --split --replace-underscore + +extract_translations: clean_translations_temp_directory + python3 i18n_scripts/translation.py --combine diff --git a/README.md b/README.md index 828a971a4..bb4b9b578 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,47 @@ Modern vision of the mobile application for the Open edX platform from Raccoon G 6. Click the **Run** button. +## Translations +### Getting translations for the app +Translations aren't included in the source code of this repository as of [OEP-58](https://docs.openedx.org/en/latest/developers/concepts/oep58.html). Therefore, they need to be pulled before testing or publishing to App Store. + +Before retrieving the translations for the app, we need to install the requirements listed in the requirements.txt file located in the i18n_scripts directory. This can be done easily by running the following make command: +```bash +make translation_requirements +``` + +Then, to get the latest translations for all languages use the following command: +```bash +make pull_translations +``` +This command runs [`atlas pull`](https://github.com/openedx/openedx-atlas) to download the latest translations files from the [openedx/openedx-translations](https://github.com/openedx/openedx-translations) repository. These files contain the latest translations for all languages. In the [openedx/openedx-translations](https://github.com/openedx/openedx-translations) repository each language's translations are saved as a single file e.g. `I18N/I18N/uk.lproj/Localization.strings` ([example](https://github.com/openedx/openedx-translations/blob/6448167e9695a921f003ff6bd8f40f006a2d6743/translations/openedx-app-ios/I18N/I18N/uk.lproj/Localizable.strings)). After these are pulled, each language's translation file is split into the App's modules e.g. `Discovery/Discovery/uk.lproj/Localization.strings`. + + After this command is run the application can load the translations by changing the device (or the emulator) language in the settings. + +### Using custom translations + +By default, the command `make pull_translations` runs [`atlas pull`](https://github.com/openedx/openedx-atlas) with no arguments which pulls transaltions from the [openedx-translations repository](https://github.com/openedx/openedx-translations). + +You can use custom translations on your fork of the openedx-translations repository by setting the following configuration parameters: + +- `--revision` (default: `"main"`): Branch or git tag to pull translations from. +- `--repository` (default: `"openedx/openedx-translations"`): GitHub repository slug. There's a feature request to [support GitLab and other providers](https://github.com/openedx/openedx-atlas/issues/20). + +Arguments can be passed via the `ATLAS_OPTIONS` environment variable as shown below: +``` bash +make ATLAS_OPTIONS='--repository=/ --revision=' pull_translations +``` +Additional arguments can be passed to `atlas pull`. Refer to the [atlas documentations ](https://github.com/openedx/openedx-atlas) for more information. + +### How to translate the app + +Translations are managed in the [open-edx/openedx-translations](https://app.transifex.com/open-edx/openedx-translations/dashboard/) Transifex project. + +To translate the app join the [Transifex project](https://app.transifex.com/open-edx/openedx-translations/dashboard/) and add your translations `openedx-app-ios` resource: https://app.transifex.com/open-edx/openedx-translations/openedx-app-ios/ (the link will start working after the [pull request #442](https://github.com/openedx/openedx-app-ios/pull/422) is merged) + +Once the resource is both 100% translated and reviewed the [Transifex integration](https://github.com/apps/transifex-integration) will automatically push it to the [openedx-translations](https://github.com/openedx/openedx-translations) repository and developers can use the translations in their app. + + ## API This project targets on the latest Open edX release and rely on the relevant mobile APIs. diff --git a/i18n_scripts/requirements.txt b/i18n_scripts/requirements.txt new file mode 100644 index 000000000..384c433ad --- /dev/null +++ b/i18n_scripts/requirements.txt @@ -0,0 +1,3 @@ +# Translation processing dependencies +openedx-atlas==0.6.1 +localizable==0.1.3 \ No newline at end of file diff --git a/i18n_scripts/translation.py b/i18n_scripts/translation.py new file mode 100644 index 000000000..5a56ca48e --- /dev/null +++ b/i18n_scripts/translation.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +""" +This script performs two jobs: + 1- Combine the English translations from all modules in the repository to the I18N directory. After the English + translation is combined, it will be pushed to the openedx-translations repository as described in OEP-58. +2- Split the pulled translation files from the openedx-translations repository into the iOS app modules. + +More detailed specifications are described in the docs/0002-atlas-translations-management.rst design doc. +""" + +import argparse +import os +import re +import sys +from collections import defaultdict +import localizable + + +def parse_arguments(): + """ + This function is the argument parser for this script. + The script takes only one of the two arguments --split or --combine. + Additionally, the --replace-underscore argument can only be used with --split. + """ + parser = argparse.ArgumentParser(description='Split or Combine translations.') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--split', action='store_true', + help='Split translations into separate files for each module and language.') + group.add_argument('--combine', action='store_true', + help='Combine the English translations from all modules into a single file.') + parser.add_argument('--replace-underscore', action='store_true', + help='Replace underscores with "-r" in language directories (only with --split).') + return parser.parse_args() + + +def get_translation_file_path(modules_dir, module_name, lang_dir, create_dirs=False): + """ + Retrieves the path of the translation file for a specified module and language directory. + + Parameters: + modules_dir (str): The path to the base directory containing all the modules. + module_name (str): The name of the module for which the translation path is being retrieved. + lang_dir (str): The name of the language directory within the module's directory. + create_dirs (bool): If True, creates the parent directories if they do not exist. Defaults to False. + + Returns: + str: The path to the module's translation file (Localizable.strings). + """ + try: + lang_dir_path = os.path.join(modules_dir, module_name, module_name, lang_dir, 'Localizable.strings') + if create_dirs: + os.makedirs(os.path.dirname(lang_dir_path), exist_ok=True) + return lang_dir_path + except Exception as e: + print(f"Error creating directory path: {e}", file=sys.stderr) + raise + + +def get_modules_to_translate(modules_dir): + """ + Retrieve the names of modules that have translation files for a specified language. + + Parameters: + modules_dir (str): The path to the directory containing all the modules. + + Returns: + list of str: A list of module names that have translation files for the specified language. + """ + try: + modules_list = [ + directory for directory in os.listdir(modules_dir) + if ( + os.path.isdir(os.path.join(modules_dir, directory)) + and os.path.isfile(get_translation_file_path(modules_dir, directory, 'en.lproj')) + and directory != 'I18N' + ) + ] + return modules_list + except FileNotFoundError as e: + print(f"Directory not found: {e}", file=sys.stderr) + raise + except PermissionError as e: + print(f"Permission denied: {e}", file=sys.stderr) + raise + + +def get_translations(modules_dir): + """ + Retrieve the translations from all modules in the modules_dir. + + Parameters: + modules_dir (str): The directory containing the modules. + + Returns: + dict: A dict containing a list of dictionaries containing the 'key', 'value', and 'comment' for each + translation line. The key of the outer dict is the name of the module where the translations are going + to be saved. + """ + translations = [] + try: + modules = get_modules_to_translate(modules_dir) + for module in modules: + translation_file = get_translation_file_path(modules_dir, module, lang_dir='en.lproj') + module_translation = localizable.parse_strings(filename=translation_file) + + translations += [ + { + 'key': f"{module}.{translation_entry['key']}", + 'value': translation_entry['value'], + 'comment': translation_entry['comment'] + } for translation_entry in module_translation + ] + except Exception as e: + print(f"Error retrieving translations: {e}", file=sys.stderr) + raise + + return {'I18N': translations} + + +def combine_translation_files(modules_dir=None): + """ + Combine translation files from different modules into a single file. + """ + try: + if not modules_dir: + modules_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + translation = get_translations(modules_dir) + write_translations_to_modules(modules_dir, 'en.lproj', translation) + except Exception as e: + print(f"Error combining translation files: {e}", file=sys.stderr) + raise + + +def get_languages_dirs(modules_dir): + """ + Retrieve directories containing language files for translation. + + Args: + modules_dir (str): The directory containing all the modules. + + Returns: + list: A list of directories containing language files for translation. Each directory represents + a specific language and ends with the '.lproj' extension. + """ + try: + lang_parent_dir = os.path.join(modules_dir, 'I18N', 'I18N') + languages_dirs = [ + directory for directory in os.listdir(lang_parent_dir) + if directory.endswith('.lproj') and directory != "en.lproj" + ] + return languages_dirs + except FileNotFoundError as e: + print(f"Directory not found: {e}", file=sys.stderr) + raise + except PermissionError as e: + print(f"Permission denied: {e}", file=sys.stderr) + raise + + +def get_translations_from_file(modules_dir, lang_dir): + """ + Get translations from the translation file in the 'I18N' directory and distribute them into the appropriate + modules' directories. + + Args: + modules_dir (str): The directory containing all the modules. + lang_dir (str): The directory containing the translation file being split. + + Returns: + dict: A dictionary containing translations split by module. The keys are module names, + and the values are lists of dictionaries, each containing the 'key', 'value', and 'comment' + for each translation entry within the module. + """ + translations = defaultdict(list) + try: + translations_file_path = get_translation_file_path(modules_dir, 'I18N', lang_dir) + lang_list = localizable.parse_strings(filename=translations_file_path) + for translation_entry in lang_list: + module_name, key_remainder = translation_entry['key'].split('.', maxsplit=1) + split_entry = { + 'key': key_remainder, + 'value': translation_entry['value'], + 'comment': translation_entry['comment'] + } + translations[module_name].append(split_entry) + except Exception as e: + print(f"Error extracting translations from file: {e}", file=sys.stderr) + raise + return translations + + +def write_translations_to_modules(modules_dir, lang_dir, modules_translations): + """ + Write translations to language files for each module. + + Args: + modules_dir (str): The directory containing all the modules. + lang_dir (str): The directory of the translation file being written. + modules_translations (dict): A dictionary containing translations for each module. + + Returns: + None + """ + for module, translation_list in modules_translations.items(): + try: + translation_file_path = get_translation_file_path(modules_dir, module, lang_dir, create_dirs=True) + with open(translation_file_path, 'w') as f: + for translation_entry in translation_list: + write_line_and_comment(f, translation_entry) + except Exception as e: + print(f"Error writing translations to file.\n Module: {module}\n Error: {e}", file=sys.stderr) + raise + + +def _escape(s): + """ + Reverse the replacements performed by _unescape() in the localizable library + """ + s = s.replace('\n', r'\n').replace('\r', r'\r').replace('"', r'\"') + return s + + +def write_line_and_comment(f, entry): + """ + Write a translation line with an optional comment to a file. + + Args: + file (file object): The file object to write to. + entry (dict): A dictionary containing the translation entry with 'key', 'value', and optional 'comment'. + + Returns: + None + """ + comment = entry.get('comment') # Retrieve the comment, if present + if comment: + f.write(f"/* {comment} */\n") + f.write(f'"{entry["key"]}" = "{_escape(entry["value"])}";\n') + + +def split_translation_files(modules_dir=None): + """ + Split translation files into separate files for each module and language. + + Args: + modules_dir (str, optional): The directory containing all the modules. If not provided, + it defaults to the parent directory of the directory containing this script. + + Returns: + None + """ + try: + if not modules_dir: + modules_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + languages_dirs = get_languages_dirs(modules_dir) + for lang_dir in languages_dirs: + translations = get_translations_from_file(modules_dir, lang_dir) + write_translations_to_modules(modules_dir, lang_dir, translations) + except Exception as e: + print(f"Error splitting translation files: {e}", file=sys.stderr) + raise + + +def replace_underscores(modules_dir=None): + try: + if not modules_dir: + modules_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + languages_dirs = get_languages_dirs(modules_dir) + + for lang_dir in languages_dirs: + try: + pattern = r'_(\w\w.lproj$)' + if re.search(pattern, lang_dir): + replacement = r'-\1' + new_name = re.sub(pattern, replacement, lang_dir) + lang_old_path = os.path.dirname(get_translation_file_path(modules_dir, 'I18N', lang_dir)) + lang_new_path = os.path.dirname(get_translation_file_path(modules_dir, 'I18N', new_name)) + + os.rename(lang_old_path, lang_new_path) + print(f"Renamed {lang_old_path} to {lang_new_path}") + + except FileNotFoundError as e: + print(f"Error: The file or directory {lang_old_path} does not exist: {e}", file=sys.stderr) + raise + except PermissionError as e: + print(f"Error: Permission denied while renaming {lang_old_path}: {e}", file=sys.stderr) + raise + except Exception as e: + print(f"Error: An unexpected error occurred while renaming {lang_old_path} to {lang_new_path}: {e}", + file=sys.stderr) + raise + + except Exception as e: + print(f"Error: An unexpected error occurred in rename_translations_files: {e}", file=sys.stderr) + raise + + +def main(): + args = parse_arguments() + if args.split: + if args.replace_underscore: + replace_underscores() + split_translation_files() + elif args.combine: + combine_translation_files() + + +if __name__ == "__main__": + main()