Update the various scripts to Transifex new API

README.md

@@ -222,6 +222,10 @@ In this case, you need to manually pull the translations from Transifex to your
    source ./venv/bin/activate
    pip install -r REQUIREMENTS.txt
    ```
+1. Install [Transifex command line client](https://github.com/transifex/cli/)
+   ```
+   curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash
+   ```
 1. Download the translated strings using the [minimize_translation script](scripts/minimize_translation.sh).
    By default this pulls all the languages.
    ```
@@ -232,15 +236,13 @@ In this case, you need to manually pull the translations from Transifex to your
    ./scripts/minimize_translation.sh -l it
    ```
 
-   IMPORTANT: to be able to pull from transifex.com, you will need a credentials file. 
-   This file should be named: ``.transifexrc`` and easiest is to put it in your home dir. 
+   IMPORTANT: to be able to pull from transifex.com, you will need a credentials file.
+   This file should be named: ``.transifexrc`` and easiest is to put it in your home dir.
    The file should contain this:
    ```
    [https://www.transifex.com]
-   hostname = https://www.transifex.com
-   password = yourtransifexpassword
-   token = 
-   username = yourtransifexusername
+   rest_hostname = https://rest.api.transifex.com
+   token = yourtransifextoken
    ```
 1. Build the docs in your language
    ```

REQUIREMENTS.txt

@@ -4,10 +4,8 @@
 Sphinx
 sphinx-intl
 sphinx_rtd_theme
-sphinx-intl[transifex]
 sphinxext-rediraffe
 sphinx_togglebutton
 sphinx_copybutton
-transifex-client
 PyYAML
 pdflatex

scripts/create_transifex_resources.sh

@@ -43,23 +43,23 @@ do
   # appending the target branch name after double "-"
   # and replacing "_", "/", ".", "\" and " " characters with "-" in the path
   # so for a file like
-  #   locale/en/LC_MESSAGES/docs/user_manual/processing/3rdParty.po in release_3.16 branch
+  #   locale/en/LC_MESSAGES/docs/user_manual/processing/3rdParty.po in release_3.22 branch
   # we will get
-  #   locale-en-lc-messages-docs-user-manual-processing-3rdparty-po--release-3-16
+  #   locale-en-lc-messages-docs-user-manual-processing-3rdparty-po--release-3-22
   RESOURCE=`echo "$POFILE--$TARGETBRANCH" | tr '[:upper:]' '[:lower:]' | sed 's,[_/ \.\\],-,g'`
   echo $RESOURCE
 
   # Populate the config file
   # When we are done in this block we should have created sections in the
   # .tx/config file that look like this:
   #
-  #   [qgis-documentation.locale-en-lc-messages-docs-user-manual-processing-3rdparty-po--release-3-16]
+  #   [o:qgis:p:qgis-documentation:r:locale-en-lc-messages-docs-user-manual-processing-3rdparty-po--release-3-22]
   #   file_filter = locale/<lang>/LC_MESSAGES/docs/user_manual/processing/3rdParty.po
   #   source_file = locale/en/LC_MESSAGES/docs/user_manual/processing/3rdParty.po
   #   source_lang = en
-  #   type = PO
+  #   type        = PO
   #
-  echo -e "[$PROJECT.$RESOURCE]\nfile_filter = $GENERICFILE\nsource_file = $POFILE\nsource_lang = en\ntype = PO\n" >> $CONFIGFILE
+  echo -e "[o:qgis:p:$PROJECT:r:$RESOURCE]\nfile_filter = $GENERICFILE\nsource_file = $POFILE\nsource_lang = en\ntype        = PO\n" >> $CONFIGFILE
 
 done
 

scripts/find_set_subst.py

@@ -102,7 +102,7 @@ def read_subst(file):
     unicode_pattern = re.compile(r"\.\. \|([\w\d\s-]+)\|\s+unicode::\s+([^\n]+)")
 
     # read substitutions file line by line searching for pattern matches
-    with open (file, "r") as f:
+    with open (file) as f:
         for line in f:
             if image_pattern.match(line) is not None:
                 # Adds new image object to dictionary

scripts/load_tx_stats.py

@@ -5,51 +5,103 @@
 from datetime import date
 
 import requests
-#import yaml
+# import yaml
 
-# This script downloads the statistics of localization of the project from Transifex.
+# This script downloads the statistics of localization of the project
+# from Transifex.
 # To be able to use it, you need to provide your user account token
-# and run `python3 scripts/load_tx_stats.py <TX_TOKEN>` from the repo main folder
+# and run `python3 scripts/load_tx_stats.py <TX_TOKEN>` from the repo
+# main folder. See https://transifex.github.io/openapi/index.html
 
-# Catch the Transifex api token value (passed as argument to the python command)
+# Catch the Transifex api token value
+# (passed as argument to the python command)
 if len(argv) > 1:
     TX_TOKEN = argv[1]
 
-#Load stats of the QGIS Documentation project from transifex
-response = requests.get('https://api.transifex.com/organizations/qgis/projects/qgis-documentation/',
-                        auth=('api', TX_TOKEN)
-                       )
-data = response.json()
-#print(data)
+headers = {'Authorization': 'Bearer ' + TX_TOKEN}
+project_id = "o:qgis:p:qgis-documentation"
 
-# Get statistics of translation for each target language
-language_rate={}
+# Load target languages of the QGIS Documentation project from transifex
+target_languages = requests.get(
+    f"https://rest.api.transifex.com/projects/{project_id}/languages",
+    headers=headers
+    )
+target_languages = target_languages.json()
+# Where we store statistics of each language
+language_rate = {}
 
 # Fetch list of languages
-for lang in data['languages']:
-    code = lang['code']
-    name = lang['name']
-    language_rate[code] = name
-
-# Fetch and store stats of interest for each target language
-for code in data['stats']:
-    language_rate[code] = {'name' : language_rate[code],
-                           'percentage' : round(data['stats'][code]['translated']['percentage']*100, 2)
-                          }
+for lang in target_languages['data']:
+    code = lang['attributes']['code']
+    language_rate[code] = {'name': lang['attributes']['name']}
+
+# Add English to the list
+language_rate['en'] = {'name': 'English'}
+# Sort by language name for a better display in docs
+language_rate = {k: v for k, v in sorted(
+    language_rate.items(),
+    key=lambda item: item[1]['name'])
+}
+# print(language_rate, ' counts ', len(language_rate))
+
+# Extract statistics for every single available languages,
+# namely their number of translated strings
+for lang in language_rate:
+    # print('LANG  ', language_rate[lang]['name'])
+    translated_strings = 0
+    total_strings = 0
+
+    resource_language_stats = requests.get(
+        f"https://rest.api.transifex.com/resource_language_stats?"
+        f"filter[project]={project_id}&filter[language]=l:{lang}",
+        headers=headers)
+    resource_language_stats = resource_language_stats.json()
+
+    all_resources = resource_language_stats['data']
+
+    # Walk through pagination
+    while 'next' in resource_language_stats['links'].keys():
+        resource_language_stats = requests.get(
+            resource_language_stats['links']['next'],
+            headers=headers
+            ).json()
+        all_resources.extend(resource_language_stats['data'])
+
+    for resource in all_resources:
+        translated_strings += resource['attributes']['translated_strings']
+        total_strings += resource['attributes']['total_strings']
+
+    language_rate[lang]['translated_strings'] = translated_strings
+    language_rate[lang]['percentage'] = round(
+        translated_strings * 100 / total_strings, 2)
+    # Keep total count of available strings in 'en' only,
+    # no need to store multiple times
+    if lang == 'en':
+        language_rate[lang]['total_strings'] = total_strings
+
+# print(language_rate)
 
 # Stats for the whole project (== English source language)
-# Number of languages declared in transifex for the project
-nb_languages = len(data['languages'])
+# Number of target translation languages declared in Transifex for the project
+nb_languages = len(language_rate) - 1
 # Total number of strings in English to translate
-totalstringcount = data['stringcount']
-# translation percentage of the whole project
-translation_ratio = round(sum([value['percentage'] for value in language_rate.values()])/nb_languages, 2)
+total_strings = language_rate['en']['total_strings']
+# translation percentage of the whole project, let's not count 'en'
+total_translated_strings = sum(
+    language_rate[lang]['translated_strings']
+    for lang in language_rate if lang != 'en')
+global_percentage = round(
+    total_translated_strings*100/(total_strings * nb_languages), 2)
+
+language_rate['en'].update(
+    {'nb_languages': nb_languages,
+     'translated_strings': total_translated_strings,
+     'percentage': global_percentage
+     }
+    )
+
+# print('all ', language_rate)
 
-language_rate['en']={'nb_languages': nb_languages,
-                     'stringcount': totalstringcount,
-                     'percentage' : translation_ratio
-                     }
-print('all', language_rate)
 
 def load_overall_stats():
     """Format statistics of translation in the project"""
@@ -60,48 +112,57 @@ def load_overall_stats():
             f"   * - Number of strings\n"
             f"     - Number of target languages\n"
             f"     - Overall Translation ratio\n"
-            f"   * - **{totalstringcount}**\n"
+            f"   * - **{total_strings}**\n"
             f"     - **{nb_languages}**\n"
-            f"     - **{translation_ratio}%**\n"
-            f"\n")
+            f"     - **{global_percentage}%**\n"
+            "\n")
 
     return text
 
-def load_lang_stats(stats, nb_columns=1):
-    """Format statistics of translated languages into a multicolumn table"""
 
-    text = (f".. list-table::\n"
-            f"   :widths: auto\n"
-            f"\n"
-            f"   * - Language\n"
-            f"     - Translation ratio (%)\n")
-    #Add more columns
-    text +=(f"     - Language\n     - Translation ratio (%)\n" * (nb_columns - 1))
+def load_lang_stats(target_langs, nb_columns=1):
+    """Format statistics of translated languages into a multi-column table"""
 
-    i=1
-    for lang in stats:
+    text = (".. list-table::\n"
+            "   :widths: auto\n"
+            "\n"
+            "   * - Language\n"
+            "     - Translation ratio (%)\n")
+    # Add more columns
+    text += (
+        "     - Language\n     - Translation ratio (%)\n" * (nb_columns - 1)
+        )
+
+    i = 1
+    for lang in target_langs:
         if lang != 'en':
             if i % nb_columns == 1 or nb_columns == 1:
-                text += (f"   * - {stats[lang]['name']}\n")
+                text += (f"   * - {target_langs[lang]['name']}\n")
             else:
-                text += (f"     - {stats[lang]['name']}\n")
+                text += (f"     - {target_langs[lang]['name']}\n")
 
-            text += (f"     - {stats[lang]['percentage']}\n")
-            i+=1
+            text += (f"     - {target_langs[lang]['percentage']}\n")
+            i += 1
 
     # Add empty cells to keep a well formatted rst table
-    # when the number of target languages is not a multiple of the number of columns
+    # when the number of target languages is not a multiple
+    # of the number of columns
     if nb_languages % nb_columns:
         remaining_cells = nb_columns - (nb_languages % nb_columns)
-        text += (f"     -\n     -\n" * remaining_cells)
+        text += ("     -\n     -\n" * remaining_cells)
 
     return text
 
+
 # Store the stats as a table in a rst file
-statsfile = path.join(path.dirname(__file__), '..','docs/user_manual/preamble/translation_stats.rst')
+statsfile = path.join(path.dirname(__file__),
+                      '..',
+                      'docs/user_manual/preamble/translation_stats.rst'
+                      )
 with open(statsfile, 'w') as f:
     f.write(f":orphan:\n\n"
-            f".. DO NOT EDIT THIS FILE DIRECTLY. It is generated automatically by\n"
+            f".. DO NOT EDIT THIS FILE DIRECTLY."
+            f" It is generated automatically by\n"
             f"   load_tx_stats.py in the scripts folder.\n\n"
             f"Statistics of translation\n"
             f"===========================\n\n"
@@ -112,13 +173,4 @@ def load_lang_stats(stats, nb_columns=1):
             f"\n\n"
             f"{load_lang_stats(language_rate, nb_columns=3)}"
             f"\n\n"
-           )
-
-
-# Output all stats to a yaml file we can parse to feed documentation based on locale
-#cfgfile = path.join(path.dirname(__file__), '..','transifex.yml')
-#with open(cfgfile, 'w') as yaml_file:
-#    yaml_file.write("#last update: {}\n\n".format(datetime.datetime.now()))
-#    yaml.dump(language_rate, yaml_file)
-
-
+            )

scripts/minimize_translation.sh

@@ -20,7 +20,7 @@ SOURCEPOFILES='locale'
 
 # Pull translations of all languages from Transifex,
 # fetching only the strings that are really translated and not the whole file
-tx pull --parallel --mode onlytranslated $LANG
+tx pull --mode onlytranslated $LANG
 
 # The onlytranslated mode actually pulls all the existing files
 # (with only the file header when no string has been translated yet)