[BB-6112] Prepare opencraft-release/maple.3 branches for configuration

CHANGELOG.md

@@ -4,6 +4,22 @@
 All notable changes to this project will be documented in this file.
 Add any new changes to the top (right below this line).
 
+ - 2022-04-06
+    - Role: edxapp
+        - Added a new `EDXAPP_COMPILE_JSI18N` variable to control whether
+          to run the `compilejsi18n` management command on edxapp deploy.
+          Defaults to false.
+
+ - 2022-04-06
+    - Role: simple_theme
+       - Added a new `SIMPLETHEME_I18N_DJANGO` setting to allow operators to provide
+         additional translations, or override existing django translations.
+
+ - 2022-02-01
+    - Role: edxapp
+        - Added a new `EDXAPP_PREPEND_LOCALE_PATHS` setting to allow operators to
+          override the default translations.
+
  - 2022-01-06
     - Role: edx_notes_api
        - Replaced `ELASTICSEARCH_URL` with `ELASTICSEARCH_DSL` in `edx_notes_api_service_config`.

playbooks/openedx_native.yml

@@ -67,6 +67,8 @@
     # For the mfe role.
     COMMON_ECOMMERCE_BASE_URL: '{{ ECOMMERCE_ECOMMERCE_URL_ROOT }}'
     ECOMMERCE_ENABLE_PAYMENT_MFE: true
+    ENABLE_UPTIME_REPORT: true
+    RETIREMENT_SERVICE_VERSION: "open-release/maple.3"
   roles:
     - role: swapfile
       SWAPFILE_SIZE: 4GB
@@ -134,3 +136,5 @@
     - role: mfe_deployer
       MFE_DEPLOY_ECOMMERCE_MFES: "{{ SANDBOX_ENABLE_ECOMMERCE }}"
     - role: mfe_flags_setup
+    - role: uptime_report
+      when: ENABLE_UPTIME_REPORT

playbooks/roles/edxapp/defaults/main.yml

@@ -626,6 +626,9 @@ EDXAPP_DCS_SESSION_COOKIE_SAMESITE_FORCE_ALL: True
 # Whether to run reindex_course on deploy
 EDXAPP_REINDEX_ALL_COURSES: false
 
+# Whether to run compilejsi18n on deploy
+EDXAPP_COMPILE_JSI18N: false
+
 # XML Course related flags
 EDXAPP_XML_FROM_GIT: false
 EDXAPP_XML_S3_BUCKET: !!null
@@ -871,6 +874,9 @@ EDXAPP_COMPREHENSIVE_THEME_DIRS:
 # list of paths to the comprehensive theme locale directories
 EDXAPP_COMPREHENSIVE_THEME_LOCALE_PATHS: []
 
+# list of paths to locale directories to load first
+EDXAPP_PREPEND_LOCALE_PATHS: []
+
 # Name of the default site theme
 EDXAPP_DEFAULT_SITE_THEME: ""
 EDXAPP_ENABLE_COMPREHENSIVE_THEMING: false
@@ -1474,6 +1480,7 @@ generic_env_config:  &edxapp_generic_env
 
   COMPREHENSIVE_THEME_DIRS: "{{ EDXAPP_COMPREHENSIVE_THEME_DIRS }}"
   COMPREHENSIVE_THEME_LOCALE_PATHS: "{{ EDXAPP_COMPREHENSIVE_THEME_LOCALE_PATHS }}"
+  PREPEND_LOCALE_PATHS: "{{ EDXAPP_PREPEND_LOCALE_PATHS }}"
   ENABLE_COMPREHENSIVE_THEMING: "{{ EDXAPP_ENABLE_COMPREHENSIVE_THEMING }}"
   DEFAULT_SITE_THEME: "{{ EDXAPP_DEFAULT_SITE_THEME }}"
   SESSION_SAVE_EVERY_REQUEST: "{{ EDXAPP_SESSION_SAVE_EVERY_REQUEST }}"

playbooks/roles/edxapp/tasks/deploy.yml

@@ -395,6 +395,15 @@
   when:
     - celery_worker is not defined
 
+- name: compile JS translations
+  shell: ". {{ edxapp_app_dir }}/edxapp_env && {{ edxapp_venv_bin }}/python ./manage.py lms --settings={{ edxapp_settings }} compilejsi18n"
+  args:
+    chdir: "{{ edxapp_code_dir }}"
+  become_user: "{{ edxapp_user }}"
+  when: EDXAPP_COMPILE_JSI18N and celery_worker is not defined
+  tags:
+    - assets
+
 # creates the supervisor jobs for the
 # service variants configured, runs
 # gather_assets and db migrations
@@ -508,16 +517,6 @@
     - manage
     - manage:db
 
-- name: reindex all courses
-  shell: ". {{ edxapp_app_dir }}/edxapp_env && {{ edxapp_venv_bin }}/python ./manage.py cms reindex_course --setup --settings={{ edxapp_settings }}"
-  args:
-    chdir: "{{ edxapp_code_dir }}"
-  become_user: "{{ common_web_user }}"
-  when: EDXAPP_REINDEX_ALL_COURSES
-  tags:
-    - install
-    - install:base
-
 - name: install cron job to run clearsessions
   cron:
     name: "clear expired Django sessions"

playbooks/roles/forum/tasks/deploy.yml

@@ -129,5 +129,12 @@
   tags:
     - deploy
 
+- name: reindex all courses
+  shell: ". {{ edxapp_app_dir }}/edxapp_env && yes | {{ edxapp_venv_bin }}/python ./manage.py cms reindex_course --all --settings={{ edxapp_settings }}"
+  args:
+    chdir: "{{ edxapp_code_dir }}"
+  become_user: "{{ common_web_user }}"
+  when: EDXAPP_REINDEX_ALL_COURSES
+
 - set_fact:
     forum_installed: true

playbooks/roles/git_clone/tasks/main.yml

@@ -69,6 +69,7 @@
 - name: Check that working tree is clean
   shell: test ! -e "{{ item }}" || git -C "{{ item }}" status --porcelain --untracked-files=no
   register: dirty_files
+  become_user: "{{ repo_owner }}"
   # Using the map here means that the items will only be the DESTINATION strings,
   # rather than the full GIT_REPOS structures, which have data we don't want to log,
   # so we don't have to use no_log on this task.
@@ -134,7 +135,7 @@
 
 - name: Run git clean after checking out code
   shell: cd {{ item.DESTINATION }} && git clean -xdf
-  become: true
+  become_user: "{{ repo_owner }}"
   with_items: "{{ GIT_REPOS }}"
   no_log: "{{ GIT_CLONE_NO_LOGGING }}"
   tags:

playbooks/roles/simple_theme/defaults/main.yml

@@ -106,3 +106,37 @@ SIMPLETHEME_STATIC_FILES_URLS: []
 #     border-top: 3px solid $main-color;
 #   }
 SIMPLETHEME_EXTRA_SASS: ""
+
+
+# Use this variable to configure django translations.
+# Note that edx-platform does not pick up translations from themes by default.
+# You will have to manually configure either `COMPREHENSIVE_THEME_LOCALE_PATHS` or
+# `PREPEND_LOCALE_PATHS` to include the path to the theme's i18n/locale folder for
+# these translations to get picked up.
+#
+# The SIMPLETHEME_I18n_DJANGO variable should contain a list of dictionaries with these keys:
+# - `lang`: the language code
+# - `domain`: the i18n domain (typically one of "django" or "djangojs")
+# - `headers`: (optional) Additional PO file headers.
+# - `messages`: Translation messages. It should be a raw string of PO formatted messages.
+#
+# Samle:
+# SIMPLETHEME_I18N_DJANGO:
+#   - lang: en
+#     domain: django
+#     headers: |
+#       "Plural-Forms: nplurals=2; plural=(n > 1);\n"
+#     messages: |
+#       msgid "my id"
+#       msgstr "my translation"
+#
+#       msgid "one"
+#       msgid_plural "many"
+#       msgstr[0] "just one"
+#       msgstr[1] "a lot"
+#   - lang: en
+#     domain: djangojs
+#     messages: |
+#       msgid "my id"
+#       msgstr "my JS translation"
+SIMPLETHEME_I18N_DJANGO: []

playbooks/roles/simple_theme/tasks/deploy.yml

@@ -137,3 +137,45 @@
       owner: "{{ edxapp_user }}"
       group: "{{ common_web_group }}"
     with_items: "{{ SIMPLETHEME_STATIC_FILES_URLS }}"
+
+# Handle translations.
+- block:
+  - name: Install needed packages
+    apt:
+      name: gettext
+      state: present
+      update_cache: true
+      cache_valid_time: 3600
+  - name: Create directories for django translations
+    file:
+      path: "{{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES"
+      state: directory
+      owner: "{{ edxapp_user }}"
+      group: "{{ common_web_group }}"
+    with_items: "{{ SIMPLETHEME_I18N_DJANGO }}"
+  - name: Make sure .po files exist
+    file:
+      path: "{{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po"
+      state: touch
+      owner: "{{ edxapp_user }}"
+      group: "{{ common_web_group }}"
+    with_items: "{{ SIMPLETHEME_I18N_DJANGO }}"
+  - name: Create temporary .po files with custom translations
+    template:
+      src: "i18n/domain.po.j2"
+      dest: "{{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po_"
+      owner: "{{ edxapp_user }}"
+      group: "{{ common_web_group }}"
+    with_items: "{{ SIMPLETHEME_I18N_DJANGO }}"
+  - name: Merge temporary .po files into default translations
+    shell: >
+      msgcat --use-first {{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po_
+      {{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po >
+      {{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po
+    with_items: "{{ SIMPLETHEME_I18N_DJANGO }}"
+  - name: Compile .po files into .mo
+    shell: >
+      msgfmt {{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po
+      -o {{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.mo
+    with_items: "{{ SIMPLETHEME_I18N_DJANGO }}"
+  when: SIMPLETHEME_I18N_DJANGO | length > 0

playbooks/roles/simple_theme/templates/i18n/domain.po.j2

@@ -0,0 +1,12 @@
+msgid ""
+msgstr ""
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: {{ item.lang }}\n"
+{% if 'headers' in item -%}
+{{ item.headers }}
+{%- endif %}
+
+{% if 'messages' in item -%}
+{{ item.messages }}
+{%- endif %}

playbooks/roles/uptime_report/defaults/main.yml

@@ -0,0 +1,24 @@
+#
+# Open edX Retirement Pipeline Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+# Deploy uptime_report
+#
+# See documentation in README.rst
+
+#
+# This file contains the variables you'll need to pass to the role, and some
+# example values.
+
+uptime_report_script: "/edx/var/upload_uptime_report.py"
+uptime_report_cron_job_days: 6
+uptime_report_cron_job_minutes: 0
+newrelic_query_key: ""
+newrelic_query_api_url: ""
+uptime_report_s3_bucket: ""
+uptime_report_s3_bucket_folder: ""

playbooks/roles/uptime_report/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role uptime_report
+#
+# Overview:
+# This role will be used to set up automatic querying of daily uptime reports
+# and upload it onto an S3 bucket
+#
+# Example play:
+#
+#
+
+- name: Create python script to upload query and upload uptime reports
+  template:
+    dest: "{{ uptime_report_script }}"
+    src: "uptime_report_script.j2"
+    mode: 0755
+    owner: root
+    group: root
+
+- name: Install cron job for automatically running the uptime report script
+  cron:
+    name: "Run uptime report script"
+    job: "{{ uptime_report_script }}"
+    hour: "{{ uptime_report_cron_job_days }}"
+    minute: "{{ uptime_report_cron_job_minutes }}"
+    day: "*"

playbooks/roles/uptime_report/templates/uptime_report_script.j2

@@ -0,0 +1,59 @@
+#!/usr/bin/python3
+
+import boto3
+import json
+import datetime
+from six.moves import urllib
+
+newrelic_query_key = '{{ newrelic_query_key }}'
+newrelic_query_api_url = '{{ newrelic_query_api_url }}'
+uptime_report_s3_bucket = '{{ uptime_report_s3_bucket }}'
+uptime_report_s3_bucket_folder = '{{ uptime_report_s3_bucket_folder }}'
+newrelic_monitor_name = 'https://{{ EDXAPP_SITE_NAME }}/'
+aws_access_key_id = '{{ AWS_S3_LOGS_ACCESS_KEY_ID }}'
+aws_secret_access_key = '{{ AWS_S3_LOGS_SECRET_KEY }}'
+
+nr_query = ("SELECT percentage(count(*), WHERE result = 'SUCCESS') as 'UPTIME' "
+            "FROM SyntheticCheck WHERE monitorName = '{0}' "
+            "FACET dateOf(timestamp), hourOf(timestamp) "
+            "SINCE '{1} 00:00:00' UNTIL '{2} 00:00:00' "
+            "LIMIT MAX")
+
+def get_uptime_data(date):
+    from_time = (date+datetime.timedelta(days=-1)).strftime("%Y-%m-%d")
+    to_time = date.strftime("%Y-%m-%d")
+    data = urllib.parse.urlencode({
+        'nrql': nr_query.format(newrelic_monitor_name, from_time, to_time)
+    })
+    headers = {'X-Query-Key': newrelic_query_key, 'Accept': 'application/json'}
+    request = urllib.request.Request(newrelic_query_api_url+'?'+data, headers=headers)
+
+    response =  urllib.request.urlopen(request)
+    data = json.loads(response.read().decode('utf-8'))
+    response.close()
+    times = []
+
+    for facet in data['facets']:
+        date, time = facet['name'][0], facet['name'][1]
+        times.append({
+            'time': datetime.datetime.strptime('{0} {1}'.format(date, time), '%B %d, %Y %H:%M').strftime("%Y-%m-%d %H:%M"),
+            'uptime_percentage': float(facet['results'][0]['result']),
+        })
+    times = sorted(times, key=lambda x: x['time'])
+    return times
+
+def upload_to_s3(uptime_data, date):
+    filepath = 'uptime/'
+    if uptime_report_s3_bucket_folder:
+        filepath = '{0}{1}/'.format(filepath, uptime_report_s3_bucket_folder)
+
+    filename = '{0}uptime-{1}.json'.format(filepath, (date+datetime.timedelta(days=-1)).strftime("%Y-%m-%d"))
+    uptime_data = json.dumps(uptime_data)
+
+    s3_client = boto3.client('s3',aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
+    s3_client.put_object(Body=uptime_data, Bucket=uptime_report_s3_bucket, Key=filename)
+
+if __name__ == "__main__":
+    today = datetime.datetime.now()
+    uptime_data = get_uptime_data(today)
+    upload_to_s3(uptime_data, today)