[SE-4860] cherry-pick configuration/pull/6586

CHANGELOG.md

@@ -16,6 +16,54 @@ Add any new changes to the top (right below this line).
    - Role: discovery
      - Replaced `ELASTICSEARCH_URL` with `ELASTICSEARCH_CLUSTER_URL` in `discovery_service_config_overrides`.
 
+ - 2021-11-30
+    - Upgrade celery to 5.2.0 and adjust CLI call parameters too
+    - Bumped single-beat to use a more supported fork of the project
+
+ - 2021-11-01
+    - Docker: edxapp
+       - Removed unnecessary `CELERY_QUEUES` overrides for LMS and Studio.
+         Instead, just use the default value of `CELERY_QUEUES` as set in
+         edx-platform's settings files.
+         Functionally, this means that in addition to the existing queues
+         that LMS and Studio defined, there is now a "low priority" queue
+         for Studio, suitable for tasks like a CourseGraph dump.
+
+ - 2021-10-20
+    - Role neo4j
+       - Upgrade Neo4j from 3.2.2 to 3.5.28.
+
+ - 2021-08-26
+    - Role neo4j
+      - Bring Neo4j role closer in with what we really deploy:
+        - Change Neo4j version from 3.2.2 to 3.3.1.
+        - Expose Bolt on 0.0.0.0:7687 with optional encryption.
+        - Enable `dbms.allow_upgrade`, which is the new name of the `dbms.allow_format_migration` key.
+        - Remove http->https redirection logic when NGINX_ENABLE_SSL is false.
+
+ - 2021-09-28
+    - Role nginx
+      - Add `NGINX_ENABLE_IPV6` configuration variable to make nginx
+        services listen on the IPv6 wildcard address (in addition to
+        the IPv4 one, where services always listen). Defaults to true.
+
+ - 2021-09-19
+    - Remove configuration for edx-certificates, as that repo and service are no longer used.
+
+ - 2021-07-29
+    - Role edxapp
+       - Add `EDXAPP_ENABLE_MONGODB_INDEXES` configuration variable to optionally set up indexes on edxapp mongodb.
+    - Role forum
+       - Add `FORUM_ENABLE_MONGODB_INDEXES` configuration variable to optionally set up indexes on forum mongodb.
+
+ - 2021-07-19
+     - Role: edx_django_service
+        - Allows writing extra requirements to an 'extra.txt' requirements file in the service's requirements directory.
+     - Role: ecommerce
+        - Adds an optional flag to write the extra requirements to an 'extra.txt' file since many of the app's setup commands
+          use tox and that creates its own environments separate from the default ecommerce virtualenv environment where the
+          `ECOMMERCE_EXTRA_REQUIREMENTS` requirements are installed.
+
  - 2021-06-17
     - Role credentials
        - Installs extra python packages specified in `CREDENTIALS_EXTRA_REQUIREMENTS` (defaults to `[]`).

playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2

@@ -17,4 +17,4 @@ export NEW_RELIC_LICENSE_KEY='{{ NEWRELIC_LICENSE_KEY }}'
 
 source {{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}_env
 # We exec so that celery is the child of supervisor and can be managed properly
-exec {{ executable }} -A ecommerce_worker worker --app ecommerce_worker.celery_app:app --concurrency={{ ECOMMERCE_WORKER_CONCURRENCY }} --loglevel=info --hostname=ecomworker.%%h --queue=ecommerce.fulfillment,ecommerce.email_marketing
+exec {{ executable }} --app ecommerce_worker.celery_app:app worker -A ecommerce_worker --concurrency={{ ECOMMERCE_WORKER_CONCURRENCY }} --loglevel=info --hostname=ecomworker.%%h --queue=ecommerce.fulfillment,ecommerce.email_marketing

playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2

@@ -10,14 +10,9 @@ directory={{ edx_django_service_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
 stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 
-command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh worker -A {{ edx_django_service_name }} --app {{ edx_django_service_name }}.celery:app --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency=1 {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }}
+command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh --app {{ edx_django_service_name }}.celery:app worker -A {{ edx_django_service_name }} --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency=1 {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }}
 killasgroup=true
 stopwaitsecs={{ w.stopwaitsecs | default(edx_django_service_default_stopwaitsecs) }}
-; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit
-; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers
-; are auto restarted even when exiting with code 0.
-; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0.
-autorestart=true
 
 {% endfor %}
 {% endfor %}
@@ -39,14 +34,9 @@ stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 {% set worker_app_name = 'course_discovery' %}
 {% endif %}
 
-command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh worker -A {{ edx_django_service_name }} --app {{ worker_app_name }}.celery:app --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }}
+command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh --app {{ worker_app_name }}.celery:app worker -A {{ edx_django_service_name }} --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }}
 killasgroup=true
 stopwaitsecs={{ w.stopwaitsecs | default(edx_django_service_default_stopwaitsecs) }}
-; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit
-; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers
-; are auto restarted even when exiting with code 0.
-; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0.
-autorestart=true
 
 {% endfor %}
 

playbooks/roles/edxapp/defaults/main.yml

@@ -178,7 +178,11 @@ EDXAPP_ENABLE_CELERY_BEAT: false
 EDXAPP_SINGLE_BEAT_LOCK_TIME: 60
 # EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL must be smaller than EDXAPP_SINGLE_BEAT_LOCK_TIME / 2
 EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL: 29
-EDXAPP_SINGLE_BEAT_VERSION: "0.4.2"
+EDXAPP_SINGLE_BEAT_REPO: "https://github.com/akachanov/single-beat.git"
+EDXAPP_SINGLE_BEAT_VERSION: "e500ac4b56756cdf96836666883af8060aaef455"
+EDXAPP_SINGLE_BEAT_USER: "{{ EDXAPP_CELERY_USER }}"
+EDXAPP_SINGLE_BEAT_PASSWORD: "{{ EDXAPP_CELERY_PASSWORD }}"
+EDXAPP_SINGLE_BEAT_IDENTIFIER: "celerybeat"
 EDXAPP_BRANCH_IO_KEY: ""
 
 EDXAPP_AUTH_USE_OPENID_PROVIDER: true

playbooks/roles/edxapp/tasks/deploy.yml

@@ -367,8 +367,7 @@
 
 - name: install single-beat to run only one celerybeat scheduler
   pip:
-    name: single-beat
-    version: "{{ EDXAPP_SINGLE_BEAT_VERSION|default(omit) }}"
+    name: "git+{{ EDXAPP_SINGLE_BEAT_REPO }}@{{ EDXAPP_SINGLE_BEAT_VERSION|default(master) }}#egg=single-beat"
     virtualenv: "{{ edxapp_venv_dir }}"
     state: present
   become_user: "{{ edxapp_user }}"

playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2

@@ -5,7 +5,7 @@
 {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %}
 source {{ edxapp_app_dir }}/edxapp_env
 {% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/single-beat ' + edxapp_venv_bin + '/celery beat' %}
+{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/single-beat ' + edxapp_venv_bin + '/celery' %}
 
 export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini"
 if command -v ec2metadata >/dev/null 2>&1; then
@@ -14,9 +14,9 @@ if command -v ec2metadata >/dev/null 2>&1; then
   export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID"
 fi
 {% else %}
-{% set executable = edxapp_venv_bin + '/single-beat' + edxapp_venv_bin + '/celery beat' %}
+{% set executable = edxapp_venv_bin + '/single-beat' + edxapp_venv_bin + '/celery' %}
 {% endif %}
 
 # We exec so that celery is the child of supervisor and can be managed properly
 
-exec {{ executable }} $@
+exec {{ executable }} --config="${SERVICE_CONFIG}" beat $@

playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2

@@ -5,7 +5,7 @@
 {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %}
 source {{ edxapp_app_dir }}/edxapp_env
 {% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/celery worker' %}
+{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/celery' %}
 
 export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini"
 if command -v ec2metadata >/dev/null 2>&1; then
@@ -14,7 +14,7 @@ if command -v ec2metadata >/dev/null 2>&1; then
   export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID"
 fi
 {% else %}
-{% set executable = edxapp_venv_bin + '/celery worker' %}
+{% set executable = edxapp_venv_bin + '/celery' %}
 {% endif %}
 
 # We exec so that celery is the child of supervisor and can be managed properly

playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2

@@ -7,14 +7,9 @@ directory={{ edxapp_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
 stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 
-command={{ edxapp_app_dir }}/worker.sh --app={{ w.service_variant }}.celery:APP --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }}
+command={{ edxapp_app_dir }}/worker.sh --app={{ w.service_variant }}.celery:APP worker --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }}
 killasgroup=true
 stopwaitsecs={{ w.stopwaitsecs | default(EDXAPP_WORKER_DEFAULT_STOPWAITSECS) }}
-; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit
-; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers
-; are auto restarted even when exiting with code 0.
-; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0.
-autorestart=true
 
 {% endfor %}
 
@@ -37,21 +32,28 @@ environment=
     EDX_REST_API_CLIENT_NAME=edx.lms.core.default,
     SINGLE_BEAT_LOCK_TIME={{ EDXAPP_SINGLE_BEAT_LOCK_TIME }},
     SINGLE_BEAT_HEARTBEAT_INTERVAL={{ EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL }},
-    SINGLE_BEAT_IDENTIFIER="celerybeat",
-    SINGLE_BEAT_REDIS_SERVER="redis://{{ EDXAPP_CELERY_BROKER_HOSTNAME }}/{{ EDXAPP_CELERY_BROKER_VHOST }}",
-    SINGLE_BEAT_WAIT_MODE="supervised"
+    SINGLE_BEAT_IDENTIFIER="{{ EDXAPP_SINGLE_BEAT_IDENTIFIER }}",
+    SINGLE_BEAT_REDIS_SERVER="redis://{{ EDXAPP_SINGLE_BEAT_USER }}:{{ EDXAPP_SINGLE_BEAT_PASSWORD }}@{{ EDXAPP_CELERY_BROKER_HOSTNAME }}/{{ EDXAPP_CELERY_BROKER_VHOST }}",
+    SINGLE_BEAT_WAIT_MODE="supervised",
+    SERVICE_CONFIG="lms.envs.{{ worker_django_settings_module }}"
 user={{ common_web_user }}
 directory={{ edxapp_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
 stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 
-command={{ edxapp_app_dir }}/beat_scheduler.sh --config=lms.envs.{{ worker_django_settings_module }} --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid"
+command={{ edxapp_app_dir }}/beat_scheduler.sh --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid"
 
 killasgroup=true
 stopwaitsecs={{ EDXAPP_WORKER_DEFAULT_STOPWAITSECS }}
-; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit
-; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers
-; are auto restarted even when exiting with code 0.
-; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0.
+
+# If multiple app servers are running, celery-beat will exit on the new app
+# server as it is already running on the old one. Once the process exited with
+# status 0, it wont restart again event the old server is deprovisioned. To
+# resolve this conflict, use `autorestart=true` to keep trying to start the
+# process. Startretries are scheduled 3 times by default; since there is no way
+# to set infinite retries, the recommended way is setting a high number. When
+# the previous server is stopped, celery-beat will start normally.
 autorestart=true
+startsecs=30
+startretries=10000
 {% endif %}

playbooks/roles/git_clone/tasks/main.yml

@@ -111,6 +111,15 @@
     - install
     - install:code
 
+- name: Run git safe.directory
+  shell: git config --global --add safe.directory {{ item.DESTINATION }}
+  become: true
+  with_items: "{{ GIT_REPOS }}"
+  no_log: "{{ GIT_CLONE_NO_LOGGING }}"
+  tags:
+    - install
+    - install:code
+
 - name: Run git clean after checking out code
   shell: cd {{ item.DESTINATION }} && git clean -xdf
   become: true