[BB-6123] cherry-pick of celery upgrade configurations

CHANGELOG.md

@@ -28,6 +28,10 @@ Add any new changes to the top (right below this line).
     - Role: discovery
        - Replaced `ELASTICSEARCH_URL` with `ELASTICSEARCH_CLUSTER_URL` in `discovery_service_config_overrides`.
 
+ - 2021-11-30
+    - Upgrade celery to 5.2.0 and adjust CLI call parameters too
+    - Bumped single-beat to use a more supported fork of the project
+
  - 2021-10-20
     - Role neo4j
        - Upgrade Neo4j from 3.3.1 to 3.5.28.

playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2

@@ -13,11 +13,6 @@ stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh worker -A {{ edx_django_service_name }} --app {{ edx_django_service_name }}.celery:app --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency=1 {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }}
 killasgroup=true
 stopwaitsecs={{ w.stopwaitsecs | default(edx_django_service_default_stopwaitsecs) }}
-; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit
-; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers
-; are auto restarted even when exiting with code 0.
-; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0.
-autorestart=true
 
 {% endfor %}
 {% endfor %}
@@ -42,11 +37,6 @@ stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh worker -A {{ edx_django_service_name }} --app {{ worker_app_name }}.celery:app --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }}
 killasgroup=true
 stopwaitsecs={{ w.stopwaitsecs | default(edx_django_service_default_stopwaitsecs) }}
-; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit
-; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers
-; are auto restarted even when exiting with code 0.
-; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0.
-autorestart=true
 
 {% endfor %}
 

playbooks/roles/edxapp/defaults/main.yml

@@ -185,7 +185,11 @@ EDXAPP_ENABLE_CELERY_BEAT: false
 EDXAPP_SINGLE_BEAT_LOCK_TIME: 60
 # EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL must be smaller than EDXAPP_SINGLE_BEAT_LOCK_TIME / 2
 EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL: 29
-EDXAPP_SINGLE_BEAT_VERSION: "0.4.2"
+EDXAPP_SINGLE_BEAT_REPO: "https://github.com/akachanov/single-beat.git"
+EDXAPP_SINGLE_BEAT_VERSION: "e500ac4b56756cdf96836666883af8060aaef455"
+EDXAPP_SINGLE_BEAT_USER: "{{ EDXAPP_CELERY_USER }}"
+EDXAPP_SINGLE_BEAT_PASSWORD: "{{ EDXAPP_CELERY_PASSWORD }}"
+EDXAPP_SINGLE_BEAT_IDENTIFIER: "celerybeat"
 EDXAPP_BRANCH_IO_KEY: ""
 
 EDXAPP_AUTH_USE_OPENID_PROVIDER: true

playbooks/roles/edxapp/tasks/deploy.yml

@@ -379,8 +379,7 @@
 
 - name: install single-beat to run only one celerybeat scheduler
   pip:
-    name: single-beat
-    version: "{{ EDXAPP_SINGLE_BEAT_VERSION|default(omit) }}"
+    name: "git+{{ EDXAPP_SINGLE_BEAT_REPO }}@{{ EDXAPP_SINGLE_BEAT_VERSION|default(master) }}#egg=single-beat"
     virtualenv: "{{ edxapp_venv_dir }}"
     state: present
   become_user: "{{ edxapp_user }}"

playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2

@@ -5,7 +5,7 @@
 {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %}
 source {{ edxapp_app_dir }}/edxapp_env
 {% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/single-beat ' + edxapp_venv_bin + '/celery beat' %}
+{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/single-beat ' + edxapp_venv_bin + '/celery' %}
 
 export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini"
 if command -v ec2metadata >/dev/null 2>&1; then
@@ -14,9 +14,9 @@ if command -v ec2metadata >/dev/null 2>&1; then
   export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID"
 fi
 {% else %}
-{% set executable = edxapp_venv_bin + '/single-beat' + edxapp_venv_bin + '/celery beat' %}
+{% set executable = edxapp_venv_bin + '/single-beat' + edxapp_venv_bin + '/celery' %}
 {% endif %}
 
 # We exec so that celery is the child of supervisor and can be managed properly
 
-exec {{ executable }} $@
+exec {{ executable }} --config="${SERVICE_CONFIG}" beat $@

playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2

@@ -5,7 +5,7 @@
 {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %}
 source {{ edxapp_app_dir }}/edxapp_env
 {% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/celery worker' %}
+{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/celery' %}
 
 export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini"
 if command -v ec2metadata >/dev/null 2>&1; then
@@ -14,7 +14,7 @@ if command -v ec2metadata >/dev/null 2>&1; then
   export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID"
 fi
 {% else %}
-{% set executable = edxapp_venv_bin + '/celery worker' %}
+{% set executable = edxapp_venv_bin + '/celery' %}
 {% endif %}
 
 # We exec so that celery is the child of supervisor and can be managed properly

playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2

@@ -7,14 +7,9 @@ directory={{ edxapp_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
 stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 
-command={{ edxapp_app_dir }}/worker.sh --app={{ w.service_variant }}.celery:APP --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }}
+command={{ edxapp_app_dir }}/worker.sh --app={{ w.service_variant }}.celery:APP worker --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }}
 killasgroup=true
 stopwaitsecs={{ w.stopwaitsecs | default(EDXAPP_WORKER_DEFAULT_STOPWAITSECS) }}
-; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit
-; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers
-; are auto restarted even when exiting with code 0.
-; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0.
-autorestart=true
 
 {% endfor %}
 
@@ -37,21 +32,28 @@ environment=
     EDX_REST_API_CLIENT_NAME=edx.lms.core.default,
     SINGLE_BEAT_LOCK_TIME={{ EDXAPP_SINGLE_BEAT_LOCK_TIME }},
     SINGLE_BEAT_HEARTBEAT_INTERVAL={{ EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL }},
-    SINGLE_BEAT_IDENTIFIER="celerybeat",
-    SINGLE_BEAT_REDIS_SERVER="redis://{{ EDXAPP_CELERY_BROKER_HOSTNAME }}/{{ EDXAPP_CELERY_BROKER_VHOST }}",
-    SINGLE_BEAT_WAIT_MODE="supervised"
+    SINGLE_BEAT_IDENTIFIER="{{ EDXAPP_SINGLE_BEAT_IDENTIFIER }}",
+    SINGLE_BEAT_REDIS_SERVER="redis://{{ EDXAPP_SINGLE_BEAT_USER }}:{{ EDXAPP_SINGLE_BEAT_PASSWORD }}@{{ EDXAPP_CELERY_BROKER_HOSTNAME }}/{{ EDXAPP_CELERY_BROKER_VHOST }}",
+    SINGLE_BEAT_WAIT_MODE="supervised",
+    SERVICE_CONFIG="lms.envs.{{ worker_django_settings_module }}"
 user={{ common_web_user }}
 directory={{ edxapp_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
 stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 
-command={{ edxapp_app_dir }}/beat_scheduler.sh --config=lms.envs.{{ worker_django_settings_module }} --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid"
+command={{ edxapp_app_dir }}/beat_scheduler.sh --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid"
 
 killasgroup=true
 stopwaitsecs={{ EDXAPP_WORKER_DEFAULT_STOPWAITSECS }}
-; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit
-; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers
-; are auto restarted even when exiting with code 0.
-; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0.
+
+# If multiple app servers are running, celery-beat will exit on the new app
+# server as it is already running on the old one. Once the process exited with
+# status 0, it wont restart again event the old server is deprovisioned. To
+# resolve this conflict, use `autorestart=true` to keep trying to start the
+# process. Startretries are scheduled 3 times by default; since there is no way
+# to set infinite retries, the recommended way is setting a high number. When
+# the previous server is stopped, celery-beat will start normally.
 autorestart=true
+startsecs=30
+startretries=10000
 {% endif %}