diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index bb8c064e30f..7f5a5979dc4 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -173,6 +173,12 @@ EDXAPP_CELERY_BROKER_VHOST: "" EDXAPP_CELERY_BROKER_USE_SSL: false EDXAPP_CELERY_EVENT_QUEUE_TTL: !!null EDXAPP_CELERY_TIMEZONE: "UTC" +EDXAPP_CELERYBEAT_SCHEDULER: "celery.beat:PersistentScheduler" +EDXAPP_ENABLE_CELERY_BEAT: false +EDXAPP_SINGLE_BEAT_LOCK_TIME: 60 +# EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL must be smaller than EDXAPP_SINGLE_BEAT_LOCK_TIME / 2 +EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL: 29 +EDXAPP_SINGLE_BEAT_VERSION: "0.4.2" EDXAPP_BRANCH_IO_KEY: "" EDXAPP_AUTH_USE_OPENID_PROVIDER: true @@ -1409,6 +1415,7 @@ generic_env_config: &edxapp_generic_env LOCATION: "{{ EDXAPP_CACHE_COURSE_STRUCTURE_MEMCACHE }}" # Default to two hours TIMEOUT: "7200" + CELERYBEAT_SCHEDULER: "{{ EDXAPP_CELERYBEAT_SCHEDULER }}" CELERY_BROKER_TRANSPORT: "{{ EDXAPP_CELERY_BROKER_TRANSPORT }}" CELERY_BROKER_HOSTNAME: "{{ EDXAPP_CELERY_BROKER_HOSTNAME }}" COMMENTS_SERVICE_URL: "{{ EDXAPP_COMMENTS_SERVICE_URL }}" diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index cad02e4cb73..e51271a6fd6 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -357,6 +357,7 @@ with_items: - "lms.sh" - "cms.sh" + - "beat_scheduler.sh" - "worker.sh" - "reload_lms_config.sh" - "reload_cms_config.sh" @@ -370,6 +371,18 @@ when: - celery_worker is not defined +- name: install single-beat to run only one celerybeat scheduler + pip: + name: single-beat + version: "{{ EDXAPP_SINGLE_BEAT_VERSION|default(omit) }}" + virtualenv: "{{ edxapp_venv_dir }}" + state: present + become_user: "{{ edxapp_user }}" + when: EDXAPP_ENABLE_CELERY_BEAT + tags: + - install + - install:app-requirements + # creates the supervisor jobs for the # service variants configured, runs # gather_assets and db migrations diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 new file mode 100644 index 00000000000..acf54f89058 --- /dev/null +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# {{ ansible_managed }} + +{% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %} +source {{ edxapp_app_dir }}/edxapp_env +{% if COMMON_ENABLE_NEWRELIC_APP %} +{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/single-beat ' + edxapp_venv_bin + '/celery beat' %} + +export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini" +if command -v ec2metadata >/dev/null 2>&1; then + INSTANCEID=$(ec2metadata --instance-id); + HOSTNAME=$(hostname) + export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID" +fi +{% else %} +{% set executable = edxapp_venv_bin + '/single-beat' + edxapp_venv_bin + '/celery beat' %} +{% endif %} + +# We exec so that celery is the child of supervisor and can be managed properly + +exec {{ executable }} $@ diff --git a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 index 949d3e1b81e..7165d51760f 100644 --- a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 @@ -20,3 +20,38 @@ autorestart=true [group:edxapp_worker] programs={%- for w in edxapp_workers %}{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}{%- if not loop.last %},{%- endif %}{%- endfor %} + + +{% if EDXAPP_ENABLE_CELERY_BEAT|bool and EDXAPP_CELERY_BROKER_TRANSPORT == "redis" %} +[program:celerybeat_scheduler] +environment= + {% if COMMON_ENABLE_NEWRELIC_APP %} + NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_WORKERS_APPNAME }}-lms, + NEW_RELIC_DISTRIBUTED_TRACING_ENABLED={{ EDXAPP_WORKERS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}, + NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }}, + {% endif -%} + LANG={{ EDXAPP_LANG }}, + PYTHONPATH={{ edxapp_code_dir }}, + SERVICE_VARIANT=lms, + BOTO_CONFIG="{{ edxapp_app_dir }}/.boto", + EDX_REST_API_CLIENT_NAME=edx.lms.core.default, + SINGLE_BEAT_LOCK_TIME={{ EDXAPP_SINGLE_BEAT_LOCK_TIME }}, + SINGLE_BEAT_HEARTBEAT_INTERVAL={{ EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL }}, + SINGLE_BEAT_IDENTIFIER="celerybeat", + SINGLE_BEAT_REDIS_SERVER="redis://{{ EDXAPP_CELERY_BROKER_HOSTNAME }}/{{ EDXAPP_CELERY_BROKER_VHOST }}", + SINGLE_BEAT_WAIT_MODE="supervised" +user={{ common_web_user }} +directory={{ edxapp_code_dir }} +stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log +stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log + +command={{ edxapp_app_dir }}/beat_scheduler.sh --config=lms.envs.{{ worker_django_settings_module }} --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid" + +killasgroup=true +stopwaitsecs={{ EDXAPP_WORKER_DEFAULT_STOPWAITSECS }} +; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit +; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers +; are auto restarted even when exiting with code 0. +; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0. +autorestart=true +{% endif %}