From 3bd3d1ea4ce0f70def33082bd16328a8dea862c3 Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Wed, 23 Jun 2021 13:53:27 +0930 Subject: [PATCH 1/2] chore: constrain kombu to support redis6 auth Constrains kombu to https://github.com/open-craft/kombu/tree/ggabor/FAL-2030-4.6.11 to allow us to use celery with multi-tenant redis. Ran `make update` to incorporate this constraint into the requirements. --- requirements/constraints.txt | 6 ++++ requirements/edx-sandbox/py35.txt | 2 +- requirements/edx-sandbox/py38.txt | 4 +-- requirements/edx/base.txt | 44 +++++++++-------------- requirements/edx/coverage.txt | 4 ++- requirements/edx/development.txt | 58 ++++++++++--------------------- requirements/edx/testing.txt | 58 ++++++++++--------------------- 7 files changed, 64 insertions(+), 112 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 330b1fedfc91..aea825e73aa7 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -8,6 +8,12 @@ # pin when possible. Writing an issue against the offending project and # linking to it here is good. +# kombu needs additional functionality to support multi-tenant redis +# This branch backports the following PRs to kombu 4.6.11, because celery 4.4.7 has requirement kombu<4.7,>=4.6.10 +# https://github.com/celery/kombu/pull/1351 +# https://github.com/celery/kombu/pull/1349 +# https://github.com/celery/kombu/pull/1376 +git+https://github.com/open-craft/kombu.git@gabor/FAL-2030-4.6.11#egg=kombu==4.6.11.1 # This file contains all common constraints for edx-repos -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt diff --git a/requirements/edx-sandbox/py35.txt b/requirements/edx-sandbox/py35.txt index d8ed570665da..adc2e8ef55f7 100644 --- a/requirements/edx-sandbox/py35.txt +++ b/requirements/edx-sandbox/py35.txt @@ -80,7 +80,7 @@ pytz==2021.1 # via matplotlib random2==1.0.1 # via -r requirements/edx-sandbox/py35.in -regex==2021.8.21 +regex==2021.8.28 # via nltk scipy==1.2.1 # via diff --git a/requirements/edx-sandbox/py38.txt b/requirements/edx-sandbox/py38.txt index d8557d48a881..690ad6105add 100644 --- a/requirements/edx-sandbox/py38.txt +++ b/requirements/edx-sandbox/py38.txt @@ -24,7 +24,7 @@ decorator==4.4.2 # via networkx joblib==1.0.1 # via nltk -kiwisolver==1.3.1 +kiwisolver==1.3.2 # via matplotlib lxml==4.5.0 # via @@ -71,7 +71,7 @@ python-dateutil==2.4.0 # matplotlib random2==1.0.1 # via -r requirements/edx-sandbox/py38.in -regex==2021.8.21 +regex==2021.8.28 # via nltk scipy==1.7.1 # via diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 1dc26e2b2fc3..11a1fd12d6fd 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -64,7 +64,7 @@ beautifulsoup4==4.9.3 # via pynliner billiard==3.6.4.0 # via celery -bleach==4.0.0 +bleach==4.1.0 # via # -r requirements/edx/base.in # django-wiki @@ -142,7 +142,6 @@ cryptography==3.4.8 # -r requirements/edx/base.in # django-fernet-fields # edx-enterprise - # py2neo # pyjwt # social-auth-core cssutils==2.3.0 @@ -300,7 +299,7 @@ django-model-utils==4.1.1 # edxval # ora2 # super-csv -django-mptt==0.13.1 +django-mptt==0.13.2 # via # -r requirements/edx/base.in # django-wiki @@ -381,8 +380,6 @@ djangorestframework==3.12.4 # super-csv djangorestframework-xml==2.0.0 # via edx-enterprise -docker==5.0.0 - # via py2neo docopt==0.6.2 # via xmodule docutils==0.17.1 @@ -474,7 +471,7 @@ edx-opaque-keys[django]==2.2.2 # xmodule edx-organizations==6.10.0 # via -r requirements/edx/base.in -edx-proctoring==3.23.9 +edx-proctoring==3.24.0 # via # -r requirements/edx/base.in # edx-proctoring-proctortrack @@ -519,8 +516,6 @@ elasticsearch==7.13.4 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # edx-search -english==2020.7.0 - # via py2neo enmerkar==0.7.1 # via enmerkar-underscore enmerkar-underscore==2.1.0 @@ -568,6 +563,8 @@ idna==3.2 # yarl inflection==0.5.1 # via drf-yasg +interchange==2021.0.3 + # via py2neo ipaddress==1.0.23 # via -r requirements/edx/base.in isodate==0.6.0 @@ -599,8 +596,10 @@ jsonfield2==3.0.3 # edx-submissions # lti-consumer-xblock # ora2 -kombu==4.6.11 - # via celery +git+https://github.com/open-craft/kombu.git@gabor/FAL-2030-4.6.11#egg=kombu==4.6.11.1 + # via + # -c requirements/edx/../constraints.txt + # celery laboratory==1.0.2 # via -r requirements/edx/base.in lazy==1.4 @@ -676,9 +675,7 @@ multidict==5.1.0 # yarl mysqlclient==2.0.3 # via -r requirements/edx/base.in -neotime==1.7.4 - # via py2neo -newrelic==6.8.0.163 +newrelic==6.8.1.164 # via # -r requirements/edx/base.in # edx-django-utils @@ -738,13 +735,11 @@ pillow==8.3.1 # edx-organizations polib==1.1.1 # via edx-i18n-tools -prompt-toolkit==3.0.20 - # via py2neo psutil==5.8.0 # via # -r requirements/edx/paver.txt # edx-django-utils -py2neo==2021.1.5 +py2neo==2021.2.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in @@ -849,10 +844,9 @@ pytz==2021.1 # event-tracking # fs # icalendar - # neotime + # interchange # olxcleaner # ora2 - # py2neo # tincan # xblock pyuca==1.2 @@ -870,7 +864,7 @@ recommender-xblock==2.0.1 # via -r requirements/edx/base.in redis==3.5.3 # via -r requirements/edx/base.in -regex==2021.8.21 +regex==2021.8.28 # via nltk requests==2.26.0 # via @@ -878,7 +872,6 @@ requests==2.26.0 # analytics-python # coreapi # django-oauth-toolkit - # docker # edx-analytics-data-api-client # edx-bulk-grades # edx-drf-extensions @@ -902,7 +895,7 @@ rest-condition==1.0.3 # via # -r requirements/edx/base.in # edx-drf-extensions -ruamel.yaml==0.17.13 +ruamel.yaml==0.17.14 # via drf-yasg ruamel.yaml.clib==0.2.6 # via ruamel.yaml @@ -923,7 +916,7 @@ semantic-version==2.8.5 # via edx-drf-extensions shapely==1.7.1 # via -r requirements/edx/base.in -simplejson==3.17.4 +simplejson==3.17.5 # via # -r requirements/edx/base.in # sailthru-client @@ -948,14 +941,13 @@ six==1.16.0 # edx-i18n-tools # edx-milestones # edx-rbac - # english # event-tracking # fs # fs-s3fs # html5lib + # interchange # isodate # libsass - # neotime # pansi # paver # py2neo @@ -1052,8 +1044,6 @@ voluptuous==0.12.1 # via ora2 watchdog==2.1.5 # via -r requirements/edx/paver.txt -wcwidth==0.2.5 - # via prompt-toolkit web-fragments==1.1.0 # via # -r requirements/edx/base.in @@ -1070,8 +1060,6 @@ webob==1.8.7 # via # xblock # xmodule -websocket-client==1.2.1 - # via docker wrapt==1.11.2 # via # -c requirements/edx/../constraints.txt diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index fdbe423d4a22..40d591f73e28 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -21,6 +21,8 @@ jinja2-pluralize==0.3.0 markupsafe==2.0.1 # via jinja2 pluggy==0.13.1 - # via diff-cover + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # diff-cover pygments==2.10.0 # via diff-cover diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index b87c064c060d..e129f3fe74ac 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -97,7 +97,7 @@ billiard==3.6.4.0 # via # -r requirements/edx/testing.txt # celery -bleach==4.0.0 +bleach==4.1.0 # via # -r requirements/edx/testing.txt # django-wiki @@ -201,7 +201,6 @@ cryptography==3.4.8 # -r requirements/edx/testing.txt # django-fernet-fields # edx-enterprise - # py2neo # pyjwt # social-auth-core cssselect==1.1.0 @@ -384,7 +383,7 @@ django-model-utils==4.1.1 # edxval # ora2 # super-csv -django-mptt==0.13.1 +django-mptt==0.13.2 # via # -r requirements/edx/testing.txt # django-wiki @@ -471,10 +470,6 @@ djangorestframework-xml==2.0.0 # via # -r requirements/edx/testing.txt # edx-enterprise -docker==5.0.0 - # via - # -r requirements/edx/testing.txt - # py2neo docopt==0.6.2 # via # -r requirements/edx/testing.txt @@ -579,7 +574,7 @@ edx-opaque-keys[django]==2.2.2 # xmodule edx-organizations==6.10.0 # via -r requirements/edx/testing.txt -edx-proctoring==3.23.9 +edx-proctoring==3.24.0 # via # -r requirements/edx/testing.txt # edx-proctoring-proctortrack @@ -631,10 +626,6 @@ elasticsearch==7.13.4 # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/edx/testing.txt # edx-search -english==2020.7.0 - # via - # -r requirements/edx/testing.txt - # py2neo enmerkar==0.7.1 # via # -r requirements/edx/testing.txt @@ -724,7 +715,7 @@ idna==3.2 # yarl imagesize==1.2.0 # via sphinx -importlib-metadata==4.6.4 +importlib-metadata==4.7.1 # via # -r requirements/edx/testing.txt # pytest-randomly @@ -740,6 +731,10 @@ iniconfig==1.1.1 # via # -r requirements/edx/testing.txt # pytest +interchange==2021.0.3 + # via + # -r requirements/edx/testing.txt + # py2neo ipaddress==1.0.23 # via -r requirements/edx/testing.txt isodate==0.6.0 @@ -793,8 +788,9 @@ jsonfield2==3.0.3 # ora2 jsonschema==3.2.0 # via sphinxcontrib-openapi -kombu==4.6.11 +git+https://github.com/open-craft/kombu.git@gabor/FAL-2030-4.6.11#egg=kombu==4.6.11.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt # celery laboratory==1.0.2 @@ -900,11 +896,7 @@ mypy-extensions==0.4.3 # via mypy mysqlclient==2.0.3 # via -r requirements/edx/testing.txt -neotime==1.7.4 - # via - # -r requirements/edx/testing.txt - # py2neo -newrelic==6.8.0.163 +newrelic==6.8.1.164 # via # -r requirements/edx/testing.txt # edx-django-utils @@ -984,6 +976,7 @@ platformdirs==2.2.0 # virtualenv pluggy==0.13.1 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/edx/testing.txt # diff-cover # pytest @@ -992,10 +985,6 @@ polib==1.1.1 # via # -r requirements/edx/testing.txt # edx-i18n-tools -prompt-toolkit==3.0.20 - # via - # -r requirements/edx/testing.txt - # py2neo psutil==5.8.0 # via # -r requirements/edx/testing.txt @@ -1008,7 +997,7 @@ py==1.10.0 # pytest # pytest-forked # tox -py2neo==2021.1.5 +py2neo==2021.2.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt @@ -1191,10 +1180,9 @@ pytz==2021.1 # event-tracking # fs # icalendar - # neotime + # interchange # olxcleaner # ora2 - # py2neo # tincan # xblock pyuca==1.2 @@ -1215,7 +1203,7 @@ recommender-xblock==2.0.1 # via -r requirements/edx/testing.txt redis==3.5.3 # via -r requirements/edx/testing.txt -regex==2021.8.21 +regex==2021.8.28 # via # -r requirements/edx/testing.txt # nltk @@ -1225,7 +1213,6 @@ requests==2.26.0 # analytics-python # coreapi # django-oauth-toolkit - # docker # edx-analytics-data-api-client # edx-bulk-grades # edx-drf-extensions @@ -1252,7 +1239,7 @@ rest-condition==1.0.3 # via # -r requirements/edx/testing.txt # edx-drf-extensions -ruamel.yaml==0.17.13 +ruamel.yaml==0.17.14 # via # -r requirements/edx/testing.txt # drf-yasg @@ -1288,7 +1275,7 @@ semantic-version==2.8.5 # edx-drf-extensions shapely==1.7.1 # via -r requirements/edx/testing.txt -simplejson==3.17.4 +simplejson==3.17.5 # via # -r requirements/edx/testing.txt # sailthru-client @@ -1317,17 +1304,16 @@ six==1.16.0 # edx-milestones # edx-rbac # edx-sphinx-theme - # english # event-tracking # freezegun # fs # fs-s3fs # html5lib # httpretty + # interchange # isodate # jsonschema # libsass - # neotime # pact-python # pansi # paver @@ -1522,10 +1508,6 @@ vulture==2.3 # via -r requirements/edx/development.in watchdog==2.1.5 # via -r requirements/edx/testing.txt -wcwidth==0.2.5 - # via - # -r requirements/edx/testing.txt - # prompt-toolkit web-fragments==1.1.0 # via # -r requirements/edx/testing.txt @@ -1544,10 +1526,6 @@ webob==1.8.7 # -r requirements/edx/testing.txt # xblock # xmodule -websocket-client==1.2.1 - # via - # -r requirements/edx/testing.txt - # docker wheel==0.37.0 # via # -r requirements/edx/pip-tools.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index aabdded096a5..a32ea458e65e 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -89,7 +89,7 @@ billiard==3.6.4.0 # via # -r requirements/edx/base.txt # celery -bleach==4.0.0 +bleach==4.1.0 # via # -r requirements/edx/base.txt # django-wiki @@ -189,7 +189,6 @@ cryptography==3.4.8 # -r requirements/edx/base.txt # django-fernet-fields # edx-enterprise - # py2neo # pyjwt # social-auth-core cssselect==1.1.0 @@ -367,7 +366,7 @@ django-model-utils==4.1.1 # edxval # ora2 # super-csv -django-mptt==0.13.1 +django-mptt==0.13.2 # via # -r requirements/edx/base.txt # django-wiki @@ -454,10 +453,6 @@ djangorestframework-xml==2.0.0 # via # -r requirements/edx/base.txt # edx-enterprise -docker==5.0.0 - # via - # -r requirements/edx/base.txt - # py2neo docopt==0.6.2 # via # -r requirements/edx/base.txt @@ -561,7 +556,7 @@ edx-opaque-keys[django]==2.2.2 # xmodule edx-organizations==6.10.0 # via -r requirements/edx/base.txt -edx-proctoring==3.23.9 +edx-proctoring==3.24.0 # via # -r requirements/edx/base.txt # edx-proctoring-proctortrack @@ -611,10 +606,6 @@ elasticsearch==7.13.4 # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/edx/base.txt # edx-search -english==2020.7.0 - # via - # -r requirements/edx/base.txt - # py2neo enmerkar==0.7.1 # via # -r requirements/edx/base.txt @@ -689,7 +680,7 @@ idna==3.2 # -r requirements/edx/base.txt # requests # yarl -importlib-metadata==4.6.4 +importlib-metadata==4.7.1 # via pytest-randomly inflect==5.3.0 # via @@ -701,6 +692,10 @@ inflection==0.5.1 # drf-yasg iniconfig==1.1.1 # via pytest +interchange==2021.0.3 + # via + # -r requirements/edx/base.txt + # py2neo ipaddress==1.0.23 # via -r requirements/edx/base.txt isodate==0.6.0 @@ -752,8 +747,9 @@ jsonfield2==3.0.3 # edx-submissions # lti-consumer-xblock # ora2 -kombu==4.6.11 +git+https://github.com/open-craft/kombu.git@gabor/FAL-2030-4.6.11#egg=kombu==4.6.11.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # celery laboratory==1.0.2 @@ -848,11 +844,7 @@ multidict==5.1.0 # yarl mysqlclient==2.0.3 # via -r requirements/edx/base.txt -neotime==1.7.4 - # via - # -r requirements/edx/base.txt - # py2neo -newrelic==6.8.0.163 +newrelic==6.8.1.164 # via # -r requirements/edx/base.txt # edx-django-utils @@ -923,6 +915,7 @@ platformdirs==2.2.0 # via virtualenv pluggy==0.13.1 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/edx/coverage.txt # diff-cover # pytest @@ -932,10 +925,6 @@ polib==1.1.1 # -r requirements/edx/base.txt # -r requirements/edx/testing.in # edx-i18n-tools -prompt-toolkit==3.0.20 - # via - # -r requirements/edx/base.txt - # py2neo psutil==5.8.0 # via # -r requirements/edx/base.txt @@ -947,7 +936,7 @@ py==1.10.0 # pytest # pytest-forked # tox -py2neo==2021.1.5 +py2neo==2021.2.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -1118,10 +1107,9 @@ pytz==2021.1 # event-tracking # fs # icalendar - # neotime + # interchange # olxcleaner # ora2 - # py2neo # tincan # xblock pyuca==1.2 @@ -1139,7 +1127,7 @@ recommender-xblock==2.0.1 # via -r requirements/edx/base.txt redis==3.5.3 # via -r requirements/edx/base.txt -regex==2021.8.21 +regex==2021.8.28 # via # -r requirements/edx/base.txt # nltk @@ -1149,7 +1137,6 @@ requests==2.26.0 # analytics-python # coreapi # django-oauth-toolkit - # docker # edx-analytics-data-api-client # edx-bulk-grades # edx-drf-extensions @@ -1175,7 +1162,7 @@ rest-condition==1.0.3 # via # -r requirements/edx/base.txt # edx-drf-extensions -ruamel.yaml==0.17.13 +ruamel.yaml==0.17.14 # via # -r requirements/edx/base.txt # drf-yasg @@ -1211,7 +1198,7 @@ semantic-version==2.8.5 # edx-drf-extensions shapely==1.7.1 # via -r requirements/edx/base.txt -simplejson==3.17.4 +simplejson==3.17.5 # via # -r requirements/edx/base.txt # sailthru-client @@ -1239,16 +1226,15 @@ six==1.16.0 # edx-lint # edx-milestones # edx-rbac - # english # event-tracking # freezegun # fs # fs-s3fs # html5lib # httpretty + # interchange # isodate # libsass - # neotime # pact-python # pansi # paver @@ -1402,10 +1388,6 @@ voluptuous==0.12.1 # ora2 watchdog==2.1.5 # via -r requirements/edx/base.txt -wcwidth==0.2.5 - # via - # -r requirements/edx/base.txt - # prompt-toolkit web-fragments==1.1.0 # via # -r requirements/edx/base.txt @@ -1424,10 +1406,6 @@ webob==1.8.7 # -r requirements/edx/base.txt # xblock # xmodule -websocket-client==1.2.1 - # via - # -r requirements/edx/base.txt - # docker wrapt==1.11.2 # via # -c requirements/edx/../constraints.txt From 3c1a43ea5c0ccf6c0f7a03d4c7bf058a7c8591ed Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Wed, 23 Jun 2021 14:13:04 +0930 Subject: [PATCH 2/2] feat: adds CELERY_BROKER_TRANSPORT_OPTIONS which can be configured from the lms/studio environment --- cms/envs/production.py | 1 + lms/envs/production.py | 1 + 2 files changed, 2 insertions(+) diff --git a/cms/envs/production.py b/cms/envs/production.py index 012c29123246..1ba06eadaf28 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -403,6 +403,7 @@ def get_env_setting(setting): BROKER_TRANSPORT_OPTIONS = { 'fanout_patterns': True, 'fanout_prefix': True, + **ENV_TOKENS.get('CELERY_BROKER_TRANSPORT_OPTIONS', {}) } # Message expiry time in seconds diff --git a/lms/envs/production.py b/lms/envs/production.py index 9ede3006ac36..823f6db57347 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -519,6 +519,7 @@ def get_env_setting(setting): BROKER_TRANSPORT_OPTIONS = { 'fanout_patterns': True, 'fanout_prefix': True, + **ENV_TOKENS.get('CELERY_BROKER_TRANSPORT_OPTIONS', {}) } # Block Structures