[SE-4304] [SE-4495] feat: add celery beat configuration to Koa

cms/celery.py

@@ -8,41 +8,8 @@
 
 import os
 
-from openedx.core.lib.celery.routers import AlternateEnvironmentRouter
 
 # Set the default Django settings module for the 'celery' program
 # and then instantiate the Celery singleton.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cms.envs.production')
 from openedx.core.lib.celery import APP  # pylint: disable=wrong-import-position,unused-import
-
-# Import after autodiscovery has had a chance to connect to the import_module signal
-# so celery doesn't miss any apps getting installed.
-from django.conf import settings  # pylint: disable=wrong-import-position,wrong-import-order
-
-
-class Router(AlternateEnvironmentRouter):
-    """
-    An implementation of AlternateEnvironmentRouter, for routing tasks to non-cms queues.
-    """
-
-    @property
-    def alternate_env_tasks(self):
-        """
-        Defines alternate environment tasks, as a dict of form { task_name: alternate_queue }
-        """
-        # The tasks below will be routed to the default lms queue.
-        return {
-            'completion_aggregator.tasks.update_aggregators': 'lms',
-            'openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache': 'lms',
-            'openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache_v2': 'lms',
-        }
-
-    @property
-    def explicit_queues(self):
-        """
-        Defines specific queues for tasks to run in (typically outside of the cms environment),
-        as a dict of form { task_name: queue_name }.
-        """
-        return {
-            'lms.djangoapps.grades.tasks.compute_all_grades_for_course': settings.POLICY_CHANGE_GRADES_ROUTING_KEY,
-        }

cms/djangoapps/contentstore/tasks.py

@@ -22,6 +22,7 @@
 from django.test import RequestFactory
 from django.utils.text import get_valid_filename
 from django.utils.translation import ugettext as _
+from edx_django_utils.monitoring import set_code_owner_attribute
 from opaque_keys.edx.keys import CourseKey
 from opaque_keys.edx.locator import LibraryLocator
 from organizations.models import OrganizationCourse
@@ -82,7 +83,8 @@ def clone_instance(instance, field_values):
     return instance
 
 
-@task()
+@task
+@set_code_owner_attribute
 def rerun_course(source_course_key_string, destination_course_key_string, user_id, fields=None):
     """
     Reruns a course in a new celery task.
@@ -168,7 +170,8 @@ def _parse_time(time_isoformat):
     ).replace(tzinfo=UTC)
 
 
-@task(routing_key=settings.UPDATE_SEARCH_INDEX_JOB_QUEUE)
+@task
+@set_code_owner_attribute
 def update_search_index(course_id, triggered_time_isoformat):
     """ Updates course search index. """
     try:
@@ -192,7 +195,8 @@ def update_search_index(course_id, triggered_time_isoformat):
         LOGGER.debug(u'Search indexing successful for complete course %s', course_id)
 
 
-@task()
+@task
+@set_code_owner_attribute
 def update_library_index(library_id, triggered_time_isoformat):
     """ Updates course search index. """
     try:

cms/djangoapps/contentstore/tests/test_course_create_rerun.py

@@ -14,6 +14,7 @@
 
 from cms.djangoapps.contentstore.tests.utils import AjaxEnabledTestClient, parse_json
 from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole
+from common.djangoapps.student.models import CourseAccessRole
 from common.djangoapps.student.tests.factories import UserFactory
 from common.djangoapps.util.organizations_helpers import add_organization, get_course_organizations
 from xmodule.course_module import CourseFields
@@ -34,6 +35,9 @@ def setUp(self):
         """
         super(TestCourseListing, self).setUp()
         # create and log in a staff user.
+        self.admin_user = UserFactory(is_staff=True)
+        self.admin_client = AjaxEnabledTestClient()
+        self.admin_client.login(username=self.admin_user.username, password='test')
         # create and log in a non-staff user
         self.user = UserFactory()
         self.factory = RequestFactory()
@@ -64,6 +68,7 @@ def tearDown(self):
         Reverse the setup
         """
         self.client.logout()
+        self.admin_client.logout()
         ModuleStoreTestCase.tearDown(self)
 
     @patch.dict('django.conf.settings.FEATURES', {'ORGANIZATIONS_APP': True})
@@ -176,3 +181,81 @@ def test_course_creation_with_org_in_system(self, store):
             course_orgs = get_course_organizations(new_course_key)
             self.assertEqual(len(course_orgs), 1)
             self.assertEqual(course_orgs[0]['short_name'], 'orgX')
+
+    @patch.dict('django.conf.settings.FEATURES', {'RESTRICT_COURSE_CREATION_TO_ORG_ROLES': True})
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_course_creation_when_user_not_in_org(self, store):
+        """
+        Tests course creation with restriction and user not registered in CourseAccessRole.
+        """
+        with modulestore().default_store(store):
+            response = self.client.ajax_post(self.course_create_rerun_url, {
+                'org': 'TestorgX',
+                'number': 'CS101',
+                'display_name': 'Course with web certs enabled',
+                'run': '2021_T1'
+            })
+            self.assertEqual(response.status_code, 400)
+            data = parse_json(response)
+            self.assertEqual(
+                data["error"],
+                'User does not have the permission to create courses in this organization'
+            )
+
+    @patch.dict('django.conf.settings.FEATURES', {'RESTRICT_COURSE_CREATION_TO_ORG_ROLES': True})
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_course_creation_when_user_in_org(self, store):
+        """
+        Tests course creation with restriction and user registered as staff.
+        """
+        staff_role = 'staff'
+        CourseAccessRole.objects.create(
+            org='TestorgX', role=staff_role, user=self.user
+        )
+        with modulestore().default_store(store):
+            response = self.client.ajax_post(self.course_create_rerun_url, {
+                'org': 'TestorgX',
+                'number': 'CS101',
+                'display_name': 'Course with web certs enabled',
+                'run': '2021_T1'
+            })
+            self.assertEqual(response.status_code, 200)
+
+    @patch.dict('django.conf.settings.FEATURES', {'RESTRICT_COURSE_CREATION_TO_ORG_ROLES': True})
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_course_creation_when_user_in_org_with_non_access_role(self, store):
+        """
+        Tests course creation with restriction and user registered as role who doesn't have the access.
+        """
+        staff_role = 'finance_admin'
+        CourseAccessRole.objects.create(
+            org='Stark', role=staff_role, user=self.user
+        )
+        with modulestore().default_store(store):
+            response = self.client.ajax_post(self.course_create_rerun_url, {
+                'org': 'Stark',
+                'number': 'AV101',
+                'display_name': 'Build Iron Man Suit',
+                'run': '2021_T1'
+            })
+            self.assertEqual(response.status_code, 400)
+            data = parse_json(response)
+            self.assertEqual(
+                data["error"],
+                'User does not have the permission to create courses in this organization'
+            )
+
+    @patch.dict('django.conf.settings.FEATURES', {'RESTRICT_COURSE_CREATION_TO_ORG_ROLES': True})
+    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
+    def test_course_creation_when_user_is_global_staff(self, store):
+        """
+        Tests course creation with restriction and user is global staff.
+        """
+        with modulestore().default_store(store):
+            response = self.admin_client.ajax_post(self.course_create_rerun_url, {
+                'org': 'Oscorp',
+                'number': 'SP101',
+                'display_name': 'Making better web',
+                'run': '2021_T1'
+            })
+            self.assertEqual(response.status_code, 200)

cms/djangoapps/contentstore/tests/test_libraries.py

@@ -26,6 +26,7 @@
     OrgLibraryUserRole,
     OrgStaffRole
 )
+from common.djangoapps.student.models import CourseAccessRole
 from common.djangoapps.student.tests.factories import UserFactory
 from common.djangoapps.xblock_django.user_service import DjangoXBlockUserService
 from xmodule.modulestore import ModuleStoreEnum
@@ -829,6 +830,77 @@ def _get_settings_html():
         self.assertNotIn('admin_lib_2', non_staff_settings_html)
 
 
+    @patch.dict('django.conf.settings.FEATURES', {'RESTRICT_COURSE_CREATION_TO_ORG_ROLES': True})
+    def test_library_creation_when_user_is_global_staff(self):
+        """
+        Tests course creation with restriction and user is global staff.
+        """
+        self._login_as_staff_user()
+        response = self.client.ajax_post(LIBRARY_REST_URL, {
+            'org': 'Oscorp',
+            'library': 'CentralLibrary',
+            'display_name': 'Making better web',
+        })
+        self.assertEqual(response.status_code, 200)
+
+    @patch.dict('django.conf.settings.FEATURES', {'RESTRICT_COURSE_CREATION_TO_ORG_ROLES': True})
+    def test_library_creation_with_normaL_user_with_no_role(self):
+        """
+        Tests course creation with restriction and user is not a global staff.
+        """
+        self._login_as_non_staff_user()
+        response = self.client.ajax_post(LIBRARY_REST_URL, {
+            'org': 'Stark',
+            'library': 'AvengerLibrary',
+            'display_name': 'Alien Science',
+        })
+        self.assertEqual(response.status_code, 400)
+        data = parse_json(response)
+        self.assertEqual(
+            data["ErrMsg"],
+            "User does not have the permission to create library in this organization"
+        )
+
+    @patch.dict('django.conf.settings.FEATURES', {'RESTRICT_COURSE_CREATION_TO_ORG_ROLES': True})
+    def test_library_creation_with_normaL_user_with_non_access_role(self):
+        """
+        Tests course creation with restriction and user doesn't have access role for org.
+        """
+        staff_role = "finance_admin"
+        self._login_as_non_staff_user()
+        CourseAccessRole.objects.create(
+            org='Stark', role=staff_role, user=self.non_staff_user
+        )
+        response = self.client.ajax_post(LIBRARY_REST_URL, {
+            'org': 'Stark',
+            'library': 'AvengerLibrary',
+            'display_name': 'Alien Science',
+        })
+        self.assertEqual(response.status_code, 400)
+        data = parse_json(response)
+        self.assertEqual(
+            data["ErrMsg"],
+            "User does not have the permission to create library in this organization"
+        )
+
+    @patch.dict('django.conf.settings.FEATURES', {'RESTRICT_COURSE_CREATION_TO_ORG_ROLES': True})
+    def test_library_creation_with_normaL_user_with_role(self):
+        """
+        Tests course creation with restriction and user has role access.
+        """
+        staff_role = "instructor"
+        self._login_as_non_staff_user()
+        CourseAccessRole.objects.create(
+            org='Stark', role=staff_role, user=self.non_staff_user
+        )
+        response = self.client.ajax_post(LIBRARY_REST_URL, {
+            'org': 'Stark',
+            'library': 'AvengerLibrary',
+            'display_name': 'Alien Science',
+        })
+        self.assertEqual(response.status_code, 200)
+
+
 @ddt.ddt
 @override_settings(SEARCH_ENGINE=None)
 class TestOverrides(LibraryTestCase):

cms/djangoapps/contentstore/views/course.py

@@ -831,6 +831,20 @@ def _create_or_rerun_course(request):
                     status=400
                 )
 
+        # Allow user to create the course only if they belong to the organisation
+        # This flag doesn't apply to Global Staff and Superusers
+        if settings.FEATURES.get('RESTRICT_COURSE_CREATION_TO_ORG_ROLES', False):
+            has_org_permission = has_studio_write_access(request.user, None, org)
+            if not has_org_permission:
+                log.exception(
+                    "User does not have the permission to create course in this organization."
+                    "User: {} Org: {} Course: {}".format(request.user.id, org, course)
+                )
+                return JsonResponse(
+                    {'error': _('User does not have the permission to create courses in this organization')},
+                    status=400
+                )
+
         fields = {'start': start}
         if display_name is not None:
             fields['display_name'] = display_name

cms/djangoapps/contentstore/views/library.py

@@ -186,6 +186,18 @@ def _create_library(request):
         library = request.json.get('number', None)
         if library is None:
             library = request.json['library']
+        # Allow user to create libraries only if they belong to the organization
+        # This flag doesn't apply to Global Staff and Superusers
+        if settings.FEATURES.get('RESTRICT_COURSE_CREATION_TO_ORG_ROLES', False):
+            has_org_permission = has_studio_write_access(request.user, None, org)
+            if not has_org_permission:
+                log.exception(
+                    "User does not have the permission to create library in this organization."
+                    "User: {} Org: {} Library: {}".format(request.user.id, org, library)
+                )
+                return JsonResponseBadRequest({
+                    "ErrMsg": _(u"User does not have the permission to create library in this organization")
+                })
         store = modulestore()
         with store.default_store(ModuleStoreEnum.Type.split):
             new_lib = store.create_library(

cms/envs/common.py

@@ -410,6 +410,21 @@
     # .. toggle_warnings: Also set settings.LIBRARY_AUTHORING_MICROFRONTEND_URL and see
     #   REDIRECT_TO_LIBRARY_AUTHORING_MICROFRONTEND for rollout.
     'ENABLE_LIBRARY_AUTHORING_MICROFRONTEND': False,
+
+    # .. toggle_name: RESTRICT_COURSE_CREATION_TO_ORG_ROLES
+    # .. toggle_implementation: DjangoSetting
+    # .. toggle_default: False
+    # .. toggle_description: Restricts users from creating courses/libraries in organisations
+    #      which they don't belong to. This flag doesn't apply to Global Staff and Superusers.
+    #      To enable, set to True.
+    #      To disable, set to False.
+    # .. toggle_category: n/a
+    # .. toggle_use_cases: open_edx
+    # .. toggle_creation_date: 2021-06-23
+    # .. toggle_expiration_date: None
+    # .. toggle_status: supported
+    'RESTRICT_COURSE_CREATION_TO_ORG_ROLES': False
+
 }
 
 ENABLE_JASMINE = False
@@ -1216,6 +1231,10 @@
 
 ################################# CELERY ######################################
 
+# Celery beat configuration
+
+CELERYBEAT_SCHEDULER = 'celery.beat:PersistentScheduler'
+
 # Message configuration
 
 CELERY_TASK_SERIALIZER = 'json'

cms/envs/production.py

@@ -136,7 +136,7 @@ def get_env_setting(setting):
     DEFAULT_PRIORITY_QUEUE: {}
 }
 
-CELERY_ROUTES = "{}celery.Router".format(QUEUE_VARIANT)
+CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task"
 
 # STATIC_URL_BASE specifies the base url to use for static files
 STATIC_URL_BASE = ENV_TOKENS.get('STATIC_URL_BASE', None)
@@ -567,3 +567,24 @@ def get_env_setting(setting):
     CORS_ALLOW_HEADERS = corsheaders_default_headers + (
         'use-jwt-cookie',
     )
+
+######################## CELERY ROTUING ########################
+
+# Celery beat configuration
+
+CELERYBEAT_SCHEDULER = ENV_TOKENS.get('CELERYBEAT_SCHEDULER', CELERYBEAT_SCHEDULER)
+
+# Defines alternate environment tasks, as a dict of form { task_name: alternate_queue }
+ALTERNATE_ENV_TASKS = {
+    'completion_aggregator.tasks.update_aggregators': 'lms',
+    'openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache': 'lms',
+    'openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache_v2': 'lms',
+}
+
+# Defines the task -> alternate worker queue to be used when routing.
+EXPLICIT_QUEUES = {
+    'lms.djangoapps.grades.tasks.compute_all_grades_for_course': {
+        'queue': POLICY_CHANGE_GRADES_ROUTING_KEY},
+    'cms.djangoapps.contentstore.tasks.update_search_index': {
+        'queue': UPDATE_SEARCH_INDEX_JOB_QUEUE},
+}

cms/envs/test.py

@@ -190,6 +190,8 @@
 
 CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION = False
 
+HIGH_PRIORITY_QUEUE = 'edx.cms.core.high'
+
 # test_status_cancel in cms/cms_user_tasks/test.py is failing without this
 # @override_setting for BROKER_URL is not working in testcase, so updating here
 BROKER_URL = 'memory://localhost/'

cms/static/js/factories/settings.js

@@ -26,7 +26,8 @@ define([
                 });
                 editor.render();
             },
-            reset: true
+            reset: true,
+            cache: false
         });
     };
 });