From 690132f70cffd65a37d7b77d25622843d57b1049 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Fri, 6 Oct 2023 11:18:36 -0400 Subject: [PATCH] test: unskip content_libraries tests which previously depended on micro-service WIP --- cms/envs/common.py | 18 +- .../lib/blockstore_db_router.py | 0 lms/envs/common.py | 68 --- openedx/core/lib/blockstore_api/__init__.py | 50 -- .../lib/blockstore_api/config/__init__.py | 13 - .../core/lib/blockstore_api/config/waffle.py | 20 - openedx/core/lib/blockstore_api/methods.py | 496 ------------------ .../core/lib/blockstore_api/tests/__init__.py | 0 openedx/core/lib/blockstore_api/tests/base.py | 42 -- .../tests/test_blockstore_api.py | 211 -------- 10 files changed, 1 insertion(+), 917 deletions(-) rename openedx/core/lib/blockstore_api/db_routers.py => cms/lib/blockstore_db_router.py (100%) delete mode 100644 openedx/core/lib/blockstore_api/__init__.py delete mode 100644 openedx/core/lib/blockstore_api/config/__init__.py delete mode 100644 openedx/core/lib/blockstore_api/config/waffle.py delete mode 100644 openedx/core/lib/blockstore_api/methods.py delete mode 100644 openedx/core/lib/blockstore_api/tests/__init__.py delete mode 100644 openedx/core/lib/blockstore_api/tests/base.py delete mode 100644 openedx/core/lib/blockstore_api/tests/test_blockstore_api.py diff --git a/cms/envs/common.py b/cms/envs/common.py index 2941087e859e..6bbac68ba5bb 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -115,7 +115,6 @@ ENTERPRISE_BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL, # Blockstore - BLOCKSTORE_USE_BLOCKSTORE_APP_API, BUNDLE_ASSET_STORAGE_SETTINGS, # Methods to derive settings @@ -295,9 +294,6 @@ # Enable content libraries (modulestore) search functionality 'ENABLE_LIBRARY_INDEX': False, - # Enable content libraries (blockstore) indexing - 'ENABLE_CONTENT_LIBRARY_INDEX': False, - # .. toggle_name: FEATURES['ALLOW_COURSE_RERUNS'] # .. toggle_implementation: DjangoSetting # .. toggle_default: True @@ -2164,7 +2160,7 @@ DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', - 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', + 'cms.lib.blockstore_db_router.BlockstoreRouter', ] ############################ Cache Configuration ############################### @@ -2598,22 +2594,10 @@ PROCTORING_SETTINGS = {} ################## BLOCKSTORE RELATED SETTINGS ######################### -BLOCKSTORE_PUBLIC_URL_ROOT = 'http://localhost:18250' -BLOCKSTORE_API_URL = 'http://localhost:18250/api/v1/' # Which of django's caches to use for storing anonymous user state for XBlocks # in the blockstore-based XBlock runtime XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' -# .. setting_name: BLOCKSTORE_BUNDLE_CACHE_TIMEOUT -# .. setting_default: 3000 -# .. setting_description: Maximum time-to-live of cached Bundles fetched from -# Blockstore, in seconds. When the values returned from Blockstore have -# TTLs of their own (such as signed S3 URLs), the maximum TTL of this cache -# must be lower than the minimum TTL of those values. -# We use a default of 3000s (50mins) because temporary URLs are often -# configured to expire after one hour. -BLOCKSTORE_BUNDLE_CACHE_TIMEOUT = 3000 - ###################### LEARNER PORTAL ################################ LEARNER_PORTAL_URL_ROOT = 'https://learner-portal-localhost:18000' diff --git a/openedx/core/lib/blockstore_api/db_routers.py b/cms/lib/blockstore_db_router.py similarity index 100% rename from openedx/core/lib/blockstore_api/db_routers.py rename to cms/lib/blockstore_db_router.py diff --git a/lms/envs/common.py b/lms/envs/common.py index 214a7d3b68bf..dbf29693dd85 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1111,25 +1111,12 @@ DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', - 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', 'edx_django_utils.db.read_replica.ReadReplicaRouter', ] ############################ Cache Configuration ############################### CACHES = { - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': ['localhost:11211'], - 'TIMEOUT': '86400', # This data should be long-lived for performance, BundleCache handles invalidation - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'OPTIONS': { - 'no_delay': True, - 'ignore_exc': True, - 'use_pooling': True, - } - }, 'course_structure_cache': { 'KEY_PREFIX': 'course_structure', 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', @@ -3299,9 +3286,6 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # For edx ace template tags 'edx_ace', - # Blockstore - 'blockstore.apps.bundles', - # MFE API 'lms.djangoapps.mfe_config_api', @@ -5128,63 +5112,11 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring ########################## MAILCHIMP SETTINGS ################################# MAILCHIMP_NEW_USER_LIST_ID = "" -########################## BLOCKSTORE ##################################### -BLOCKSTORE_PUBLIC_URL_ROOT = 'http://localhost:18250' -BLOCKSTORE_API_URL = 'http://localhost:18250/api/v1/' - -# Disable the Blockstore app API by default. -# See openedx.core.lib.blockstore_api.config for details. -BLOCKSTORE_USE_BLOCKSTORE_APP_API = False - # .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE # .. setting_default: default # .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' -# .. setting_name: BLOCKSTORE_BUNDLE_CACHE_TIMEOUT -# .. setting_default: 3000 -# .. setting_description: Maximum time-to-live of cached Bundles fetched from -# Blockstore, in seconds. When the values returned from Blockstore have -# TTLs of their own (such as signed S3 URLs), the maximum TTL of this cache -# must be lower than the minimum TTL of those values. -# We use a default of 3000s (50mins) because temporary URLs are often -# configured to expire after one hour. -BLOCKSTORE_BUNDLE_CACHE_TIMEOUT = 3000 - -# .. setting_name: BUNDLE_ASSET_URL_STORAGE_KEY -# .. setting_default: None -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_SECRET` is -# set, and `boto3` is installed, this is used as an AWS IAM access key for -# generating signed, read-only URLs for blockstore assets stored in S3. -# Otherwise, URLs are generated based on the default storage configuration. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_URL_STORAGE_KEY = None - -# .. setting_name: BUNDLE_ASSET_URL_STORAGE_SECRET -# .. setting_default: None -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is -# set, and `boto3` is installed, this is used as an AWS IAM secret key for -# generating signed, read-only URLs for blockstore assets stored in S3. -# Otherwise, URLs are generated based on the default storage configuration. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_URL_STORAGE_SECRET = None - -# .. setting_name: BUNDLE_ASSET_STORAGE_SETTINGS -# .. setting_default: dict, appropriate for file system storage. -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is -# set, and `boto3` is installed, this provides the bucket name and location for blockstore assets stored in S3. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_STORAGE_SETTINGS = dict( - # Backend storage - # STORAGE_CLASS='storages.backends.s3boto3.S3Boto3Storage', - # STORAGE_KWARGS=dict(bucket='bundle-asset-bucket', location='/path-to-bundles/'), - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), -) - SYSLOG_SERVER = '' FEEDBACK_SUBMISSION_EMAIL = '' GITHUB_REPO_ROOT = '/edx/var/edxapp/data' diff --git a/openedx/core/lib/blockstore_api/__init__.py b/openedx/core/lib/blockstore_api/__init__.py deleted file mode 100644 index 50b352578cdf..000000000000 --- a/openedx/core/lib/blockstore_api/__init__.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -API Client for Blockstore - -This API does not do any caching; consider using BundleCache or (in -openedx.core.djangolib.blockstore_cache) together with these API methods for -improved performance. -""" -from blockstore.apps.api.data import ( - BundleFileData, -) -from blockstore.apps.api.exceptions import ( - CollectionNotFound, - BundleNotFound, - DraftNotFound, - BundleVersionNotFound, - BundleFileNotFound, - BundleStorageError, -) -from .methods import ( - # Collections: - get_collection, - create_collection, - update_collection, - delete_collection, - # Bundles: - get_bundles, - get_bundle, - create_bundle, - update_bundle, - delete_bundle, - # Drafts: - get_draft, - get_or_create_bundle_draft, - write_draft_file, - set_draft_link, - commit_draft, - delete_draft, - # Bundles or drafts: - get_bundle_files, - get_bundle_files_dict, - get_bundle_file_metadata, - get_bundle_file_data, - get_bundle_version, - get_bundle_version_files, - # Links: - get_bundle_links, - get_bundle_version_links, - # Misc: - force_browser_url, -) diff --git a/openedx/core/lib/blockstore_api/config/__init__.py b/openedx/core/lib/blockstore_api/config/__init__.py deleted file mode 100644 index 4cd2999d2e32..000000000000 --- a/openedx/core/lib/blockstore_api/config/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -Helper method to indicate when the blockstore app API is enabled. -""" -from django.conf import settings -from .waffle import BLOCKSTORE_USE_BLOCKSTORE_APP_API # pylint: disable=invalid-django-waffle-import - - -def use_blockstore_app(): - """ - Use the Blockstore app API if the settings say to (e.g. in test) - or if the waffle switch is enabled. - """ - return settings.BLOCKSTORE_USE_BLOCKSTORE_APP_API or BLOCKSTORE_USE_BLOCKSTORE_APP_API.is_enabled() diff --git a/openedx/core/lib/blockstore_api/config/waffle.py b/openedx/core/lib/blockstore_api/config/waffle.py deleted file mode 100644 index 192d4d5c3ab2..000000000000 --- a/openedx/core/lib/blockstore_api/config/waffle.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Toggles for blockstore. -""" - -from edx_toggles.toggles import WaffleSwitch - -# .. toggle_name: blockstore.use_blockstore_app_api -# .. toggle_implementation: WaffleSwitch -# .. toggle_default: False -# .. toggle_description: Enable to use the installed blockstore app's Python API directly instead of the -# external blockstore service REST API. -# The blockstore REST API is used by default. -# .. toggle_use_cases: temporary, open_edx -# .. toggle_creation_date: 2022-01-13 -# .. toggle_target_removal_date: None -# .. toggle_tickets: TNL-8705, BD-14 -# .. toggle_warning: This temporary feature toggle does not have a target removal date. -BLOCKSTORE_USE_BLOCKSTORE_APP_API = WaffleSwitch( - 'blockstore.use_blockstore_app_api', __name__ -) diff --git a/openedx/core/lib/blockstore_api/methods.py b/openedx/core/lib/blockstore_api/methods.py deleted file mode 100644 index 86b4730efe1b..000000000000 --- a/openedx/core/lib/blockstore_api/methods.py +++ /dev/null @@ -1,496 +0,0 @@ -""" -API Client methods for working with Blockstore bundles and drafts -""" - -import base64 -from functools import wraps -from urllib.parse import urlencode -from uuid import UUID - -import dateutil.parser -from django.conf import settings -from django.core.exceptions import ImproperlyConfigured -import requests - -from blockstore.apps.api.data import ( - BundleData, - CollectionData, - DraftData, - BundleVersionData, - BundleFileData, - DraftFileData, - BundleLinkData, - DraftLinkData, - Dependency, -) -from blockstore.apps.api.exceptions import ( - NotFound, - CollectionNotFound, - BundleNotFound, - DraftNotFound, - BundleFileNotFound, -) -import blockstore.apps.api.methods as blockstore_api_methods - -from .config import use_blockstore_app - - -def toggle_blockstore_api(func): - """ - Decorator function to toggle usage of the Blockstore service - and the in-built Blockstore app dependency. - """ - @wraps(func) - def wrapper(*args, **kwargs): - if use_blockstore_app(): - return getattr(blockstore_api_methods, func.__name__)(*args, **kwargs) - return func(*args, **kwargs) - return wrapper - - -def api_url(*path_parts): - if not settings.BLOCKSTORE_API_URL or not settings.BLOCKSTORE_API_URL.endswith('/api/v1/'): - raise ImproperlyConfigured('BLOCKSTORE_API_URL must be set and should end with /api/v1/') - return settings.BLOCKSTORE_API_URL + '/'.join(path_parts) - - -def api_request(method, url, **kwargs): - """ - Helper method for making a request to the Blockstore REST API - """ - if not settings.BLOCKSTORE_API_AUTH_TOKEN: - raise ImproperlyConfigured("Cannot use Blockstore unless BLOCKSTORE_API_AUTH_TOKEN is set.") - kwargs.setdefault('headers', {})['Authorization'] = f"Token {settings.BLOCKSTORE_API_AUTH_TOKEN}" - response = requests.request(method, url, **kwargs) - if response.status_code == 404: - raise NotFound - response.raise_for_status() - if response.status_code == 204: - return None # No content - return response.json() - - -def _collection_from_response(data): - """ - Given data about a Collection returned by any blockstore REST API, convert it to - a CollectionData instance. - """ - return CollectionData(uuid=UUID(data['uuid']), title=data['title']) - - -def _bundle_from_response(data): - """ - Given data about a Bundle returned by any blockstore REST API, convert it to - a BundleData instance. - """ - return BundleData( - uuid=UUID(data['uuid']), - title=data['title'], - description=data['description'], - slug=data['slug'], - # drafts: Convert from a dict of URLs to a dict of UUIDs: - drafts={draft_name: UUID(url.split('/')[-1]) for (draft_name, url) in data['drafts'].items()}, - # versions field: take the last one and convert it from URL to an int - # i.e.: [..., 'https://blockstore/api/v1/bundle_versions/bundle_uuid,15'] -> 15 - latest_version=int(data['versions'][-1].split(',')[-1]) if data['versions'] else 0, - ) - - -def _bundle_version_from_response(data): - """ - Given data about a BundleVersion returned by any blockstore REST API, convert it to - a BundleVersionData instance. - """ - return BundleVersionData( - bundle_uuid=UUID(data['bundle_uuid']), - version=data.get('version', 0), - change_description=data['change_description'], - created_at=dateutil.parser.parse(data['snapshot']['created_at']), - files={ - path: BundleFileData(path=path, **filedata) - for path, filedata in data['snapshot']['files'].items() - }, - links={ - name: BundleLinkData( - name=name, - direct=Dependency(**link["direct"]), - indirect=[Dependency(**ind) for ind in link["indirect"]], - ) - for name, link in data['snapshot']['links'].items() - } - ) - - -def _draft_from_response(data): - """ - Given data about a Draft returned by any blockstore REST API, convert it to - a DraftData instance. - """ - return DraftData( - uuid=UUID(data['uuid']), - bundle_uuid=UUID(data['bundle_uuid']), - name=data['name'], - created_at=dateutil.parser.parse(data['staged_draft']['created_at']), - updated_at=dateutil.parser.parse(data['staged_draft']['updated_at']), - files={ - path: DraftFileData(path=path, **file) - for path, file in data['staged_draft']['files'].items() - }, - links={ - name: DraftLinkData( - name=name, - direct=Dependency(**link["direct"]), - indirect=[Dependency(**ind) for ind in link["indirect"]], - modified=link["modified"], - ) - for name, link in data['staged_draft']['links'].items() - } - ) - - -@toggle_blockstore_api -def get_collection(collection_uuid): - """ - Retrieve metadata about the specified collection - - Raises CollectionNotFound if the collection does not exist - """ - assert isinstance(collection_uuid, UUID) - try: - data = api_request('get', api_url('collections', str(collection_uuid))) - except NotFound: - raise CollectionNotFound(f"Collection {collection_uuid} does not exist.") # lint-amnesty, pylint: disable=raise-missing-from - return _collection_from_response(data) - - -@toggle_blockstore_api -def create_collection(title): - """ - Create a new collection. - """ - result = api_request('post', api_url('collections'), json={"title": title}) - return _collection_from_response(result) - - -@toggle_blockstore_api -def update_collection(collection_uuid, title): - """ - Update a collection's title - """ - assert isinstance(collection_uuid, UUID) - data = {"title": title} - result = api_request('patch', api_url('collections', str(collection_uuid)), json=data) - return _collection_from_response(result) - - -@toggle_blockstore_api -def delete_collection(collection_uuid): - """ - Delete a collection - """ - assert isinstance(collection_uuid, UUID) - api_request('delete', api_url('collections', str(collection_uuid))) - - -@toggle_blockstore_api -def get_bundles(uuids=None, text_search=None): - """ - Get the details of all bundles. - """ - query_params = {} - data = {} - if uuids: - # Potentially we could have a lot of libraries which will lead to 414 error (Request-URI Too Long) - # if sending uuids in the query_params. So we have to use the request data instead. - data = {'uuid': ','.join(map(str, uuids))} - if text_search: - query_params['text_search'] = text_search - version_url = api_url('bundles') + '?' + urlencode(query_params) - response = api_request('get', version_url, json=data) - # build bundle from response, convert map object to list and return - return [_bundle_from_response(item) for item in response] - - -@toggle_blockstore_api -def get_bundle(bundle_uuid): - """ - Retrieve metadata about the specified bundle - - Raises BundleNotFound if the bundle does not exist - """ - assert isinstance(bundle_uuid, UUID) - try: - data = api_request('get', api_url('bundles', str(bundle_uuid))) - except NotFound: - raise BundleNotFound(f"Bundle {bundle_uuid} does not exist.") # lint-amnesty, pylint: disable=raise-missing-from - return _bundle_from_response(data) - - -@toggle_blockstore_api -def create_bundle(collection_uuid, slug, title="New Bundle", description=""): - """ - Create a new bundle. - - Note that description is currently required. - """ - result = api_request('post', api_url('bundles'), json={ - "collection_uuid": str(collection_uuid), - "slug": slug, - "title": title, - "description": description, - }) - return _bundle_from_response(result) - - -@toggle_blockstore_api -def update_bundle(bundle_uuid, **fields): - """ - Update a bundle's title, description, slug, or collection. - """ - assert isinstance(bundle_uuid, UUID) - data = {} - # Most validation will be done by Blockstore, so we don't worry too much about data validation - for str_field in ("title", "description", "slug"): - if str_field in fields: - data[str_field] = fields.pop(str_field) - if "collection_uuid" in fields: - data["collection_uuid"] = str(fields.pop("collection_uuid")) - if fields: - raise ValueError(f"Unexpected extra fields passed " - f"to update_bundle: {fields.keys()}") - result = api_request('patch', api_url('bundles', str(bundle_uuid)), json=data) - return _bundle_from_response(result) - - -@toggle_blockstore_api -def delete_bundle(bundle_uuid): - """ - Delete a bundle - """ - assert isinstance(bundle_uuid, UUID) - api_request('delete', api_url('bundles', str(bundle_uuid))) - - -@toggle_blockstore_api -def get_draft(draft_uuid): - """ - Retrieve metadata about the specified draft. - If you don't know the draft's UUID, look it up using get_bundle() - """ - assert isinstance(draft_uuid, UUID) - try: - data = api_request('get', api_url('drafts', str(draft_uuid))) - except NotFound: - raise DraftNotFound(f"Draft does not exist: {draft_uuid}") # lint-amnesty, pylint: disable=raise-missing-from - return _draft_from_response(data) - - -@toggle_blockstore_api -def get_or_create_bundle_draft(bundle_uuid, draft_name): - """ - Retrieve metadata about the specified draft. - """ - bundle = get_bundle(bundle_uuid) - try: - return get_draft(bundle.drafts[draft_name]) # pylint: disable=unsubscriptable-object - except KeyError: - # The draft doesn't exist yet, so create it: - response = api_request('post', api_url('drafts'), json={ - "bundle_uuid": str(bundle_uuid), - "name": draft_name, - }) - # The result of creating a draft doesn't include all the fields we want, so retrieve it now: - return get_draft(UUID(response["uuid"])) - - -@toggle_blockstore_api -def commit_draft(draft_uuid): - """ - Commit all of the pending changes in the draft, creating a new version of - the associated bundle. - - Does not return any value. - """ - api_request('post', api_url('drafts', str(draft_uuid), 'commit')) - - -@toggle_blockstore_api -def delete_draft(draft_uuid): - """ - Delete the specified draft, removing any staged changes/files/deletes. - - Does not return any value. - """ - api_request('delete', api_url('drafts', str(draft_uuid))) - - -@toggle_blockstore_api -def get_bundle_version(bundle_uuid, version_number): - """ - Get the details of the specified bundle version - """ - if version_number == 0: - return None - version_url = api_url('bundle_versions', str(bundle_uuid) + ',' + str(version_number)) - return _bundle_version_from_response(api_request('get', version_url)) - - -@toggle_blockstore_api -def get_bundle_version_files(bundle_uuid, version_number): - """ - Get a list of the files in the specified bundle version - """ - if version_number == 0: - return [] - version_info = get_bundle_version(bundle_uuid, version_number) - return list(version_info.files.values()) - - -@toggle_blockstore_api -def get_bundle_version_links(bundle_uuid, version_number): - """ - Get a dictionary of the links in the specified bundle version - """ - if version_number == 0: - return {} - version_info = get_bundle_version(bundle_uuid, version_number) - return version_info.links - - -@toggle_blockstore_api -def get_bundle_files_dict(bundle_uuid, use_draft=None): - """ - Get a dict of all the files in the specified bundle. - - Returns a dict where the keys are the paths (strings) and the values are - BundleFileData or DraftFileData tuples. - """ - bundle = get_bundle(bundle_uuid) - if use_draft and use_draft in bundle.drafts: # pylint: disable=unsupported-membership-test - draft_uuid = bundle.drafts[use_draft] # pylint: disable=unsubscriptable-object - return get_draft(draft_uuid).files - elif not bundle.latest_version: - # This bundle has no versions so definitely does not contain any files - return {} - else: - return {file_meta.path: file_meta for file_meta in get_bundle_version_files(bundle_uuid, bundle.latest_version)} - - -@toggle_blockstore_api -def get_bundle_files(bundle_uuid, use_draft=None): - """ - Get an iterator over all the files in the specified bundle or draft. - """ - return get_bundle_files_dict(bundle_uuid, use_draft).values() - - -@toggle_blockstore_api -def get_bundle_links(bundle_uuid, use_draft=None): - """ - Get a dict of all the links in the specified bundle. - - Returns a dict where the keys are the link names (strings) and the values - are BundleLinkData or DraftLinkData tuples. - """ - bundle = get_bundle(bundle_uuid) - if use_draft and use_draft in bundle.drafts: # pylint: disable=unsupported-membership-test - draft_uuid = bundle.drafts[use_draft] # pylint: disable=unsubscriptable-object - return get_draft(draft_uuid).links - elif not bundle.latest_version: - # This bundle has no versions so definitely does not contain any links - return {} - else: - return get_bundle_version_links(bundle_uuid, bundle.latest_version) - - -@toggle_blockstore_api -def get_bundle_file_metadata(bundle_uuid, path, use_draft=None): - """ - Get the metadata of the specified file. - """ - assert isinstance(bundle_uuid, UUID) - files_dict = get_bundle_files_dict(bundle_uuid, use_draft=use_draft) - try: - return files_dict[path] - except KeyError: - raise BundleFileNotFound( # lint-amnesty, pylint: disable=raise-missing-from - f"Bundle {bundle_uuid} (draft: {use_draft}) does not contain a file {path}" - ) - - -@toggle_blockstore_api -def get_bundle_file_data(bundle_uuid, path, use_draft=None): - """ - Read all the data in the given bundle file and return it as a - binary string. - - Do not use this for large files! - """ - metadata = get_bundle_file_metadata(bundle_uuid, path, use_draft) - with requests.get(metadata.url, stream=True) as r: - return r.content - - -@toggle_blockstore_api -def write_draft_file(draft_uuid, path, contents): - """ - Create or overwrite the file at 'path' in the specified draft with the given - contents. To delete a file, pass contents=None. - - If you don't know the draft's UUID, look it up using - get_or_create_bundle_draft() - - Does not return anything. - """ - api_request('patch', api_url('drafts', str(draft_uuid)), json={ - 'files': { - path: _encode_str_for_draft(contents) if contents is not None else None, - }, - }) - - -@toggle_blockstore_api -def set_draft_link(draft_uuid, link_name, bundle_uuid, version): - """ - Create or replace the link with the given name in the specified draft so - that it points to the specified bundle version. To delete a link, pass - bundle_uuid=None, version=None. - - If you don't know the draft's UUID, look it up using - get_or_create_bundle_draft() - - Does not return anything. - """ - api_request('patch', api_url('drafts', str(draft_uuid)), json={ - 'links': { - link_name: {"bundle_uuid": str(bundle_uuid), "version": version} if bundle_uuid is not None else None, - }, - }) - - -def _encode_str_for_draft(input_str): - """ - Given a string, return UTF-8 representation that is then base64 encoded. - """ - if isinstance(input_str, str): - binary = input_str.encode('utf8') - else: - binary = input_str - return base64.b64encode(binary) - - -@toggle_blockstore_api -def force_browser_url(blockstore_file_url): - """ - Ensure that the given devstack URL is a URL accessible from the end user's browser. - """ - # Hack: on some devstacks, we must necessarily use different URLs for - # accessing Blockstore file data from within and outside of docker - # containers, but Blockstore has no way of knowing which case any particular - # request is for. So it always returns a URL suitable for use from within - # the container. Only this edxapp can transform the URL at the last second, - # knowing that in this case it's going to the user's browser and not being - # read by edxapp. - # In production, the same S3 URLs get used for internal and external access - # so this hack is not necessary. - return blockstore_file_url.replace('http://edx.devstack.blockstore:', 'http://localhost:') diff --git a/openedx/core/lib/blockstore_api/tests/__init__.py b/openedx/core/lib/blockstore_api/tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/openedx/core/lib/blockstore_api/tests/base.py b/openedx/core/lib/blockstore_api/tests/base.py deleted file mode 100644 index 0888e8c81dc7..000000000000 --- a/openedx/core/lib/blockstore_api/tests/base.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Common code for tests that work with Blockstore -""" -from unittest import mock, skipUnless -from urllib.parse import urlparse - -from django.conf import settings -from django.test.client import RequestFactory - -# Decorators for tests that require the blockstore service/app -requires_blockstore = skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server") - -requires_blockstore_app = skipUnless(settings.BLOCKSTORE_USE_BLOCKSTORE_APP_API, "Requires blockstore app") - - -class BlockstoreAppTestMixin: - """ - Sets up the environment for tests to be run using the installed Blockstore app. - """ - def setUp(self): - """ - Ensure there's an active request, so that bundle file URLs can be made absolute. - """ - super().setUp() - - # Patch the blockstore get_current_request to use our live_server_url - mock.patch('blockstore.apps.api.methods.get_current_request', - mock.Mock(return_value=self._get_current_request())).start() - self.addCleanup(mock.patch.stopall) - - def _get_current_request(self): - """ - Returns a request object using the live_server_url, if available. - """ - request_args = {} - if hasattr(self, 'live_server_url'): - live_server_url = urlparse(self.live_server_url) - name, port = live_server_url.netloc.split(':') - request_args['SERVER_NAME'] = name - request_args['SERVER_PORT'] = port or '80' - request_args['wsgi.url_scheme'] = live_server_url.scheme - return RequestFactory().request(**request_args) diff --git a/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py b/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py deleted file mode 100644 index 9c15e16668d0..000000000000 --- a/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py +++ /dev/null @@ -1,211 +0,0 @@ -""" -Tests for xblock_utils.py -""" - -from uuid import UUID - -import pytest -from django.test import TestCase - -from openedx.core.lib import blockstore_api as api -from openedx.core.lib.blockstore_api.tests.base import ( - BlockstoreAppTestMixin, - requires_blockstore, - requires_blockstore_app, -) - -# A fake UUID that won't represent any real bundle/draft/collection: -BAD_UUID = UUID('12345678-0000-0000-0000-000000000000') - - -class BlockstoreApiClientTestMixin: - """ - Tests for the Blockstore API Client. - - The goal of these tests is not to test that Blockstore works correctly, but - that the API client can interact with it and all the API client methods - work. - """ - - # Collections - - def test_nonexistent_collection(self): - """ Request a collection that doesn't exist -> CollectionNotFound """ - with pytest.raises(api.CollectionNotFound): - api.get_collection(BAD_UUID) - - def test_collection_crud(self): - """ Create, Fetch, Update, and Delete a Collection """ - title = "Fire 🔥 Collection" - # Create: - coll = api.create_collection(title) - assert coll.title == title - assert isinstance(coll.uuid, UUID) - # Fetch: - coll2 = api.get_collection(coll.uuid) - assert coll == coll2 - # Update: - new_title = "Air 🌀 Collection" - coll3 = api.update_collection(coll.uuid, title=new_title) - assert coll3.title == new_title - coll4 = api.get_collection(coll.uuid) - assert coll4.title == new_title - # Delete: - api.delete_collection(coll.uuid) - with pytest.raises(api.CollectionNotFound): - api.get_collection(coll.uuid) - - # Bundles - - def test_nonexistent_bundle(self): - """ Request a bundle that doesn't exist -> BundleNotFound """ - with pytest.raises(api.BundleNotFound): - api.get_bundle(BAD_UUID) - - def test_bundle_crud(self): - """ Create, Fetch, Update, and Delete a Bundle """ - coll = api.create_collection("Test Collection") - args = { - "title": "Water 💧 Bundle", - "slug": "h2o", - "description": "Sploosh", - } - # Create: - bundle = api.create_bundle(coll.uuid, **args) - for attr, value in args.items(): - assert getattr(bundle, attr) == value - assert isinstance(bundle.uuid, UUID) - # Fetch: - bundle2 = api.get_bundle(bundle.uuid) - assert bundle == bundle2 - # Update: - new_description = "Water Nation Bending Lessons" - bundle3 = api.update_bundle(bundle.uuid, description=new_description) - assert bundle3.description == new_description - bundle4 = api.get_bundle(bundle.uuid) - assert bundle4.description == new_description - # Delete: - api.delete_bundle(bundle.uuid) - with pytest.raises(api.BundleNotFound): - api.get_bundle(bundle.uuid) - - # Drafts, files, and reading/writing file contents: - - def test_nonexistent_draft(self): - """ Request a draft that doesn't exist -> DraftNotFound """ - with pytest.raises(api.DraftNotFound): - api.get_draft(BAD_UUID) - - def test_drafts_and_files(self): - """ - Test creating, reading, writing, committing, and reverting drafts and - files. - """ - coll = api.create_collection("Test Collection") - bundle = api.create_bundle(coll.uuid, title="Earth 🗿 Bundle", slug="earth", description="another test bundle") - # Create a draft - draft = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") - assert draft.bundle_uuid == bundle.uuid - assert draft.name == 'test-draft' - assert draft.updated_at.year >= 2019 - # And retrieve it again: - draft2 = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") - assert draft == draft2 - # Also test retrieving using get_draft - draft3 = api.get_draft(draft.uuid) - assert draft == draft3 - - # Write a file into the bundle: - api.write_draft_file(draft.uuid, "test.txt", b"initial version") - # Now the file should be visible in the draft: - draft_contents = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - assert draft_contents == b'initial version' - api.commit_draft(draft.uuid) - - # Write a new version into the draft: - api.write_draft_file(draft.uuid, "test.txt", b"modified version") - published_contents = api.get_bundle_file_data(bundle.uuid, "test.txt") - assert published_contents == b'initial version' - draft_contents2 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - assert draft_contents2 == b'modified version' - # Now delete the draft: - api.delete_draft(draft.uuid) - draft_contents3 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - # Confirm the file is now reset: - assert draft_contents3 == b'initial version' - - # Finaly, test the get_bundle_file* methods: - file_info1 = api.get_bundle_file_metadata(bundle.uuid, "test.txt") - assert file_info1.path == 'test.txt' - assert file_info1.size == len(b'initial version') - assert file_info1.hash_digest == 'a45a5c6716276a66c4005534a51453ab16ea63c4' - - assert list(api.get_bundle_files(bundle.uuid)) == [file_info1] - assert api.get_bundle_files_dict(bundle.uuid) == {'test.txt': file_info1} - - # Links - - def test_links(self): - """ - Test operations involving bundle links. - """ - coll = api.create_collection("Test Collection") - # Create two library bundles and a course bundle: - lib1_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib1") - lib1_draft = api.get_or_create_bundle_draft(lib1_bundle.uuid, draft_name="test-draft") - lib2_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib2") - lib2_draft = api.get_or_create_bundle_draft(lib2_bundle.uuid, draft_name="other-draft") - course_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="course") - course_draft = api.get_or_create_bundle_draft(course_bundle.uuid, draft_name="test-draft") - - # To create links, we need valid BundleVersions, which requires having committed at least one change: - api.write_draft_file(lib1_draft.uuid, "lib1-data.txt", "hello world") - api.commit_draft(lib1_draft.uuid) # Creates version 1 - api.write_draft_file(lib2_draft.uuid, "lib2-data.txt", "hello world") - api.commit_draft(lib2_draft.uuid) # Creates version 1 - - # Lib2 has no links: - assert not api.get_bundle_links(lib2_bundle.uuid) - - # Create a link from lib2 to lib1 - link1_name = "lib2_to_lib1" - api.set_draft_link(lib2_draft.uuid, link1_name, lib1_bundle.uuid, version=1) - # Now confirm the link exists in the draft: - lib2_draft_links = api.get_bundle_links(lib2_bundle.uuid, use_draft=lib2_draft.name) - assert link1_name in lib2_draft_links - assert lib2_draft_links[link1_name].direct.bundle_uuid == lib1_bundle.uuid - assert lib2_draft_links[link1_name].direct.version == 1 - # Now commit the change to lib2: - api.commit_draft(lib2_draft.uuid) # Creates version 2 - - # Now create a link from course to lib2 - link2_name = "course_to_lib2" - api.set_draft_link(course_draft.uuid, link2_name, lib2_bundle.uuid, version=2) - api.commit_draft(course_draft.uuid) - - # And confirm the link exists in the resulting bundle version: - course_links = api.get_bundle_links(course_bundle.uuid) - assert link2_name in course_links - assert course_links[link2_name].direct.bundle_uuid == lib2_bundle.uuid - assert course_links[link2_name].direct.version == 2 - # And since the links go course->lib2->lib1, course has an indirect link to lib1: - assert course_links[link2_name].indirect[0].bundle_uuid == lib1_bundle.uuid - assert course_links[link2_name].indirect[0].version == 1 - - # Finally, test deleting a link from course's draft: - api.set_draft_link(course_draft.uuid, link2_name, None, None) - assert not api.get_bundle_links(course_bundle.uuid, use_draft=course_draft.name) - - -@requires_blockstore -class BlockstoreServiceApiClientTest(BlockstoreApiClientTestMixin, TestCase): - """ - Test the Blockstore API Client, using the standalone Blockstore service. - """ - - -@requires_blockstore_app -class BlockstoreAppApiClientTest(BlockstoreApiClientTestMixin, BlockstoreAppTestMixin, TestCase): - """ - Test the Blockstore API Client, using the installed Blockstore app. - """