From c2cc5e8c756703d861dc8bdad81873fd27d15f5a Mon Sep 17 00:00:00 2001 From: Usman Khalid <2200617@gmail.com> Date: Thu, 12 Nov 2020 02:57:58 +0500 Subject: [PATCH 1/6] Remove tagstore. Tagstore is not being used. Systems using blockstore for content storage should develop their own tagging system appropritate for their needs. --- .coveragerc | 1 - Makefile | 9 +- .../mysql_unicode/migrations/0001_initial.py | 1 - blockstore/settings/base.py | 1 - .../admin/tagstore_django/list_item.html | 19 - .../admin/tagstore_django/tag_hierarchy.html | 12 - .../tagstore_django/taxonomy/change_form.html | 5 - blockstore/urls.py | 1 - decisions/0003-tagstore-removal.rst | 33 + tagstore.json | 809 ------------------ tagstore/README.rst | 99 --- tagstore/__init__.py | 4 - tagstore/backends/__init__.py | 0 tagstore/backends/django.py | 191 ----- tagstore/backends/tagstore_django/admin.py | 60 -- .../migrations/0001_initial.py | 2 +- .../migrations/0002_delete_all_app_models.py | 34 + tagstore/backends/tagstore_django/models.py | 120 --- .../tagstore_django/templatetags/__init__.py | 0 .../templatetags/tagstore_admin.py | 17 - tagstore/backends/tests/__init__.py | 0 tagstore/constants.py | 4 - tagstore/models/__init__.py | 7 - tagstore/models/entity.py | 7 - tagstore/models/taxonomy.py | 73 -- tagstore/models/user.py | 9 - tagstore/mypy.ini | 7 - tagstore/tagstore.py | 150 ---- tagstore/tagstore_rest/__init__.py | 0 tagstore/tagstore_rest/routers.py | 71 -- tagstore/tagstore_rest/urls.py | 13 - tagstore/tagstore_rest/v1/__init__.py | 0 .../tagstore_rest/v1/serializers/__init__.py | 0 .../tagstore_rest/v1/serializers/entities.py | 43 - tagstore/tagstore_rest/v1/tests/__init__.py | 1 - .../v1/tests/test_serializers.py | 87 -- tagstore/tagstore_rest/v1/tests/test_views.py | 125 --- tagstore/tagstore_rest/v1/urls.py | 16 - tagstore/tagstore_rest/v1/views/__init__.py | 0 tagstore/tagstore_rest/v1/views/entities.py | 125 --- 40 files changed, 72 insertions(+), 2084 deletions(-) delete mode 100644 blockstore/templates/admin/tagstore_django/list_item.html delete mode 100644 blockstore/templates/admin/tagstore_django/tag_hierarchy.html delete mode 100644 blockstore/templates/admin/tagstore_django/taxonomy/change_form.html create mode 100644 decisions/0003-tagstore-removal.rst delete mode 100644 tagstore.json delete mode 100644 tagstore/README.rst delete mode 100644 tagstore/__init__.py delete mode 100644 tagstore/backends/__init__.py delete mode 100644 tagstore/backends/django.py delete mode 100644 tagstore/backends/tagstore_django/admin.py create mode 100644 tagstore/backends/tagstore_django/migrations/0002_delete_all_app_models.py delete mode 100644 tagstore/backends/tagstore_django/models.py delete mode 100644 tagstore/backends/tagstore_django/templatetags/__init__.py delete mode 100644 tagstore/backends/tagstore_django/templatetags/tagstore_admin.py delete mode 100644 tagstore/backends/tests/__init__.py delete mode 100644 tagstore/constants.py delete mode 100644 tagstore/models/__init__.py delete mode 100644 tagstore/models/entity.py delete mode 100644 tagstore/models/taxonomy.py delete mode 100644 tagstore/models/user.py delete mode 100644 tagstore/mypy.ini delete mode 100644 tagstore/tagstore.py delete mode 100644 tagstore/tagstore_rest/__init__.py delete mode 100644 tagstore/tagstore_rest/routers.py delete mode 100644 tagstore/tagstore_rest/urls.py delete mode 100644 tagstore/tagstore_rest/v1/__init__.py delete mode 100644 tagstore/tagstore_rest/v1/serializers/__init__.py delete mode 100644 tagstore/tagstore_rest/v1/serializers/entities.py delete mode 100644 tagstore/tagstore_rest/v1/tests/__init__.py delete mode 100644 tagstore/tagstore_rest/v1/tests/test_serializers.py delete mode 100644 tagstore/tagstore_rest/v1/tests/test_views.py delete mode 100644 tagstore/tagstore_rest/v1/urls.py delete mode 100644 tagstore/tagstore_rest/v1/views/__init__.py delete mode 100644 tagstore/tagstore_rest/v1/views/entities.py diff --git a/.coveragerc b/.coveragerc index fc5aebd1..7e05e92d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,7 +3,6 @@ branch = True data_file = .coverage source= blockstore - tagstore omit = blockstore/settings* blockstore/conf* diff --git a/Makefile b/Makefile index d7f1372a..73b70d54 100644 --- a/Makefile +++ b/Makefile @@ -83,10 +83,10 @@ static: ## Collect static files ${VENV_BIN}/python manage.py collectstatic --noinput test: clean ## Run tests and generate coverage report - ${VENV_BIN}/coverage run ./manage.py test blockstore tagstore --settings=blockstore.settings.test + ${VENV_BIN}/coverage run ./manage.py test blockstore --settings=blockstore.settings.test ${VENV_BIN}/coverage html ${VENV_BIN}/coverage xml - ${VENV_BIN}/diff-cover coverage.xml --html-report diff-cover.html --compare-branch origin/master + ${VENV_BIN}/diff-cover coverage.xml --html-report diff-cover.html easyserver: dev.up dev.provision # Start and provision a Blockstore container and run the server until CTRL-C, then stop it # Now run blockstore until the user hits CTRL-C: @@ -108,9 +108,8 @@ html_coverage: ## Generate HTML coverage report ${VENV_BIN}/coverage html quality: ## Run quality checks - ${VENV_BIN}/pycodestyle --config=pycodestyle blockstore tagstore *.py - ${VENV_BIN}/pylint --django-settings-module=blockstore.settings.test --rcfile=pylintrc blockstore tagstore *.py - ${VENV_BIN}/mypy --config-file tagstore/mypy.ini tagstore + ${VENV_BIN}/pycodestyle --config=pycodestyle blockstore *.py + ${VENV_BIN}/pylint --django-settings-module=blockstore.settings.test --rcfile=pylintrc blockstore *.py validate: test quality ## Run tests and quality checks diff --git a/blockstore/apps/mysql_unicode/migrations/0001_initial.py b/blockstore/apps/mysql_unicode/migrations/0001_initial.py index 96b6fe2b..02cb3e47 100644 --- a/blockstore/apps/mysql_unicode/migrations/0001_initial.py +++ b/blockstore/apps/mysql_unicode/migrations/0001_initial.py @@ -21,7 +21,6 @@ class Migration(migrations.Migration): run_before = [ ('contenttypes', '0001_initial'), ('core', '0001_initial'), - ('tagstore_django', '0001_initial'), ] operations = [ diff --git a/blockstore/settings/base.py b/blockstore/settings/base.py index fe1638ad..7e7a0ed0 100644 --- a/blockstore/settings/base.py +++ b/blockstore/settings/base.py @@ -58,7 +58,6 @@ def root(*x): 'blockstore.apps.api', 'blockstore.apps.bundles.apps.BundlesConfig', 'tagstore.backends.tagstore_django', - 'tagstore.tagstore_rest', ) INSTALLED_APPS += THIRD_PARTY_APPS diff --git a/blockstore/templates/admin/tagstore_django/list_item.html b/blockstore/templates/admin/tagstore_django/list_item.html deleted file mode 100644 index cba79b1e..00000000 --- a/blockstore/templates/admin/tagstore_django/list_item.html +++ /dev/null @@ -1,19 +0,0 @@ -{# Template to display a hierarchical list of tags on the admin page #} -{% load i18n %} - -{% for tag in taglist %} -
  • - {{ tag.name }} -   - {#   #} - {# 🗙 #} - - {% if tag.children|length %} - - {% endif %} -
    • -{% endfor %} diff --git a/blockstore/templates/admin/tagstore_django/tag_hierarchy.html b/blockstore/templates/admin/tagstore_django/tag_hierarchy.html deleted file mode 100644 index 6bc7f645..00000000 --- a/blockstore/templates/admin/tagstore_django/tag_hierarchy.html +++ /dev/null @@ -1,12 +0,0 @@ -{# Template to display list of taxonomies (and tags) on the admin page #} -{% load i18n %} - diff --git a/blockstore/templates/admin/tagstore_django/taxonomy/change_form.html b/blockstore/templates/admin/tagstore_django/taxonomy/change_form.html deleted file mode 100644 index 81a31cfe..00000000 --- a/blockstore/templates/admin/tagstore_django/taxonomy/change_form.html +++ /dev/null @@ -1,5 +0,0 @@ -{% extends "admin/change_form.html" %} -{% load i18n %} -{% load tagstore_admin %} - -{% block sidebar %}{% tag_hierarchy object_id %}{% endblock %} diff --git a/blockstore/urls.py b/blockstore/urls.py index cb8f8eaa..a0ae69c8 100644 --- a/blockstore/urls.py +++ b/blockstore/urls.py @@ -41,7 +41,6 @@ url(r'^api-auth/', include((oauth2_urlpatterns, 'auth_backends'), namespace='rest_framework')), url(r'^auto_auth/$', core_views.AutoAuth.as_view(), name='auto_auth'), url(r'^health/$', core_views.health, name='health'), - url(r'^tagstore/', include('tagstore.tagstore_rest.urls', namespace='tagstore')), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) urlpatterns += make_docs_urls(api_info) diff --git a/decisions/0003-tagstore-removal.rst b/decisions/0003-tagstore-removal.rst new file mode 100644 index 00000000..7c5808d1 --- /dev/null +++ b/decisions/0003-tagstore-removal.rst @@ -0,0 +1,33 @@ +Status +====== + +Accepted + + +Context +======= + +During the initial development of Blockstore for the LabXchange project, a tagging service called tagstore was developed to allow tagging XBlock content stored in Blockstore. + +However, the current approach of having XBlock content in bundles and tags in the database has at least two limitations: + +1. Changes to tags are applied immediately and do not support the draft-publish workflow. +2. For course export-import workflows, a mechanism will need to be developed to include the tags data in the bundle anyway. + +For architectural simplicitiy and to meet rapidly evolving product needs a custom tagging system was added to the LabXchange backend instead of trying to make tagstore cover a wide range of use cases. + +Decision +======== + +Tagstore code will be removed from the blockstore repo. If Studio or other services need the ability to tag XBlock or other content a tagging system appropriate for the product needs can be developed. + + +Consequences +============ + +Since the tagstore is not being used there are no changes needed anywhere else. + +Background +========== + +* `Open edX Tagging Service Proposal `_ diff --git a/tagstore.json b/tagstore.json deleted file mode 100644 index 4ad92f1d..00000000 --- a/tagstore.json +++ /dev/null @@ -1,809 +0,0 @@ -[ - { - "model": "tagstore_django.entity", - "pk": 1, - "fields": { - "entity_type": "xblock", - "external_id": "block-v1:OpenCraft+B+1+type@drag-and-drop-v2+block@971aadaaaa5a4886b37fee3ced711b30", - "tags": [ - 65, - 146, - 145, - 147, - 148, - 150, - 152, - 153 - ] - } - }, - { - "model": "tagstore_django.entity", - "pk": 11, - "fields": { - "entity_type": "pathway", - "external_id": "cb292b23-f56f-468b-a3ad-3256425a5fee", - "tags": [ - 65, - 146, - 145, - 147, - 148, - 150, - 152, - 153 - ] - } - }, - { - "model": "tagstore_django.entity", - "pk": 12, - "fields": { - "entity_type": "image", - "external_id": "block-v1:edX+DemoX+Demo_Course+type@vertical+block@867dddb6f55d410caaa9c1eb9c6743ec", - "tags": [ - 65, - 146, - 145, - 147, - 148, - 150, - 152, - 153 - ] - } - }, - { - "model": "tagstore_django.taxonomy", - "pk": 1, - "fields": { - "name": "Subject Area", - "owner": null - } - }, - { - "model": "tagstore_django.taxonomy", - "pk": 50, - "fields": { - "name": "Depth of Knowledge", - "owner": null - } - }, - { - "model": "tagstore_django.taxonomy", - "pk": 54, - "fields": { - "name": "temporary", - "owner": null - } - }, - { - "model": "tagstore_django.tag", - "pk": 35, - "fields": { - "taxonomy": 1, - "name": "Physical sciences", - "path": "1:Physical sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 36, - "fields": { - "taxonomy": 1, - "name": "Earth and environmental sciences", - "path": "1:Earth and environmental sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 37, - "fields": { - "taxonomy": 1, - "name": "Biological sciences", - "path": "1:Biological sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 38, - "fields": { - "taxonomy": 1, - "name": "Health sciences", - "path": "1:Health sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 39, - "fields": { - "taxonomy": 1, - "name": "Scientific community and society", - "path": "1:Scientific community and society:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 40, - "fields": { - "taxonomy": 1, - "name": "Chemistry", - "path": "1:Physical sciences:Chemistry:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 41, - "fields": { - "taxonomy": 1, - "name": "Materials science", - "path": "1:Physical sciences:Materials science:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 42, - "fields": { - "taxonomy": 1, - "name": "Physics", - "path": "1:Physical sciences:Physics:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 43, - "fields": { - "taxonomy": 1, - "name": "Nanoscience and technology", - "path": "1:Physical sciences:Nanoscience and technology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 44, - "fields": { - "taxonomy": 1, - "name": "Optics and photonics", - "path": "1:Physical sciences:Optics and photonics:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 45, - "fields": { - "taxonomy": 1, - "name": "Astronomy and planetary science", - "path": "1:Physical sciences:Astronomy and planetary science:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 46, - "fields": { - "taxonomy": 1, - "name": "Energy science and technology", - "path": "1:Physical sciences:Energy science and technology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 47, - "fields": { - "taxonomy": 1, - "name": "Engineering", - "path": "1:Physical sciences:Engineering:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 48, - "fields": { - "taxonomy": 1, - "name": "Mathematics and computing", - "path": "1:Physical sciences:Mathematics and computing:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 52, - "fields": { - "taxonomy": 1, - "name": "Ecology", - "path": "1:Earth and environmental sciences:Ecology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 53, - "fields": { - "taxonomy": 1, - "name": "Climate sciences", - "path": "1:Earth and environmental sciences:Climate sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 54, - "fields": { - "taxonomy": 1, - "name": "Planetary science", - "path": "1:Earth and environmental sciences:Planetary science:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 55, - "fields": { - "taxonomy": 1, - "name": "Solid Earth sciences", - "path": "1:Earth and environmental sciences:Solid Earth sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 56, - "fields": { - "taxonomy": 1, - "name": "Environmental sciences", - "path": "1:Earth and environmental sciences:Environmental sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 57, - "fields": { - "taxonomy": 1, - "name": "Biogeochemistry", - "path": "1:Earth and environmental sciences:Biogeochemistry:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 58, - "fields": { - "taxonomy": 1, - "name": "Ocean sciences", - "path": "1:Earth and environmental sciences:Ocean sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 59, - "fields": { - "taxonomy": 1, - "name": "Environmental social sciences", - "path": "1:Earth and environmental sciences:Environmental social sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 60, - "fields": { - "taxonomy": 1, - "name": "Hydrology", - "path": "1:Earth and environmental sciences:Hydrology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 61, - "fields": { - "taxonomy": 1, - "name": "Natural hazards", - "path": "1:Earth and environmental sciences:Natural hazards:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 62, - "fields": { - "taxonomy": 1, - "name": "Space physics", - "path": "1:Earth and environmental sciences:Space physics:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 63, - "fields": { - "taxonomy": 1, - "name": "Limnology", - "path": "1:Earth and environmental sciences:Limnology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 64, - "fields": { - "taxonomy": 1, - "name": "Biological techniques", - "path": "1:Biological sciences:Biological techniques:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 65, - "fields": { - "taxonomy": 1, - "name": "Cell biology", - "path": "1:Biological sciences:Cell biology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 152, - "fields": { - "taxonomy": 1, - "name": "PCR", - "path": "1:Biological sciences:PCR:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 153, - "fields": { - "taxonomy": 1, - "name": "immunology", - "path": "1:Biological sciences:immunology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 66, - "fields": { - "taxonomy": 1, - "name": "Biochemistry", - "path": "1:Biological sciences:Biochemistry:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 67, - "fields": { - "taxonomy": 1, - "name": "Chemical biology", - "path": "1:Biological sciences:Chemical biology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 68, - "fields": { - "taxonomy": 1, - "name": "Molecular biology", - "path": "1:Biological sciences:Molecular biology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 69, - "fields": { - "taxonomy": 1, - "name": "Biotechnology", - "path": "1:Biological sciences:Biotechnology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 70, - "fields": { - "taxonomy": 1, - "name": "Genetics", - "path": "1:Biological sciences:Genetics:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 71, - "fields": { - "taxonomy": 1, - "name": "Neuroscience", - "path": "1:Biological sciences:Neuroscience:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 73, - "fields": { - "taxonomy": 1, - "name": "Microbiology", - "path": "1:Biological sciences:Microbiology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 74, - "fields": { - "taxonomy": 1, - "name": "Cancer", - "path": "1:Biological sciences:Cancer:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 75, - "fields": { - "taxonomy": 1, - "name": "Developmental biology", - "path": "1:Biological sciences:Developmental biology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 76, - "fields": { - "taxonomy": 1, - "name": "Biophysics", - "path": "1:Biological sciences:Biophysics:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 77, - "fields": { - "taxonomy": 1, - "name": "Physiology", - "path": "1:Biological sciences:Physiology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 78, - "fields": { - "taxonomy": 1, - "name": "Structural biology", - "path": "1:Biological sciences:Structural biology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 79, - "fields": { - "taxonomy": 1, - "name": "Evolution", - "path": "1:Biological sciences:Evolution:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 80, - "fields": { - "taxonomy": 1, - "name": "Computational biology and bioinformatics", - "path": "1:Biological sciences:Computational biology and bioinformatics:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 81, - "fields": { - "taxonomy": 1, - "name": "Systems biology", - "path": "1:Biological sciences:Systems biology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 82, - "fields": { - "taxonomy": 1, - "name": "Drug discovery", - "path": "1:Biological sciences:Drug discovery:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 83, - "fields": { - "taxonomy": 1, - "name": "Plant sciences", - "path": "1:Biological sciences:Plant sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 84, - "fields": { - "taxonomy": 1, - "name": "Stem cells", - "path": "1:Biological sciences:Stem cells:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 85, - "fields": { - "taxonomy": 1, - "name": "Zoology", - "path": "1:Biological sciences:Zoology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 86, - "fields": { - "taxonomy": 1, - "name": "Psychology", - "path": "1:Biological sciences:Psychology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 87, - "fields": { - "taxonomy": 1, - "name": "Diseases", - "path": "1:Health sciences:Diseases:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 88, - "fields": { - "taxonomy": 1, - "name": "Oncology", - "path": "1:Health sciences:Oncology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 89, - "fields": { - "taxonomy": 1, - "name": "Pathogenesis", - "path": "1:Health sciences:Pathogenesis:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 90, - "fields": { - "taxonomy": 1, - "name": "Anatomy", - "path": "1:Health sciences:Anatomy:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 91, - "fields": { - "taxonomy": 1, - "name": "Medical research", - "path": "1:Health sciences:Medical research:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 92, - "fields": { - "taxonomy": 1, - "name": "Health care", - "path": "1:Health sciences:Health care:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 93, - "fields": { - "taxonomy": 1, - "name": "Neurology", - "path": "1:Health sciences:Neurology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 94, - "fields": { - "taxonomy": 1, - "name": "Gastroenterology", - "path": "1:Health sciences:Gastroenterology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 95, - "fields": { - "taxonomy": 1, - "name": "Cardiology", - "path": "1:Health sciences:Cardiology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 96, - "fields": { - "taxonomy": 1, - "name": "Endocrinology", - "path": "1:Health sciences:Endocrinology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 97, - "fields": { - "taxonomy": 1, - "name": "Rheumatology", - "path": "1:Health sciences:Rheumatology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 98, - "fields": { - "taxonomy": 1, - "name": "Urology", - "path": "1:Health sciences:Urology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 99, - "fields": { - "taxonomy": 1, - "name": "Nephrology", - "path": "1:Health sciences:Nephrology:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 100, - "fields": { - "taxonomy": 1, - "name": "Biomarkers", - "path": "1:Health sciences:Biomarkers:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 101, - "fields": { - "taxonomy": 1, - "name": "Signs and symptoms", - "path": "1:Health sciences:Signs and symptoms:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 102, - "fields": { - "taxonomy": 1, - "name": "Molecular medicine", - "path": "1:Health sciences:Molecular medicine:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 103, - "fields": { - "taxonomy": 1, - "name": "Risk factors", - "path": "1:Health sciences:Risk factors:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 104, - "fields": { - "taxonomy": 1, - "name": "Social sciences", - "path": "1:Scientific community and society:Social sciences:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 105, - "fields": { - "taxonomy": 1, - "name": "Business and industry", - "path": "1:Scientific community and society:Business and industry:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 106, - "fields": { - "taxonomy": 1, - "name": "Scientific community", - "path": "1:Scientific community and society:Scientific community:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 107, - "fields": { - "taxonomy": 1, - "name": "Agriculture", - "path": "1:Scientific community and society:Agriculture:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 108, - "fields": { - "taxonomy": 1, - "name": "Energy and society", - "path": "1:Scientific community and society:Energy and society:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 109, - "fields": { - "taxonomy": 1, - "name": "Forestry", - "path": "1:Scientific community and society:Forestry:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 110, - "fields": { - "taxonomy": 1, - "name": "Water resources", - "path": "1:Scientific community and society:Water resources:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 111, - "fields": { - "taxonomy": 1, - "name": "Developing world", - "path": "1:Scientific community and society:Developing world:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 112, - "fields": { - "taxonomy": 1, - "name": "Geography", - "path": "1:Scientific community and society:Geography:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 145, - "fields": { - "taxonomy": 54, - "name": "language-en", - "path": "language-en:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 146, - "fields": { - "taxonomy": 54, - "name": "subtitle-en", - "path": "subtitle-en:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 147, - "fields": { - "taxonomy": 54, - "name": "subtitle-zh", - "path": "subtitle-zh:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 148, - "fields": { - "taxonomy": 50, - "name": "0", - "path": "0:" - } - }, - { - "model": "tagstore_django.tag", - "pk": 150, - "fields": { - "taxonomy": 54, - "name": "license-CC-BY-SA-4.0 Remix and Share", - "path": "license-CC-BY-SA-4.0 Remix and Share:" - } - } -] diff --git a/tagstore/README.rst b/tagstore/README.rst deleted file mode 100644 index 87bbfc65..00000000 --- a/tagstore/README.rst +++ /dev/null @@ -1,99 +0,0 @@ -Tagstore -======== - -Tagstore is a system for tagging entities. For example, a common use case would be applying difficulty tags like "easy", "medium", or "hard" to XBlocks (learnable content components) that are stored in Blockstore. - -Tagstore holds collections of tags called Taxonomies. The simplest taxonomy is a set of tags; the set {"easy", "medium", "hard"} is an example of a taxonomy for content difficulty levels. The tags in a taxonomy can optionally be hierarchical, so that they exist in a tree with parent-child relationships (e.g. all dogs are mammals, all mammals are animals). This is designed to support learning outcome hierarchies in particular. - -Tagstore is part of Blockstore but has been designed to be easily separable should the need arise. - -Features --------- - -* Python 3 API with type hints -* Allows any "entity" to be tagged, where an entity could be a user, a block, a collection, etc. -* Allows rich searching for entities by tags. e.g. "Find all large animals" will return an entity that was tagged with "large" and "dog", since it knows that the "dog" tag is a type of "animal" tag. -* Designed to support multiple tag storage backends, although the current version of Tagstore only includes a Django ORM backend, which stores tags in MySQL/PostgeSQL/SQLite. (A reasonably complete Neo4j backend implementation also existed in an early version and can found at https://github.com/open-craft/blockstore/commit/714b22f8456fb3b509aca54f59dc96de060d36fe) - -Non-features ------------- - -* Does not allow any tag to be in two different places in the same taxonomy (i.e. a tag cannot have two parents, nor can the same child tag appear in two different places in the hierarchy) -* Does not allow other types of relationships between tags other than organizing them into a hierarchy (no support for arbitrary relationships like "dog is similar to wolf"; such advanced graph relationships - which enable other types of taxonomies and fuzzy searches - could be added later but would mean we can't use SQL backends) -* Does not implement "private tags" (user A applies tag T to entity E, but only user A sees that tag). However, applications that use Tagstore may add an authorization/permissions layer to allow for private or hidden taxonomies. -* Does not allow manipulating tag hierarchies once they are created, other than by adding new tags to the tree. i.e. you cannot remove tags from a hierarchy, nor change their position in the tree etc. We assume that hierarchical tags will usually be created via import/export of externally developed taxonomies. - -API Example ------------ - -Here is an example of using the Tagstore API:: - - from tagstore.backends.django import DjangoTagstore - from tagstore.models import EntityId - tagstore = DjangoTagstore() - - # Create a biology taxonomy: - biology = tagstore.create_taxonomy("Biology", owner_id=None) - plant = biology.add_tag('plant') - conifer = biology.add_tag('conifer', parent_tag=plant) - cypress = biology.add_tag('cypress', parent_tag=conifer) - pine = biology.add_tag('pine', parent_tag=conifer) - aster = biology.add_tag('aster', parent_tag=plant) - - # Print tag hierarchy tree: - depths = {} - for (tag, parent) in biology.list_tags_hierarchically(): - depths[tag] = depths[parent] + 1 if parent else 0 - print((" " * depths[tag]) + tag.name) - # The resulting hierarchy that gets printed out is: - # plant - # aster - # conifer - # cypress - # pine - - # Create a "sizes" taxonomy: - sizes = tagstore.create_taxonomy("sizes", owner_id=None) - small = sizes.add_tag('small') - med = sizes.add_tag('med') - large = sizes.add_tag('large') - - # Tag some entities: - dandelion = EntityId(entity_type='thing', external_id='dandelion') - tagstore.add_tag_to(small, dandelion) - tagstore.add_tag_to(aster, dandelion) - redwood = EntityId(entity_type='thing', external_id='redwood') - tagstore.add_tag_to(large, redwood) - tagstore.add_tag_to(cypress, redwood) - - # Find all asters - set(tagstore.get_entities_tagged_with(aster)) - # result: {dandelion} - - # plants - set(tagstore.get_entities_tagged_with(plant)) - # result: {dandelion, redwood} - - # small plants - set(tagstore.get_entities_tagged_with_all({plant, small})) - # result: {dandelion} - - # plants, with no tag inheritance - set(tagstore.get_entities_tagged_with(plant, include_child_tags=False)) - # result: set() - - # conifers - set(tagstore.get_entities_tagged_with(conifer)) - # result: {redwood} - - # plants starting with "d" - set(tagstore.get_entities_tagged_with( - plant, entity_types=['thing'], external_id_prefix='d' - )) - # result: {dandelion} - - -Future Features ---------------- - -* REST API? diff --git a/tagstore/__init__.py b/tagstore/__init__.py deleted file mode 100644 index 55c87ff4..00000000 --- a/tagstore/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -Convenience module to make importing Tagstore easier. -""" -from .tagstore import Tagstore diff --git a/tagstore/backends/__init__.py b/tagstore/backends/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tagstore/backends/django.py b/tagstore/backends/django.py deleted file mode 100644 index 32793f40..00000000 --- a/tagstore/backends/django.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -Django ORM tag storage backend. -""" -from typing import Iterator, List, Optional, Set, Tuple -from django.db.models import Q, Subquery - -from .tagstore_django.models import Entity as EntityModel, Tag as TagModel, Taxonomy as TaxonomyModel - -from .. import Tagstore -from ..models import EntityId, Tag, TaxonomyId, Taxonomy, UserId - - -class DjangoTagstore(Tagstore): - """ - Django tag storage backend. - """ - - def create_taxonomy(self, name: str, owner_id: Optional[UserId] = None) -> Taxonomy: - """ Create a new taxonomy with the specified name and owner. """ - owner_obj = None - if owner_id is not None: - (owner_obj, _created) = EntityModel.objects.get_or_create( - entity_type=owner_id.entity_type, - external_id=owner_id.external_id, - ) - obj = TaxonomyModel.objects.create(name=name, owner=owner_obj) - return Taxonomy(uid=obj.id, name=name, owner_id=owner_id, tagstore=self) - - def get_taxonomy(self, taxonomy_uid: TaxonomyId) -> Optional[Taxonomy]: - try: - tax = TaxonomyModel.objects.get(pk=taxonomy_uid) - except TaxonomyModel.DoesNotExist: - return None - return tax.as_tuple(self) - - def _add_tag_to_taxonomy(self, taxonomy_uid: TaxonomyId, name: str, parent_tag: Optional[str] = None) -> str: - if parent_tag: - # Check the parent tag: - try: - pt = TagModel.objects.get(taxonomy_id=taxonomy_uid, name=parent_tag) - except TagModel.DoesNotExist as err: - raise ValueError("Invalid parent tag.") from err - path = TagModel.make_path(taxonomy_uid, name, pt.path) - else: - path = TagModel.make_path(taxonomy_uid, name) - db_tag, created = TagModel.objects.get_or_create( - taxonomy_id=taxonomy_uid, - name=name, - defaults={'path': path}, - ) - if not created: - if db_tag.path.lower() != path.lower(): - raise ValueError("That tag already exists with a different parent tag.") - return db_tag.name - - def get_tag_in_taxonomy(self, name: str, taxonomy_uid: TaxonomyId) -> Optional[Tag]: - """ - If a tag with the specified name (case insensitive) exists in this taxonomy, get it. - - Otherwise returns None. - """ - try: - tag_obj = TagModel.objects.get(taxonomy_id=taxonomy_uid, name=name) - return Tag(taxonomy_uid=taxonomy_uid, name=tag_obj.name) - except TagModel.DoesNotExist: - return None - - def list_tags_in_taxonomy(self, taxonomy_uid: TaxonomyId) -> Iterator[Tag]: - for tag in TagModel.objects.filter(taxonomy_id=taxonomy_uid).order_by('name'): - yield Tag(taxonomy_uid=taxonomy_uid, name=tag.name) - - def list_tags_in_taxonomy_hierarchically(self, taxonomy_uid: TaxonomyId) -> Iterator[Tuple[Tag, Tag]]: - """ - Get a list of all tags in the given taxonomy, in hierarchical and alphabetical order. - - Returns tuples of (Tag, parent_tag) where parent_tag is the parent tag. This method - guarantees that parent tags will be returned before their child tags. - """ - for tag in TagModel.objects.filter(taxonomy_id=taxonomy_uid).order_by('path'): - yield (Tag(taxonomy_uid=taxonomy_uid, name=tag.name), tag.parent_tag_tuple) - - def get_tags_in_taxonomy_hierarchically_as_dict(self, taxonomy_uid: TaxonomyId) -> dict: - """ - Get all tags in the given taxonomy as nested dictionaries. - - Returns a dictionary. An example is {'children': [ - {'name': 'mammal', 'id': 57, 'children': [ - {'name': 'cow', 'id': 58, 'children': []} - ]} - ]}. - """ - root = {'children': []} # type: ignore - all_nodes = {None: root} - taxonomy_uid_as_int = int(taxonomy_uid) - for tag in TagModel.objects.filter(taxonomy_id=taxonomy_uid_as_int).order_by('path'): - node = {'name': tag.name, 'id': tag.id, 'children': []} - as_tuple = Tag(taxonomy_uid=taxonomy_uid_as_int, name=tag.name) - all_nodes[as_tuple] = node # type: ignore - all_nodes[tag.parent_tag_tuple]['children'].append(node) - return root - - def list_tags_in_taxonomy_containing(self, taxonomy_uid: TaxonomyId, text: str) -> Iterator[Tag]: - for tag in TagModel.objects.filter(taxonomy_id=taxonomy_uid, name__icontains=text).order_by('name'): - yield Tag(taxonomy_uid=taxonomy_uid, name=tag.name) - - # Tagging Entities ########################## - - def add_tag_to(self, tag: Tag, *entity_ids: EntityId) -> None: - """ - Add the specified tag to the specified entity/entities. - - Will be a no-op if the tag is already applied. - """ - tag_model = TagModel.objects.get(taxonomy_id=tag.taxonomy_uid, name=tag.name) - for entity in entity_ids: - (em, _created) = EntityModel.objects.get_or_create( - entity_type=entity.entity_type, - external_id=entity.external_id, - ) - em.tags.add(tag_model) - - def remove_tag_from(self, tag: Tag, *entity_ids: EntityId) -> None: - """ - Remove the specified tag from the specified entity/entities - - Will be a no-op if the entities do not have that tag. - """ - tag = TagModel.objects.get(taxonomy_id=tag.taxonomy_uid, name=tag.name) - # This could be optimized to a single DB query, but that's probably not necessary - for eid in entity_ids: - try: - EntityModel.objects.get(entity_type=eid.entity_type, external_id=eid.external_id).tags.remove(tag) - except EntityModel.DoesNotExist: - pass - - def get_tags_applied_to(self, *entity_ids: EntityId) -> Set[Tag]: - """ Get the set of unique tags applied to any of the specified entity IDs """ - entity_filter = Q() - for eid in entity_ids: - q = Q(entity_type=eid.entity_type) & Q(external_id=eid.external_id) - entity_filter = entity_filter | q - entities = EntityModel.objects.filter(entity_filter) - tags = TagModel.objects.filter(entity__id__in=Subquery(entities.values('id'))) - tags_found = set() - for tag in tags: - tags_found.add(Tag(taxonomy_uid=tag.taxonomy_id, name=tag.name)) - return tags_found - - # Searching Entities ########################## - - def get_entities_tagged_with_all( - self, - tags: Set[Tag], - entity_types: Optional[List[str]] = None, - external_id_prefix: Optional[str] = None, - entity_ids: Optional[List[EntityId]] = None, # use this to filter a list of entity IDs by tag - include_child_tags=True, # For hierarchical taxonomies, include child tags - # (e.g. search for "Animal" will return results tagged only with "Dog") - ) -> Iterator[EntityId]: - - if not tags: - raise ValueError("tags must contain at least one Tag") - - entities = EntityModel.objects.all() # We start with the all() queryset, and filter it down. - - if include_child_tags: - # Convert the set of tags to a set of materialized paths: - tags_filter = Q() - for tag in tags: - tags_filter = tags_filter | (Q(taxonomy_id=tag.taxonomy_uid) & Q(name=tag.name)) - paths = TagModel.objects.filter(tags_filter).values_list('path', flat=True) - for path in paths: - entities = entities.filter(tags__path__startswith=path) - else: - for tag in tags: - entities = entities.filter(tags__taxonomy_id=tag.taxonomy_uid, tags__name=tag.name) - - if entity_types is not None: - entities = entities.filter(entity_type__in=entity_types) - - if external_id_prefix is not None: - entities = entities.filter(external_id__startswith=external_id_prefix) - - if entity_ids is not None: - addl_filter = Q() - for eid in entity_ids: - addl_filter = addl_filter | (Q(entity_type=eid.entity_type) & Q(external_id=eid.external_id)) - entities = entities.filter(addl_filter) - - for e in entities: - yield EntityId(entity_type=e.entity_type, external_id=e.external_id) diff --git a/tagstore/backends/tagstore_django/admin.py b/tagstore/backends/tagstore_django/admin.py deleted file mode 100644 index acee90ce..00000000 --- a/tagstore/backends/tagstore_django/admin.py +++ /dev/null @@ -1,60 +0,0 @@ -""" Admin for tags. """ - -from django import forms -from django.contrib import admin - -from tagstore.backends.django import DjangoTagstore - -from .models import Taxonomy, Tag, Entity, MAX_CHAR_FIELD_LENGTH - - -class CustomTagAdminForm(forms.ModelForm): - """ Sets an extra field `parent` which does not exist in Tag. """ - parent = forms.CharField(max_length=MAX_CHAR_FIELD_LENGTH, required=False) - - def __init__(self, *args, **kwargs): - """ Adds the parent name, if it exists, to the edit tag form. """ - parent_tag = '' - if 'instance' in kwargs and kwargs['instance'] is not None: - if kwargs['instance'].parent_tag_tuple: - parent_tag = kwargs['instance'].parent_tag_tuple.name - super().__init__(*args, **kwargs) - self.fields['parent'].widget.attrs.update({'value': parent_tag}) - - -class TagAdmin(admin.ModelAdmin): - """ Controls display and saving of Tag model objects. """ - readonly_fields = ('path',) - search_fields = ('path',) - form = CustomTagAdminForm - - def has_change_permission(self, request, obj=None): - """ Makes Tag objects uneditable. """ - return False - - def save_model(self, request, obj, form, change): - """ Uses the tagstore API to save new tags to the database. """ - tagstore = DjangoTagstore() - taxonomy = form.cleaned_data['taxonomy'] - name = form.cleaned_data['name'] - parent_tag_str = form.cleaned_data['parent'] - parent = tagstore.get_tag_in_taxonomy(parent_tag_str, taxonomy.id) - try: - tagstore.add_tag_to_taxonomy(name, taxonomy.id, parent) - except ValueError: - pass - - # def delete_model(self, request, obj): - # """ TODO: Uses the tagstore API to delete tags from the database. """ - # super(TagAdmin, self).delete_model(request, obj) - - -class EntityAdmin(admin.ModelAdmin): - """ Controls display and saving of Entity model objects. """ - list_display = ('entity_type', 'external_id') - search_fields = ('entity_type', 'external_id') - - -admin.site.register(Tag, TagAdmin) -admin.site.register(Entity, EntityAdmin) -admin.site.register(Taxonomy) diff --git a/tagstore/backends/tagstore_django/migrations/0001_initial.py b/tagstore/backends/tagstore_django/migrations/0001_initial.py index 65511b02..ddb97e0c 100644 --- a/tagstore/backends/tagstore_django/migrations/0001_initial.py +++ b/tagstore/backends/tagstore_django/migrations/0001_initial.py @@ -67,4 +67,4 @@ class Migration(migrations.Migration): name='entity', unique_together={('entity_type', 'external_id')}, ), - ] + ] \ No newline at end of file diff --git a/tagstore/backends/tagstore_django/migrations/0002_delete_all_app_models.py b/tagstore/backends/tagstore_django/migrations/0002_delete_all_app_models.py new file mode 100644 index 00000000..9a8d1310 --- /dev/null +++ b/tagstore/backends/tagstore_django/migrations/0002_delete_all_app_models.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.24 on 2021-08-24 13:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('tagstore_django', '0001_initial'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='tag', + unique_together=None, + ), + migrations.RemoveField( + model_name='tag', + name='taxonomy', + ), + migrations.RemoveField( + model_name='taxonomy', + name='owner', + ), + migrations.DeleteModel( + name='Entity', + ), + migrations.DeleteModel( + name='Tag', + ), + migrations.DeleteModel( + name='Taxonomy', + ), + ] diff --git a/tagstore/backends/tagstore_django/models.py b/tagstore/backends/tagstore_django/models.py deleted file mode 100644 index d1b42ff5..00000000 --- a/tagstore/backends/tagstore_django/models.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -Tagstore backend that uses the django ORM -""" -from typing import Optional - -from django.db import models - -from tagstore import Tagstore -from tagstore.models import EntityId, Taxonomy as TaxonomyTuple, UserId, Tag as TagTuple - -# If MySQL is configured to use utf8mb4 (correct utf8), indexed -# columns have a max length of 191. Until Django supports limiting index -# length to 191 characters, we need to limit the value length to below -# 191 characters, for any column that might be indexed. -# (https://code.djangoproject.com/ticket/18392#comment:3) -MAX_CHAR_FIELD_LENGTH = 180 - - -class Entity(models.Model): - """ - An entity that can be tagged. - """ - id = models.BigAutoField(primary_key=True) - entity_type = models.CharField(max_length=MAX_CHAR_FIELD_LENGTH) - external_id = models.CharField(max_length=MAX_CHAR_FIELD_LENGTH) - - tags = models.ManyToManyField('Tag') - - class Meta: - unique_together = ( - ('entity_type', 'external_id'), - ) - db_table = 'tagstore_entity' - verbose_name_plural = 'Entities' - - @property - def as_tuple(self) -> EntityId: - return EntityId(entity_type=self.entity_type, external_id=self.external_id) - - def __str__(self) -> str: - return "%s %s" % (self.entity_type, self.external_id) - - -class Taxonomy(models.Model): - """ - A taxonomy is a collection of tags, some of which may be organized into - a hierarchy. - """ - id = models.BigAutoField(primary_key=True) - name = models.CharField(max_length=MAX_CHAR_FIELD_LENGTH) - owner = models.ForeignKey(Entity, null=True, blank=True, on_delete=models.SET_NULL) - - class Meta: - db_table = 'tagstore_taxonomy' - verbose_name_plural = 'Taxonomies' - - def as_tuple(self, tagstore: Tagstore) -> TaxonomyTuple: - owner_id = UserId(self.owner.as_tuple) if self.owner is not None else None - return TaxonomyTuple(uid=self.id, name=self.name, owner_id=owner_id, tagstore=tagstore) - - def __str__(self) -> str: - return self.name - - -class Tag(models.Model): - """ - A tag within a taxonomy - """ - id = models.BigAutoField(primary_key=True) - taxonomy = models.ForeignKey(Taxonomy, null=False, on_delete=models.CASCADE) - # The tag string, like "good problem". - name = models.CharField(max_length=MAX_CHAR_FIELD_LENGTH, db_column='tag') - # Materialized path. Always ends with ":". - # A simple tag like "good-problem" would have a path of "good-problem:" - # A tag like "mammal" that is a child of "animal" would have a path of - # "animal:mammal:". Tags are not allowed to contain the ":" character - # so no escaping is necessary. - path = models.CharField(max_length=MAX_CHAR_FIELD_LENGTH, db_index=True) - - PATH_SEP = ':' # Character used to separate tags - - class Meta: - db_table = 'tagstore_tag' - ordering = ('name', ) - unique_together = ( - ('taxonomy', 'name'), - # Note that (taxonomy, path) is also unique but we don't bother - # with an index for that. - ) - - @classmethod - def make_path(cls, taxonomy_id: int, name: str, parent_path: str = '') -> str: - """ - Return the full 'materialized path' for use in the path field. - - make_path(15, 'easy') -> '15:easy:' - make_path(200, 'Lion', 'animal:mammal:') -> '200:animal:mammal:lion:' - """ - prefix = str(taxonomy_id) + cls.PATH_SEP - if parent_path: - assert parent_path.startswith(prefix) - return parent_path + name + cls.PATH_SEP - else: - return prefix + name + cls.PATH_SEP - - @property - def parent_tag_tuple(self) -> Optional[TagTuple]: - """ - Get the Tag tuple of this tag's parent, or None if it has no parent - - This model's 'path' field might look like '200:animal:mammal:lion:' - in which case parent_tag will return 'mammal' - """ - parts = self.path.split(self.PATH_SEP) - if len(parts) <= 3: - return None - return TagTuple(taxonomy_uid=self.taxonomy_id, name=parts[-3]) - - def __str__(self) -> str: - return self.name diff --git a/tagstore/backends/tagstore_django/templatetags/__init__.py b/tagstore/backends/tagstore_django/templatetags/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tagstore/backends/tagstore_django/templatetags/tagstore_admin.py b/tagstore/backends/tagstore_django/templatetags/tagstore_admin.py deleted file mode 100644 index 8743a7ef..00000000 --- a/tagstore/backends/tagstore_django/templatetags/tagstore_admin.py +++ /dev/null @@ -1,17 +0,0 @@ -""" Template tags for Django admin. """ -from django import template - -from tagstore.backends.django import DjangoTagstore - -register = template.Library() - - -@register.inclusion_tag('admin/tagstore_django/tag_hierarchy.html') -def tag_hierarchy(taxonomy_uid): - """ Renders a hierarchical view in HTML of Tag objects. """ - tagstore = DjangoTagstore() - if taxonomy_uid: - tags = tagstore.get_tags_in_taxonomy_hierarchically_as_dict(taxonomy_uid) - return {'tags': tags, 'taxonomy_uid': taxonomy_uid} - else: - return None diff --git a/tagstore/backends/tests/__init__.py b/tagstore/backends/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tagstore/constants.py b/tagstore/constants.py deleted file mode 100644 index ec647d14..00000000 --- a/tagstore/constants.py +++ /dev/null @@ -1,4 +0,0 @@ -""" Constants for tagstore """ - -# Special UID in blockstore for non-hierarchical, free form tags -FREEFORM_TAXONOMY_UID = -100 diff --git a/tagstore/models/__init__.py b/tagstore/models/__init__.py deleted file mode 100644 index 6fabd9fa..00000000 --- a/tagstore/models/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Convenience module to allow easier importing of models -""" -from .entity import EntityId -from .tag import Tag -from .taxonomy import TaxonomyId, Taxonomy -from .user import UserId diff --git a/tagstore/models/entity.py b/tagstore/models/entity.py deleted file mode 100644 index a73678f7..00000000 --- a/tagstore/models/entity.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -This is how we model any external entity that can be tagged. -""" -from collections import namedtuple - - -EntityId = namedtuple('EntityId', ['entity_type', 'external_id']) diff --git a/tagstore/models/taxonomy.py b/tagstore/models/taxonomy.py deleted file mode 100644 index 7f010d68..00000000 --- a/tagstore/models/taxonomy.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -A taxonomy is a collection of tags that can be applied to content. -""" -from collections import namedtuple -from typing import Iterator, NewType, Optional, Tuple - -from .tag import Tag - - -TaxonomyId = NewType('TaxonomyId', int) - -TaxonomyPrivate = namedtuple('TaxonomyPrivate', ['uid', 'name', 'owner_id', 'tagstore']) - - -class Taxonomy(TaxonomyPrivate): - """ - A taxonomy is a collection of tags that can be applied to content. - - Is a NamedTuple for performance, simplicity, and immutability, but - we can change it to a full class at some point if needed. - - uid: TaxonomyId - name: str - owner_id: Optional[UserId] - - tagstore: Any # Type is Tagstore but we can't define that without circular import - """ - - # Convenience methods: - - def add_tag(self, name: str, parent_tag: Optional[Tag] = None) -> Tag: - """ - Add the specified tag to this given taxonomy, and return it. - - If a Tag already exists in the taxonomy with the given name (case-insensitive) - and the given parent, then that Tag is returned and no changes are made. - - Will raise a ValueError if the specified taxonomy or parent doesn't exist. - Will raise a ValueError if trying to add a child tag that - already exists anywhere in the taxonomy. - """ - return self.tagstore.add_tag_to_taxonomy(name, self.uid, parent_tag) - - def get_tag(self, tag: str) -> Optional[Tag]: - """ - If a tag with the specified name (case insensitive) exists in this taxonomy, get it. - - Otherwise returns None. - """ - return self.tagstore.get_tag_in_taxonomy(tag, self.uid) - - def list_tags(self) -> Iterator[Tag]: - """ - Get a (flattened) list of all tags in the given taxonomy, in alphabetical order. - """ - return self.tagstore.list_tags_in_taxonomy(self.uid) - - def list_tags_hierarchically(self) -> Iterator[Tuple[Tag, Tag]]: - """ - Get a list of all tags in the given taxonomy, in hierarchical and alphabetical order. - - Returns tuples of (Tag, parent_tag). - This method guarantees that parent tags will be returned before their child tags. - """ - return self.tagstore.list_tags_in_taxonomy_hierarchically(self.uid) - - def list_tags_containing(self, text: str) -> Iterator[Tag]: - """ - Get a (flattened) list of all tags in the given taxonomy that contain the given string - (case insensitive). This is intended to be used for auto-complete when users tag content - by typing tags into a text field, for example. - """ - return self.tagstore.list_tags_in_taxonomy_containing(self.uid, text) diff --git a/tagstore/models/user.py b/tagstore/models/user.py deleted file mode 100644 index b00368cf..00000000 --- a/tagstore/models/user.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -User-related types -""" - -from typing import NewType - -from .entity import EntityId - -UserId = NewType('UserId', EntityId) diff --git a/tagstore/mypy.ini b/tagstore/mypy.ini deleted file mode 100644 index 3e7cb11f..00000000 --- a/tagstore/mypy.ini +++ /dev/null @@ -1,7 +0,0 @@ -[mypy] -# Don't warn about django modules being untyped: -ignore_missing_imports = True - -[mypy-tagstore.backends.tagstore_django.migrations.*] -ignore_errors = True - diff --git a/tagstore/tagstore.py b/tagstore/tagstore.py deleted file mode 100644 index 4f15dedf..00000000 --- a/tagstore/tagstore.py +++ /dev/null @@ -1,150 +0,0 @@ -""" -A system for storing and retrieving tags related to Blockstore entities -""" - -from typing import Iterator, List, Optional, Set, Tuple - -from .models import EntityId, Tag, TaxonomyId, Taxonomy, UserId - - -class Tagstore: - """ - Python API to store and retrieve tags and taxonomies - - This class defines the API only, and the implementation is done - by a backend subclass. - """ - - # Taxonomy CRUD ########################## - - def create_taxonomy(self, name: str, owner_id: Optional[UserId] = None) -> Taxonomy: - """ Create a new taxonomy with the specified name and owner. """ - raise NotImplementedError() - - def get_taxonomy(self, taxonomy_uid: TaxonomyId) -> Optional[Taxonomy]: - """ Get metadata about the given taxonomy """ - raise NotImplementedError() - - def add_tag_to_taxonomy(self, name: str, taxonomy_uid: TaxonomyId, parent_tag: Optional[Tag] = None) -> Tag: - """ - Add the specified tag to the given taxonomy, and retuns it. - - If a Tag already exists in the taxonomy with the given name (case-insensitive) - and the given parent, then that Tag is returned and no changes are made. - - Will raise a ValueError if the specified taxonomy or parent doesn't exist. - Will raise a ValueError if trying to add a child tag that - already exists but with a different parent tag. - - Subclasses should implement this by overriding _add_tag_to_taxonomy() - """ - if not isinstance(name, str) or len(name) < 1: - raise ValueError("Tag name must be a (non-empty) string.") - - if name != name.strip(): - raise ValueError("Tag name cannot start or end with whitespace.") - - if any(char in name for char in ':,;\n\r\\'): - raise ValueError("Tag name contains an invalid character.") - - parent_tag_str = None - if parent_tag is not None: - if parent_tag.taxonomy_uid != taxonomy_uid: - raise ValueError("A tag cannot have a parent from another taxonomy") - parent_tag_str = parent_tag.name - - final_name = self._add_tag_to_taxonomy(taxonomy_uid=taxonomy_uid, name=name, parent_tag=parent_tag_str) - return Tag(taxonomy_uid=taxonomy_uid, name=final_name) - - def _add_tag_to_taxonomy(self, taxonomy_uid: TaxonomyId, name: str, parent_tag: Optional[str] = None) -> str: - """ - Subclasses should override this method to implement adding tags to a taxonomy. - - It should return the 'name' value of the newly created tag, or the existing tag - if a tag already exists. - """ - raise NotImplementedError() - - def get_tag_in_taxonomy(self, name: str, taxonomy_uid: TaxonomyId) -> Optional[Tag]: - """ - If a tag with the specified name (case insensitive) exists in this taxonomy, get it. - - Otherwise returns None. - """ - raise NotImplementedError() - - def list_tags_in_taxonomy(self, taxonomy_uid: TaxonomyId) -> Iterator[Tag]: - """ - Get a (flattened) list of all tags in the given taxonomy, in alphabetical order. - """ - raise NotImplementedError() - yield None # Required to make this non-implementation also a generator. pylint: disable=unreachable - - def list_tags_in_taxonomy_hierarchically(self, taxonomy_uid: TaxonomyId) -> Iterator[Tuple[Tag, Tag]]: - """ - Get a list of all tags in the given taxonomy, in hierarchical and alphabetical order. - - Returns tuples of (Tag, parent_tag). - This method guarantees that parent tags will be returned before their child tags. - """ - raise NotImplementedError() - yield None # Required to make this non-implementation also a generator. pylint: disable=unreachable - - def list_tags_in_taxonomy_containing(self, taxonomy_uid: TaxonomyId, text: str) -> Iterator[Tag]: - """ - Get a (flattened) list of all tags in the given taxonomy that contain the given string - (case insensitive). This is intended to be used for auto-complete when users tag content - by typing tags into a text field, for example. - """ - raise NotImplementedError() - yield None # Required to make this non-implementation also a generator. pylint: disable=unreachable - - # Tagging Entities ########################## - - def add_tag_to(self, tag: Tag, *entity_ids: EntityId) -> None: - """ - Add the specified tag to the specified entity/entities. - - Will be a no-op if the tag is already applied. - """ - raise NotImplementedError() - - def remove_tag_from(self, tag: Tag, *entity_ids: EntityId) -> None: - """ - Remove the specified tag from the specified entity/entities - - Will be a no-op if the entities do not have that tag. - """ - raise NotImplementedError() - - def get_tags_applied_to(self, *entity_ids: EntityId) -> Set[Tag]: - """ Get the set of unique tags applied to any of the specified entity IDs """ - raise NotImplementedError() - - # Searching Entities ########################## - - def get_entities_tagged_with(self, tag: Tag, **kwargs) -> Iterator[EntityId]: - """ - Get an iterator over all entities that have been tagged with the given tag. - - Also accepts the same filtering keyword arguments as - get_entities_tagged_with_all() - """ - # Subclasses do not generally need to override this method. - yield from self.get_entities_tagged_with_all({tag}, **kwargs) - - def get_entities_tagged_with_all( - self, - tags: Set[Tag], - entity_types: Optional[List[str]] = None, - external_id_prefix: Optional[str] = None, - entity_ids: Optional[List[EntityId]] = None, # use this to filter a list of entity IDs by tag - include_child_tags=True, # For hierarchical taxonomies, include child tags - # (e.g. search for "Animal" will return results tagged only with "Dog") - ) -> Iterator[EntityId]: - """ - Method for searching/filtering for entities that have all the specified tags - and match all of the specified conditions - """ - raise NotImplementedError() - yield None # Required to make this non-implementation also a generator. pylint: disable=unreachable diff --git a/tagstore/tagstore_rest/__init__.py b/tagstore/tagstore_rest/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tagstore/tagstore_rest/routers.py b/tagstore/tagstore_rest/routers.py deleted file mode 100644 index 86145eeb..00000000 --- a/tagstore/tagstore_rest/routers.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Routers for Tagstore API. -""" - -from rest_framework_nested import routers -from rest_framework.routers import Route - - -class EntityRouter(routers.DefaultRouter): - """ - Custom routing for the Entity model. - """ - routes = [ - Route( - url=r'^{prefix}/$', - mapping={'get': 'list'}, - name='{basename}-list', - detail=False, - initkwargs={'suffix': 'List'} - ), - Route( - url=r'^{prefix}/(?P[^/.]+)/{lookup}/$', - mapping={'get': 'retrieve'}, - name='{basename}-detail', - detail=True, - initkwargs={'suffix': 'Detail'} - ), - Route( - url=r'^{prefix}/(?P[^/.]+)/{lookup}/tags/$', - mapping={'get': 'tags', 'post': 'update_tags'}, - name='{basename}-tags', - detail=True, - initkwargs={'suffix': 'Tags'} - ), - ] - - def get_lookup_regex(self, viewset, lookup_prefix=''): - """ - Given a viewset, return the portion of URL regex that is used - to match against a single instance. - - This method adds support for comma-delimited composite keys. If a resource - requires those, set lookup_fields, lookup_url_kwargs and lookup_value_regexes - instead of the singular versions. - - Note that lookup_prefix is not used directly inside REST rest_framework - itself, but is required in order to nicely support nested router - implementations, such as drf-nested-routers. - - https://github.com/alanjds/drf-nested-routers - """ - - # drf-nested-routers assumes that lookup will have length > 0 and so - # always appends "_" to it to create the lookup_prefix. If lookup_prefix - # is only "_" we reset it to "". - if lookup_prefix == '_': - lookup_prefix = '' - - lookup_fields = getattr(viewset, 'lookup_fields', None) - if lookup_fields: - lookup_url_kwargs = getattr(viewset, 'lookup_url_kwargs', lookup_fields) - lookup_value_regexes = getattr(viewset, 'lookup_value_regexes', len(lookup_fields) * ['[^/.]+']) - return ','.join([ - '(?P<{lookup_prefix}{lookup_url_kwarg}>{lookup_value_regex})'.format( - lookup_prefix=lookup_prefix, - lookup_url_kwarg=lookup_url_kwarg, - lookup_value_regex=lookup_value_regex, - ) for (lookup_url_kwarg, lookup_value_regex) in zip(lookup_url_kwargs, lookup_value_regexes) - ]) - - return super().get_lookup_regex(viewset, lookup_prefix=lookup_prefix) diff --git a/tagstore/tagstore_rest/urls.py b/tagstore/tagstore_rest/urls.py deleted file mode 100644 index ad1cfb0c..00000000 --- a/tagstore/tagstore_rest/urls.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -Root API URLs for Tagstore. - -All API URLs should be versioned, so urlpatterns should only -contain namespaces for the active versions of the API. -""" -from django.conf.urls import url, include - -app_name = 'tagstore' - -urlpatterns = [ - url(r'^api/v1/', include('tagstore.tagstore_rest.v1.urls', namespace='apiv1')), -] diff --git a/tagstore/tagstore_rest/v1/__init__.py b/tagstore/tagstore_rest/v1/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tagstore/tagstore_rest/v1/serializers/__init__.py b/tagstore/tagstore_rest/v1/serializers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tagstore/tagstore_rest/v1/serializers/entities.py b/tagstore/tagstore_rest/v1/serializers/entities.py deleted file mode 100644 index 68a3a0d1..00000000 --- a/tagstore/tagstore_rest/v1/serializers/entities.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Serializers for Entities. -""" - -from rest_framework import serializers - -from tagstore.backends.tagstore_django.models import Entity - - -class EntitySerializer(serializers.ModelSerializer): - """ - Serializer for the Entity model. - """ - - class Meta: - - model = Entity - - fields = ( - 'id', - 'entity_type', - 'external_id', - ) - - -class TagByTaxonomySerializer(serializers.Serializer): - """ - Serializer for the Tag model by Taxonomy. - """ - # pylint: disable=abstract-method - - taxonomy_uid = serializers.IntegerField() - taxonomy_name = serializers.CharField() - tag = serializers.CharField() - - -class EntityTagSerializer(serializers.Serializer): - """ - Serializer for the Tag model by Entity. - """ - # pylint: disable=abstract-method - - tags = TagByTaxonomySerializer(many=True, required=False) diff --git a/tagstore/tagstore_rest/v1/tests/__init__.py b/tagstore/tagstore_rest/v1/tests/__init__.py deleted file mode 100644 index d2557275..00000000 --- a/tagstore/tagstore_rest/v1/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Create your tests in sub-packages prefixed with "test_" (e.g. test_views). diff --git a/tagstore/tagstore_rest/v1/tests/test_serializers.py b/tagstore/tagstore_rest/v1/tests/test_serializers.py deleted file mode 100644 index 2277affb..00000000 --- a/tagstore/tagstore_rest/v1/tests/test_serializers.py +++ /dev/null @@ -1,87 +0,0 @@ -""" Tests for api v1 serializers. """ - -from django.test import TestCase -from rest_framework.test import APIRequestFactory - -from tagstore.backends.tagstore_django.models import Entity - -from ..serializers.entities import EntitySerializer, EntityTagSerializer - - -class SerializerBaseTestCase(TestCase): - """ - Base class for serializer tests. - """ - - def setUp(self): - - super().setUp() - - self.request = APIRequestFactory().get('/') - self.request.query_params = {} - self.context = { - 'request': self.request - } - - -class EntitySerializerTestCase(SerializerBaseTestCase): - """ - Tests for the EntitySerializer - """ - - def test_entity_serializer_data(self): - - entity = Entity( - entity_type='xblock', - external_id='some-resource-uri', - ) - - entity_serializer = EntitySerializer( - entity, context=self.context, - ) - - self.assertSequenceEqual(list(entity_serializer.data.keys()), [ - 'id', 'entity_type', 'external_id', - ]) - - self.assertEqual(entity_serializer.data['entity_type'], 'xblock') - self.assertEqual(entity_serializer.data['external_id'], 'some-resource-uri') - - -class EntityTagSerializerTestCase(TestCase): - """ - Tests for the EntityTagSerializer - """ - - def setUp(self): - - super().setUp() - - self.entity_tags = { - 'tags': [ - { - 'taxonomy_uid': 7, - 'taxonomy_name': 'Subject Area', - 'tag': 'Biochemistry' - }, - { - 'taxonomy_uid': 9, - 'taxonomy_name': 'License', - 'tag': 'CC-BY-SA-4.0' - } - ] - } - - def test_entity_tag_serializer_data(self): - - entity_tag_serializer = EntityTagSerializer( - self.entity_tags - ) - - self.assertSequenceEqual(list(entity_tag_serializer.data.keys()), ['tags']) - self.assertSequenceEqual(list(entity_tag_serializer.data['tags'][0].keys()), [ - 'taxonomy_uid', 'taxonomy_name', 'tag', - ]) - - self.assertEqual(entity_tag_serializer.data['tags'][0]['taxonomy_uid'], 7) - self.assertEqual(entity_tag_serializer.data['tags'][0]['taxonomy_name'], 'Subject Area') diff --git a/tagstore/tagstore_rest/v1/tests/test_views.py b/tagstore/tagstore_rest/v1/tests/test_views.py deleted file mode 100644 index 682aa00a..00000000 --- a/tagstore/tagstore_rest/v1/tests/test_views.py +++ /dev/null @@ -1,125 +0,0 @@ -""" Tests for api v1 views. """ -from urllib.parse import urlencode - -from django.contrib.auth import get_user_model -from django.test import TestCase -from rest_framework.authtoken.models import Token -from rest_framework.reverse import reverse -from rest_framework.test import APIClient - -from tagstore.backends.django import DjangoTagstore -from tagstore.models import EntityId - -User = get_user_model() - - -class ViewsBaseTestCase(TestCase): - """ Base class for tests. """ - - def setUp(self): - - super().setUp() - - self.client = APIClient() - test_user = User.objects.create(username='test-service-user') - token = Token.objects.create(user=test_user) - self.client.credentials(HTTP_AUTHORIZATION='Token ' + token.key) - - self.tagstore = DjangoTagstore() - - self.taxonomy = self.tagstore.create_taxonomy("This Taxonomy") - self.tags = [ - self.taxonomy.add_tag('tag1'), - self.taxonomy.add_tag('tag2'), - ] - - self.entity = EntityId(entity_type='xblock', external_id='some-resource-uri') - for t in self.tags: - self.tagstore.add_tag_to(t, self.entity) - - def response(self, view_name, kwargs=None, method='get', query_params=None, - expected_response_code=200, body=None, **method_kwargs): - """ - Returns a response with the wsgi_request containing any given query params. - """ - url = reverse(view_name, kwargs=kwargs) - if query_params: - url = '{}?{}'.format(url, urlencode(query_params)) - - response = getattr(self.client, method)(url, body, **method_kwargs) - self.assertEqual(response.status_code, expected_response_code) - return response - - -class EntityTagViewSetTestCase(ViewsBaseTestCase): - """ Tests for EntityTagViewSet. """ - - def test_list(self): - - response = self.response('tagstore:apiv1:entity-tags', kwargs={ - 'pk': self.entity.external_id, - 'entity_type': self.entity.entity_type, - }) - - self.assertIn('tags', response.data) - self.assertTrue(isinstance(response.data['tags'], list)) - self.assertEqual(len(response.data['tags']), 2) - self.assertIn('taxonomy_uid', response.data['tags'][0]) - self.assertIn('taxonomy_name', response.data['tags'][0]) - self.assertIn('tag', response.data['tags'][0]) - self.assertEqual(response.data['tags'][0]['tag'], 'tag1') - - def test_query_params_for_taxonomy(self): - - response = self.response('tagstore:apiv1:entity-tags', kwargs={ - 'pk': self.entity.external_id, - 'entity_type': self.entity.entity_type, - }, query_params={'taxonomies': self.taxonomy.uid}) - - self.assertTrue(isinstance(response.data['tags'], list)) - self.assertEqual(len(response.data['tags']), 2) - - def test_query_params_for_taxonomy_by_name(self): - - response = self.response('tagstore:apiv1:entity-tags', kwargs={ - 'pk': self.entity.external_id, - 'entity_type': self.entity.entity_type, - }, query_params={'taxonomies': self.taxonomy.name}) - - self.assertTrue(isinstance(response.data['tags'], list)) - self.assertEqual(len(response.data['tags']), 2) - - def test_post(self): - body = { - 'tags': ['bunch', 'of', 'silly', 'tags']} - - response = self.response('tagstore:apiv1:entity-tags', kwargs={ - 'pk': self.entity.external_id, - 'entity_type': self.entity.entity_type, - }, expected_response_code=201, body=body, method='post') - - self.assertTrue(isinstance(response.data['tags'], list)) - self.assertEqual(len(response.data['tags']), 6) - - def test_post_with_complex_tag_json(self): - body = { - 'tags': [ - { - 'taxonomy_uid': self.taxonomy.uid, - 'tag': 'test', - }, - { - 'taxonomy_uid': self.taxonomy.uid, - 'tag': 'test3', - 'parent': 'test', - } - ]} - - response = self.response('tagstore:apiv1:entity-tags', kwargs={ - 'pk': self.entity.external_id, - 'entity_type': self.entity.entity_type, - }, expected_response_code=201, body=body, method='post', format='json') - - self.assertTrue(isinstance(response.data['tags'], list)) - self.assertEqual(len(response.data['tags']), 4) - self.assertTrue(any(x['tag'] == 'test' for x in response.data['tags'])) diff --git a/tagstore/tagstore_rest/v1/urls.py b/tagstore/tagstore_rest/v1/urls.py deleted file mode 100644 index a3f37a7c..00000000 --- a/tagstore/tagstore_rest/v1/urls.py +++ /dev/null @@ -1,16 +0,0 @@ -""" API v1 URLs for Tagstore. """ - -from django.conf.urls import url, include - -from ..routers import EntityRouter -from .views.entities import EntityViewSet - -root_router = EntityRouter() - -app_name = 'tagstore' - -root_router.register(r'entities', EntityViewSet) - -urlpatterns = [ - url(r'^', include(root_router.urls)), -] diff --git a/tagstore/tagstore_rest/v1/views/__init__.py b/tagstore/tagstore_rest/v1/views/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tagstore/tagstore_rest/v1/views/entities.py b/tagstore/tagstore_rest/v1/views/entities.py deleted file mode 100644 index 000c2115..00000000 --- a/tagstore/tagstore_rest/v1/views/entities.py +++ /dev/null @@ -1,125 +0,0 @@ -''' -Views for Tags and Taxonomies. -''' -import logging - -from rest_framework import viewsets -from rest_framework.generics import get_object_or_404 -from rest_framework.response import Response - -from tagstore.backends.tagstore_django.models import ( - Entity, - Tag, - Taxonomy -) -from tagstore.backends.django import DjangoTagstore -from tagstore.models.taxonomy import TaxonomyId - -from tagstore.constants import FREEFORM_TAXONOMY_UID - -from ..serializers.entities import EntitySerializer, EntityTagSerializer - -logger = logging.getLogger(__name__) - - -class EntityViewSet(viewsets.ViewSet): - ''' - ViewSet for Entity model and its tags. - ''' - - queryset = Entity.objects.all() - serializer_class = EntitySerializer - - def list(self, request, *args, **kwargs): - ''' - Get a list of all entities. - ''' - serializer = EntitySerializer(self.queryset, many=True) - return Response(serializer.data) - - def retrieve(self, request, pk=None, entity_type=None): - ''' - Get a single entity. - ''' - entity = get_object_or_404(Entity, external_id=pk, entity_type=entity_type) - serializer = EntitySerializer(entity) - return Response(serializer.data) - - def _convert(self, tag: Tag) -> dict: - ''' - Prepare the tag for serialization. - ''' - return { - 'taxonomy_uid': tag.taxonomy.id, - 'taxonomy_name': tag.taxonomy.name, - 'tag': tag.name, - } - - def serialize_tags(self, entity, taxonomies=None): - ''' - Serialize an entity's tags, optionally filtered by taxonomy. - ''' - if taxonomies and taxonomies[0]: - try: - queryset = entity.tags.filter(taxonomy__in=taxonomies) - except ValueError: - queryset = entity.tags.filter(taxonomy__name__in=taxonomies) - else: - queryset = entity.tags.all() - - extracted = [self._convert(tag) for tag in queryset] - return EntityTagSerializer({'tags': extracted}) - - def tags(self, request, entity_type=None, pk=None): - ''' - Get a list of all tags belonging to an entity. - - These can be optionally filtered by `taxonomy` params. - ''' - entity = get_object_or_404(Entity, external_id=pk, entity_type=entity_type) - taxonomies = request.GET.get('taxonomies', '').split(',') - - serializer = self.serialize_tags(entity, taxonomies) - return Response(serializer.data) - - def update_tags(self, request, entity_type=None, pk=None): - ''' - Update tags belonging to an entity. - ''' - tagstore = DjangoTagstore() - - entity_obj = get_object_or_404(Entity, external_id=pk, entity_type=entity_type) - freeform_obj, _ = Taxonomy.objects.get_or_create(id=FREEFORM_TAXONOMY_UID, name='FreeForm') - freeform = TaxonomyId(freeform_obj.id) - - if not request.data.get('tags', ''): - return Response(status=204) - - for tag in request.data['tags']: - # a single free form tag - if isinstance(tag, str): - _tag = tagstore.get_tag_in_taxonomy(tag, freeform) - if not _tag: - _tag = tagstore.add_tag_to_taxonomy(tag, freeform) - tagstore.add_tag_to(_tag, entity_obj.as_tuple) - - # a tag within a taxonomy - elif isinstance(tag, dict): - taxonomy_uid = tag.get('taxonomy_uid', None) - tag_name = tag.get('tag', None) - tag_parent = tag.get('parent', None) - - if not taxonomy_uid or not tag_name: - continue - - tx_obj, _ = Taxonomy.objects.get_or_create(id=taxonomy_uid) - - tx = TaxonomyId(tx_obj.id) - p = tagstore.get_tag_in_taxonomy(tag_parent, tx) - _tag = tagstore.get_tag_in_taxonomy(tag_name, tx) - if not _tag: - _tag = tagstore.add_tag_to_taxonomy(tag_name, tx, p) - tagstore.add_tag_to(_tag, entity_obj.as_tuple) - - serializer = self.serialize_tags(entity_obj) - return Response(serializer.data, status=201) From 296d19b41a1180add870967058ebb7f69c8b676d Mon Sep 17 00:00:00 2001 From: Usman Khalid <2200617@gmail.com> Date: Sun, 15 Nov 2020 06:50:01 +0500 Subject: [PATCH 2/6] Move apps/api to apps/rest_api. --- blockstore/apps/{api => rest_api}/__init__.py | 0 blockstore/apps/{api => rest_api}/constants.py | 0 blockstore/apps/{api => rest_api}/permissions.py | 0 blockstore/apps/{api => rest_api}/relations.py | 0 blockstore/apps/{api => rest_api}/routers.py | 0 blockstore/apps/{api => rest_api}/urls.py | 2 +- blockstore/apps/{api => rest_api}/v1/__init__.py | 0 blockstore/apps/{api => rest_api}/v1/serializers/__init__.py | 0 blockstore/apps/{api => rest_api}/v1/serializers/bundles.py | 0 .../apps/{api => rest_api}/v1/serializers/collections.py | 0 blockstore/apps/{api => rest_api}/v1/serializers/drafts.py | 0 blockstore/apps/{api => rest_api}/v1/tests/__init__.py | 0 blockstore/apps/{api => rest_api}/v1/tests/helpers.py | 0 .../apps/{api => rest_api}/v1/tests/test_authorization.py | 2 +- blockstore/apps/{api => rest_api}/v1/tests/test_contract.py | 2 +- blockstore/apps/{api => rest_api}/v1/urls.py | 0 blockstore/apps/{api => rest_api}/v1/views/__init__.py | 0 blockstore/apps/{api => rest_api}/v1/views/bundles.py | 0 blockstore/apps/{api => rest_api}/v1/views/collections.py | 0 blockstore/apps/{api => rest_api}/v1/views/drafts.py | 0 blockstore/settings/base.py | 4 ++-- blockstore/urls.py | 2 +- 22 files changed, 6 insertions(+), 6 deletions(-) rename blockstore/apps/{api => rest_api}/__init__.py (100%) rename blockstore/apps/{api => rest_api}/constants.py (100%) rename blockstore/apps/{api => rest_api}/permissions.py (100%) rename blockstore/apps/{api => rest_api}/relations.py (100%) rename blockstore/apps/{api => rest_api}/routers.py (100%) rename blockstore/apps/{api => rest_api}/urls.py (74%) rename blockstore/apps/{api => rest_api}/v1/__init__.py (100%) rename blockstore/apps/{api => rest_api}/v1/serializers/__init__.py (100%) rename blockstore/apps/{api => rest_api}/v1/serializers/bundles.py (100%) rename blockstore/apps/{api => rest_api}/v1/serializers/collections.py (100%) rename blockstore/apps/{api => rest_api}/v1/serializers/drafts.py (100%) rename blockstore/apps/{api => rest_api}/v1/tests/__init__.py (100%) rename blockstore/apps/{api => rest_api}/v1/tests/helpers.py (100%) rename blockstore/apps/{api => rest_api}/v1/tests/test_authorization.py (97%) rename blockstore/apps/{api => rest_api}/v1/tests/test_contract.py (99%) rename blockstore/apps/{api => rest_api}/v1/urls.py (100%) rename blockstore/apps/{api => rest_api}/v1/views/__init__.py (100%) rename blockstore/apps/{api => rest_api}/v1/views/bundles.py (100%) rename blockstore/apps/{api => rest_api}/v1/views/collections.py (100%) rename blockstore/apps/{api => rest_api}/v1/views/drafts.py (100%) diff --git a/blockstore/apps/api/__init__.py b/blockstore/apps/rest_api/__init__.py similarity index 100% rename from blockstore/apps/api/__init__.py rename to blockstore/apps/rest_api/__init__.py diff --git a/blockstore/apps/api/constants.py b/blockstore/apps/rest_api/constants.py similarity index 100% rename from blockstore/apps/api/constants.py rename to blockstore/apps/rest_api/constants.py diff --git a/blockstore/apps/api/permissions.py b/blockstore/apps/rest_api/permissions.py similarity index 100% rename from blockstore/apps/api/permissions.py rename to blockstore/apps/rest_api/permissions.py diff --git a/blockstore/apps/api/relations.py b/blockstore/apps/rest_api/relations.py similarity index 100% rename from blockstore/apps/api/relations.py rename to blockstore/apps/rest_api/relations.py diff --git a/blockstore/apps/api/routers.py b/blockstore/apps/rest_api/routers.py similarity index 100% rename from blockstore/apps/api/routers.py rename to blockstore/apps/rest_api/routers.py diff --git a/blockstore/apps/api/urls.py b/blockstore/apps/rest_api/urls.py similarity index 74% rename from blockstore/apps/api/urls.py rename to blockstore/apps/rest_api/urls.py index 12ba0a3e..8ccdd356 100644 --- a/blockstore/apps/api/urls.py +++ b/blockstore/apps/rest_api/urls.py @@ -9,5 +9,5 @@ app_name = 'blockstore' urlpatterns = [ - url(r'^v1/', include('blockstore.apps.api.v1.urls', namespace='v1')), + url(r'^v1/', include('blockstore.apps.rest_api.v1.urls', namespace='v1')), ] diff --git a/blockstore/apps/api/v1/__init__.py b/blockstore/apps/rest_api/v1/__init__.py similarity index 100% rename from blockstore/apps/api/v1/__init__.py rename to blockstore/apps/rest_api/v1/__init__.py diff --git a/blockstore/apps/api/v1/serializers/__init__.py b/blockstore/apps/rest_api/v1/serializers/__init__.py similarity index 100% rename from blockstore/apps/api/v1/serializers/__init__.py rename to blockstore/apps/rest_api/v1/serializers/__init__.py diff --git a/blockstore/apps/api/v1/serializers/bundles.py b/blockstore/apps/rest_api/v1/serializers/bundles.py similarity index 100% rename from blockstore/apps/api/v1/serializers/bundles.py rename to blockstore/apps/rest_api/v1/serializers/bundles.py diff --git a/blockstore/apps/api/v1/serializers/collections.py b/blockstore/apps/rest_api/v1/serializers/collections.py similarity index 100% rename from blockstore/apps/api/v1/serializers/collections.py rename to blockstore/apps/rest_api/v1/serializers/collections.py diff --git a/blockstore/apps/api/v1/serializers/drafts.py b/blockstore/apps/rest_api/v1/serializers/drafts.py similarity index 100% rename from blockstore/apps/api/v1/serializers/drafts.py rename to blockstore/apps/rest_api/v1/serializers/drafts.py diff --git a/blockstore/apps/api/v1/tests/__init__.py b/blockstore/apps/rest_api/v1/tests/__init__.py similarity index 100% rename from blockstore/apps/api/v1/tests/__init__.py rename to blockstore/apps/rest_api/v1/tests/__init__.py diff --git a/blockstore/apps/api/v1/tests/helpers.py b/blockstore/apps/rest_api/v1/tests/helpers.py similarity index 100% rename from blockstore/apps/api/v1/tests/helpers.py rename to blockstore/apps/rest_api/v1/tests/helpers.py diff --git a/blockstore/apps/api/v1/tests/test_authorization.py b/blockstore/apps/rest_api/v1/tests/test_authorization.py similarity index 97% rename from blockstore/apps/api/v1/tests/test_authorization.py rename to blockstore/apps/rest_api/v1/tests/test_authorization.py index 022f1b66..cdcdb63b 100644 --- a/blockstore/apps/api/v1/tests/test_authorization.py +++ b/blockstore/apps/rest_api/v1/tests/test_authorization.py @@ -1,7 +1,7 @@ """ Tests to check that the API is only accessible by superusers or authorized application service users. See - blockstore.apps.api.permissions.IsSuperUserOrAuthorizedApplication + blockstore.apps.rest_api.permissions.IsSuperUserOrAuthorizedApplication for details. """ from django.contrib.auth import get_user_model diff --git a/blockstore/apps/api/v1/tests/test_contract.py b/blockstore/apps/rest_api/v1/tests/test_contract.py similarity index 99% rename from blockstore/apps/api/v1/tests/test_contract.py rename to blockstore/apps/rest_api/v1/tests/test_contract.py index 70b9b8a3..39d6a841 100644 --- a/blockstore/apps/api/v1/tests/test_contract.py +++ b/blockstore/apps/rest_api/v1/tests/test_contract.py @@ -26,7 +26,7 @@ from rest_framework.test import APIClient from blockstore.apps.bundles.tests.storage_utils import isolate_test_storage -from blockstore.apps.api.constants import UUID4_REGEX +from blockstore.apps.rest_api.constants import UUID4_REGEX from .helpers import ( create_bundle_with_history, encode_str_for_draft, response_str_file, response_data ) diff --git a/blockstore/apps/api/v1/urls.py b/blockstore/apps/rest_api/v1/urls.py similarity index 100% rename from blockstore/apps/api/v1/urls.py rename to blockstore/apps/rest_api/v1/urls.py diff --git a/blockstore/apps/api/v1/views/__init__.py b/blockstore/apps/rest_api/v1/views/__init__.py similarity index 100% rename from blockstore/apps/api/v1/views/__init__.py rename to blockstore/apps/rest_api/v1/views/__init__.py diff --git a/blockstore/apps/api/v1/views/bundles.py b/blockstore/apps/rest_api/v1/views/bundles.py similarity index 100% rename from blockstore/apps/api/v1/views/bundles.py rename to blockstore/apps/rest_api/v1/views/bundles.py diff --git a/blockstore/apps/api/v1/views/collections.py b/blockstore/apps/rest_api/v1/views/collections.py similarity index 100% rename from blockstore/apps/api/v1/views/collections.py rename to blockstore/apps/rest_api/v1/views/collections.py diff --git a/blockstore/apps/api/v1/views/drafts.py b/blockstore/apps/rest_api/v1/views/drafts.py similarity index 100% rename from blockstore/apps/api/v1/views/drafts.py rename to blockstore/apps/rest_api/v1/views/drafts.py diff --git a/blockstore/settings/base.py b/blockstore/settings/base.py index 7e7a0ed0..f0537b27 100644 --- a/blockstore/settings/base.py +++ b/blockstore/settings/base.py @@ -55,8 +55,8 @@ def root(*x): PROJECT_APPS = ( 'blockstore.apps.mysql_unicode', 'blockstore.apps.core', - 'blockstore.apps.api', 'blockstore.apps.bundles.apps.BundlesConfig', + 'blockstore.apps.rest_api', 'tagstore.backends.tagstore_django', ) @@ -236,7 +236,7 @@ def root(*x): 'DEFAULT_PERMISSION_CLASSES': ( # Only superusers or authorized applications that authenticate with a # token are allowed to use the API. - 'blockstore.apps.api.permissions.IsSuperUserOrAuthorizedApplication', + 'blockstore.apps.rest_api.permissions.IsSuperUserOrAuthorizedApplication', ), } diff --git a/blockstore/urls.py b/blockstore/urls.py index a0ae69c8..27de581e 100644 --- a/blockstore/urls.py +++ b/blockstore/urls.py @@ -36,7 +36,7 @@ urlpatterns = oauth2_urlpatterns + [ url(r'^admin/', admin.site.urls), - url(r'^api/', include('blockstore.apps.api.urls', namespace='api')), + url(r'^api/', include('blockstore.apps.rest_api.urls', namespace='api')), # Use the same auth views for all logins, including those originating from the browseable API. url(r'^api-auth/', include((oauth2_urlpatterns, 'auth_backends'), namespace='rest_framework')), url(r'^auto_auth/$', core_views.AutoAuth.as_view(), name='auto_auth'), From 563039bea2ff4df56e3325283463e9ab7f6172fe Mon Sep 17 00:00:00 2001 From: Usman Khalid <2200617@gmail.com> Date: Sun, 15 Nov 2020 07:03:41 +0500 Subject: [PATCH 3/6] Copy edx-platform/openedx/core/lib/blockstore_api to blockstore/apps/api. --- blockstore/apps/api/__init__.py | 57 +++ blockstore/apps/api/apps.py | 9 + blockstore/apps/api/exceptions.py | 31 ++ blockstore/apps/api/methods.py | 433 ++++++++++++++++++ blockstore/apps/api/models.py | 98 ++++ blockstore/apps/api/tests/__init__.py | 0 .../apps/api/tests/test_blockstore_api.py | 195 ++++++++ 7 files changed, 823 insertions(+) create mode 100644 blockstore/apps/api/__init__.py create mode 100644 blockstore/apps/api/apps.py create mode 100644 blockstore/apps/api/exceptions.py create mode 100644 blockstore/apps/api/methods.py create mode 100644 blockstore/apps/api/models.py create mode 100644 blockstore/apps/api/tests/__init__.py create mode 100644 blockstore/apps/api/tests/test_blockstore_api.py diff --git a/blockstore/apps/api/__init__.py b/blockstore/apps/api/__init__.py new file mode 100644 index 00000000..483e97c2 --- /dev/null +++ b/blockstore/apps/api/__init__.py @@ -0,0 +1,57 @@ +""" +API Client for Blockstore + +This API does not do any caching; consider using BundleCache or (in +openedx.core.djangolib.blockstore_cache) together with these API methods for +improved performance. +""" +from .models import ( + Collection, + Bundle, + Draft, + BundleFile, + DraftFile, + LinkReference, + LinkDetails, + DraftLinkDetails, +) +from .methods import ( + # Collections: + get_collection, + create_collection, + update_collection, + delete_collection, + # Bundles: + get_bundles, + get_bundle, + create_bundle, + update_bundle, + delete_bundle, + # Drafts: + get_draft, + get_or_create_bundle_draft, + write_draft_file, + set_draft_link, + commit_draft, + delete_draft, + # Bundles or drafts: + get_bundle_files, + get_bundle_files_dict, + get_bundle_file_metadata, + get_bundle_file_data, + get_bundle_version, + get_bundle_version_files, + # Links: + get_bundle_links, + get_bundle_version_links, + # Misc: + force_browser_url, +) +from .exceptions import ( + BlockstoreException, + CollectionNotFound, + BundleNotFound, + DraftNotFound, + BundleFileNotFound, + BundleStorageError, +) diff --git a/blockstore/apps/api/apps.py b/blockstore/apps/api/apps.py new file mode 100644 index 00000000..a1a673fb --- /dev/null +++ b/blockstore/apps/api/apps.py @@ -0,0 +1,9 @@ +""" AppConfig for API app. """ + +from django.apps import AppConfig + + +class ApiConfig(AppConfig): + name = 'blockstore.apps.api' + label = 'blockstore_apps_api' + verbose_name = "Blockstore API" diff --git a/blockstore/apps/api/exceptions.py b/blockstore/apps/api/exceptions.py new file mode 100644 index 00000000..b58251d3 --- /dev/null +++ b/blockstore/apps/api/exceptions.py @@ -0,0 +1,31 @@ +""" +Exceptions that may be raised by the Blockstore API +""" + + +class BlockstoreException(Exception): + pass + + +class NotFound(BlockstoreException): + pass + + +class CollectionNotFound(NotFound): + pass + + +class BundleNotFound(NotFound): + pass + + +class DraftNotFound(NotFound): + pass + + +class BundleFileNotFound(NotFound): + pass + + +class BundleStorageError(BlockstoreException): + pass diff --git a/blockstore/apps/api/methods.py b/blockstore/apps/api/methods.py new file mode 100644 index 00000000..1e39b80e --- /dev/null +++ b/blockstore/apps/api/methods.py @@ -0,0 +1,433 @@ +""" +API Client methods for working with Blockstore bundles and drafts +""" + +import base64 +from urllib.parse import urlencode +from uuid import UUID + +import dateutil.parser +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +import requests +import six + +from .models import ( + Bundle, + Collection, + Draft, + BundleFile, + DraftFile, + LinkDetails, + LinkReference, + DraftLinkDetails, +) +from .exceptions import ( + NotFound, + CollectionNotFound, + BundleNotFound, + DraftNotFound, + BundleFileNotFound, +) + + +def api_url(*path_parts): + if not settings.BLOCKSTORE_API_URL or not settings.BLOCKSTORE_API_URL.endswith('/api/v1/'): + raise ImproperlyConfigured('BLOCKSTORE_API_URL must be set and should end with /api/v1/') + return settings.BLOCKSTORE_API_URL + '/'.join(path_parts) + + +def api_request(method, url, **kwargs): + """ + Helper method for making a request to the Blockstore REST API + """ + if not settings.BLOCKSTORE_API_AUTH_TOKEN: + raise ImproperlyConfigured("Cannot use Blockstore unless BLOCKSTORE_API_AUTH_TOKEN is set.") + kwargs.setdefault('headers', {})['Authorization'] = "Token {}".format(settings.BLOCKSTORE_API_AUTH_TOKEN) + response = requests.request(method, url, **kwargs) + if response.status_code == 404: + raise NotFound + response.raise_for_status() + if response.status_code == 204: + return None # No content + return response.json() + + +def _collection_from_response(data): + """ + Given data about a Collection returned by any blockstore REST API, convert it to + a Collection instance. + """ + return Collection(uuid=UUID(data['uuid']), title=data['title']) + + +def _bundle_from_response(data): + """ + Given data about a Bundle returned by any blockstore REST API, convert it to + a Bundle instance. + """ + return Bundle( + uuid=UUID(data['uuid']), + title=data['title'], + description=data['description'], + slug=data['slug'], + # drafts: Convert from a dict of URLs to a dict of UUIDs: + drafts={draft_name: UUID(url.split('/')[-1]) for (draft_name, url) in data['drafts'].items()}, + # versions field: take the last one and convert it from URL to an int + # i.e.: [..., 'https://blockstore/api/v1/bundle_versions/bundle_uuid,15'] -> 15 + latest_version=int(data['versions'][-1].split(',')[-1]) if data['versions'] else 0, + ) + + +def _draft_from_response(data): + """ + Given data about a Draft returned by any blockstore REST API, convert it to + a Draft instance. + """ + return Draft( + uuid=UUID(data['uuid']), + bundle_uuid=UUID(data['bundle_uuid']), + name=data['name'], + updated_at=dateutil.parser.parse(data['staged_draft']['updated_at']), + files={ + path: DraftFile(path=path, **file) + for path, file in data['staged_draft']['files'].items() + }, + links={ + name: DraftLinkDetails( + name=name, + direct=LinkReference(**link["direct"]), + indirect=[LinkReference(**ind) for ind in link["indirect"]], + modified=link["modified"], + ) + for name, link in data['staged_draft']['links'].items() + } + ) + + +def get_collection(collection_uuid): + """ + Retrieve metadata about the specified collection + + Raises CollectionNotFound if the collection does not exist + """ + assert isinstance(collection_uuid, UUID) + try: + data = api_request('get', api_url('collections', str(collection_uuid))) + except NotFound: + raise CollectionNotFound("Collection {} does not exist.".format(collection_uuid)) + return _collection_from_response(data) + + +def create_collection(title): + """ + Create a new collection. + """ + result = api_request('post', api_url('collections'), json={"title": title}) + return _collection_from_response(result) + + +def update_collection(collection_uuid, title): + """ + Update a collection's title + """ + assert isinstance(collection_uuid, UUID) + data = {"title": title} + result = api_request('patch', api_url('collections', str(collection_uuid)), json=data) + return _collection_from_response(result) + + +def delete_collection(collection_uuid): + """ + Delete a collection + """ + assert isinstance(collection_uuid, UUID) + api_request('delete', api_url('collections', str(collection_uuid))) + + +def get_bundles(uuids=None, text_search=None): + """ + Get the details of all bundles + """ + query_params = {} + if uuids: + query_params['uuid'] = ','.join(map(str, uuids)) + if text_search: + query_params['text_search'] = text_search + version_url = api_url('bundles') + '?' + urlencode(query_params) + response = api_request('get', version_url) + # build bundle from response, convert map object to list and return + return [_bundle_from_response(item) for item in response] + + +def get_bundle(bundle_uuid): + """ + Retrieve metadata about the specified bundle + + Raises BundleNotFound if the bundle does not exist + """ + assert isinstance(bundle_uuid, UUID) + try: + data = api_request('get', api_url('bundles', str(bundle_uuid))) + except NotFound: + raise BundleNotFound("Bundle {} does not exist.".format(bundle_uuid)) + return _bundle_from_response(data) + + +def create_bundle(collection_uuid, slug, title="New Bundle", description=""): + """ + Create a new bundle. + + Note that description is currently required. + """ + result = api_request('post', api_url('bundles'), json={ + "collection_uuid": str(collection_uuid), + "slug": slug, + "title": title, + "description": description, + }) + return _bundle_from_response(result) + + +def update_bundle(bundle_uuid, **fields): + """ + Update a bundle's title, description, slug, or collection. + """ + assert isinstance(bundle_uuid, UUID) + data = {} + # Most validation will be done by Blockstore, so we don't worry too much about data validation + for str_field in ("title", "description", "slug"): + if str_field in fields: + data[str_field] = fields.pop(str_field) + if "collection_uuid" in fields: + data["collection_uuid"] = str(fields.pop("collection_uuid")) + if fields: + raise ValueError("Unexpected extra fields passed to update_bundle: {}".format(fields.keys())) + result = api_request('patch', api_url('bundles', str(bundle_uuid)), json=data) + return _bundle_from_response(result) + + +def delete_bundle(bundle_uuid): + """ + Delete a bundle + """ + assert isinstance(bundle_uuid, UUID) + api_request('delete', api_url('bundles', str(bundle_uuid))) + + +def get_draft(draft_uuid): + """ + Retrieve metadata about the specified draft. + If you don't know the draft's UUID, look it up using get_bundle() + """ + assert isinstance(draft_uuid, UUID) + try: + data = api_request('get', api_url('drafts', str(draft_uuid))) + except NotFound: + raise DraftNotFound("Draft does not exist: {}".format(draft_uuid)) + return _draft_from_response(data) + + +def get_or_create_bundle_draft(bundle_uuid, draft_name): + """ + Retrieve metadata about the specified draft. + """ + bundle = get_bundle(bundle_uuid) + try: + return get_draft(bundle.drafts[draft_name]) # pylint: disable=unsubscriptable-object + except KeyError: + # The draft doesn't exist yet, so create it: + response = api_request('post', api_url('drafts'), json={ + "bundle_uuid": str(bundle_uuid), + "name": draft_name, + }) + # The result of creating a draft doesn't include all the fields we want, so retrieve it now: + return get_draft(UUID(response["uuid"])) + + +def commit_draft(draft_uuid): + """ + Commit all of the pending changes in the draft, creating a new version of + the associated bundle. + + Does not return any value. + """ + api_request('post', api_url('drafts', str(draft_uuid), 'commit')) + + +def delete_draft(draft_uuid): + """ + Delete the specified draft, removing any staged changes/files/deletes. + + Does not return any value. + """ + api_request('delete', api_url('drafts', str(draft_uuid))) + + +def get_bundle_version(bundle_uuid, version_number): + """ + Get the details of the specified bundle version + """ + if version_number == 0: + return None + version_url = api_url('bundle_versions', str(bundle_uuid) + ',' + str(version_number)) + return api_request('get', version_url) + + +def get_bundle_version_files(bundle_uuid, version_number): + """ + Get a list of the files in the specified bundle version + """ + if version_number == 0: + return [] + version_info = get_bundle_version(bundle_uuid, version_number) + return [BundleFile(path=path, **file_metadata) for path, file_metadata in version_info["snapshot"]["files"].items()] + + +def get_bundle_version_links(bundle_uuid, version_number): + """ + Get a dictionary of the links in the specified bundle version + """ + if version_number == 0: + return {} + version_info = get_bundle_version(bundle_uuid, version_number) + return { + name: LinkDetails( + name=name, + direct=LinkReference(**link["direct"]), + indirect=[LinkReference(**ind) for ind in link["indirect"]], + ) + for name, link in version_info['snapshot']['links'].items() + } + + +def get_bundle_files_dict(bundle_uuid, use_draft=None): + """ + Get a dict of all the files in the specified bundle. + + Returns a dict where the keys are the paths (strings) and the values are + BundleFile or DraftFile tuples. + """ + bundle = get_bundle(bundle_uuid) + if use_draft and use_draft in bundle.drafts: # pylint: disable=unsupported-membership-test + draft_uuid = bundle.drafts[use_draft] # pylint: disable=unsubscriptable-object + return get_draft(draft_uuid).files + elif not bundle.latest_version: + # This bundle has no versions so definitely does not contain any files + return {} + else: + return {file_meta.path: file_meta for file_meta in get_bundle_version_files(bundle_uuid, bundle.latest_version)} + + +def get_bundle_files(bundle_uuid, use_draft=None): + """ + Get an iterator over all the files in the specified bundle or draft. + """ + return get_bundle_files_dict(bundle_uuid, use_draft).values() + + +def get_bundle_links(bundle_uuid, use_draft=None): + """ + Get a dict of all the links in the specified bundle. + + Returns a dict where the keys are the link names (strings) and the values + are LinkDetails or DraftLinkDetails tuples. + """ + bundle = get_bundle(bundle_uuid) + if use_draft and use_draft in bundle.drafts: # pylint: disable=unsupported-membership-test + draft_uuid = bundle.drafts[use_draft] # pylint: disable=unsubscriptable-object + return get_draft(draft_uuid).links + elif not bundle.latest_version: + # This bundle has no versions so definitely does not contain any links + return {} + else: + return get_bundle_version_links(bundle_uuid, bundle.latest_version) + + +def get_bundle_file_metadata(bundle_uuid, path, use_draft=None): + """ + Get the metadata of the specified file. + """ + assert isinstance(bundle_uuid, UUID) + files_dict = get_bundle_files_dict(bundle_uuid, use_draft=use_draft) + try: + return files_dict[path] + except KeyError: + raise BundleFileNotFound( + "Bundle {} (draft: {}) does not contain a file {}".format(bundle_uuid, use_draft, path) + ) + + +def get_bundle_file_data(bundle_uuid, path, use_draft=None): + """ + Read all the data in the given bundle file and return it as a + binary string. + + Do not use this for large files! + """ + metadata = get_bundle_file_metadata(bundle_uuid, path, use_draft) + with requests.get(metadata.url, stream=True) as r: + return r.content + + +def write_draft_file(draft_uuid, path, contents): + """ + Create or overwrite the file at 'path' in the specified draft with the given + contents. To delete a file, pass contents=None. + + If you don't know the draft's UUID, look it up using + get_or_create_bundle_draft() + + Does not return anything. + """ + api_request('patch', api_url('drafts', str(draft_uuid)), json={ + 'files': { + path: encode_str_for_draft(contents) if contents is not None else None, + }, + }) + + +def set_draft_link(draft_uuid, link_name, bundle_uuid, version): + """ + Create or replace the link with the given name in the specified draft so + that it points to the specified bundle version. To delete a link, pass + bundle_uuid=None, version=None. + + If you don't know the draft's UUID, look it up using + get_or_create_bundle_draft() + + Does not return anything. + """ + api_request('patch', api_url('drafts', str(draft_uuid)), json={ + 'links': { + link_name: {"bundle_uuid": str(bundle_uuid), "version": version} if bundle_uuid is not None else None, + }, + }) + + +def encode_str_for_draft(input_str): + """ + Given a string, return UTF-8 representation that is then base64 encoded. + """ + if isinstance(input_str, six.text_type): + binary = input_str.encode('utf8') + else: + binary = input_str + return base64.b64encode(binary) + + +def force_browser_url(blockstore_file_url): + """ + Ensure that the given URL Blockstore is a URL accessible from the end user's + browser. + """ + # Hack: on some devstacks, we must necessarily use different URLs for + # accessing Blockstore file data from within and outside of docker + # containers, but Blockstore has no way of knowing which case any particular + # request is for. So it always returns a URL suitable for use from within + # the container. Only this edxapp can transform the URL at the last second, + # knowing that in this case it's going to the user's browser and not being + # read by edxapp. + # In production, the same S3 URLs get used for internal and external access + # so this hack is not necessary. + return blockstore_file_url.replace('http://edx.devstack.blockstore:', 'http://localhost:') diff --git a/blockstore/apps/api/models.py b/blockstore/apps/api/models.py new file mode 100644 index 00000000..995571e2 --- /dev/null +++ b/blockstore/apps/api/models.py @@ -0,0 +1,98 @@ +""" +Data models used for Blockstore API Client +""" + +from datetime import datetime +from uuid import UUID + +import attr +import six + + +def _convert_to_uuid(value): + if not isinstance(value, UUID): + return UUID(value) + return value + + +@attr.s(frozen=True) +class Collection(object): + """ + Metadata about a blockstore collection + """ + uuid = attr.ib(type=UUID, converter=_convert_to_uuid) + title = attr.ib(type=six.text_type) + + +@attr.s(frozen=True) +class Bundle(object): + """ + Metadata about a blockstore bundle + """ + uuid = attr.ib(type=UUID, converter=_convert_to_uuid) + title = attr.ib(type=six.text_type) + description = attr.ib(type=six.text_type) + slug = attr.ib(type=six.text_type) + drafts = attr.ib(type=dict) # Dict of drafts, where keys are the draft names and values are draft UUIDs + # Note that if latest_version is 0, it means that no versions yet exist + latest_version = attr.ib(type=int, validator=attr.validators.instance_of(int)) + + +@attr.s(frozen=True) +class Draft(object): + """ + Metadata about a blockstore draft + """ + uuid = attr.ib(type=UUID, converter=_convert_to_uuid) + bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid) + name = attr.ib(type=six.text_type) + updated_at = attr.ib(type=datetime, validator=attr.validators.instance_of(datetime)) + files = attr.ib(type=dict) + links = attr.ib(type=dict) + + +@attr.s(frozen=True) +class BundleFile(object): + """ + Metadata about a file in a blockstore bundle or draft. + """ + path = attr.ib(type=six.text_type) + size = attr.ib(type=int) + url = attr.ib(type=six.text_type) + hash_digest = attr.ib(type=six.text_type) + + +@attr.s(frozen=True) +class DraftFile(BundleFile): + """ + Metadata about a file in a blockstore draft. + """ + modified = attr.ib(type=bool) # Was this file modified in the draft? + + +@attr.s(frozen=True) +class LinkReference(object): + """ + A pointer to a specific BundleVersion + """ + bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid) + version = attr.ib(type=int) + snapshot_digest = attr.ib(type=six.text_type) + + +@attr.s(frozen=True) +class LinkDetails(object): + """ + Details about a specific link in a BundleVersion or Draft + """ + name = attr.ib(type=str) + direct = attr.ib(type=LinkReference) + indirect = attr.ib(type=list) # List of LinkReference objects + + +@attr.s(frozen=True) +class DraftLinkDetails(LinkDetails): + """ + Details about a specific link in a Draft + """ + modified = attr.ib(type=bool) diff --git a/blockstore/apps/api/tests/__init__.py b/blockstore/apps/api/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/blockstore/apps/api/tests/test_blockstore_api.py b/blockstore/apps/api/tests/test_blockstore_api.py new file mode 100644 index 00000000..fc773d3b --- /dev/null +++ b/blockstore/apps/api/tests/test_blockstore_api.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +""" +Tests for xblock_utils.py +""" + +import unittest +from uuid import UUID + +from django.conf import settings +from openedx.core.lib import blockstore_api as api + +# A fake UUID that won't represent any real bundle/draft/collection: +BAD_UUID = UUID('12345678-0000-0000-0000-000000000000') + + +@unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server") +class BlockstoreApiClientTest(unittest.TestCase): + """ + Test for the Blockstore API Client. + + The goal of these tests is not to test that Blockstore works correctly, but + that the API client can interact with it and all the API client methods + work. + """ + + # Collections + + def test_nonexistent_collection(self): + """ Request a collection that doesn't exist -> CollectionNotFound """ + with self.assertRaises(api.CollectionNotFound): + api.get_collection(BAD_UUID) + + def test_collection_crud(self): + """ Create, Fetch, Update, and Delete a Collection """ + title = "Fire 🔥 Collection" + # Create: + coll = api.create_collection(title) + self.assertEqual(coll.title, title) + self.assertIsInstance(coll.uuid, UUID) + # Fetch: + coll2 = api.get_collection(coll.uuid) + self.assertEqual(coll, coll2) + # Update: + new_title = "Air 🌀 Collection" + coll3 = api.update_collection(coll.uuid, title=new_title) + self.assertEqual(coll3.title, new_title) + coll4 = api.get_collection(coll.uuid) + self.assertEqual(coll4.title, new_title) + # Delete: + api.delete_collection(coll.uuid) + with self.assertRaises(api.CollectionNotFound): + api.get_collection(coll.uuid) + + # Bundles + + def test_nonexistent_bundle(self): + """ Request a bundle that doesn't exist -> BundleNotFound """ + with self.assertRaises(api.BundleNotFound): + api.get_bundle(BAD_UUID) + + def test_bundle_crud(self): + """ Create, Fetch, Update, and Delete a Bundle """ + coll = api.create_collection("Test Collection") + args = { + "title": "Water 💧 Bundle", + "slug": "h2o", + "description": "Sploosh", + } + # Create: + bundle = api.create_bundle(coll.uuid, **args) + for attr, value in args.items(): + self.assertEqual(getattr(bundle, attr), value) + self.assertIsInstance(bundle.uuid, UUID) + # Fetch: + bundle2 = api.get_bundle(bundle.uuid) + self.assertEqual(bundle, bundle2) + # Update: + new_description = "Water Nation Bending Lessons" + bundle3 = api.update_bundle(bundle.uuid, description=new_description) + self.assertEqual(bundle3.description, new_description) + bundle4 = api.get_bundle(bundle.uuid) + self.assertEqual(bundle4.description, new_description) + # Delete: + api.delete_bundle(bundle.uuid) + with self.assertRaises(api.BundleNotFound): + api.get_bundle(bundle.uuid) + + # Drafts, files, and reading/writing file contents: + + def test_nonexistent_draft(self): + """ Request a draft that doesn't exist -> DraftNotFound """ + with self.assertRaises(api.DraftNotFound): + api.get_draft(BAD_UUID) + + def test_drafts_and_files(self): + """ + Test creating, reading, writing, committing, and reverting drafts and + files. + """ + coll = api.create_collection("Test Collection") + bundle = api.create_bundle(coll.uuid, title="Earth 🗿 Bundle", slug="earth", description="another test bundle") + # Create a draft + draft = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") + self.assertEqual(draft.bundle_uuid, bundle.uuid) + self.assertEqual(draft.name, "test-draft") + self.assertGreaterEqual(draft.updated_at.year, 2019) + # And retrieve it again: + draft2 = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") + self.assertEqual(draft, draft2) + # Also test retrieving using get_draft + draft3 = api.get_draft(draft.uuid) + self.assertEqual(draft, draft3) + + # Write a file into the bundle: + api.write_draft_file(draft.uuid, "test.txt", b"initial version") + # Now the file should be visible in the draft: + draft_contents = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) + self.assertEqual(draft_contents, b"initial version") + api.commit_draft(draft.uuid) + + # Write a new version into the draft: + api.write_draft_file(draft.uuid, "test.txt", b"modified version") + published_contents = api.get_bundle_file_data(bundle.uuid, "test.txt") + self.assertEqual(published_contents, b"initial version") + draft_contents2 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) + self.assertEqual(draft_contents2, b"modified version") + # Now delete the draft: + api.delete_draft(draft.uuid) + draft_contents3 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) + # Confirm the file is now reset: + self.assertEqual(draft_contents3, b"initial version") + + # Finaly, test the get_bundle_file* methods: + file_info1 = api.get_bundle_file_metadata(bundle.uuid, "test.txt") + self.assertEqual(file_info1.path, "test.txt") + self.assertEqual(file_info1.size, len(b"initial version")) + self.assertEqual(file_info1.hash_digest, "a45a5c6716276a66c4005534a51453ab16ea63c4") + + self.assertEqual(list(api.get_bundle_files(bundle.uuid)), [file_info1]) + self.assertEqual(api.get_bundle_files_dict(bundle.uuid), { + "test.txt": file_info1, + }) + + # Links + + def test_links(self): + """ + Test operations involving bundle links. + """ + coll = api.create_collection("Test Collection") + # Create two library bundles and a course bundle: + lib1_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib1") + lib1_draft = api.get_or_create_bundle_draft(lib1_bundle.uuid, draft_name="test-draft") + lib2_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib2") + lib2_draft = api.get_or_create_bundle_draft(lib2_bundle.uuid, draft_name="other-draft") + course_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="course") + course_draft = api.get_or_create_bundle_draft(course_bundle.uuid, draft_name="test-draft") + + # To create links, we need valid BundleVersions, which requires having committed at least one change: + api.write_draft_file(lib1_draft.uuid, "lib1-data.txt", "hello world") + api.commit_draft(lib1_draft.uuid) # Creates version 1 + api.write_draft_file(lib2_draft.uuid, "lib2-data.txt", "hello world") + api.commit_draft(lib2_draft.uuid) # Creates version 1 + + # Lib2 has no links: + self.assertFalse(api.get_bundle_links(lib2_bundle.uuid)) + + # Create a link from lib2 to lib1 + link1_name = "lib2_to_lib1" + api.set_draft_link(lib2_draft.uuid, link1_name, lib1_bundle.uuid, version=1) + # Now confirm the link exists in the draft: + lib2_draft_links = api.get_bundle_links(lib2_bundle.uuid, use_draft=lib2_draft.name) + self.assertIn(link1_name, lib2_draft_links) + self.assertEqual(lib2_draft_links[link1_name].direct.bundle_uuid, lib1_bundle.uuid) + self.assertEqual(lib2_draft_links[link1_name].direct.version, 1) + # Now commit the change to lib2: + api.commit_draft(lib2_draft.uuid) # Creates version 2 + + # Now create a link from course to lib2 + link2_name = "course_to_lib2" + api.set_draft_link(course_draft.uuid, link2_name, lib2_bundle.uuid, version=2) + api.commit_draft(course_draft.uuid) + + # And confirm the link exists in the resulting bundle version: + course_links = api.get_bundle_links(course_bundle.uuid) + self.assertIn(link2_name, course_links) + self.assertEqual(course_links[link2_name].direct.bundle_uuid, lib2_bundle.uuid) + self.assertEqual(course_links[link2_name].direct.version, 2) + # And since the links go course->lib2->lib1, course has an indirect link to lib1: + self.assertEqual(course_links[link2_name].indirect[0].bundle_uuid, lib1_bundle.uuid) + self.assertEqual(course_links[link2_name].indirect[0].version, 1) + + # Finally, test deleting a link from course's draft: + api.set_draft_link(course_draft.uuid, link2_name, None, None) + self.assertFalse(api.get_bundle_links(course_bundle.uuid, use_draft=course_draft.name)) From cb6a37bfb1775a1905a8f36378b5abad222fbcbe Mon Sep 17 00:00:00 2001 From: Usman Khalid <2200617@gmail.com> Date: Sun, 15 Nov 2020 06:35:13 +0500 Subject: [PATCH 4/6] Make the apps installable in edx-platform. --- CHANGELOG.rst | 25 +++++ blockstore/__init__.py | 5 + blockstore/apps/bundles/apps.py | 1 + .../apps/bundles/migrations/0001_initial.py | 4 +- .../bundles/migrations/0002_create_drafts.py | 4 +- pycodestyle | 2 +- pylintrc | 2 +- pylintrc_tweaks | 2 +- setup.cfg | 8 ++ setup.py | 92 +++++++++++++++++++ 10 files changed, 138 insertions(+), 7 deletions(-) create mode 100644 CHANGELOG.rst create mode 100644 setup.cfg create mode 100755 setup.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 00000000..ded0c5a8 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,25 @@ +Change Log +---------- + +.. + All enhancements and patches to blockstore will be documented + in this file. It adheres to the structure of https://keepachangelog.com/ , + but in reStructuredText instead of Markdown (for ease of incorporation into + Sphinx documentation and the PyPI description). + + This project adheres to Semantic Versioning (https://semver.org/). + +.. There should always be an "Unreleased" section for changes pending release. + +Unreleased +~~~~~~~~~~ + +* + +[1.0.0] - 2020-11-11 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Added +_____ + +* First release on PyPI. diff --git a/blockstore/__init__.py b/blockstore/__init__.py index e69de29b..904a3e31 100644 --- a/blockstore/__init__.py +++ b/blockstore/__init__.py @@ -0,0 +1,5 @@ +""" +Blockstore is a system for storing educational content. +""" + +__version__ = '1.0.0' diff --git a/blockstore/apps/bundles/apps.py b/blockstore/apps/bundles/apps.py index 089a64c1..64cb491d 100644 --- a/blockstore/apps/bundles/apps.py +++ b/blockstore/apps/bundles/apps.py @@ -5,4 +5,5 @@ class BundlesConfig(AppConfig): name = 'blockstore.apps.bundles' + label = 'blockstore_apps_bundles' verbose_name = "Blockstore Bundles" diff --git a/blockstore/apps/bundles/migrations/0001_initial.py b/blockstore/apps/bundles/migrations/0001_initial.py index 14c4e9f1..828ce344 100644 --- a/blockstore/apps/bundles/migrations/0001_initial.py +++ b/blockstore/apps/bundles/migrations/0001_initial.py @@ -38,7 +38,7 @@ class Migration(migrations.Migration): ('version_num', models.PositiveIntegerField(editable=False)), ('snapshot_digest', models.CharField(db_index=True, editable=False, max_length=40)), ('change_description', models.TextField(blank=True, max_length=1000)), - ('bundle', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='versions', related_query_name='version', to='bundles.Bundle')), + ('bundle', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='versions', related_query_name='version', to='blockstore_apps_bundles.Bundle')), ], ), migrations.CreateModel( @@ -52,7 +52,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='bundle', name='collection', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='bundles', related_query_name='bundle', to='bundles.Collection'), + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='bundles', related_query_name='bundle', to='blockstore_apps_bundles.Collection'), ), migrations.AlterUniqueTogether( name='bundleversion', diff --git a/blockstore/apps/bundles/migrations/0002_create_drafts.py b/blockstore/apps/bundles/migrations/0002_create_drafts.py index a67bfc26..4ac283b4 100644 --- a/blockstore/apps/bundles/migrations/0002_create_drafts.py +++ b/blockstore/apps/bundles/migrations/0002_create_drafts.py @@ -9,7 +9,7 @@ class Migration(migrations.Migration): dependencies = [ - ('bundles', '0001_initial'), + ('blockstore_apps_bundles', '0001_initial'), ] operations = [ @@ -19,7 +19,7 @@ class Migration(migrations.Migration): ('id', models.BigAutoField(primary_key=True, serialize=False)), ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), ('name', models.CharField(max_length=180)), - ('bundle', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='drafts', related_query_name='draft', to='bundles.Bundle')), + ('bundle', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='drafts', related_query_name='draft', to='blockstore_apps_bundles.Bundle')), ], ), migrations.AlterUniqueTogether( diff --git a/pycodestyle b/pycodestyle index 4a990ca4..75f22cd4 100644 --- a/pycodestyle +++ b/pycodestyle @@ -1,3 +1,3 @@ [pycodestyle] max-line-length = 120 -exclude = */migrations/*, wsgi.py +exclude = */migrations/*, private.py, wsgi.py diff --git a/pylintrc b/pylintrc index e2423d20..d2a32f09 100644 --- a/pylintrc +++ b/pylintrc @@ -64,7 +64,7 @@ # # ------------------------------ [MASTER] -ignore = ,migrations, settings, wsgi.py +ignore = ,migrations, settings, setup.py, wsgi.py persistent = yes load-plugins = edx_lint.pylint,pylint_django,pylint_celery diff --git a/pylintrc_tweaks b/pylintrc_tweaks index 60554765..fa89789b 100644 --- a/pylintrc_tweaks +++ b/pylintrc_tweaks @@ -1,5 +1,5 @@ [MASTER] -ignore+= ,migrations, settings, wsgi.py +ignore+= ,migrations, settings, setup.py, wsgi.py [BASIC] const-rgx = (([A-Z_][A-Z0-9_]*)|(__.*__)|log|urlpatterns|logger|User)$ diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..9a902de5 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,8 @@ +[isort] +include_trailing_comma = True +indent = ' ' +line_length = 120 +multi_line_output = 3 + +[wheel] +universal = 1 diff --git a/setup.py b/setup.py new file mode 100755 index 00000000..ce723a81 --- /dev/null +++ b/setup.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +""" +Package metadata for blockstore. +""" +import os +import re +import sys + +from setuptools import setup + + +def get_version(*file_paths): + """ + Extract the version string from the file. + + Input: + - file_paths: relative path fragments to file with + version string + """ + filename = os.path.join(os.path.dirname(__file__), *file_paths) + version_file = open(filename).read() + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError('Unable to find version string.') + + +def load_requirements(*requirements_paths): + """ + Load all requirements from the specified requirements files. + + Returns: + list: Requirements file relative path strings + """ + requirements = set() + for path in requirements_paths: + requirements.update( + line.split('#')[0].strip() for line in open(path).readlines() + if is_requirement(line.strip()) + ) + return list(requirements) + + +def is_requirement(line): + """ + Return True if the requirement line is a package requirement. + + Returns: + bool: True if the line is not blank, a comment, a URL, or + an included file + """ + return line and not line.startswith(('-r', '#', '-e', 'git+', '-c')) + + +VERSION = get_version('blockstore', '__init__.py') + +if sys.argv[-1] == 'tag': + print("Tagging the version on github:") + os.system("git tag -a %s -m 'version %s'" % (VERSION, VERSION)) + os.system("git push --tags") + sys.exit() + +README = open(os.path.join(os.path.dirname(__file__), 'README.rst')).read() +CHANGELOG = open(os.path.join(os.path.dirname(__file__), 'CHANGELOG.rst')).read() + +setup( + name='blockstore', + version=VERSION, + description="""Blockstore is a storage system for learning content in Open edX.""", + long_description=README + '\n\n' + CHANGELOG, + url='https://github.com/edx/blockstore', + packages=[ + 'blockstore', + ], + include_package_data=True, + install_requires=load_requirements('requirements/base.in'), + python_requires=">=3.8", + license="AGPL 3.0", + zip_safe=False, + keywords='Python edx', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Framework :: Django', + 'Framework :: Django :: 2.2', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', + 'Natural Language :: English', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.8', + ], +) From bf2aba871934a8dc02a13f532a76ab5373452e24 Mon Sep 17 00:00:00 2001 From: Usman Khalid <2200617@gmail.com> Date: Sun, 31 Jan 2021 18:03:05 +0500 Subject: [PATCH 5/6] Update implementation of Python API to work directly with the models. --- blockstore/apps/api/__init__.py | 28 +- blockstore/apps/api/{models.py => data.py} | 44 +- blockstore/apps/api/exceptions.py | 8 + blockstore/apps/api/methods.py | 534 ++++++++++-------- .../apps/api/tests/test_blockstore_api.py | 41 +- 5 files changed, 359 insertions(+), 296 deletions(-) rename blockstore/apps/api/{models.py => data.py} (72%) diff --git a/blockstore/apps/api/__init__.py b/blockstore/apps/api/__init__.py index 483e97c2..2960ad24 100644 --- a/blockstore/apps/api/__init__.py +++ b/blockstore/apps/api/__init__.py @@ -5,15 +5,16 @@ openedx.core.djangolib.blockstore_cache) together with these API methods for improved performance. """ -from .models import ( - Collection, - Bundle, - Draft, - BundleFile, - DraftFile, - LinkReference, - LinkDetails, - DraftLinkDetails, +from .data import ( + CollectionData, + BundleData, + BundleVersionData, + DraftData, + BundleFileData, + DraftFileData, + Dependency, + BundleLinkData, + DraftLinkData, ) from .methods import ( # Collections: @@ -29,29 +30,26 @@ delete_bundle, # Drafts: get_draft, - get_or_create_bundle_draft, + get_or_create_draft, write_draft_file, set_draft_link, commit_draft, delete_draft, # Bundles or drafts: get_bundle_files, - get_bundle_files_dict, get_bundle_file_metadata, get_bundle_file_data, get_bundle_version, - get_bundle_version_files, # Links: get_bundle_links, - get_bundle_version_links, - # Misc: - force_browser_url, ) from .exceptions import ( BlockstoreException, CollectionNotFound, BundleNotFound, + BundleVersionNotFound, DraftNotFound, + DraftHasNoChangesToCommit, BundleFileNotFound, BundleStorageError, ) diff --git a/blockstore/apps/api/models.py b/blockstore/apps/api/data.py similarity index 72% rename from blockstore/apps/api/models.py rename to blockstore/apps/api/data.py index 995571e2..a7fee6df 100644 --- a/blockstore/apps/api/models.py +++ b/blockstore/apps/api/data.py @@ -8,6 +8,8 @@ import attr import six +from blockstore.apps.bundles.links import Dependency + def _convert_to_uuid(value): if not isinstance(value, UUID): @@ -16,7 +18,7 @@ def _convert_to_uuid(value): @attr.s(frozen=True) -class Collection(object): +class CollectionData: """ Metadata about a blockstore collection """ @@ -25,7 +27,7 @@ class Collection(object): @attr.s(frozen=True) -class Bundle(object): +class BundleData: """ Metadata about a blockstore bundle """ @@ -39,20 +41,34 @@ class Bundle(object): @attr.s(frozen=True) -class Draft(object): +class BundleVersionData: + """ + Metadata about a blockstore bundle version. + """ + bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid) + version = attr.ib(type=int, validator=attr.validators.instance_of(int)) + change_description = attr.ib(type=six.text_type) + created_at = attr.ib(type=datetime, validator=attr.validators.instance_of(datetime)) + files = attr.ib(type=dict) + links = attr.ib(type=dict) + + +@attr.s(frozen=True) +class DraftData: """ Metadata about a blockstore draft """ uuid = attr.ib(type=UUID, converter=_convert_to_uuid) bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid) name = attr.ib(type=six.text_type) + created_at = attr.ib(type=datetime, validator=attr.validators.instance_of(datetime)) updated_at = attr.ib(type=datetime, validator=attr.validators.instance_of(datetime)) files = attr.ib(type=dict) links = attr.ib(type=dict) @attr.s(frozen=True) -class BundleFile(object): +class BundleFileData: """ Metadata about a file in a blockstore bundle or draft. """ @@ -63,7 +79,7 @@ class BundleFile(object): @attr.s(frozen=True) -class DraftFile(BundleFile): +class DraftFileData(BundleFileData): """ Metadata about a file in a blockstore draft. """ @@ -71,27 +87,17 @@ class DraftFile(BundleFile): @attr.s(frozen=True) -class LinkReference(object): - """ - A pointer to a specific BundleVersion - """ - bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - version = attr.ib(type=int) - snapshot_digest = attr.ib(type=six.text_type) - - -@attr.s(frozen=True) -class LinkDetails(object): +class BundleLinkData: """ Details about a specific link in a BundleVersion or Draft """ name = attr.ib(type=str) - direct = attr.ib(type=LinkReference) - indirect = attr.ib(type=list) # List of LinkReference objects + direct_dependency = attr.ib(type=Dependency) + indirect_dependencies = attr.ib(type=list) # List of Dependency objects @attr.s(frozen=True) -class DraftLinkDetails(LinkDetails): +class DraftLinkData(BundleLinkData): """ Details about a specific link in a Draft """ diff --git a/blockstore/apps/api/exceptions.py b/blockstore/apps/api/exceptions.py index b58251d3..baa3b828 100644 --- a/blockstore/apps/api/exceptions.py +++ b/blockstore/apps/api/exceptions.py @@ -19,10 +19,18 @@ class BundleNotFound(NotFound): pass +class BundleVersionNotFound(NotFound): + pass + + class DraftNotFound(NotFound): pass +class DraftHasNoChangesToCommit(Exception): + pass + + class BundleFileNotFound(NotFound): pass diff --git a/blockstore/apps/api/methods.py b/blockstore/apps/api/methods.py index 1e39b80e..ad305955 100644 --- a/blockstore/apps/api/methods.py +++ b/blockstore/apps/api/methods.py @@ -3,175 +3,144 @@ """ import base64 -from urllib.parse import urlencode -from uuid import UUID - -import dateutil.parser -from django.conf import settings -from django.core.exceptions import ImproperlyConfigured -import requests -import six - -from .models import ( - Bundle, - Collection, - Draft, - BundleFile, - DraftFile, - LinkDetails, - LinkReference, - DraftLinkDetails, + +from django.db.models import Q +from rest_framework import serializers + +from blockstore.apps.bundles import models +from blockstore.apps.bundles.links import LinkCycleError +from blockstore.apps.bundles.store import DraftRepo, SnapshotRepo +from blockstore.apps.rest_api.v1.serializers.drafts import ( + DraftFileUpdateSerializer, +) + +from .data import ( + BundleData, + BundleVersionData, + CollectionData, + DraftData, + BundleFileData, + DraftFileData, + BundleLinkData, + DraftLinkData, ) from .exceptions import ( - NotFound, CollectionNotFound, BundleNotFound, + BundleVersionNotFound, DraftNotFound, + DraftHasNoChangesToCommit, BundleFileNotFound, ) -def api_url(*path_parts): - if not settings.BLOCKSTORE_API_URL or not settings.BLOCKSTORE_API_URL.endswith('/api/v1/'): - raise ImproperlyConfigured('BLOCKSTORE_API_URL must be set and should end with /api/v1/') - return settings.BLOCKSTORE_API_URL + '/'.join(path_parts) - - -def api_request(method, url, **kwargs): +def _collection_data_from_model(collection_model): """ - Helper method for making a request to the Blockstore REST API + Create and return CollectionData from collection model. """ - if not settings.BLOCKSTORE_API_AUTH_TOKEN: - raise ImproperlyConfigured("Cannot use Blockstore unless BLOCKSTORE_API_AUTH_TOKEN is set.") - kwargs.setdefault('headers', {})['Authorization'] = "Token {}".format(settings.BLOCKSTORE_API_AUTH_TOKEN) - response = requests.request(method, url, **kwargs) - if response.status_code == 404: - raise NotFound - response.raise_for_status() - if response.status_code == 204: - return None # No content - return response.json() + return CollectionData(uuid=collection_model.uuid, title=collection_model.title) -def _collection_from_response(data): +def _get_collection_model(collection_uuid): """ - Given data about a Collection returned by any blockstore REST API, convert it to - a Collection instance. + Get collection model from UUID. + + Raises CollectionNotFound if the collection does not exist. """ - return Collection(uuid=UUID(data['uuid']), title=data['title']) + try: + collection_model = models.Collection.objects.get(uuid=collection_uuid) + except models.Collection.DoesNotExist as exc: + raise CollectionNotFound("Collection {} does not exist.".format(collection_uuid)) from exc + return collection_model -def _bundle_from_response(data): +def get_collection(collection_uuid): """ - Given data about a Bundle returned by any blockstore REST API, convert it to - a Bundle instance. + Retrieve data about the specified collection. + + Raises CollectionNotFound if collection with UUID does not exist. """ - return Bundle( - uuid=UUID(data['uuid']), - title=data['title'], - description=data['description'], - slug=data['slug'], - # drafts: Convert from a dict of URLs to a dict of UUIDs: - drafts={draft_name: UUID(url.split('/')[-1]) for (draft_name, url) in data['drafts'].items()}, - # versions field: take the last one and convert it from URL to an int - # i.e.: [..., 'https://blockstore/api/v1/bundle_versions/bundle_uuid,15'] -> 15 - latest_version=int(data['versions'][-1].split(',')[-1]) if data['versions'] else 0, - ) + collection_model = _get_collection_model(collection_uuid) + return _collection_data_from_model(collection_model) -def _draft_from_response(data): +def create_collection(title): """ - Given data about a Draft returned by any blockstore REST API, convert it to - a Draft instance. + Create a new collection. """ - return Draft( - uuid=UUID(data['uuid']), - bundle_uuid=UUID(data['bundle_uuid']), - name=data['name'], - updated_at=dateutil.parser.parse(data['staged_draft']['updated_at']), - files={ - path: DraftFile(path=path, **file) - for path, file in data['staged_draft']['files'].items() - }, - links={ - name: DraftLinkDetails( - name=name, - direct=LinkReference(**link["direct"]), - indirect=[LinkReference(**ind) for ind in link["indirect"]], - modified=link["modified"], - ) - for name, link in data['staged_draft']['links'].items() - } - ) + collection_model = models.Collection(title=title) + collection_model.save() + return _collection_data_from_model(collection_model) -def get_collection(collection_uuid): +def update_collection(collection_uuid, title): """ - Retrieve metadata about the specified collection - - Raises CollectionNotFound if the collection does not exist + Update a collection's title. """ - assert isinstance(collection_uuid, UUID) - try: - data = api_request('get', api_url('collections', str(collection_uuid))) - except NotFound: - raise CollectionNotFound("Collection {} does not exist.".format(collection_uuid)) - return _collection_from_response(data) + collection_model = _get_collection_model(collection_uuid) + collection_model.title = title + collection_model.save() + return _collection_data_from_model(collection_model) -def create_collection(title): +def delete_collection(collection_uuid): """ - Create a new collection. + Delete a collection. """ - result = api_request('post', api_url('collections'), json={"title": title}) - return _collection_from_response(result) + collection_model = _get_collection_model(collection_uuid) + collection_model.delete() -def update_collection(collection_uuid, title): +def _bundle_data_from_model(bundle_model): """ - Update a collection's title + Create and return BundleData from bundle model. """ - assert isinstance(collection_uuid, UUID) - data = {"title": title} - result = api_request('patch', api_url('collections', str(collection_uuid)), json=data) - return _collection_from_response(result) + latest_bundle_version_model = bundle_model.get_bundle_version() + return BundleData( + uuid=bundle_model.uuid, + title=bundle_model.title, + description=bundle_model.description, + slug=bundle_model.slug, + drafts={draft.name: draft.uuid for draft in bundle_model.drafts.all()}, + latest_version=latest_bundle_version_model.version_num if latest_bundle_version_model else 0, + ) -def delete_collection(collection_uuid): +def _get_bundle_model(bundle_uuid): """ - Delete a collection + Get Bundle model from UUID. + + Raises BundleNotFound if bundle with UUID does not exist. """ - assert isinstance(collection_uuid, UUID) - api_request('delete', api_url('collections', str(collection_uuid))) + try: + bundle_model = models.Bundle.objects.get(uuid=bundle_uuid) + except models.Bundle.DoesNotExist as exc: + raise BundleNotFound("Bundle {} does not exist.".format(bundle_uuid)) from exc + return bundle_model def get_bundles(uuids=None, text_search=None): """ - Get the details of all bundles + Get the details of all bundles. """ - query_params = {} + bundles_queryset = models.Bundle.objects.all() if uuids: - query_params['uuid'] = ','.join(map(str, uuids)) + bundles_queryset = bundles_queryset.filter(uuid__in=uuids) if text_search: - query_params['text_search'] = text_search - version_url = api_url('bundles') + '?' + urlencode(query_params) - response = api_request('get', version_url) - # build bundle from response, convert map object to list and return - return [_bundle_from_response(item) for item in response] + bundles_queryset = bundles_queryset.filter( + Q(title__icontains=text_search) | Q(description__icontains=text_search) | Q(slug__icontains=text_search) + ) + return [_bundle_data_from_model(bundle_model) for bundle_model in bundles_queryset] def get_bundle(bundle_uuid): """ - Retrieve metadata about the specified bundle + Retrieve data about the specified bundle. - Raises BundleNotFound if the bundle does not exist + Raises BundleNotFound if bundle with UUID does not exist. """ - assert isinstance(bundle_uuid, UUID) - try: - data = api_request('get', api_url('bundles', str(bundle_uuid))) - except NotFound: - raise BundleNotFound("Bundle {} does not exist.".format(bundle_uuid)) - return _bundle_from_response(data) + bundle_model = _get_bundle_model(bundle_uuid) + return _bundle_data_from_model(bundle_model) def create_bundle(collection_uuid, slug, title="New Bundle", description=""): @@ -180,69 +149,116 @@ def create_bundle(collection_uuid, slug, title="New Bundle", description=""): Note that description is currently required. """ - result = api_request('post', api_url('bundles'), json={ - "collection_uuid": str(collection_uuid), - "slug": slug, - "title": title, - "description": description, - }) - return _bundle_from_response(result) + collection_model = _get_collection_model(collection_uuid) + bundle_model = models.Bundle( + title=title, + collection=collection_model, + slug=slug, + description=description, + ) + bundle_model.save() + return _bundle_data_from_model(bundle_model) def update_bundle(bundle_uuid, **fields): """ Update a bundle's title, description, slug, or collection. """ - assert isinstance(bundle_uuid, UUID) - data = {} - # Most validation will be done by Blockstore, so we don't worry too much about data validation + bundle_model = _get_bundle_model(bundle_uuid) for str_field in ("title", "description", "slug"): if str_field in fields: - data[str_field] = fields.pop(str_field) + setattr(bundle_model, str_field, fields.pop(str_field)) if "collection_uuid" in fields: - data["collection_uuid"] = str(fields.pop("collection_uuid")) + collection_uuid = fields.pop("collection_uuid") + collection_model = _get_collection_model(collection_uuid) + bundle_model.collection = collection_model if fields: raise ValueError("Unexpected extra fields passed to update_bundle: {}".format(fields.keys())) - result = api_request('patch', api_url('bundles', str(bundle_uuid)), json=data) - return _bundle_from_response(result) + + bundle_model.save() + return _bundle_data_from_model(bundle_model) def delete_bundle(bundle_uuid): """ - Delete a bundle + Delete a bundle. + """ + bundle_model = _get_bundle_model(bundle_uuid) + bundle_model.delete() + + +def _draft_data_from_model(draft_model): + """ + Create and return DraftData from draft model. + """ + draft_repo = DraftRepo(SnapshotRepo()) + staged_draft = draft_model.staged_draft + + return DraftData( + uuid=draft_model.uuid, + bundle_uuid=draft_model.bundle.uuid, + name=draft_model.name, + created_at=draft_model.staged_draft.created_at, + updated_at=draft_model.staged_draft.updated_at, + files={ + path: DraftFileData( + path=path, + size=file_info.size, + url=draft_repo.url(staged_draft, path), # TODO + hash_digest=file_info.hash_digest, + modified=path in draft_model.staged_draft.files_to_overwrite, + ) + for path, file_info in staged_draft.files.items() + }, + links={ + link.name: DraftLinkData( + name=link.name, + direct_dependency=link.direct_dependency, + indirect_dependencies=link.indirect_dependencies, + modified=link.name in staged_draft.links_to_overwrite.modified_set, + ) + for link in staged_draft.composed_links() + } + ) + + +def _get_draft_model(draft_uuid): + """ + Get Draft model from UUID. + + Raises DraftNotFound if draft with UUID does not exist. """ - assert isinstance(bundle_uuid, UUID) - api_request('delete', api_url('bundles', str(bundle_uuid))) + try: + draft_model = models.Draft.objects.get(uuid=draft_uuid) + except models.Draft.DoesNotExist as exc: + raise DraftNotFound("Draft {} does not exist.".format(draft_uuid)) from exc + return draft_model def get_draft(draft_uuid): """ - Retrieve metadata about the specified draft. + Retrieve data about the specified draft. + If you don't know the draft's UUID, look it up using get_bundle() """ - assert isinstance(draft_uuid, UUID) - try: - data = api_request('get', api_url('drafts', str(draft_uuid))) - except NotFound: - raise DraftNotFound("Draft does not exist: {}".format(draft_uuid)) - return _draft_from_response(data) + draft_model = _get_draft_model(draft_uuid) + return _draft_data_from_model(draft_model) -def get_or_create_bundle_draft(bundle_uuid, draft_name): +def get_or_create_draft(bundle_uuid, draft_name): """ - Retrieve metadata about the specified draft. + Retrieve data about the specified draft, creating a new one if it does not exist yet. """ - bundle = get_bundle(bundle_uuid) try: - return get_draft(bundle.drafts[draft_name]) # pylint: disable=unsubscriptable-object - except KeyError: - # The draft doesn't exist yet, so create it: - response = api_request('post', api_url('drafts'), json={ - "bundle_uuid": str(bundle_uuid), - "name": draft_name, - }) - # The result of creating a draft doesn't include all the fields we want, so retrieve it now: - return get_draft(UUID(response["uuid"])) + draft_model = models.Draft.objects.get(bundle__uuid=bundle_uuid, name=draft_name) + except models.Draft.DoesNotExist: + bundle_model = _get_bundle_model(bundle_uuid) + draft_model = models.Draft( + bundle=bundle_model, + name=draft_name, + ) + draft_model.save() + return _draft_data_from_model(draft_model) def commit_draft(draft_uuid): @@ -252,7 +268,16 @@ def commit_draft(draft_uuid): Does not return any value. """ - api_request('post', api_url('drafts', str(draft_uuid), 'commit')) + draft_repo = DraftRepo(SnapshotRepo()) + staged_draft = draft_repo.get(draft_uuid) + + if not staged_draft.files_to_overwrite and not staged_draft.links_to_overwrite: + raise DraftHasNoChangesToCommit("Draft {} does not have any changes to commit.".format(draft_uuid)) + + new_snapshot, _updated_draft = draft_repo.commit(staged_draft) + models.BundleVersion.create_new_version( + new_snapshot.bundle_uuid, new_snapshot.hash_digest + ) def delete_draft(draft_uuid): @@ -261,69 +286,85 @@ def delete_draft(draft_uuid): Does not return any value. """ - api_request('delete', api_url('drafts', str(draft_uuid))) + draft_model = _get_draft_model(draft_uuid) + draft_repo = DraftRepo(SnapshotRepo()) + draft_repo.delete(draft_uuid) + draft_model.delete() -def get_bundle_version(bundle_uuid, version_number): +def _bundle_version_data_from_model(bundle_version_model): # TODO """ - Get the details of the specified bundle version + Create and return BundleVersionData from bundle version model. """ - if version_number == 0: - return None - version_url = api_url('bundle_versions', str(bundle_uuid) + ',' + str(version_number)) - return api_request('get', version_url) + snapshot = bundle_version_model.snapshot() + snapshot_repo = SnapshotRepo() + return BundleVersionData( + bundle_uuid=bundle_version_model.bundle.uuid, + version=bundle_version_model.version_num, + change_description=bundle_version_model.change_description, + created_at=snapshot.created_at, + files={ + path: BundleFileData( + path=path, + url=snapshot_repo.url(snapshot, path), + size=file_info.size, + hash_digest=file_info.hash_digest.hex(), + ) for path, file_info in snapshot.files.items() + }, + links={ + link.name: BundleLinkData( + name=link.name, + direct_dependency=link.direct_dependency, + indirect_dependencies=link.indirect_dependencies, + ) + for link in snapshot.links + }, + ) -def get_bundle_version_files(bundle_uuid, version_number): + +def _get_bundle_version_model(bundle_uuid, version_number=None): """ - Get a list of the files in the specified bundle version + Get BundleVersion from bundle UUID and version number. + + If version_number is None, returns the latest bundle version of the bundle. """ - if version_number == 0: - return [] - version_info = get_bundle_version(bundle_uuid, version_number) - return [BundleFile(path=path, **file_metadata) for path, file_metadata in version_info["snapshot"]["files"].items()] + filter_kwargs = { + 'bundle__uuid': bundle_uuid + } + if version_number: + filter_kwargs['version_num'] = version_number + bundle_version_model = models.BundleVersion.objects.filter(**filter_kwargs).order_by('-version_num').first() + if bundle_version_model is None: + raise BundleVersionNotFound("Bundle Version {},{} does not exist.".format(bundle_uuid, version_number)) + return bundle_version_model -def get_bundle_version_links(bundle_uuid, version_number): + +def get_bundle_version(bundle_uuid, version_number=None): """ - Get a dictionary of the links in the specified bundle version + Get the details of the specified bundle version """ - if version_number == 0: - return {} - version_info = get_bundle_version(bundle_uuid, version_number) - return { - name: LinkDetails( - name=name, - direct=LinkReference(**link["direct"]), - indirect=[LinkReference(**ind) for ind in link["indirect"]], - ) - for name, link in version_info['snapshot']['links'].items() - } + bundle_version_model = _get_bundle_version_model(bundle_uuid, version_number) + return _bundle_version_data_from_model(bundle_version_model) -def get_bundle_files_dict(bundle_uuid, use_draft=None): +def get_bundle_files(bundle_uuid, use_draft=None): """ Get a dict of all the files in the specified bundle. Returns a dict where the keys are the paths (strings) and the values are - BundleFile or DraftFile tuples. - """ - bundle = get_bundle(bundle_uuid) - if use_draft and use_draft in bundle.drafts: # pylint: disable=unsupported-membership-test - draft_uuid = bundle.drafts[use_draft] # pylint: disable=unsubscriptable-object - return get_draft(draft_uuid).files - elif not bundle.latest_version: - # This bundle has no versions so definitely does not contain any files - return {} - else: - return {file_meta.path: file_meta for file_meta in get_bundle_version_files(bundle_uuid, bundle.latest_version)} - - -def get_bundle_files(bundle_uuid, use_draft=None): + BundleFileData or DraftFileData tuples. """ - Get an iterator over all the files in the specified bundle or draft. - """ - return get_bundle_files_dict(bundle_uuid, use_draft).values() + if use_draft: + try: + draft_model = models.Draft.objects.get(bundle__uuid=bundle_uuid, name=use_draft) + except models.Draft.DoesNotExist: + pass + else: + return _draft_data_from_model(draft_model).files + + return get_bundle_version(bundle_uuid).files def get_bundle_links(bundle_uuid, use_draft=None): @@ -333,29 +374,28 @@ def get_bundle_links(bundle_uuid, use_draft=None): Returns a dict where the keys are the link names (strings) and the values are LinkDetails or DraftLinkDetails tuples. """ - bundle = get_bundle(bundle_uuid) - if use_draft and use_draft in bundle.drafts: # pylint: disable=unsupported-membership-test - draft_uuid = bundle.drafts[use_draft] # pylint: disable=unsubscriptable-object - return get_draft(draft_uuid).links - elif not bundle.latest_version: - # This bundle has no versions so definitely does not contain any links - return {} - else: - return get_bundle_version_links(bundle_uuid, bundle.latest_version) + if use_draft: + try: + draft_model = models.Draft.objects.get(bundle__uuid=bundle_uuid, name=use_draft) + except models.Draft.DoesNotExist: + pass + else: + return _draft_data_from_model(draft_model).links + + return get_bundle_version(bundle_uuid).links def get_bundle_file_metadata(bundle_uuid, path, use_draft=None): """ Get the metadata of the specified file. """ - assert isinstance(bundle_uuid, UUID) - files_dict = get_bundle_files_dict(bundle_uuid, use_draft=use_draft) + files_dict = get_bundle_files(bundle_uuid, use_draft=use_draft) try: return files_dict[path] - except KeyError: + except KeyError as exc: raise BundleFileNotFound( "Bundle {} (draft: {}) does not contain a file {}".format(bundle_uuid, use_draft, path) - ) + ) from exc def get_bundle_file_data(bundle_uuid, path, use_draft=None): @@ -365,9 +405,24 @@ def get_bundle_file_data(bundle_uuid, path, use_draft=None): Do not use this for large files! """ - metadata = get_bundle_file_metadata(bundle_uuid, path, use_draft) - with requests.get(metadata.url, stream=True) as r: - return r.content + + if use_draft: + try: + draft_model = models.Draft.objects.get(bundle__uuid=bundle_uuid, name=use_draft) + except models.Draft.DoesNotExist: + pass + else: + draft_repo = DraftRepo(SnapshotRepo()) + staged_draft = draft_model.staged_draft + with draft_repo.open(staged_draft, path) as file: + return file.read() + + bundle_version_model = _get_bundle_version_model(bundle_uuid) + + snapshot_repo = SnapshotRepo() + snapshot = bundle_version_model.snapshot() + with snapshot_repo.open(snapshot, path) as file: + return file.read() def write_draft_file(draft_uuid, path, contents): @@ -380,11 +435,21 @@ def write_draft_file(draft_uuid, path, contents): Does not return anything. """ - api_request('patch', api_url('drafts', str(draft_uuid)), json={ + data = { 'files': { - path: encode_str_for_draft(contents) if contents is not None else None, + path: _encode_str_for_draft(contents) if contents is not None else None, }, - }) + } + serializer = DraftFileUpdateSerializer(data=data) + serializer.is_valid(raise_exception=True) + files_to_write = serializer.validated_data['files'] + dependencies_to_write = serializer.validated_data['links'] + + draft_repo = DraftRepo(SnapshotRepo()) + try: + draft_repo.update(draft_uuid, files_to_write, dependencies_to_write) + except LinkCycleError as exc: + raise serializers.ValidationError("Link cycle detected: Cannot create draft.") from exc def set_draft_link(draft_uuid, link_name, bundle_uuid, version): @@ -398,36 +463,29 @@ def set_draft_link(draft_uuid, link_name, bundle_uuid, version): Does not return anything. """ - api_request('patch', api_url('drafts', str(draft_uuid)), json={ + data = { 'links': { link_name: {"bundle_uuid": str(bundle_uuid), "version": version} if bundle_uuid is not None else None, }, - }) + } + serializer = DraftFileUpdateSerializer(data=data) + serializer.is_valid(raise_exception=True) + files_to_write = serializer.validated_data['files'] + dependencies_to_write = serializer.validated_data['links'] + + draft_repo = DraftRepo(SnapshotRepo()) + try: + draft_repo.update(draft_uuid, files_to_write, dependencies_to_write) + except LinkCycleError as exc: + raise serializers.ValidationError("Link cycle detected: Cannot create draft.") from exc -def encode_str_for_draft(input_str): +def _encode_str_for_draft(input_str): """ Given a string, return UTF-8 representation that is then base64 encoded. """ - if isinstance(input_str, six.text_type): + if isinstance(input_str, str): binary = input_str.encode('utf8') else: binary = input_str return base64.b64encode(binary) - - -def force_browser_url(blockstore_file_url): - """ - Ensure that the given URL Blockstore is a URL accessible from the end user's - browser. - """ - # Hack: on some devstacks, we must necessarily use different URLs for - # accessing Blockstore file data from within and outside of docker - # containers, but Blockstore has no way of knowing which case any particular - # request is for. So it always returns a URL suitable for use from within - # the container. Only this edxapp can transform the URL at the last second, - # knowing that in this case it's going to the user's browser and not being - # read by edxapp. - # In production, the same S3 URLs get used for internal and external access - # so this hack is not necessary. - return blockstore_file_url.replace('http://edx.devstack.blockstore:', 'http://localhost:') diff --git a/blockstore/apps/api/tests/test_blockstore_api.py b/blockstore/apps/api/tests/test_blockstore_api.py index fc773d3b..a904dd93 100644 --- a/blockstore/apps/api/tests/test_blockstore_api.py +++ b/blockstore/apps/api/tests/test_blockstore_api.py @@ -1,26 +1,20 @@ # -*- coding: utf-8 -*- """ -Tests for xblock_utils.py +Tests for the api. """ -import unittest from uuid import UUID -from django.conf import settings -from openedx.core.lib import blockstore_api as api +from django.test import TestCase +from blockstore.apps import api # A fake UUID that won't represent any real bundle/draft/collection: BAD_UUID = UUID('12345678-0000-0000-0000-000000000000') -@unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server") -class BlockstoreApiClientTest(unittest.TestCase): +class BlockstoreApiTest(TestCase): """ - Test for the Blockstore API Client. - - The goal of these tests is not to test that Blockstore works correctly, but - that the API client can interact with it and all the API client methods - work. + Tests for the Blockstore API. """ # Collections @@ -100,12 +94,12 @@ def test_drafts_and_files(self): coll = api.create_collection("Test Collection") bundle = api.create_bundle(coll.uuid, title="Earth 🗿 Bundle", slug="earth", description="another test bundle") # Create a draft - draft = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") + draft = api.get_or_create_draft(bundle.uuid, draft_name="test-draft") self.assertEqual(draft.bundle_uuid, bundle.uuid) self.assertEqual(draft.name, "test-draft") self.assertGreaterEqual(draft.updated_at.year, 2019) # And retrieve it again: - draft2 = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") + draft2 = api.get_or_create_draft(bundle.uuid, draft_name="test-draft") self.assertEqual(draft, draft2) # Also test retrieving using get_draft draft3 = api.get_draft(draft.uuid) @@ -136,8 +130,7 @@ def test_drafts_and_files(self): self.assertEqual(file_info1.size, len(b"initial version")) self.assertEqual(file_info1.hash_digest, "a45a5c6716276a66c4005534a51453ab16ea63c4") - self.assertEqual(list(api.get_bundle_files(bundle.uuid)), [file_info1]) - self.assertEqual(api.get_bundle_files_dict(bundle.uuid), { + self.assertEqual(api.get_bundle_files(bundle.uuid), { "test.txt": file_info1, }) @@ -150,11 +143,11 @@ def test_links(self): coll = api.create_collection("Test Collection") # Create two library bundles and a course bundle: lib1_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib1") - lib1_draft = api.get_or_create_bundle_draft(lib1_bundle.uuid, draft_name="test-draft") + lib1_draft = api.get_or_create_draft(lib1_bundle.uuid, draft_name="test-draft") lib2_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib2") - lib2_draft = api.get_or_create_bundle_draft(lib2_bundle.uuid, draft_name="other-draft") + lib2_draft = api.get_or_create_draft(lib2_bundle.uuid, draft_name="other-draft") course_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="course") - course_draft = api.get_or_create_bundle_draft(course_bundle.uuid, draft_name="test-draft") + course_draft = api.get_or_create_draft(course_bundle.uuid, draft_name="test-draft") # To create links, we need valid BundleVersions, which requires having committed at least one change: api.write_draft_file(lib1_draft.uuid, "lib1-data.txt", "hello world") @@ -171,8 +164,8 @@ def test_links(self): # Now confirm the link exists in the draft: lib2_draft_links = api.get_bundle_links(lib2_bundle.uuid, use_draft=lib2_draft.name) self.assertIn(link1_name, lib2_draft_links) - self.assertEqual(lib2_draft_links[link1_name].direct.bundle_uuid, lib1_bundle.uuid) - self.assertEqual(lib2_draft_links[link1_name].direct.version, 1) + self.assertEqual(lib2_draft_links[link1_name].direct_dependency.bundle_uuid, lib1_bundle.uuid) + self.assertEqual(lib2_draft_links[link1_name].direct_dependency.version, 1) # Now commit the change to lib2: api.commit_draft(lib2_draft.uuid) # Creates version 2 @@ -184,11 +177,11 @@ def test_links(self): # And confirm the link exists in the resulting bundle version: course_links = api.get_bundle_links(course_bundle.uuid) self.assertIn(link2_name, course_links) - self.assertEqual(course_links[link2_name].direct.bundle_uuid, lib2_bundle.uuid) - self.assertEqual(course_links[link2_name].direct.version, 2) + self.assertEqual(course_links[link2_name].direct_dependency.bundle_uuid, lib2_bundle.uuid) + self.assertEqual(course_links[link2_name].direct_dependency.version, 2) # And since the links go course->lib2->lib1, course has an indirect link to lib1: - self.assertEqual(course_links[link2_name].indirect[0].bundle_uuid, lib1_bundle.uuid) - self.assertEqual(course_links[link2_name].indirect[0].version, 1) + self.assertEqual(course_links[link2_name].indirect_dependencies[0].bundle_uuid, lib1_bundle.uuid) + self.assertEqual(course_links[link2_name].indirect_dependencies[0].version, 1) # Finally, test deleting a link from course's draft: api.set_draft_link(course_draft.uuid, link2_name, None, None) From 16a32201f0faa445560f1ab4cdc6d957dec20c42 Mon Sep 17 00:00:00 2001 From: Usman Khalid <2200617@gmail.com> Date: Thu, 2 Sep 2021 00:24:18 +0500 Subject: [PATCH 6/6] Fix diff-cover. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 73b70d54..9c7736b3 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ test: clean ## Run tests and generate coverage report ${VENV_BIN}/coverage run ./manage.py test blockstore --settings=blockstore.settings.test ${VENV_BIN}/coverage html ${VENV_BIN}/coverage xml - ${VENV_BIN}/diff-cover coverage.xml --html-report diff-cover.html + ${VENV_BIN}/diff-cover coverage.xml --html-report diff-cover.html --compare-branch=origin/master easyserver: dev.up dev.provision # Start and provision a Blockstore container and run the server until CTRL-C, then stop it # Now run blockstore until the user hits CTRL-C: