Started REST API v1

.devcontainer/devcontainer.json

@@ -17,7 +17,7 @@
 	// The optional 'workspaceFolder' property is the path VS Code should open by default when
 	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
 	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
-	
+
 	// Features to add to the dev container. More info: https://containers.dev/features.
 	// "features": {},
 
@@ -50,7 +50,7 @@
 	// "shutdownAction": "none",
 
 	// Uncomment the next line to run commands after the container is created.
-	"postCreateCommand": "cd frontend; yarn; cd ../application; make setup; make import-mock-data",
+	"postCreateCommand": "cd frontend; yarn; cd ../application/; pip install -e .[test,dev]; make setup; make import-mock-data",
 
 	// Configure tool-specific properties.
 	"customizations": {
@@ -68,7 +68,7 @@
 							"type": "node",
 							"request": "launch",
 							"runtimeExecutable": "yarn",
-							"cwd": "${workspaceFolder}/frontend",
+							"cwd": "${workspaceFolder:Frontend}/",
 							"runtimeArgs": [
 								"dev"
 							]
@@ -82,7 +82,18 @@
 							],
 							"django": true,
 							"autoStartBrowser": false,
-							"program": "${workspaceFolder}/application/src/manage.py"
+							"program": "${workspaceFolder:Backend}/src/manage.py"
+						},
+						{
+							"name": "TIRA Backend Tests",
+							"type": "python",
+							"request": "launch",
+							"program": "${workspaceFolder:Backend}/src/manage.py",
+							"cwd": "${workspaceFolder:Backend}/test",
+							"args": [ "test", "--failfast", "--settings=settings_test" ],
+							"django": true,
+							"env": { "PYTHONPATH": ":../src:.", "DJANGO_SETTINGS_MODULE": "settings_test" },
+							"justMyCode": false
 						}
 					],
 					"compounds": [
@@ -109,7 +120,8 @@
 				"ms-python.isort",
 				"ms-python.black-formatter",
 				"ms-python.flake8",
-				"ms-python.mypy-type-checker"
+				"ms-python.mypy-type-checker",
+				"42Crunch.vscode-openapi",
 			]
 		}
 	},

.github/workflows/run-all-tests.yml

@@ -21,7 +21,7 @@ jobs:
         run: |
           # Create a dummy DISRAPTOR_API_KEY
           sudo bash -c 'mkdir -p "/etc/discourse/" && echo "I am so secret" > "/etc/discourse/client-api-key"'
-          pip3 install -r requirements.txt
+          pip3 install -e .[dev,test]
           make setup
           make tests
 

Dockerfile.dev

@@ -43,13 +43,9 @@ apt-get update && apt-get install -y python3 python3-pip python3-dev pkg-config
     libpcre3-dev
 pip3 install black flake8 isort mypy
 EOF
-COPY application/requirements.txt /tmp/application/requirements.txt
 RUN <<EOF
 # Create a dummy secret
 mkdir -p "/etc/discourse/" && echo "I am so secret" > "/etc/discourse/client-api-key"
-# Setup the application
-cd /tmp/application
-pip3 install -r requirements.txt
 EOF
 
 ########################################################################################################################

README.md

@@ -12,7 +12,7 @@
    <img alt="Current Release" src="https://img.shields.io/github/release/tira-io/tira.svg"/>
    </a>
    <a href="https://tira.io">
-   <img alt="Deployment" src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fwww.tira.io%2Fapi%2Fv1%2F&query=%24.version&prefix=v.&label=tira.io"/>
+   <img alt="Deployment" src="https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fwww.tira.io%2Finfo&query=%24.version&prefix=v.&label=tira.io"/>
    </a>
    <a href="https://github.com/tira-io/tira/actions/workflows/run-all-tests.yml">
    <img alt="Tests" src="https://github.com/tira-io/tira/actions/workflows/run-all-tests.yml/badge.svg"/>

application/setup.cfg

@@ -1,3 +1,48 @@
+
+[options]
+python_requires = >=3.9
+include_package_data = True
+packages = find:
+install_requires =
+    grpcio>=1.53.2
+    # grpcio-tools==1.36.1  # still needed?
+    protobuf<4.0dev
+    pyuwsgi
+    Django
+    pyyaml
+    requests
+    randomname
+    tqdm
+    mysql
+    mysqlclient
+    django-webpack-loader==0.6.0
+    python-gitlab
+    GitPython
+    python-slugify
+    ir-datasets@git+https://github.com/allenai/ir_datasets
+    diffir@git+https://github.com/mam10eks/diffir
+    pandas
+    markdown
+    PyGithub==1.59.1
+    ghapi
+    django-extensions
+    discourse-client-in-disraptor==0.0.8
+    tira>=0.0.97
+    huggingface-hub
+    djangorestframework==3.15.1
+    django-filter==24.2
+    djangorestframework-jsonapi==7.0.0
+
+[options.extras_require]
+test =
+    mockito
+    parameterized
+    approvaltests==7.3.0
+dev =
+    coverage
+    coverage-badge
+
+
 [flake8]
 max-line-length = 120
 extend-ignore = E203

application/src/django_admin/settings.py

@@ -70,12 +70,14 @@
 
 INSTALLED_APPS = [
     "tira.apps.TiraConfig",
-    "django.contrib.admin",
     "django.contrib.auth",
     "django.contrib.contenttypes",
     "django.contrib.sessions",
     "django.contrib.messages",
     "django.contrib.staticfiles",
+    "django_filters",
+    "rest_framework",
+    "rest_framework_json_api",
     "webpack_loader",
 ]
 
@@ -89,6 +91,11 @@
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
 ]
 
+REST_FRAMEWORK = {
+    "DEFAULT_AUTHENTICATION_CLASSES": ("tira.authentication.TrustedHeaderAuthentication",),
+    "DEFAULT_FILTER_BACKENDS": ("rest_framework_json_api.django_filters.DjangoFilterBackend",),
+}
+
 ROOT_URLCONF = "django_admin.urls"
 
 TEMPLATES = [

application/src/django_admin/urls.py

@@ -16,10 +16,8 @@
 
 from django.conf import settings
 from django.conf.urls.static import static
-from django.contrib import admin
 from django.urls import include, path
 
 urlpatterns = [
-    path("admin/", admin.site.urls),
     path("", include("tira.urls")),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

application/src/tira/authentication.py

@@ -475,3 +475,72 @@ def user_is_organizer_for_endpoint(
 
 
 auth = Authentication(authentication_source=settings.DEPLOYMENT)
+
+
+"""
+Trusted Header Authentication implementation to integrate with Django
+"""
+
+from typing import NamedTuple
+
+from django.contrib.auth.models import AnonymousUser
+from rest_framework import authentication
+
+_DISRAPTOR_APP_SECRET_KEY = os.getenv("DISRAPTOR_APP_SECRET_KEY")
+
+
+class User(NamedTuple):
+    username: str
+    is_staff: bool
+
+
+class TiraGuest(AnonymousUser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.username = "guest"
+        self._groups: list[str] = []
+        self.is_staff = False
+
+    def __str__(self) -> str:
+        return self.username
+
+    @property
+    def is_anonymous(self):
+        return True
+
+    @property
+    def is_authenticated(self):
+        return False
+
+
+class TiraUser(AnonymousUser):
+    def __init__(self, username: str, groups: list[str]) -> None:
+        super().__init__()
+        self.username = username
+        self._groups = groups
+        self.is_staff = "admins" in groups or "tira_reviewer" in groups
+
+    def __str__(self) -> str:
+        return self.username
+
+    @property
+    def is_anonymous(self):
+        return False
+
+    @property
+    def is_authenticated(self):
+        return True
+
+
+class TrustedHeaderAuthentication(authentication.BaseAuthentication):
+
+    def authenticate(self, request) -> tuple[User, None]:
+        if not request.headers.get("X-Disraptor-App-Secret-Key", None) == _DISRAPTOR_APP_SECRET_KEY:
+            return HttpResponseNotAllowed("Access forbidden.")
+        username = request.headers.get("X-Disraptor-User")
+        groups = request.headers.get("X-Disraptor-Groups")
+        grouplist = [] if not groups else groups.split(",")
+        if not username:
+            return (TiraGuest(), None)
+
+        return (TiraUser(username, grouplist), None)

application/src/tira/endpoints/misc.py

@@ -0,0 +1,44 @@
+"""
+This file contains miscellaneous and **unversioned** endpoints (e.g., the /health or /info).
+"""
+
+from django.urls import path
+from rest_framework import status
+from rest_framework.decorators import api_view
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+# TODO: this does not work so I hardcoded for now
+# from tira import __version__ as tira_version
+
+tira_version = "0.0.136"
+rest_api_version = "v1.0.0-draft"
+
+
+@api_view(["GET"])
+def health_endpoint(request: Request) -> Response:
+    """
+    The /health endpoint returns 2xx on success (currently 204 because we don't respond with any content). It can be
+    used to check if the REST-API is served.
+    """
+    return Response(status=status.HTTP_204_NO_CONTENT)
+
+
+@api_view(["GET"])
+def info_endpoint(request: Request) -> Response:
+    """
+    The /info endpoint contains general information about the running server (e.g., the version of TIRA that is
+    running). Do not add any sensitive information to this endpoint as it is **public**!
+    """
+    return Response(
+        {
+            "version": tira_version,
+            "restApiVersion": rest_api_version,
+        }
+    )
+
+
+endpoints = [
+    path("health", health_endpoint),
+    path("info", info_endpoint),
+]

application/src/tira/endpoints/v1/__init__.py

@@ -0,0 +1,17 @@
+from django.urls import include, path
+
+from ._datasets import endpoints as dataset_endpoints
+from ._evaluations import endpoints as evaluation_endpoints
+from ._organizers import endpoints as organizer_endpoints
+from ._runs import endpoints as run_endpoints
+from ._tasks import endpoints as task_endpoints
+from ._user import endpoints as user_endpoints
+
+endpoints = [
+    path("datasets/", include(dataset_endpoints)),
+    path("evaluations/", include(evaluation_endpoints)),
+    path("organizers/", include(organizer_endpoints)),
+    path("runs/", include(run_endpoints)),
+    path("tasks/", include(task_endpoints)),
+    path("user/", include(user_endpoints)),
+]

application/src/tira/endpoints/v1/_datasets.py

@@ -0,0 +1,43 @@
+from django.urls import path
+from rest_framework import pagination
+from rest_framework.permissions import IsAdminUser
+from rest_framework.serializers import CharField, ModelSerializer
+from rest_framework_json_api.views import ModelViewSet
+
+from ... import model as modeldb
+from ._tasks import TaskSerializer
+
+
+class DatasetSerializer(ModelSerializer):
+    id = CharField(source="dataset_id")
+    default_task = TaskSerializer()
+
+    class Meta:
+        model = modeldb.Dataset
+        fields = [
+            "id",
+            "default_task",
+            "display_name",
+            "evaluator",
+            "is_confidential",
+            "is_deprecated",
+            "data_server",
+            "released",
+            "default_upload_name",
+            "created",
+            "last_modified",
+        ]
+
+
+class _DatasetView(ModelViewSet):
+    queryset = modeldb.Dataset.objects.all()
+    serializer_class = DatasetSerializer
+    pagination_class = pagination.CursorPagination
+    lookup_field = "dataset_id"
+    permission_classes = [IsAdminUser]  # TODO: set to something sensible
+
+
+endpoints = [
+    path("", _DatasetView.as_view({"get": "list"})),
+    path("<str:dataset_id>/", _DatasetView.as_view({"get": "retrieve", "delete": "destroy"})),
+]