From b8e9c6017f5d51c01f0165f1928e4891dfe6538f Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 3 Sep 2024 17:30:56 +0200 Subject: [PATCH] renamed application module to tira_app and updated production container --- .github/workflows/publish.yml | 4 +- application/.gitignore | 5 +- application/Dockerfile.prod | 69 ++++++++++++++---- application/setup.cfg | 17 ++++- application/setup.py | 3 + application/src/django_admin/settings.py | 10 ++- application/src/django_admin/urls.py | 2 +- application/src/run_grpc_server.py | 2 +- .../src/{tira => tira_app}/__init__.py | 0 application/src/{tira => tira_app}/admin.py | 2 +- application/src/{tira => tira_app}/apps.py | 3 +- .../src/{tira => tira_app}/authentication.py | 2 +- application/src/{tira => tira_app}/checks.py | 3 +- .../{tira => tira_app}/data/FileDatabase.py | 4 +- .../{tira => tira_app}/data/HybridDatabase.py | 8 +- .../src/{tira => tira_app}/data/data.py | 7 +- .../{tira => tira_app}/endpoints/admin_api.py | 10 +-- .../src/{tira => tira_app}/endpoints/aha | Bin .../{tira => tira_app}/endpoints/data_api.py | 12 +-- .../endpoints/diffir_api.py | 6 +- .../src/{tira => tira_app}/endpoints/misc.py | 0 .../endpoints/organizer_api.py | 4 +- .../{tira => tira_app}/endpoints/serp_api.py | 6 +- .../endpoints/stdout_beautifier.py | 0 .../endpoints/v1/__init__.py | 0 .../endpoints/v1/_datasets.py | 0 .../endpoints/v1/_evaluations.py | 0 .../endpoints/v1/_organizers.py | 0 .../{tira => tira_app}/endpoints/v1/_runs.py | 0 .../{tira => tira_app}/endpoints/v1/_tasks.py | 0 .../{tira => tira_app}/endpoints/v1/_user.py | 0 .../{tira => tira_app}/endpoints/vm_api.py | 20 ++--- .../src/{tira => tira_app}/git_runner.py | 10 +-- .../git_runner_integration.py | 10 +-- .../{tira => tira_app}/grpc/grpc_server.py | 6 +- .../grpc/test_grpc_host_client.py | 2 +- .../grpc/test_grpc_host_server.py | 4 +- .../src/{tira => tira_app}/grpc_client.py | 3 +- .../huggingface_hub_integration.py | 3 +- .../{tira => tira_app}/ir_datasets_loader.py | 4 +- .../commands/archive_runs_to_zenodo.py | 4 +- .../management/commands/cache_daemon.py | 6 +- .../management/commands/dump_tira.py | 0 .../management/commands/git_runner_cli.py | 10 +-- .../management/commands/grpc_mock_host.py | 4 +- .../management/commands/grpc_server.py | 4 +- .../management/commands/index_model.py | 2 +- .../commands/ir_datasets_loader_cli.py | 2 +- .../management/commands/irds_cli.sh | 0 .../management/commands/playground.py | 4 +- .../management/commands/run_develop.py | 0 .../management/commands/run_mockup.py | 6 +- .../management/commands/run_to_evaluations.py | 2 +- application/src/{tira => tira_app}/model.py | 0 .../src/{tira => tira_app}/permissions.py | 0 .../proto/TiraClientWebMessages_pb2.py | 0 .../proto/TiraClientWebMessages_pb2_grpc.py | 0 .../proto/TiraHostMessages_pb2.py | 0 .../proto/TiraHostMessages_pb2_grpc.py | 0 .../src/{tira => tira_app}/proto/__init__.py | 0 .../{tira => tira_app}/proto/tira_host_pb2.py | 0 .../proto/tira_host_pb2_grpc.py | 0 .../proto/tira_messages_pb2.py | 0 .../proto/tira_to_web_pb2.py | 0 .../proto/tira_to_web_pb2_grpc.py | 0 .../{ => tira_app/res}/tirex-components.yml | 0 .../static/tira/css/tira-style.css | 0 .../static/tira/img/background1.jpg | Bin .../static/tira/img/background2.jpg | Bin .../static/tira/img/google-icon.png | Bin .../static/tira/img/ia-icon.png | Bin .../static/tira/img/image-licence.txt | 0 .../static/tira/img/logo-tira-32x32.png | 0 .../tira/img/logo-tira-40x40-transparent.png | Bin .../static/tira/img/logo-tira-40x40.png | Bin .../static/tira/img/zenodo-icon.png | Bin .../static/tira/js/review.js | 0 .../templates/tira/base.html | 0 .../tira/git-repo-template/Dockerfile | 0 .../tira/git-repo-template/README.md | 0 .../tira/git-repo-template/github-action.yml | 0 .../tira/git-repo-template/script.py | 0 .../tira/git_task_repository_gitlab_ci.yml | 0 .../tira/git_task_repository_readme.md | 0 .../tira/git_user_repository_readme.md | 0 .../templates/tira/navbar.html | 0 .../templates/tira/tira_git_cmd.py | 0 .../templates/tira/tira_git_cmd.sh | 0 .../templates/tira/tira_git_makefile | 0 .../templates/tira/tira_git_tutorial.ipynb | 0 .../src/{tira => tira_app}/tests/tests.py | 0 .../src/{tira => tira_app}/tira_data.py | 2 +- .../src/{tira => tira_app}/tira_model.py | 8 +- application/src/{tira => tira_app}/urls.py | 0 application/src/{tira => tira_app}/util.py | 7 +- application/src/{tira => tira_app}/views.py | 3 +- .../test/auth_tests/test_is_admin_for_task.py | 2 +- .../test_evaluations_for_vm.py | 4 +- .../test_rendering_of_job_file.py | 2 +- .../test_hf_mounts_are_parsed.py | 2 +- ..._multi_step_jobs_with_alternative_input.py | 4 +- ...st_multi_step_jobs_with_multiple_inputs.py | 2 +- .../test_creation_of_dataset_id.py | 2 +- application/test/settings_test.py | 9 ++- application/test/utils_for_testing.py | 6 +- pipelines/src/django_tira_git/settings.py | 2 +- 106 files changed, 198 insertions(+), 142 deletions(-) create mode 100644 application/setup.py rename application/src/{tira => tira_app}/__init__.py (100%) rename application/src/{tira => tira_app}/admin.py (89%) rename application/src/{tira => tira_app}/apps.py (61%) rename application/src/{tira => tira_app}/authentication.py (99%) rename application/src/{tira => tira_app}/checks.py (99%) rename application/src/{tira => tira_app}/data/FileDatabase.py (99%) rename application/src/{tira => tira_app}/data/HybridDatabase.py (99%) rename application/src/{tira => tira_app}/data/data.py (98%) rename application/src/{tira => tira_app}/endpoints/admin_api.py (98%) rename application/src/{tira => tira_app}/endpoints/aha (100%) rename application/src/{tira => tira_app}/endpoints/data_api.py (98%) rename application/src/{tira => tira_app}/endpoints/diffir_api.py (97%) rename application/src/{tira => tira_app}/endpoints/misc.py (100%) rename application/src/{tira => tira_app}/endpoints/organizer_api.py (93%) rename application/src/{tira => tira_app}/endpoints/serp_api.py (92%) rename application/src/{tira => tira_app}/endpoints/stdout_beautifier.py (100%) rename application/src/{tira => tira_app}/endpoints/v1/__init__.py (100%) rename application/src/{tira => tira_app}/endpoints/v1/_datasets.py (100%) rename application/src/{tira => tira_app}/endpoints/v1/_evaluations.py (100%) rename application/src/{tira => tira_app}/endpoints/v1/_organizers.py (100%) rename application/src/{tira => tira_app}/endpoints/v1/_runs.py (100%) rename application/src/{tira => tira_app}/endpoints/v1/_tasks.py (100%) rename application/src/{tira => tira_app}/endpoints/v1/_user.py (100%) rename application/src/{tira => tira_app}/endpoints/vm_api.py (98%) rename application/src/{tira => tira_app}/git_runner.py (90%) rename application/src/{tira => tira_app}/git_runner_integration.py (99%) rename application/src/{tira => tira_app}/grpc/grpc_server.py (98%) rename application/src/{tira => tira_app}/grpc/test_grpc_host_client.py (98%) rename application/src/{tira => tira_app}/grpc/test_grpc_host_server.py (99%) rename application/src/{tira => tira_app}/grpc_client.py (99%) rename application/src/{tira => tira_app}/huggingface_hub_integration.py (99%) rename application/src/{tira => tira_app}/ir_datasets_loader.py (99%) rename application/src/{tira => tira_app}/management/commands/archive_runs_to_zenodo.py (99%) rename application/src/{tira => tira_app}/management/commands/cache_daemon.py (97%) rename application/src/{tira => tira_app}/management/commands/dump_tira.py (100%) rename application/src/{tira => tira_app}/management/commands/git_runner_cli.py (98%) rename application/src/{tira => tira_app}/management/commands/grpc_mock_host.py (91%) rename application/src/{tira => tira_app}/management/commands/grpc_server.py (91%) rename application/src/{tira => tira_app}/management/commands/index_model.py (89%) rename application/src/{tira => tira_app}/management/commands/ir_datasets_loader_cli.py (98%) rename application/src/{tira => tira_app}/management/commands/irds_cli.sh (100%) rename application/src/{tira => tira_app}/management/commands/playground.py (93%) rename application/src/{tira => tira_app}/management/commands/run_develop.py (100%) rename application/src/{tira => tira_app}/management/commands/run_mockup.py (91%) rename application/src/{tira => tira_app}/management/commands/run_to_evaluations.py (96%) rename application/src/{tira => tira_app}/model.py (100%) rename application/src/{tira => tira_app}/permissions.py (100%) rename application/src/{tira => tira_app}/proto/TiraClientWebMessages_pb2.py (100%) rename application/src/{tira => tira_app}/proto/TiraClientWebMessages_pb2_grpc.py (100%) rename application/src/{tira => tira_app}/proto/TiraHostMessages_pb2.py (100%) rename application/src/{tira => tira_app}/proto/TiraHostMessages_pb2_grpc.py (100%) rename application/src/{tira => tira_app}/proto/__init__.py (100%) rename application/src/{tira => tira_app}/proto/tira_host_pb2.py (100%) rename application/src/{tira => tira_app}/proto/tira_host_pb2_grpc.py (100%) rename application/src/{tira => tira_app}/proto/tira_messages_pb2.py (100%) rename application/src/{tira => tira_app}/proto/tira_to_web_pb2.py (100%) rename application/src/{tira => tira_app}/proto/tira_to_web_pb2_grpc.py (100%) rename application/src/{ => tira_app/res}/tirex-components.yml (100%) rename application/src/{tira => tira_app}/static/tira/css/tira-style.css (100%) rename application/src/{tira => tira_app}/static/tira/img/background1.jpg (100%) rename application/src/{tira => tira_app}/static/tira/img/background2.jpg (100%) rename application/src/{tira => tira_app}/static/tira/img/google-icon.png (100%) rename application/src/{tira => tira_app}/static/tira/img/ia-icon.png (100%) rename application/src/{tira => tira_app}/static/tira/img/image-licence.txt (100%) rename application/src/{tira => tira_app}/static/tira/img/logo-tira-32x32.png (100%) rename application/src/{tira => tira_app}/static/tira/img/logo-tira-40x40-transparent.png (100%) rename application/src/{tira => tira_app}/static/tira/img/logo-tira-40x40.png (100%) rename application/src/{tira => tira_app}/static/tira/img/zenodo-icon.png (100%) rename application/src/{tira => tira_app}/static/tira/js/review.js (100%) rename application/src/{tira => tira_app}/templates/tira/base.html (100%) rename application/src/{tira => tira_app}/templates/tira/git-repo-template/Dockerfile (100%) rename application/src/{tira => tira_app}/templates/tira/git-repo-template/README.md (100%) rename application/src/{tira => tira_app}/templates/tira/git-repo-template/github-action.yml (100%) rename application/src/{tira => tira_app}/templates/tira/git-repo-template/script.py (100%) rename application/src/{tira => tira_app}/templates/tira/git_task_repository_gitlab_ci.yml (100%) rename application/src/{tira => tira_app}/templates/tira/git_task_repository_readme.md (100%) rename application/src/{tira => tira_app}/templates/tira/git_user_repository_readme.md (100%) rename application/src/{tira => tira_app}/templates/tira/navbar.html (100%) rename application/src/{tira => tira_app}/templates/tira/tira_git_cmd.py (100%) rename application/src/{tira => tira_app}/templates/tira/tira_git_cmd.sh (100%) rename application/src/{tira => tira_app}/templates/tira/tira_git_makefile (100%) rename application/src/{tira => tira_app}/templates/tira/tira_git_tutorial.ipynb (100%) rename application/src/{tira => tira_app}/tests/tests.py (100%) rename application/src/{tira => tira_app}/tira_data.py (98%) rename application/src/{tira => tira_app}/tira_model.py (99%) rename application/src/{tira => tira_app}/urls.py (100%) rename application/src/{tira => tira_app}/util.py (98%) rename application/src/{tira => tira_app}/views.py (99%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 10ae719ad..655b169bc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -50,8 +50,8 @@ jobs: - name: Publish tira-backend image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: - context: ${{ github.workspace }}/application + context: ${{ github.workspace }}/ file: ${{ github.workspace }}/application/Dockerfile.prod push: true tags: ${{ steps.meta-backend.outputs.tags }} - labels: ${{ steps.meta-backend.outputs.labels }} \ No newline at end of file + labels: ${{ steps.meta-backend.outputs.labels }} diff --git a/application/.gitignore b/application/.gitignore index 347232bbe..117dbbb48 100644 --- a/application/.gitignore +++ b/application/.gitignore @@ -3,11 +3,10 @@ src/tira_vm_states.sqlite3 src/venv src/__init__.py src/config -src/tira/migrations/*.py +src/tira_app/migrations/*.py test/test-database -src/tira/frontend/node_modules/ -src/tira/static/tira/dist/* +src/tira_app/static/tira/dist/* package-lock.json test/tira-root/ **/*.received.txt diff --git a/application/Dockerfile.prod b/application/Dockerfile.prod index 82514118d..d772de178 100644 --- a/application/Dockerfile.prod +++ b/application/Dockerfile.prod @@ -5,13 +5,14 @@ ######################################################################################################################## # Production Container # ######################################################################################################################## -FROM debian:stable-slim +FROM debian:stable-slim AS build ENV TZ=Europe/Berlin -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -RUN apt-get update && apt-get install -y locales -RUN echo "en_US.UTF-8 UTF-8" | tee -a /etc/locale.gen && locale-gen +RUN < /etc/timezone +apt-get update && apt-get install -y locales +echo "en_US.UTF-8 UTF-8" | tee -a /etc/locale.gen && locale-gen +EOF ######################################################################################################################## # Create User # @@ -24,16 +25,17 @@ RUN useradd -ms /bin/bash tira ######################################################################################################################## USER tira WORKDIR /tira/ -COPY --chown=tira:tira . ./ +COPY --chown=tira:tira ./application ./ +COPY --chown=tira:tira ./python-client ../python-client ######################################################################################################################## # Install Python and Dependencies # ######################################################################################################################## USER root -ENV PIP_BREAK_SYSTEM_PACKAGES 1 +ENV PIP_BREAK_SYSTEM_PACKAGES=1 # For faster build of GRPCIO -ENV GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS 16 +ENV GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=16 RUN < /etc/timezone +apt-get update && apt-get install -y locales +echo "en_US.UTF-8 UTF-8" | tee -a /etc/locale.gen && locale-gen + +apt-get update +# Tools +apt-get install -y python3 python3-pip +export PIP_BREAK_SYSTEM_PACKAGES=1; pip3 install uwsgi +# +mkdir -p /tira/application/src +chown -R tira:tira /tira +EOF ######################################################################################################################## # Final Configuration Stuff # ######################################################################################################################## USER tira -ENV HF_HOME=/mnt/ceph/tira/data/publicly-shared-datasets/huggingface/ +# CONFIGURE THIS ENVIRONMENT VARIABLE IN YOUR DOCKER-COMPOSE FILE +ENV HF_HOME=/home/tira/data/publicly-shared-datasets/huggingface/ +ENV PATH=/home/tira/.local/bin:$PATH + +EXPOSE 80 -CMD uwsgi --uid 1010 --gid 1010 --ini /tira/src/uwsgi.ini +# TODO: at some point it probably makes sense, not to use /tira/application/src as a working directory anymore +CMD ["uwsgi", "--strict", "--master", "--enable-threads", "--module", "django_admin.wsgi:application", "--chdir", "/tira/application/src", "--processes", "50", "--http-socket", ":80", "--vacuum", "--max-requests", "5000"] \ No newline at end of file diff --git a/application/setup.cfg b/application/setup.cfg index 3be880370..d60832e9d 100644 --- a/application/setup.cfg +++ b/application/setup.cfg @@ -1,8 +1,12 @@ +[metadata] +name = tira_app [options] python_requires = >=3.9 include_package_data = True -packages = find: +package_dir = + = src +packages = find_namespace: install_requires = grpcio>=1.53.2 # grpcio-tools==1.36.1 # still needed? @@ -23,7 +27,7 @@ install_requires = PyGithub==1.59.1 django-extensions discourse-client-in-disraptor==0.0.8 - tira>=0.0.97 + # tira>=0.0.97 huggingface-hub djangorestframework==3.15.1 django-filter==24.2 @@ -44,6 +48,15 @@ dev = isort mypy +[options.packages.find] +where = src +include = + tira_app + tira_app.* + django_admin + +[options.package_data] +tira_app.res = *.yml [flake8] max-line-length = 120 diff --git a/application/setup.py b/application/setup.py new file mode 100644 index 000000000..606849326 --- /dev/null +++ b/application/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/application/src/django_admin/settings.py b/application/src/django_admin/settings.py index cb84e7347..0151b1fdf 100644 --- a/application/src/django_admin/settings.py +++ b/application/src/django_admin/settings.py @@ -10,6 +10,7 @@ https://docs.djangoproject.com/en/3.1/ref/settings/ """ +import importlib.resources as resources import logging import os from pathlib import Path @@ -69,7 +70,7 @@ # Application definition INSTALLED_APPS = [ - "tira.apps.TiraConfig", + "tira_app.apps.TiraConfig", "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", @@ -90,7 +91,7 @@ ] REST_FRAMEWORK = { - "DEFAULT_AUTHENTICATION_CLASSES": ("tira.authentication.TrustedHeaderAuthentication",), + "DEFAULT_AUTHENTICATION_CLASSES": ("tira_app.authentication.TrustedHeaderAuthentication",), "DEFAULT_FILTER_BACKENDS": ("rest_framework_json_api.django_filters.DjangoFilterBackend",), } @@ -335,8 +336,9 @@ def logger_config(log_dir: Path): } } -# FIXME: I don't close my file handle :(((((((( -TIREX_COMPONENTS = yaml.load(open(BASE_DIR / "tirex-components.yml").read(), Loader=yaml.FullLoader) +TIREX_COMPONENTS = yaml.load( + (resources.files("tira_app.res") / "tirex-components.yml").read_bytes(), Loader=yaml.FullLoader +) # Logging ld = Path(custom_settings.get("logging_dir", TIRA_ROOT / "log" / "tira-application")) diff --git a/application/src/django_admin/urls.py b/application/src/django_admin/urls.py index fb5bb717f..02850af19 100644 --- a/application/src/django_admin/urls.py +++ b/application/src/django_admin/urls.py @@ -17,5 +17,5 @@ from django.urls import include, path urlpatterns = [ - path("", include("tira.urls")), + path("", include("tira_app.urls")), ] diff --git a/application/src/run_grpc_server.py b/application/src/run_grpc_server.py index de9d3b581..b1aa43ddd 100644 --- a/application/src/run_grpc_server.py +++ b/application/src/run_grpc_server.py @@ -2,7 +2,7 @@ import django -from tira.grpc import grpc_server +from tira_app.grpc import grpc_server os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_admin.settings") diff --git a/application/src/tira/__init__.py b/application/src/tira_app/__init__.py similarity index 100% rename from application/src/tira/__init__.py rename to application/src/tira_app/__init__.py diff --git a/application/src/tira/admin.py b/application/src/tira_app/admin.py similarity index 89% rename from application/src/tira/admin.py rename to application/src/tira_app/admin.py index f2aadbf6e..8cf462a55 100644 --- a/application/src/tira/admin.py +++ b/application/src/tira_app/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -import tira.model as modeldb +from . import model as modeldb # Register your models here. diff --git a/application/src/tira/apps.py b/application/src/tira_app/apps.py similarity index 61% rename from application/src/tira/apps.py rename to application/src/tira_app/apps.py index 36a51708e..926187884 100644 --- a/application/src/tira/apps.py +++ b/application/src/tira_app/apps.py @@ -2,4 +2,5 @@ class TiraConfig(AppConfig): - name = "tira" + name = "tira_app" + label = "tira" diff --git a/application/src/tira/authentication.py b/application/src/tira_app/authentication.py similarity index 99% rename from application/src/tira/authentication.py rename to application/src/tira_app/authentication.py index 7118380ca..0b5587583 100644 --- a/application/src/tira/authentication.py +++ b/application/src/tira_app/authentication.py @@ -8,7 +8,7 @@ from django.http import HttpRequest, HttpResponseNotAllowed from slugify import slugify -import tira.tira_model as model +from . import tira_model as model logger = logging.getLogger(__name__) diff --git a/application/src/tira/checks.py b/application/src/tira_app/checks.py similarity index 99% rename from application/src/tira/checks.py rename to application/src/tira_app/checks.py index c659bc02a..854acee0d 100644 --- a/application/src/tira/checks.py +++ b/application/src/tira_app/checks.py @@ -6,8 +6,7 @@ from django.shortcuts import redirect from django.urls import resolve -import tira.tira_model as model - +from . import tira_model as model from .authentication import auth logger = logging.getLogger("tira") diff --git a/application/src/tira/data/FileDatabase.py b/application/src/tira_app/data/FileDatabase.py similarity index 99% rename from application/src/tira/data/FileDatabase.py rename to application/src/tira_app/data/FileDatabase.py index b1bed39d6..9a687075e 100644 --- a/application/src/tira/data/FileDatabase.py +++ b/application/src/tira_app/data/FileDatabase.py @@ -10,8 +10,8 @@ from django.conf import settings from google.protobuf.text_format import Parse -from tira.proto import TiraClientWebMessages_pb2 as modelpb -from tira.util import TiraModelWriteError, auto_reviewer, extract_year_from_dataset_id +from ..proto import TiraClientWebMessages_pb2 as modelpb +from ..util import TiraModelWriteError, auto_reviewer, extract_year_from_dataset_id logger = logging.getLogger("tira") diff --git a/application/src/tira/data/HybridDatabase.py b/application/src/tira_app/data/HybridDatabase.py similarity index 99% rename from application/src/tira/data/HybridDatabase.py rename to application/src/tira_app/data/HybridDatabase.py index 404d5a643..0ee0d8dff 100644 --- a/application/src/tira/data/HybridDatabase.py +++ b/application/src/tira_app/data/HybridDatabase.py @@ -12,10 +12,9 @@ from django.db import IntegrityError from google.protobuf.text_format import Parse -import tira.data.data as dbops -import tira.model as modeldb -from tira.proto import TiraClientWebMessages_pb2 as modelpb -from tira.util import ( +from .. import model as modeldb +from ..proto import TiraClientWebMessages_pb2 as modelpb +from ..util import ( TiraModelIntegrityError, TiraModelWriteError, auto_reviewer, @@ -24,6 +23,7 @@ link_to_discourse_team, now, ) +from . import data as dbops logger = logging.getLogger("tira_db") diff --git a/application/src/tira/data/data.py b/application/src/tira_app/data/data.py similarity index 98% rename from application/src/tira/data/data.py rename to application/src/tira_app/data/data.py index b63dadf4a..f57b3fcc9 100644 --- a/application/src/tira/data/data.py +++ b/application/src/tira_app/data/data.py @@ -3,14 +3,13 @@ """ import logging -from pathlib import Path from google.protobuf.text_format import Parse from tqdm import tqdm -import tira.model as modeldb -from tira.proto import TiraClientWebMessages_pb2 as modelpb -from tira.util import auto_reviewer, extract_year_from_dataset_id +from .. import model as modeldb +from ..proto import TiraClientWebMessages_pb2 as modelpb +from ..util import auto_reviewer, extract_year_from_dataset_id logger = logging.getLogger("tira") diff --git a/application/src/tira/endpoints/admin_api.py b/application/src/tira_app/endpoints/admin_api.py similarity index 98% rename from application/src/tira/endpoints/admin_api.py rename to application/src/tira_app/endpoints/admin_api.py index 7dcb039e0..4f5820399 100644 --- a/application/src/tira/endpoints/admin_api.py +++ b/application/src/tira_app/endpoints/admin_api.py @@ -11,11 +11,11 @@ from django.core.serializers.json import DjangoJSONEncoder from django.http import JsonResponse -import tira.tira_model as model -from tira.authentication import auth -from tira.checks import check_conditional_permissions, check_permissions, check_resources_exist -from tira.git_runner import check_that_git_integration_is_valid -from tira.ir_datasets_loader import run_irds_command +from .. import tira_model as model +from ..authentication import auth +from ..checks import check_conditional_permissions, check_permissions, check_resources_exist +from ..git_runner import check_that_git_integration_is_valid +from ..ir_datasets_loader import run_irds_command logger = logging.getLogger("tira") logger.info("ajax_routes: Logger active") diff --git a/application/src/tira/endpoints/aha b/application/src/tira_app/endpoints/aha similarity index 100% rename from application/src/tira/endpoints/aha rename to application/src/tira_app/endpoints/aha diff --git a/application/src/tira/endpoints/data_api.py b/application/src/tira_app/endpoints/data_api.py similarity index 98% rename from application/src/tira/endpoints/data_api.py rename to application/src/tira_app/endpoints/data_api.py index a33463b63..6ba47371c 100644 --- a/application/src/tira/endpoints/data_api.py +++ b/application/src/tira_app/endpoints/data_api.py @@ -14,12 +14,12 @@ from django.http import HttpResponse, JsonResponse from slugify import slugify -import tira.tira_model as model -from tira.authentication import auth -from tira.checks import check_permissions, check_resources_exist -from tira.tira_data import get_run_file_list, get_run_runtime, get_stderr, get_stdout, get_tira_log -from tira.util import link_to_discourse_team -from tira.views import _add_user_vms_to_context, add_context +from .. import tira_model as model +from ..authentication import auth +from ..checks import check_permissions, check_resources_exist +from ..tira_data import get_run_file_list, get_run_runtime, get_stderr, get_stdout, get_tira_log +from ..util import link_to_discourse_team +from ..views import _add_user_vms_to_context, add_context include_navigation = False diff --git a/application/src/tira/endpoints/diffir_api.py b/application/src/tira_app/endpoints/diffir_api.py similarity index 97% rename from application/src/tira/endpoints/diffir_api.py rename to application/src/tira_app/endpoints/diffir_api.py index 61d287a19..164af8a45 100644 --- a/application/src/tira/endpoints/diffir_api.py +++ b/application/src/tira_app/endpoints/diffir_api.py @@ -6,9 +6,9 @@ from django.conf import settings from django.http import HttpResponse, JsonResponse -import tira.tira_model as model -from tira.checks import check_permissions -from tira.views import add_context +from .. import tira_model as model +from ..checks import check_permissions +from ..views import add_context logger = logging.getLogger("tira") diff --git a/application/src/tira/endpoints/misc.py b/application/src/tira_app/endpoints/misc.py similarity index 100% rename from application/src/tira/endpoints/misc.py rename to application/src/tira_app/endpoints/misc.py diff --git a/application/src/tira/endpoints/organizer_api.py b/application/src/tira_app/endpoints/organizer_api.py similarity index 93% rename from application/src/tira/endpoints/organizer_api.py rename to application/src/tira_app/endpoints/organizer_api.py index 955eabd95..7b3bc07d5 100644 --- a/application/src/tira/endpoints/organizer_api.py +++ b/application/src/tira_app/endpoints/organizer_api.py @@ -2,8 +2,8 @@ from django.http import JsonResponse -import tira.tira_model as model -from tira.checks import check_conditional_permissions, check_permissions, check_resources_exist +from .. import tira_model as model +from ..checks import check_conditional_permissions, check_permissions, check_resources_exist include_navigation = False diff --git a/application/src/tira/endpoints/serp_api.py b/application/src/tira_app/endpoints/serp_api.py similarity index 92% rename from application/src/tira/endpoints/serp_api.py rename to application/src/tira_app/endpoints/serp_api.py index 836db5be2..06360a717 100644 --- a/application/src/tira/endpoints/serp_api.py +++ b/application/src/tira_app/endpoints/serp_api.py @@ -4,9 +4,9 @@ from django.conf import settings from django.http import HttpResponse, JsonResponse -from tira.checks import check_permissions -from tira.endpoints.diffir_api import doc_file_for_run -from tira.views import add_context +from ..checks import check_permissions +from ..endpoints.diffir_api import doc_file_for_run +from ..views import add_context logger = logging.getLogger("tira") diff --git a/application/src/tira/endpoints/stdout_beautifier.py b/application/src/tira_app/endpoints/stdout_beautifier.py similarity index 100% rename from application/src/tira/endpoints/stdout_beautifier.py rename to application/src/tira_app/endpoints/stdout_beautifier.py diff --git a/application/src/tira/endpoints/v1/__init__.py b/application/src/tira_app/endpoints/v1/__init__.py similarity index 100% rename from application/src/tira/endpoints/v1/__init__.py rename to application/src/tira_app/endpoints/v1/__init__.py diff --git a/application/src/tira/endpoints/v1/_datasets.py b/application/src/tira_app/endpoints/v1/_datasets.py similarity index 100% rename from application/src/tira/endpoints/v1/_datasets.py rename to application/src/tira_app/endpoints/v1/_datasets.py diff --git a/application/src/tira/endpoints/v1/_evaluations.py b/application/src/tira_app/endpoints/v1/_evaluations.py similarity index 100% rename from application/src/tira/endpoints/v1/_evaluations.py rename to application/src/tira_app/endpoints/v1/_evaluations.py diff --git a/application/src/tira/endpoints/v1/_organizers.py b/application/src/tira_app/endpoints/v1/_organizers.py similarity index 100% rename from application/src/tira/endpoints/v1/_organizers.py rename to application/src/tira_app/endpoints/v1/_organizers.py diff --git a/application/src/tira/endpoints/v1/_runs.py b/application/src/tira_app/endpoints/v1/_runs.py similarity index 100% rename from application/src/tira/endpoints/v1/_runs.py rename to application/src/tira_app/endpoints/v1/_runs.py diff --git a/application/src/tira/endpoints/v1/_tasks.py b/application/src/tira_app/endpoints/v1/_tasks.py similarity index 100% rename from application/src/tira/endpoints/v1/_tasks.py rename to application/src/tira_app/endpoints/v1/_tasks.py diff --git a/application/src/tira/endpoints/v1/_user.py b/application/src/tira_app/endpoints/v1/_user.py similarity index 100% rename from application/src/tira/endpoints/v1/_user.py rename to application/src/tira_app/endpoints/v1/_user.py diff --git a/application/src/tira/endpoints/vm_api.py b/application/src/tira_app/endpoints/vm_api.py similarity index 98% rename from application/src/tira/endpoints/vm_api.py rename to application/src/tira_app/endpoints/vm_api.py index 613ce691b..d614436e5 100644 --- a/application/src/tira/endpoints/vm_api.py +++ b/application/src/tira_app/endpoints/vm_api.py @@ -12,13 +12,13 @@ from grpc import RpcError, StatusCode from markdown import markdown -import tira.tira_model as model -from tira.authentication import auth -from tira.checks import check_conditional_permissions, check_permissions, check_resources_exist -from tira.grpc_client import GrpcClient -from tira.model import EvaluationLog, TransitionLog -from tira.util import get_tira_id, link_to_discourse_team, reroute_host -from tira.views import add_context +from .. import tira_model as model +from ..authentication import auth +from ..checks import check_conditional_permissions, check_permissions, check_resources_exist +from ..grpc_client import GrpcClient +from ..model import EvaluationLog, TransitionLog +from ..util import get_tira_id, link_to_discourse_team, reroute_host +from ..views import add_context include_navigation = False @@ -161,7 +161,7 @@ def docker_software_details(request, context, vm_id, docker_software_id): @check_permissions def huggingface_model_mounts(request, vm_id, hf_model): - from tira.huggingface_hub_integration import huggingface_model_mounts, snapshot_download_hf_model + from ..huggingface_hub_integration import huggingface_model_mounts, snapshot_download_hf_model context = {"hf_model_available": False, "hf_model_for_vm": vm_id} @@ -569,7 +569,7 @@ def docker_software_add(request, task_id, vm_id): if data.get("mount_hf_model"): try: - from tira.huggingface_hub_integration import huggingface_model_mounts + from ..huggingface_hub_integration import huggingface_model_mounts mounts = huggingface_model_mounts(data.get("mount_hf_model")) model.add_docker_software_mounts(new_docker_software, mounts) @@ -925,7 +925,7 @@ def run_details(request, task_id, vm_id, run_id): ) elif "upload_id" in run and run["upload_id"]: - import tira.model as modeldb + from .. import model as modeldb run_upload = modeldb.Upload.objects.filter(vm__vm_id=vm_id, id=run["upload_id"]).get() vm_id_from_run = run_upload.vm.vm_id diff --git a/application/src/tira/git_runner.py b/application/src/tira_app/git_runner.py similarity index 90% rename from application/src/tira/git_runner.py rename to application/src/tira_app/git_runner.py index 39883afd3..df8207aee 100644 --- a/application/src/tira/git_runner.py +++ b/application/src/tira_app/git_runner.py @@ -6,7 +6,7 @@ def all_git_runners(): - from tira.tira_model import model + from .tira_model import model ret = [] for git_integration in model.all_git_integrations(return_dict=True): @@ -20,8 +20,8 @@ def all_git_runners(): def check_that_git_integration_is_valid(namespace_url, private_token): - import tira.model as modeldb - from tira.tira_model import model + from . import model as modeldb + from .tira_model import model git_integration = {"namespace_url": namespace_url, "private_token": private_token} @@ -48,7 +48,7 @@ def check_that_git_integration_is_valid(namespace_url, private_token): def get_git_runner(git_integration): - from tira.git_runner_integration import GithubRunner, GitLabRunner + from .git_runner_integration import GithubRunner, GitLabRunner if not git_integration or "namespace_url" not in git_integration: return None @@ -68,6 +68,6 @@ def get_git_runner(git_integration): def get_git_runner_for_software_integration(): - from tira.git_runner_integration import GithubRunner + from .git_runner_integration import GithubRunner return GithubRunner(settings.GITHUB_TOKEN) diff --git a/application/src/tira/git_runner_integration.py b/application/src/tira_app/git_runner_integration.py similarity index 99% rename from application/src/tira/git_runner_integration.py rename to application/src/tira_app/git_runner_integration.py index 356cc9724..213cdca21 100644 --- a/application/src/tira/git_runner_integration.py +++ b/application/src/tira_app/git_runner_integration.py @@ -21,8 +21,8 @@ from slugify import slugify from tqdm import tqdm -from tira.grpc_client import new_transaction -from tira.model import EvaluationLog, TransactionLog +from .grpc_client import new_transaction +from .model import EvaluationLog, TransactionLog logger = logging.getLogger("tira") @@ -493,7 +493,7 @@ def yield_all_running_pipelines(self, git_repository_id): raise ValueError("ToDo: Implement.") def archive_software(self, working_directory, software_definition, download_images, persist_images, upload_images): - from tira.util import docker_image_details, run_cmd + from .util import docker_image_details, run_cmd image = ( software_definition["TIRA_EVALUATION_IMAGE_TO_EXECUTE"] @@ -562,7 +562,7 @@ def archive_repository( ): from django.template.loader import render_to_string - from tira.tira_model import get_dataset, get_docker_software, get_docker_softwares_with_runs + from .tira_model import get_dataset, get_docker_software, get_docker_softwares_with_runs softwares = set() evaluations = set() @@ -1232,7 +1232,7 @@ def extract_job_configuration(self, gl_project, branch): software_from_db = {"display_name": "Evaluate Run", "image": "evaluator", "command": "evaluator"} else: try: - from tira.tira_model import model + from .tira_model import model software_from_db = model.get_docker_software(int(ret["TIRA_SOFTWARE_ID"].split("docker-software-")[-1])) except Exception as e: diff --git a/application/src/tira/grpc/grpc_server.py b/application/src/tira_app/grpc/grpc_server.py similarity index 98% rename from application/src/tira/grpc/grpc_server.py rename to application/src/tira_app/grpc/grpc_server.py index 1c7a305a6..7b97a2fc9 100644 --- a/application/src/tira/grpc/grpc_server.py +++ b/application/src/tira_app/grpc/grpc_server.py @@ -5,9 +5,9 @@ import grpc from django.conf import settings -import tira.tira_model as model -from tira.model import EvaluationLog, TransactionLog, TransitionLog -from tira.proto import tira_host_pb2, tira_host_pb2_grpc +from .. import tira_model as model +from ..model import EvaluationLog, TransactionLog, TransitionLog +from ..proto import tira_host_pb2, tira_host_pb2_grpc grpc_port = settings.APPLICATION_GRPC_PORT diff --git a/application/src/tira/grpc/test_grpc_host_client.py b/application/src/tira_app/grpc/test_grpc_host_client.py similarity index 98% rename from application/src/tira/grpc/test_grpc_host_client.py rename to application/src/tira_app/grpc/test_grpc_host_client.py index c9a5cf0e9..74e117ea2 100644 --- a/application/src/tira/grpc/test_grpc_host_client.py +++ b/application/src/tira_app/grpc/test_grpc_host_client.py @@ -1,6 +1,6 @@ import grpc -from tira.proto import tira_host_pb2, tira_host_pb2_grpc +from ..proto import tira_host_pb2, tira_host_pb2_grpc class TestGrpcHostClient: diff --git a/application/src/tira/grpc/test_grpc_host_server.py b/application/src/tira_app/grpc/test_grpc_host_server.py similarity index 99% rename from application/src/tira/grpc/test_grpc_host_server.py rename to application/src/tira_app/grpc/test_grpc_host_server.py index 8027bbe39..bd5a60766 100644 --- a/application/src/tira/grpc/test_grpc_host_server.py +++ b/application/src/tira_app/grpc/test_grpc_host_server.py @@ -9,8 +9,8 @@ import grpc -from tira.grpc.test_grpc_host_client import TestGrpcHostClient -from tira.proto import tira_host_pb2, tira_host_pb2_grpc +from tira_app.grpc.test_grpc_host_client import TestGrpcHostClient +from tira_app.proto import tira_host_pb2, tira_host_pb2_grpc VIRTUAL_MACHINES: dict[str, "DummyVirtualMachine"] = {} diff --git a/application/src/tira/grpc_client.py b/application/src/tira_app/grpc_client.py similarity index 99% rename from application/src/tira/grpc_client.py rename to application/src/tira_app/grpc_client.py index d4761314f..6a7449050 100644 --- a/application/src/tira/grpc_client.py +++ b/application/src/tira_app/grpc_client.py @@ -10,8 +10,7 @@ from django.conf import settings from google.protobuf.empty_pb2 import Empty -from tira.model import EvaluationLog, TransactionLog - +from .model import EvaluationLog, TransactionLog from .proto import tira_host_pb2, tira_host_pb2_grpc logger = logging.getLogger("tira") diff --git a/application/src/tira/huggingface_hub_integration.py b/application/src/tira_app/huggingface_hub_integration.py similarity index 99% rename from application/src/tira/huggingface_hub_integration.py rename to application/src/tira_app/huggingface_hub_integration.py index 4724d6eb7..4c92a1115 100644 --- a/application/src/tira/huggingface_hub_integration.py +++ b/application/src/tira_app/huggingface_hub_integration.py @@ -1,10 +1,9 @@ from typing import Iterable, Optional +import tira.io_utils as tira_cli_io_utils from huggingface_hub import HFCacheInfo, scan_cache_dir, snapshot_download from huggingface_hub.constants import HF_HOME -import tira.io_utils as tira_cli_io_utils - HF_CACHE: Optional[HFCacheInfo] = None diff --git a/application/src/tira/ir_datasets_loader.py b/application/src/tira_app/ir_datasets_loader.py similarity index 99% rename from application/src/tira/ir_datasets_loader.py rename to application/src/tira_app/ir_datasets_loader.py index 226f2fb48..067e2b479 100644 --- a/application/src/tira/ir_datasets_loader.py +++ b/application/src/tira_app/ir_datasets_loader.py @@ -12,8 +12,8 @@ def run_irds_command(task_id, dataset_id, image, command, output_dir, truth_command, truth_output_dir): - from tira.tira_model import model - from tira.util import run_cmd_as_documented_background_process + from .tira_model import model + from .util import run_cmd_as_documented_background_process irds_root = model.custom_irds_datasets_path / task_id / dataset_id command = command.replace("$outputDir", "/output-tira-tmp/") diff --git a/application/src/tira/management/commands/archive_runs_to_zenodo.py b/application/src/tira_app/management/commands/archive_runs_to_zenodo.py similarity index 99% rename from application/src/tira/management/commands/archive_runs_to_zenodo.py rename to application/src/tira_app/management/commands/archive_runs_to_zenodo.py index 910e30712..f02ef92f7 100644 --- a/application/src/tira/management/commands/archive_runs_to_zenodo.py +++ b/application/src/tira_app/management/commands/archive_runs_to_zenodo.py @@ -4,8 +4,8 @@ from django.core.management.base import BaseCommand from tqdm import tqdm -from tira.endpoints.data_api import model -from tira.views import zip_run, zip_runs +from ...endpoints.data_api import model +from ...views import zip_run, zip_runs def md5(filename): diff --git a/application/src/tira/management/commands/cache_daemon.py b/application/src/tira_app/management/commands/cache_daemon.py similarity index 97% rename from application/src/tira/management/commands/cache_daemon.py rename to application/src/tira_app/management/commands/cache_daemon.py index 21ff57155..c4f386bf2 100644 --- a/application/src/tira/management/commands/cache_daemon.py +++ b/application/src/tira_app/management/commands/cache_daemon.py @@ -6,11 +6,11 @@ from django.core.management import call_command from django.core.management.base import BaseCommand -import tira.tira_model as model +from ... import tira_model as model logger = logging.getLogger("cache_daemon") -from tira.git_runner import all_git_runners -from tira.tira_model import get_all_reranking_datasets, get_git_integration +from ...git_runner import all_git_runners +from ...tira_model import get_all_reranking_datasets, get_git_integration class Command(BaseCommand): diff --git a/application/src/tira/management/commands/dump_tira.py b/application/src/tira_app/management/commands/dump_tira.py similarity index 100% rename from application/src/tira/management/commands/dump_tira.py rename to application/src/tira_app/management/commands/dump_tira.py diff --git a/application/src/tira/management/commands/git_runner_cli.py b/application/src/tira_app/management/commands/git_runner_cli.py similarity index 98% rename from application/src/tira/management/commands/git_runner_cli.py rename to application/src/tira_app/management/commands/git_runner_cli.py index 551be04de..831afe867 100644 --- a/application/src/tira/management/commands/git_runner_cli.py +++ b/application/src/tira_app/management/commands/git_runner_cli.py @@ -6,13 +6,13 @@ from slugify import slugify from tqdm import tqdm -from tira.tira_model import ( +from ...tira_model import ( add_input_run_id_to_all_rerank_runs, create_re_rank_output_on_dataset, get_git_integration, load_refresh_timestamp_for_cache_key, ) -from tira.util import get_tira_id +from ...util import get_tira_id logger = logging.getLogger("tira") @@ -59,7 +59,7 @@ def run_command_stop_job_and_clean_up(self, options, git_runner): git_runner.stop_job_and_clean_up(options["stop_job_and_clean_up"], options["user_id"], options["run_id"]) def archive_repository_add_images_from_git_repo(self, options): - import tira.model as modeldb + from ... import model as modeldb with open(options["archive_repository_add_images_from_git_repo"], "r") as f: for line in tqdm(f): @@ -83,8 +83,8 @@ def archive_repository_add_images_from_git_repo(self, options): software.save() def archive_docker_software(self, approach, git_runner): - import tira.model as modeldb - from tira.util import docker_image_details + from ... import model as modeldb + from ...util import docker_image_details task_id, vm_id, name = approach.split("/") software = modeldb.DockerSoftware.objects.filter( diff --git a/application/src/tira/management/commands/grpc_mock_host.py b/application/src/tira_app/management/commands/grpc_mock_host.py similarity index 91% rename from application/src/tira/management/commands/grpc_mock_host.py rename to application/src/tira_app/management/commands/grpc_mock_host.py index 03f68216b..f25a61766 100644 --- a/application/src/tira/management/commands/grpc_mock_host.py +++ b/application/src/tira_app/management/commands/grpc_mock_host.py @@ -7,8 +7,8 @@ from django.conf import settings from django.core.management.base import BaseCommand -from tira.grpc.test_grpc_host_server import TiraHostService -from tira.proto import tira_host_pb2_grpc +from ...grpc.test_grpc_host_server import TiraHostService +from ...proto import tira_host_pb2_grpc grpc_host_port = settings.HOST_GRPC_PORT diff --git a/application/src/tira/management/commands/grpc_server.py b/application/src/tira_app/management/commands/grpc_server.py similarity index 91% rename from application/src/tira/management/commands/grpc_server.py rename to application/src/tira_app/management/commands/grpc_server.py index e7d7a524a..19f59ecdf 100644 --- a/application/src/tira/management/commands/grpc_server.py +++ b/application/src/tira_app/management/commands/grpc_server.py @@ -7,8 +7,8 @@ from django.conf import settings from django.core.management.base import BaseCommand -from tira.grpc.grpc_server import TiraApplicationService -from tira.proto import tira_host_pb2_grpc +from ...grpc.grpc_server import TiraApplicationService +from ...proto import tira_host_pb2_grpc grpc_port = settings.APPLICATION_GRPC_PORT listen_addr = f"[::]:{grpc_port}" diff --git a/application/src/tira/management/commands/index_model.py b/application/src/tira_app/management/commands/index_model.py similarity index 89% rename from application/src/tira/management/commands/index_model.py rename to application/src/tira_app/management/commands/index_model.py index 5df293c79..e4ddddd18 100644 --- a/application/src/tira/management/commands/index_model.py +++ b/application/src/tira_app/management/commands/index_model.py @@ -16,6 +16,6 @@ def handle(self, *args, **options): call_command("makemigrations") call_command("makemigrations", "tira") call_command("migrate") - from tira.data.HybridDatabase import HybridDatabase + from ...data.HybridDatabase import HybridDatabase HybridDatabase().create_model() diff --git a/application/src/tira/management/commands/ir_datasets_loader_cli.py b/application/src/tira_app/management/commands/ir_datasets_loader_cli.py similarity index 98% rename from application/src/tira/management/commands/ir_datasets_loader_cli.py rename to application/src/tira_app/management/commands/ir_datasets_loader_cli.py index 5287586d8..968904bab 100644 --- a/application/src/tira/management/commands/ir_datasets_loader_cli.py +++ b/application/src/tira_app/management/commands/ir_datasets_loader_cli.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand -from tira.ir_datasets_loader import IrDatasetsLoader +from ...ir_datasets_loader import IrDatasetsLoader logger = logging.getLogger("tira") diff --git a/application/src/tira/management/commands/irds_cli.sh b/application/src/tira_app/management/commands/irds_cli.sh similarity index 100% rename from application/src/tira/management/commands/irds_cli.sh rename to application/src/tira_app/management/commands/irds_cli.sh diff --git a/application/src/tira/management/commands/playground.py b/application/src/tira_app/management/commands/playground.py similarity index 93% rename from application/src/tira/management/commands/playground.py rename to application/src/tira_app/management/commands/playground.py index 10aa7aa51..9be832c3e 100644 --- a/application/src/tira/management/commands/playground.py +++ b/application/src/tira_app/management/commands/playground.py @@ -7,7 +7,7 @@ class Command(BaseCommand): """Runs some playground command.""" def handle(self, *args, **options): - from tira.git_runner import all_git_runners + from ...git_runner import all_git_runners g = all_git_runners() assert len(g) == 1 @@ -28,7 +28,7 @@ def handle(self, *args, **options): # context = {'user_id': 'mf2'} # print(add_registration(request, context, 'ir-lab-jena-leipzig-sose-2023', 'del-me-maik')) - # from tira.ir_datasets_loader import run_irds_command + # from ...ir_datasets_loader import run_irds_command # run_irds_command('tmp-test-maik', 'pssda', 'webis/tira-ir-datasets-starter:0.0.45-pangram', '/irds_cli.sh --skip_qrels true --ir_datasets_id pangrams --output_dataset_path $outputDir', '/tmp/sda-1/1/') # run_irds_command('tmp-test-maik', 'pssda', 'webis/tira-ir-datasets-starter:0.0.45-pangram', '/irds_cli.sh --skip_documents true --ir_datasets_id pangrams --output_dataset_truth_path $outputDir', '/tmp/sda-1/2/') diff --git a/application/src/tira/management/commands/run_develop.py b/application/src/tira_app/management/commands/run_develop.py similarity index 100% rename from application/src/tira/management/commands/run_develop.py rename to application/src/tira_app/management/commands/run_develop.py diff --git a/application/src/tira/management/commands/run_mockup.py b/application/src/tira_app/management/commands/run_mockup.py similarity index 91% rename from application/src/tira/management/commands/run_mockup.py rename to application/src/tira_app/management/commands/run_mockup.py index caa6a0de7..774bf499a 100644 --- a/application/src/tira/management/commands/run_mockup.py +++ b/application/src/tira_app/management/commands/run_mockup.py @@ -7,9 +7,9 @@ from django.core.management import call_command from django.core.management.base import BaseCommand -from tira.grpc.grpc_server import TiraApplicationService -from tira.grpc.test_grpc_host_server import TiraHostService -from tira.proto import tira_host_pb2_grpc +from ...grpc.grpc_server import TiraApplicationService +from ...grpc.test_grpc_host_server import TiraHostService +from ...proto import tira_host_pb2_grpc grpc_app_port = settings.APPLICATION_GRPC_PORT grpc_host_port = settings.HOST_GRPC_PORT diff --git a/application/src/tira/management/commands/run_to_evaluations.py b/application/src/tira_app/management/commands/run_to_evaluations.py similarity index 96% rename from application/src/tira/management/commands/run_to_evaluations.py rename to application/src/tira_app/management/commands/run_to_evaluations.py index 7cfe5ca78..fd64a0cca 100644 --- a/application/src/tira/management/commands/run_to_evaluations.py +++ b/application/src/tira_app/management/commands/run_to_evaluations.py @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand -import tira.tira_model as model +from ... import tira_model as model class Command(BaseCommand): diff --git a/application/src/tira/model.py b/application/src/tira_app/model.py similarity index 100% rename from application/src/tira/model.py rename to application/src/tira_app/model.py diff --git a/application/src/tira/permissions.py b/application/src/tira_app/permissions.py similarity index 100% rename from application/src/tira/permissions.py rename to application/src/tira_app/permissions.py diff --git a/application/src/tira/proto/TiraClientWebMessages_pb2.py b/application/src/tira_app/proto/TiraClientWebMessages_pb2.py similarity index 100% rename from application/src/tira/proto/TiraClientWebMessages_pb2.py rename to application/src/tira_app/proto/TiraClientWebMessages_pb2.py diff --git a/application/src/tira/proto/TiraClientWebMessages_pb2_grpc.py b/application/src/tira_app/proto/TiraClientWebMessages_pb2_grpc.py similarity index 100% rename from application/src/tira/proto/TiraClientWebMessages_pb2_grpc.py rename to application/src/tira_app/proto/TiraClientWebMessages_pb2_grpc.py diff --git a/application/src/tira/proto/TiraHostMessages_pb2.py b/application/src/tira_app/proto/TiraHostMessages_pb2.py similarity index 100% rename from application/src/tira/proto/TiraHostMessages_pb2.py rename to application/src/tira_app/proto/TiraHostMessages_pb2.py diff --git a/application/src/tira/proto/TiraHostMessages_pb2_grpc.py b/application/src/tira_app/proto/TiraHostMessages_pb2_grpc.py similarity index 100% rename from application/src/tira/proto/TiraHostMessages_pb2_grpc.py rename to application/src/tira_app/proto/TiraHostMessages_pb2_grpc.py diff --git a/application/src/tira/proto/__init__.py b/application/src/tira_app/proto/__init__.py similarity index 100% rename from application/src/tira/proto/__init__.py rename to application/src/tira_app/proto/__init__.py diff --git a/application/src/tira/proto/tira_host_pb2.py b/application/src/tira_app/proto/tira_host_pb2.py similarity index 100% rename from application/src/tira/proto/tira_host_pb2.py rename to application/src/tira_app/proto/tira_host_pb2.py diff --git a/application/src/tira/proto/tira_host_pb2_grpc.py b/application/src/tira_app/proto/tira_host_pb2_grpc.py similarity index 100% rename from application/src/tira/proto/tira_host_pb2_grpc.py rename to application/src/tira_app/proto/tira_host_pb2_grpc.py diff --git a/application/src/tira/proto/tira_messages_pb2.py b/application/src/tira_app/proto/tira_messages_pb2.py similarity index 100% rename from application/src/tira/proto/tira_messages_pb2.py rename to application/src/tira_app/proto/tira_messages_pb2.py diff --git a/application/src/tira/proto/tira_to_web_pb2.py b/application/src/tira_app/proto/tira_to_web_pb2.py similarity index 100% rename from application/src/tira/proto/tira_to_web_pb2.py rename to application/src/tira_app/proto/tira_to_web_pb2.py diff --git a/application/src/tira/proto/tira_to_web_pb2_grpc.py b/application/src/tira_app/proto/tira_to_web_pb2_grpc.py similarity index 100% rename from application/src/tira/proto/tira_to_web_pb2_grpc.py rename to application/src/tira_app/proto/tira_to_web_pb2_grpc.py diff --git a/application/src/tirex-components.yml b/application/src/tira_app/res/tirex-components.yml similarity index 100% rename from application/src/tirex-components.yml rename to application/src/tira_app/res/tirex-components.yml diff --git a/application/src/tira/static/tira/css/tira-style.css b/application/src/tira_app/static/tira/css/tira-style.css similarity index 100% rename from application/src/tira/static/tira/css/tira-style.css rename to application/src/tira_app/static/tira/css/tira-style.css diff --git a/application/src/tira/static/tira/img/background1.jpg b/application/src/tira_app/static/tira/img/background1.jpg similarity index 100% rename from application/src/tira/static/tira/img/background1.jpg rename to application/src/tira_app/static/tira/img/background1.jpg diff --git a/application/src/tira/static/tira/img/background2.jpg b/application/src/tira_app/static/tira/img/background2.jpg similarity index 100% rename from application/src/tira/static/tira/img/background2.jpg rename to application/src/tira_app/static/tira/img/background2.jpg diff --git a/application/src/tira/static/tira/img/google-icon.png b/application/src/tira_app/static/tira/img/google-icon.png similarity index 100% rename from application/src/tira/static/tira/img/google-icon.png rename to application/src/tira_app/static/tira/img/google-icon.png diff --git a/application/src/tira/static/tira/img/ia-icon.png b/application/src/tira_app/static/tira/img/ia-icon.png similarity index 100% rename from application/src/tira/static/tira/img/ia-icon.png rename to application/src/tira_app/static/tira/img/ia-icon.png diff --git a/application/src/tira/static/tira/img/image-licence.txt b/application/src/tira_app/static/tira/img/image-licence.txt similarity index 100% rename from application/src/tira/static/tira/img/image-licence.txt rename to application/src/tira_app/static/tira/img/image-licence.txt diff --git a/application/src/tira/static/tira/img/logo-tira-32x32.png b/application/src/tira_app/static/tira/img/logo-tira-32x32.png similarity index 100% rename from application/src/tira/static/tira/img/logo-tira-32x32.png rename to application/src/tira_app/static/tira/img/logo-tira-32x32.png diff --git a/application/src/tira/static/tira/img/logo-tira-40x40-transparent.png b/application/src/tira_app/static/tira/img/logo-tira-40x40-transparent.png similarity index 100% rename from application/src/tira/static/tira/img/logo-tira-40x40-transparent.png rename to application/src/tira_app/static/tira/img/logo-tira-40x40-transparent.png diff --git a/application/src/tira/static/tira/img/logo-tira-40x40.png b/application/src/tira_app/static/tira/img/logo-tira-40x40.png similarity index 100% rename from application/src/tira/static/tira/img/logo-tira-40x40.png rename to application/src/tira_app/static/tira/img/logo-tira-40x40.png diff --git a/application/src/tira/static/tira/img/zenodo-icon.png b/application/src/tira_app/static/tira/img/zenodo-icon.png similarity index 100% rename from application/src/tira/static/tira/img/zenodo-icon.png rename to application/src/tira_app/static/tira/img/zenodo-icon.png diff --git a/application/src/tira/static/tira/js/review.js b/application/src/tira_app/static/tira/js/review.js similarity index 100% rename from application/src/tira/static/tira/js/review.js rename to application/src/tira_app/static/tira/js/review.js diff --git a/application/src/tira/templates/tira/base.html b/application/src/tira_app/templates/tira/base.html similarity index 100% rename from application/src/tira/templates/tira/base.html rename to application/src/tira_app/templates/tira/base.html diff --git a/application/src/tira/templates/tira/git-repo-template/Dockerfile b/application/src/tira_app/templates/tira/git-repo-template/Dockerfile similarity index 100% rename from application/src/tira/templates/tira/git-repo-template/Dockerfile rename to application/src/tira_app/templates/tira/git-repo-template/Dockerfile diff --git a/application/src/tira/templates/tira/git-repo-template/README.md b/application/src/tira_app/templates/tira/git-repo-template/README.md similarity index 100% rename from application/src/tira/templates/tira/git-repo-template/README.md rename to application/src/tira_app/templates/tira/git-repo-template/README.md diff --git a/application/src/tira/templates/tira/git-repo-template/github-action.yml b/application/src/tira_app/templates/tira/git-repo-template/github-action.yml similarity index 100% rename from application/src/tira/templates/tira/git-repo-template/github-action.yml rename to application/src/tira_app/templates/tira/git-repo-template/github-action.yml diff --git a/application/src/tira/templates/tira/git-repo-template/script.py b/application/src/tira_app/templates/tira/git-repo-template/script.py similarity index 100% rename from application/src/tira/templates/tira/git-repo-template/script.py rename to application/src/tira_app/templates/tira/git-repo-template/script.py diff --git a/application/src/tira/templates/tira/git_task_repository_gitlab_ci.yml b/application/src/tira_app/templates/tira/git_task_repository_gitlab_ci.yml similarity index 100% rename from application/src/tira/templates/tira/git_task_repository_gitlab_ci.yml rename to application/src/tira_app/templates/tira/git_task_repository_gitlab_ci.yml diff --git a/application/src/tira/templates/tira/git_task_repository_readme.md b/application/src/tira_app/templates/tira/git_task_repository_readme.md similarity index 100% rename from application/src/tira/templates/tira/git_task_repository_readme.md rename to application/src/tira_app/templates/tira/git_task_repository_readme.md diff --git a/application/src/tira/templates/tira/git_user_repository_readme.md b/application/src/tira_app/templates/tira/git_user_repository_readme.md similarity index 100% rename from application/src/tira/templates/tira/git_user_repository_readme.md rename to application/src/tira_app/templates/tira/git_user_repository_readme.md diff --git a/application/src/tira/templates/tira/navbar.html b/application/src/tira_app/templates/tira/navbar.html similarity index 100% rename from application/src/tira/templates/tira/navbar.html rename to application/src/tira_app/templates/tira/navbar.html diff --git a/application/src/tira/templates/tira/tira_git_cmd.py b/application/src/tira_app/templates/tira/tira_git_cmd.py similarity index 100% rename from application/src/tira/templates/tira/tira_git_cmd.py rename to application/src/tira_app/templates/tira/tira_git_cmd.py diff --git a/application/src/tira/templates/tira/tira_git_cmd.sh b/application/src/tira_app/templates/tira/tira_git_cmd.sh similarity index 100% rename from application/src/tira/templates/tira/tira_git_cmd.sh rename to application/src/tira_app/templates/tira/tira_git_cmd.sh diff --git a/application/src/tira/templates/tira/tira_git_makefile b/application/src/tira_app/templates/tira/tira_git_makefile similarity index 100% rename from application/src/tira/templates/tira/tira_git_makefile rename to application/src/tira_app/templates/tira/tira_git_makefile diff --git a/application/src/tira/templates/tira/tira_git_tutorial.ipynb b/application/src/tira_app/templates/tira/tira_git_tutorial.ipynb similarity index 100% rename from application/src/tira/templates/tira/tira_git_tutorial.ipynb rename to application/src/tira_app/templates/tira/tira_git_tutorial.ipynb diff --git a/application/src/tira/tests/tests.py b/application/src/tira_app/tests/tests.py similarity index 100% rename from application/src/tira/tests/tests.py rename to application/src/tira_app/tests/tests.py diff --git a/application/src/tira/tira_data.py b/application/src/tira_app/tira_data.py similarity index 98% rename from application/src/tira/tira_data.py rename to application/src/tira_app/tira_data.py index eef5c2afc..e0bcf9b0d 100644 --- a/application/src/tira/tira_data.py +++ b/application/src/tira_app/tira_data.py @@ -3,7 +3,7 @@ from django.conf import settings -from tira.endpoints.stdout_beautifier import beautify_ansi_text +from .endpoints.stdout_beautifier import beautify_ansi_text logger = logging.getLogger("tira") diff --git a/application/src/tira/tira_model.py b/application/src/tira_app/tira_model.py similarity index 99% rename from application/src/tira/tira_model.py rename to application/src/tira_app/tira_model.py index 607f9aa32..bf96f9242 100644 --- a/application/src/tira/tira_model.py +++ b/application/src/tira_app/tira_model.py @@ -16,9 +16,9 @@ from django.db import connections, router from slugify import slugify -from tira.data.HybridDatabase import HybridDatabase -from tira.git_runner import get_git_runner, get_git_runner_for_software_integration -from tira.util import get_tira_id, register_run +from .data.HybridDatabase import HybridDatabase +from .git_runner import get_git_runner, get_git_runner_for_software_integration +from .util import get_tira_id, register_run logger = logging.getLogger("tira") @@ -662,7 +662,7 @@ def all_allowed_task_teams(task_id): def user_is_registered(task_id, request): - from tira.authentication import auth + from .authentication import auth allowed_task_teams = all_allowed_task_teams(task_id) user_vm_ids = [i.strip() for i in auth.get_vm_ids(request) if i.strip()] diff --git a/application/src/tira/urls.py b/application/src/tira_app/urls.py similarity index 100% rename from application/src/tira/urls.py rename to application/src/tira_app/urls.py diff --git a/application/src/tira/util.py b/application/src/tira_app/util.py similarity index 98% rename from application/src/tira/util.py rename to application/src/tira_app/util.py index ab767c6aa..c6e707cf2 100644 --- a/application/src/tira/util.py +++ b/application/src/tira_app/util.py @@ -5,8 +5,7 @@ from django.conf import settings -from tira import tira_model - +from . import tira_model from .proto import TiraClientWebMessages_pb2 as modelpb logger = logging.getLogger("tira") @@ -132,7 +131,7 @@ def __run_cmd_as_documented_background_process(cmds, process_id, descriptions, c from subprocess import STDOUT, Popen from time import sleep - import tira.model as modeldb + from . import model as modeldb with tempfile.NamedTemporaryFile() as file: file.write("".encode("utf8")) @@ -165,7 +164,7 @@ def run_cmd_as_documented_background_process(cmd, vm_id, task_id, title, descrip import json import threading - import tira.model as modeldb + from . import model as modeldb process_id = modeldb.BackendProcess.objects.create( vm_id=vm_id, task_id=task_id, cmd=json.dumps(cmd), title=title diff --git a/application/src/tira/views.py b/application/src/tira_app/views.py similarity index 99% rename from application/src/tira/views.py rename to application/src/tira_app/views.py index f6d4bf1ab..b200081a5 100644 --- a/application/src/tira/views.py +++ b/application/src/tira_app/views.py @@ -14,8 +14,7 @@ from django.template.loader import render_to_string from django.utils.safestring import mark_safe -import tira.tira_model as model - +from . import tira_model as model from .authentication import auth from .checks import check_conditional_permissions, check_permissions, check_resources_exist diff --git a/application/test/auth_tests/test_is_admin_for_task.py b/application/test/auth_tests/test_is_admin_for_task.py index 16a2056bf..88b5a9d35 100644 --- a/application/test/auth_tests/test_is_admin_for_task.py +++ b/application/test/auth_tests/test_is_admin_for_task.py @@ -1,7 +1,7 @@ from django.test import TestCase from utils_for_testing import mock_request, set_up_tira_environment -from tira.authentication import auth +from tira_app.authentication import auth submit_url_task_1 = "submit/task-of-organizer-1" overview_url_task_1 = "task-overview/task-of-organizer-1" diff --git a/application/test/evaluation_api_integration_tests/test_evaluations_for_vm.py b/application/test/evaluation_api_integration_tests/test_evaluations_for_vm.py index 00339d85c..59763fe92 100644 --- a/application/test/evaluation_api_integration_tests/test_evaluations_for_vm.py +++ b/application/test/evaluation_api_integration_tests/test_evaluations_for_vm.py @@ -4,8 +4,8 @@ from django.test import TestCase from utils_for_testing import method_for_url_pattern, mock_request, now, set_up_tira_filesystem -import tira.model as modeldb -from tira.tira_model import model as tira_model +import tira_app.model as modeldb +from tira_app.tira_model import model as tira_model url = "api/evaluations-of-vm//" evaluations_function = method_for_url_pattern(url) diff --git a/application/test/git_runner_integration_tests/test_rendering_of_job_file.py b/application/test/git_runner_integration_tests/test_rendering_of_job_file.py index f1a9436c1..715d7ceec 100644 --- a/application/test/git_runner_integration_tests/test_rendering_of_job_file.py +++ b/application/test/git_runner_integration_tests/test_rendering_of_job_file.py @@ -2,7 +2,7 @@ from django.test import TestCase -from tira.git_runner_integration import GitRunner +from tira_app.git_runner_integration import GitRunner class TestRenderingOfJobFile(TestCase): diff --git a/application/test/hf_mount_model_tests/test_hf_mounts_are_parsed.py b/application/test/hf_mount_model_tests/test_hf_mounts_are_parsed.py index cf0640e8a..c4b4bc18b 100644 --- a/application/test/hf_mount_model_tests/test_hf_mounts_are_parsed.py +++ b/application/test/hf_mount_model_tests/test_hf_mounts_are_parsed.py @@ -2,7 +2,7 @@ from huggingface_hub import snapshot_download -from tira.huggingface_hub_integration import _hf_repos, huggingface_model_mounts +from tira_app.huggingface_hub_integration import _hf_repos, huggingface_model_mounts class TestHfMountsAreParsed(unittest.TestCase): diff --git a/application/test/multi_step_job_tests/test_multi_step_jobs_with_alternative_input.py b/application/test/multi_step_job_tests/test_multi_step_jobs_with_alternative_input.py index 6e1629cb3..2c4fd9030 100644 --- a/application/test/multi_step_job_tests/test_multi_step_jobs_with_alternative_input.py +++ b/application/test/multi_step_job_tests/test_multi_step_jobs_with_alternative_input.py @@ -3,8 +3,8 @@ from django.test import TestCase from utils_for_testing import set_up_tira_environment -import tira.model as modeldb -from tira.tira_model import latest_output_of_software_on_dataset +import tira_app.model as modeldb +from tira_app.tira_model import latest_output_of_software_on_dataset # Used for some tests now = datetime.now().strftime("%Y%m%d") diff --git a/application/test/multi_step_job_tests/test_multi_step_jobs_with_multiple_inputs.py b/application/test/multi_step_job_tests/test_multi_step_jobs_with_multiple_inputs.py index 7ff25764a..43dc655ed 100644 --- a/application/test/multi_step_job_tests/test_multi_step_jobs_with_multiple_inputs.py +++ b/application/test/multi_step_job_tests/test_multi_step_jobs_with_multiple_inputs.py @@ -1,7 +1,7 @@ from django.test import TestCase from utils_for_testing import set_up_tira_environment, software_non_public, software_public, software_with_inputs -from tira.tira_model import model +from tira_app.tira_model import model PARTICIPANT = "PARTICIPANT-FOR-TEST-1" TASK = "shared-task-1" diff --git a/application/test/organizer_api_integration_tests/test_creation_of_dataset_id.py b/application/test/organizer_api_integration_tests/test_creation_of_dataset_id.py index 9e240a9f0..e13b151da 100644 --- a/application/test/organizer_api_integration_tests/test_creation_of_dataset_id.py +++ b/application/test/organizer_api_integration_tests/test_creation_of_dataset_id.py @@ -3,7 +3,7 @@ from django.test import TestCase from utils_for_testing import set_up_tira_environment -from tira.tira_model import model +from tira_app.tira_model import model now = datetime.now().strftime("%Y%m%d") diff --git a/application/test/settings_test.py b/application/test/settings_test.py index 1a880fecc..8eb860615 100644 --- a/application/test/settings_test.py +++ b/application/test/settings_test.py @@ -10,6 +10,7 @@ https://docs.djangoproject.com/en/3.1/ref/settings/ """ +import importlib.resources as resources import logging import os from pathlib import Path @@ -61,7 +62,7 @@ # Application definition INSTALLED_APPS = [ - "tira.apps.TiraConfig", + "tira_app.apps.TiraConfig", "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", @@ -83,7 +84,7 @@ ] REST_FRAMEWORK = { - "DEFAULT_AUTHENTICATION_CLASSES": ("tira.authentication.TrustedHeaderAuthentication",), + "DEFAULT_AUTHENTICATION_CLASSES": ("tira_app.authentication.TrustedHeaderAuthentication",), "DEFAULT_FILTER_BACKENDS": ("rest_framework_json_api.django_filters.DjangoFilterBackend",), } @@ -304,7 +305,9 @@ def logger_config(log_dir: Path): USE_TZ = True -TIREX_COMPONENTS = yaml.load(open(BASE_DIR / "src" / "tirex-components.yml").read(), Loader=yaml.FullLoader) +TIREX_COMPONENTS = yaml.load( + (resources.files("tira_app.res") / "tirex-components.yml").read_bytes(), Loader=yaml.FullLoader +) GIT_CI_AVAILABLE_RESOURCES = { "small-resources": { diff --git a/application/test/utils_for_testing.py b/application/test/utils_for_testing.py index 6ed4eb4f8..c3beaff82 100644 --- a/application/test/utils_for_testing.py +++ b/application/test/utils_for_testing.py @@ -16,9 +16,9 @@ from django.urls import URLPattern, URLResolver from rest_framework.test import APIRequestFactory -import tira.model as modeldb -from tira.authentication import TrustedHeaderAuthentication -from tira.tira_model import model as tira_model +import tira_app.model as modeldb +from tira_app.authentication import TrustedHeaderAuthentication +from tira_app.tira_model import model as tira_model auth_backend = TrustedHeaderAuthentication() # There must be a way to get this from rest_framework right? diff --git a/pipelines/src/django_tira_git/settings.py b/pipelines/src/django_tira_git/settings.py index 1989fb2db..63a27b07b 100644 --- a/pipelines/src/django_tira_git/settings.py +++ b/pipelines/src/django_tira_git/settings.py @@ -66,7 +66,7 @@ # Application definition INSTALLED_APPS = [ - "tira.apps.TiraConfig", + "tira_app.apps.TiraConfig", "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes",