renamed application module to tira_app and updated production container

.github/workflows/publish.yml

@@ -50,8 +50,8 @@ jobs:
       - name: Publish tira-backend image
         uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
         with:
-          context: ${{ github.workspace }}/application
+          context: ${{ github.workspace }}/
           file: ${{ github.workspace }}/application/Dockerfile.prod
           push: true
           tags: ${{ steps.meta-backend.outputs.tags }}
-          labels: ${{ steps.meta-backend.outputs.labels }}
+          labels: ${{ steps.meta-backend.outputs.labels }}

application/.gitignore

@@ -3,11 +3,10 @@ src/tira_vm_states.sqlite3
 src/venv
 src/__init__.py
 src/config
-src/tira/migrations/*.py
+src/tira_app/migrations/*.py
 test/test-database
 
-src/tira/frontend/node_modules/
-src/tira/static/tira/dist/*
+src/tira_app/static/tira/dist/*
 package-lock.json
 test/tira-root/
 **/*.received.txt

application/Dockerfile.prod

@@ -5,13 +5,14 @@
 ########################################################################################################################
 # Production Container                                                                                                 #
 ########################################################################################################################
-FROM debian:stable-slim
+FROM debian:stable-slim AS build
 
 ENV TZ=Europe/Berlin
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-
-RUN apt-get update && apt-get install -y locales
-RUN echo "en_US.UTF-8 UTF-8" | tee -a /etc/locale.gen && locale-gen
+RUN <<EOF
+ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+apt-get update && apt-get install -y locales
+echo "en_US.UTF-8 UTF-8" | tee -a /etc/locale.gen && locale-gen
+EOF
 
 ########################################################################################################################
 # Create User                                                                                                          #
@@ -24,16 +25,17 @@ RUN useradd -ms /bin/bash tira
 ########################################################################################################################
 USER tira
 WORKDIR /tira/
-COPY --chown=tira:tira . ./
+COPY --chown=tira:tira ./application ./
+COPY --chown=tira:tira ./python-client ../python-client
 
 
 ########################################################################################################################
 # Install Python and Dependencies                                                                                      #
 ########################################################################################################################
 USER root
-ENV PIP_BREAK_SYSTEM_PACKAGES 1
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
 # For faster build of GRPCIO
-ENV GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS 16
+ENV GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=16
 
 RUN <<EOF
 apt-get update
@@ -42,27 +44,66 @@ apt-get install -y python3 python3-pip git pkg-config
 # Dependencies (stuff needed by at least one python package)
 # For MariaDB
 apt-get install -y default-libmysqlclient-dev
+EOF
 
-python3 -m pip install -r requirements.txt
+USER tira
+RUN <<EOF
+# Install dependencies
+pip3 install --no-cache-dir --user ../python-client .[test]
 EOF
 
 USER root
-RUN cp ./src/tira/management/commands/irds_cli.sh /irds_cli.sh
+RUN cp ./src/tira_app/management/commands/irds_cli.sh /irds_cli.sh
 
 USER tira
 RUN <<EOF
+export PATH="/home/tira/.local/bin:$PATH"
 python3 manage.py collectstatic
-chmod +x ./src/tira/endpoints/aha
+chmod +x ./src/tira_app/endpoints/aha
 
 # Run tests
-./test/run_all_tests.sh
+cd test
+pytest
 EOF
 
+########################################################################################################################
+# Production Container                                                                                                 #
+########################################################################################################################
+FROM debian:stable-slim
+
+
+########################################################################################################################
+# Create User                                                                                                          #
+########################################################################################################################
+RUN useradd -ms /bin/bash tira
+
+
+COPY --from=build --chown=tira:tira /home/tira/.local /home/tira/.local
+COPY --from=build /irds_cli.sh /irds_cli.sh
+
+RUN <<EOF
+ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+apt-get update && apt-get install -y locales
+echo "en_US.UTF-8 UTF-8" | tee -a /etc/locale.gen && locale-gen
+
+apt-get update
+# Tools
+apt-get install -y python3 python3-pip
+export PIP_BREAK_SYSTEM_PACKAGES=1; pip3 install uwsgi
+#
+mkdir -p /tira/application/src
+chown -R tira:tira /tira
+EOF
 
 ########################################################################################################################
 # Final Configuration Stuff                                                                                            #
 ########################################################################################################################
 USER tira
-ENV HF_HOME=/mnt/ceph/tira/data/publicly-shared-datasets/huggingface/
+# CONFIGURE THIS ENVIRONMENT VARIABLE IN YOUR DOCKER-COMPOSE FILE
+ENV HF_HOME=/home/tira/data/publicly-shared-datasets/huggingface/
+ENV PATH=/home/tira/.local/bin:$PATH
+
+EXPOSE 80
 
-CMD uwsgi --uid 1010 --gid 1010 --ini /tira/src/uwsgi.ini
+# TODO: at some point it probably makes sense, not to use /tira/application/src as a working directory anymore
+CMD ["uwsgi", "--strict", "--master", "--enable-threads", "--module", "django_admin.wsgi:application", "--chdir", "/tira/application/src", "--processes", "50", "--http-socket", ":80", "--vacuum", "--max-requests", "5000"]

application/setup.cfg

@@ -1,8 +1,12 @@
+[metadata]
+name = tira_app
 
 [options]
 python_requires = >=3.9
 include_package_data = True
-packages = find:
+package_dir =
+    = src
+packages = find_namespace:
 install_requires =
     grpcio>=1.53.2
     # grpcio-tools==1.36.1  # still needed?
@@ -23,7 +27,7 @@ install_requires =
     PyGithub==1.59.1
     django-extensions
     discourse-client-in-disraptor==0.0.8
-    tira>=0.0.97
+    # tira>=0.0.97
     huggingface-hub
     djangorestframework==3.15.1
     django-filter==24.2
@@ -44,6 +48,15 @@ dev =
     isort
     mypy
 
+[options.packages.find]
+where = src
+include =
+    tira_app
+    tira_app.*
+    django_admin
+
+[options.package_data]
+tira_app.res = *.yml
 
 [flake8]
 max-line-length = 120

application/setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()

application/src/django_admin/settings.py

@@ -10,6 +10,7 @@
 https://docs.djangoproject.com/en/3.1/ref/settings/
 """
 
+import importlib.resources as resources
 import logging
 import os
 from pathlib import Path
@@ -69,7 +70,7 @@
 # Application definition
 
 INSTALLED_APPS = [
-    "tira.apps.TiraConfig",
+    "tira_app.apps.TiraConfig",
     "django.contrib.auth",
     "django.contrib.contenttypes",
     "django.contrib.sessions",
@@ -90,7 +91,7 @@
 ]
 
 REST_FRAMEWORK = {
-    "DEFAULT_AUTHENTICATION_CLASSES": ("tira.authentication.TrustedHeaderAuthentication",),
+    "DEFAULT_AUTHENTICATION_CLASSES": ("tira_app.authentication.TrustedHeaderAuthentication",),
     "DEFAULT_FILTER_BACKENDS": ("rest_framework_json_api.django_filters.DjangoFilterBackend",),
 }
 
@@ -335,8 +336,9 @@ def logger_config(log_dir: Path):
     }
 }
 
-# FIXME: I don't close my file handle :((((((((
-TIREX_COMPONENTS = yaml.load(open(BASE_DIR / "tirex-components.yml").read(), Loader=yaml.FullLoader)
+TIREX_COMPONENTS = yaml.load(
+    (resources.files("tira_app.res") / "tirex-components.yml").read_bytes(), Loader=yaml.FullLoader
+)
 
 # Logging
 ld = Path(custom_settings.get("logging_dir", TIRA_ROOT / "log" / "tira-application"))

application/src/django_admin/urls.py

@@ -17,5 +17,5 @@
 from django.urls import include, path
 
 urlpatterns = [
-    path("", include("tira.urls")),
+    path("", include("tira_app.urls")),
 ]

application/src/run_grpc_server.py

@@ -2,7 +2,7 @@
 
 import django
 
-from tira.grpc import grpc_server
+from tira_app.grpc import grpc_server
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_admin.settings")
 

application/src/tira/admin.py → application/src/tira_app/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 
-import tira.model as modeldb
+from . import model as modeldb
 
 # Register your models here.
 

application/src/tira/apps.py → application/src/tira_app/apps.py

@@ -2,4 +2,5 @@
 
 
 class TiraConfig(AppConfig):
-    name = "tira"
+    name = "tira_app"
+    label = "tira"

application/src/tira/authentication.py → application/src/tira_app/authentication.py

@@ -8,7 +8,7 @@
 from django.http import HttpRequest, HttpResponseNotAllowed
 from slugify import slugify
 
-import tira.tira_model as model
+from . import tira_model as model
 
 logger = logging.getLogger(__name__)
 

application/src/tira/checks.py → application/src/tira_app/checks.py

@@ -6,8 +6,7 @@
 from django.shortcuts import redirect
 from django.urls import resolve
 
-import tira.tira_model as model
-
+from . import tira_model as model
 from .authentication import auth
 
 logger = logging.getLogger("tira")

application/src/tira/data/FileDatabase.py → application/src/tira_app/data/FileDatabase.py

@@ -10,8 +10,8 @@
 from django.conf import settings
 from google.protobuf.text_format import Parse
 
-from tira.proto import TiraClientWebMessages_pb2 as modelpb
-from tira.util import TiraModelWriteError, auto_reviewer, extract_year_from_dataset_id
+from ..proto import TiraClientWebMessages_pb2 as modelpb
+from ..util import TiraModelWriteError, auto_reviewer, extract_year_from_dataset_id
 
 logger = logging.getLogger("tira")
 

application/src/tira/data/HybridDatabase.py → application/src/tira_app/data/HybridDatabase.py

@@ -12,10 +12,9 @@
 from django.db import IntegrityError
 from google.protobuf.text_format import Parse
 
-import tira.data.data as dbops
-import tira.model as modeldb
-from tira.proto import TiraClientWebMessages_pb2 as modelpb
-from tira.util import (
+from .. import model as modeldb
+from ..proto import TiraClientWebMessages_pb2 as modelpb
+from ..util import (
     TiraModelIntegrityError,
     TiraModelWriteError,
     auto_reviewer,
@@ -24,6 +23,7 @@
     link_to_discourse_team,
     now,
 )
+from . import data as dbops
 
 logger = logging.getLogger("tira_db")
 

application/src/tira/data/data.py → application/src/tira_app/data/data.py

@@ -3,14 +3,13 @@
 """
 
 import logging
-from pathlib import Path
 
 from google.protobuf.text_format import Parse
 from tqdm import tqdm
 
-import tira.model as modeldb
-from tira.proto import TiraClientWebMessages_pb2 as modelpb
-from tira.util import auto_reviewer, extract_year_from_dataset_id
+from .. import model as modeldb
+from ..proto import TiraClientWebMessages_pb2 as modelpb
+from ..util import auto_reviewer, extract_year_from_dataset_id
 
 logger = logging.getLogger("tira")
 

application/src/tira/endpoints/admin_api.py → application/src/tira_app/endpoints/admin_api.py

@@ -11,11 +11,11 @@
 from django.core.serializers.json import DjangoJSONEncoder
 from django.http import JsonResponse
 
-import tira.tira_model as model
-from tira.authentication import auth
-from tira.checks import check_conditional_permissions, check_permissions, check_resources_exist
-from tira.git_runner import check_that_git_integration_is_valid
-from tira.ir_datasets_loader import run_irds_command
+from .. import tira_model as model
+from ..authentication import auth
+from ..checks import check_conditional_permissions, check_permissions, check_resources_exist
+from ..git_runner import check_that_git_integration_is_valid
+from ..ir_datasets_loader import run_irds_command
 
 logger = logging.getLogger("tira")
 logger.info("ajax_routes: Logger active")

application/src/tira/endpoints/data_api.py → application/src/tira_app/endpoints/data_api.py

@@ -14,12 +14,12 @@
 from django.http import HttpResponse, JsonResponse
 from slugify import slugify
 
-import tira.tira_model as model
-from tira.authentication import auth
-from tira.checks import check_permissions, check_resources_exist
-from tira.tira_data import get_run_file_list, get_run_runtime, get_stderr, get_stdout, get_tira_log
-from tira.util import link_to_discourse_team
-from tira.views import _add_user_vms_to_context, add_context
+from .. import tira_model as model
+from ..authentication import auth
+from ..checks import check_permissions, check_resources_exist
+from ..tira_data import get_run_file_list, get_run_runtime, get_stderr, get_stdout, get_tira_log
+from ..util import link_to_discourse_team
+from ..views import _add_user_vms_to_context, add_context
 
 include_navigation = False
 

application/src/tira/endpoints/diffir_api.py → application/src/tira_app/endpoints/diffir_api.py

@@ -6,9 +6,9 @@
 from django.conf import settings
 from django.http import HttpResponse, JsonResponse
 
-import tira.tira_model as model
-from tira.checks import check_permissions
-from tira.views import add_context
+from .. import tira_model as model
+from ..checks import check_permissions
+from ..views import add_context
 
 logger = logging.getLogger("tira")
 

application/src/tira/endpoints/organizer_api.py → application/src/tira_app/endpoints/organizer_api.py

@@ -2,8 +2,8 @@
 
 from django.http import JsonResponse
 
-import tira.tira_model as model
-from tira.checks import check_conditional_permissions, check_permissions, check_resources_exist
+from .. import tira_model as model
+from ..checks import check_conditional_permissions, check_permissions, check_resources_exist
 
 include_navigation = False
 

application/src/tira/endpoints/serp_api.py → application/src/tira_app/endpoints/serp_api.py

@@ -4,9 +4,9 @@
 from django.conf import settings
 from django.http import HttpResponse, JsonResponse
 
-from tira.checks import check_permissions
-from tira.endpoints.diffir_api import doc_file_for_run
-from tira.views import add_context
+from ..checks import check_permissions
+from ..endpoints.diffir_api import doc_file_for_run
+from ..views import add_context
 
 logger = logging.getLogger("tira")