diff --git a/application/.dockerignore b/application/.dockerignore index 048f0304..9fb602b6 100644 --- a/application/.dockerignore +++ b/application/.dockerignore @@ -1,7 +1,304 @@ -venv/ +config +!config/*.docker.yml +mock-data + +.dockerignore .gitignore Dockerfile.* Makefile README.md -mock-data -config/*.dev.yml \ No newline at end of file + +test/tira-root + + +# Gitignore contentn +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +#build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. + .idea/artifacts + .idea/compiler.xml + .idea/jarRepositories.xml + .idea/modules.xml + .idea/*.iml + .idea/modules + *.iml + *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser +tira.iml + +### Eclipse ### +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + +### Eclipse Patch ### +# Spring Boot Tooling +.sts4-cache/ + +# End of https://www.toptal.com/developers/gitignore/api/eclipse + +/bin/ +conf/disraptor.properties +*.swp +/info/ +tira-web/tira-web/lib/ + +# Mac +.DS_Store + +# TextMate +frontend/.editorconfig + diff --git a/application/Dockerfile.prod b/application/Dockerfile.prod index 6ad3b47c..097de373 100644 --- a/application/Dockerfile.prod +++ b/application/Dockerfile.prod @@ -2,8 +2,10 @@ # At some point, Dockerfile.application* and Dockerfile.basis should be obsolete and can be deleted. If you read this in # the future and that is the case, then "Hello Future Person" and also please delete these dockerfiles. +# !!! This Dockerfile needs to be build from the project root and NOT the application folder !!! + ######################################################################################################################## -# Production Container # +# Build Container # ######################################################################################################################## FROM debian:stable-slim AS build @@ -19,7 +21,6 @@ EOF ######################################################################################################################## RUN useradd -ms /bin/bash tira - ######################################################################################################################## # Copy all neccessary files over # ######################################################################################################################## @@ -28,13 +29,12 @@ WORKDIR /tira/ COPY --chown=tira:tira ./application ./ COPY --chown=tira:tira ./python-client ../python-client - ######################################################################################################################## # Install Python and Dependencies # ######################################################################################################################## USER root ENV PIP_BREAK_SYSTEM_PACKAGES=1 -# For faster build of GRPCIO +# For faster build of GRPCIO (TODO: remove when GRPC is not used anymore) ENV GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=16 RUN < /etc/timezone apt-get update && apt-get install -y locales echo "en_US.UTF-8 UTF-8" | tee -a /etc/locale.gen && locale-gen -apt-get update # Tools -apt-get install -y python3 python3-pip -export PIP_BREAK_SYSTEM_PACKAGES=1; pip3 install uwsgi +apt-get install -y python3 # mkdir -p /tira/application/src chown -R tira:tira /tira @@ -100,9 +95,10 @@ EOF # Final Configuration Stuff # ######################################################################################################################## USER tira -# CONFIGURE THIS ENVIRONMENT VARIABLE IN YOUR DOCKER-COMPOSE FILE -ENV HF_HOME=/home/tira/data/publicly-shared-datasets/huggingface/ ENV PATH=/home/tira/.local/bin:$PATH +# CONFIGURE THE FOLLOWING ENVIRONMENT VARIABLES IN YOUR DOCKER-COMPOSE FILE +ENV HF_HOME=/home/tira/data/publicly-shared-datasets/huggingface/ +ENV TIRA_CONFIG=/tira/config/tira-application-config.docker.yml EXPOSE 80 diff --git a/application/config/tira-application-config.dev.yml b/application/config/tira-application-config.dev.yml index b9586363..bb777536 100644 --- a/application/config/tira-application-config.dev.yml +++ b/application/config/tira-application-config.dev.yml @@ -2,9 +2,9 @@ debug: true allowed_hosts: - "127.0.0.1" -django_secret: 'not-so-secret' +django_secret: "not-so-secret" # --- -# tira_root: /mnt/ceph/tira +tira_root: !ENV ${TIRA_ROOT:/tira} # deployment = {disraptor} # TODO: remove deployment configuration key deployment: disraptor @@ -17,10 +17,10 @@ grpc_host: local host_grpc_port: 50051 application_grpc_port: 50052 database: - engine: django.db.backends.sqlite3 # django.db.backends.mysql or django.db.backends.sqlite3 - name: tira # when backend is sqlite, this will be the name of the database below TIRA_ROOT/state - user: tira # ignored when using sqlite3 - password: TODO-ENTER-PASSWORD # ignored when using sqlite3 - host: tira-mariadb # ignored when using sqlite3 - port: 3306 # ignored when using sqlite3 + engine: django.db.backends.sqlite3 # django.db.backends.mysql or django.db.backends.sqlite3 + name: tira # when backend is sqlite, this will be the name of the database below TIRA_ROOT/state + user: tira # ignored when using sqlite3 + password: TODO-ENTER-PASSWORD # ignored when using sqlite3 + host: tira-mariadb # ignored when using sqlite3 + port: 3306 # ignored when using sqlite3 github_token: TODO-ENTER-TOKEN diff --git a/application/config/tira-application-config.docker.yml b/application/config/tira-application-config.docker.yml index 9f13b3cc..14e16c88 100644 --- a/application/config/tira-application-config.docker.yml +++ b/application/config/tira-application-config.docker.yml @@ -2,9 +2,9 @@ debug: true allowed_hosts: - "127.0.0.1" -django_secret: 'not-so-secret' +django_secret: "not-so-secret" # --- -# tira_root: /mnt/ceph/tira +tira_root: !ENV ${TIRA_ROOT:/tira} # deployment = {disraptor} deployment: disraptor # disraptor_secret_file: /etc/discourse/client-api-key @@ -16,10 +16,10 @@ grpc_host: local host_grpc_port: 50051 application_grpc_port: 50052 database: - engine: django.db.backends.sqlite3 # django.db.backends.mysql or django.db.backends.sqlite3 - name: tira # when backend is sqlite, this will be the name of the database below TIRA_ROOT/state - user: tira # ignored when using sqlite3 - password: TODO-ENTER-PASSWORD # ignored when using sqlite3 - host: tira-mariadb # ignored when using sqlite3 - port: 3306 # ignored when using sqlite3 + engine: django.db.backends.sqlite3 # django.db.backends.mysql or django.db.backends.sqlite3 + name: tira # when backend is sqlite, this will be the name of the database below TIRA_ROOT/state + user: tira # ignored when using sqlite3 + password: TODO-ENTER-PASSWORD # ignored when using sqlite3 + host: tira-mariadb # ignored when using sqlite3 + port: 3306 # ignored when using sqlite3 diff --git a/application/pyproject.toml b/application/pyproject.toml index 30388f6b..fd76035b 100644 --- a/application/pyproject.toml +++ b/application/pyproject.toml @@ -31,4 +31,5 @@ pythonpath = ["./src", "./test"] python_files = "test_*.py" [tool.pytest_env] -HF_HOME = "./tira-root/huggingface" \ No newline at end of file +HF_HOME = "./tira-root/huggingface" +TIRA_ROOT = "./tira-root" \ No newline at end of file diff --git a/application/setup.cfg b/application/setup.cfg index d60832e9..eb8b835c 100644 --- a/application/setup.cfg +++ b/application/setup.cfg @@ -32,6 +32,7 @@ install_requires = djangorestframework==3.15.1 django-filter==24.2 djangorestframework-jsonapi==7.0.0 + pyaml-env==1.2.1 [options.extras_require] test = @@ -47,6 +48,8 @@ dev = flake8 isort mypy +deploy = + uwsgi [options.packages.find] where = src diff --git a/application/src/django_admin/settings.py b/application/src/django_admin/settings.py index 0151b1fd..108de29a 100644 --- a/application/src/django_admin/settings.py +++ b/application/src/django_admin/settings.py @@ -15,15 +15,19 @@ import os from pathlib import Path -import yaml +from pyaml_env import parse_config # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent custom_settings = {} -for cfg in (BASE_DIR / "config").glob("*.yml"): - print(f"Load settings from {cfg}.") - custom_settings.update(yaml.load(open(cfg, "r").read(), Loader=yaml.FullLoader)) +# for cfg in (BASE_DIR / "config").glob("*.yml"): +# print(f"Load settings from {cfg}.") +# custom_settings.update(yaml.load(open(cfg, "r").read(), Loader=yaml.FullLoader)) +cfgpath = os.environ.get("TIRA_CONFIG", str(BASE_DIR / "config" / "tira-application-config.dev.yml")) +logging.info(f"Load settings from {cfgpath}.") +config = parse_config(cfgpath, default_value=None) +custom_settings.update(config) if "database" not in custom_settings: custom_settings["database"] = {} diff --git a/application/test/settings_test.py b/application/test/settings_test.py index 8eb86061..6084e6dc 100644 --- a/application/test/settings_test.py +++ b/application/test/settings_test.py @@ -16,13 +16,19 @@ from pathlib import Path import yaml +from pyaml_env import parse_config # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent custom_settings = {} -for cfg in (BASE_DIR / "config").glob("*.yml"): - custom_settings.update(yaml.load(open(cfg, "r").read(), Loader=yaml.FullLoader)) +# for cfg in (BASE_DIR / "config").glob("*.yml"): +# print(f"Load settings from {cfg}.") +# custom_settings.update(yaml.load(open(cfg, "r").read(), Loader=yaml.FullLoader)) +cfgpath = os.environ.get("TIRA_CONFIG", str(BASE_DIR / "config" / "tira-application-config.dev.yml")) +logging.info(f"Load settings from {cfgpath}.") +config = parse_config(cfgpath, default_value=None) +custom_settings.update(config) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/