diff --git a/application/Dockerfile.prod b/application/Dockerfile.prod index d72a12bc..d661ac22 100644 --- a/application/Dockerfile.prod +++ b/application/Dockerfile.prod @@ -94,10 +94,6 @@ apt-get -qq install -y mariadb-client # mkdir -p /tira/application/src chown -R tira:tira /tira - -# Allow binding port 80 -apt-get -qq install libcap2-bin -setcap CAP_NET_BIND_SERVICE=+eip /home/tira/.local/bin/uwsgi EOF @@ -124,7 +120,7 @@ ENV HF_HOME=/home/tira/data/publicly-shared-datasets/huggingface/ ENV TIRA_CONFIG=/tira/config/tira-application-config.yml ENV TIRA_DEBUG=false -EXPOSE 80 +EXPOSE 8080 # TODO: at some point it probably makes sense, not to use /tira/application/src as a working directory anymore CMD ["uwsgi", "--strict", "--master", "--enable-threads", "--module", "django_admin.wsgi:application", "--chdir", "/tira/application/src", "--processes", "50", "--http-socket", ":80", "--vacuum", "--max-requests", "5000"] diff --git a/application/config/tira-application-config.yml b/application/config/tira-application-config.yml index 742f631a..bdb93a62 100644 --- a/application/config/tira-application-config.yml +++ b/application/config/tira-application-config.yml @@ -1,24 +1,61 @@ -# ---task/celebrity-profiling/user/hodge20a +# For your convenience, we marked secrets with [SECRET]. Make sure, you change these +# values from their defaults! + +########################################################################################## +# TIRA # +########################################################################################## +# Enables debug mode. This means more verbose output in the console and for the REST-API. +# Settings this value to true in production is a security risk! debug: !ENV ${TIRA_DEBUG:false} -allowed_hosts: - - "*" -django_secret: !ENV ${DJANGO_SECRET:change-me!} -# --- + tira_root: !ENV ${TIRA_ROOT:/tira} # The directory where logs are written to. Defaults to TIRA_ROOT/log/tira-application # logging_dir: /mnt/ceph/tira/log/tira-application -# grpc_host can be local or remote. If local, it will call localhost (i.e. for testing). If remote, it will call the vm-host -# When developing, set this option to local, otherwise you might accidentally remote-control the live-vms. -grpc_host: local -host_grpc_port: 50051 -application_grpc_port: 50052 + +# [SECRET] +github_token: !ENV ${TIRA_GITHUB_TOKEN} + +########################################################################################## +# Database # +########################################################################################## database: - engine: !ENV ${TIRA_DB_ENGINE:django.db.backends.sqlite3} # django.db.backends.mysql or django.db.backends.sqlite3 - name: !ENV ${TIRA_DB_NAME:tira} # when backend is sqlite, this will be the name of the database below TIRA_ROOT/state + # django.db.backends.mysql or django.db.backends.sqlite3 + engine: !ENV ${TIRA_DB_ENGINE:django.db.backends.sqlite3} + # when backend is sqlite, this will be the name of the database below TIRA_ROOT/state + name: !ENV ${TIRA_DB_NAME:tira} user: !ENV ${TIRA_DB_USER:tira} # ignored when using sqlite3 password: !ENV ${TIRA_DB_PASSWORD} # ignored when using sqlite3 host: !ENV ${TIRA_DB_HOST:tira-mariadb} # ignored when using sqlite3 port: !ENV ${TIRA_DB_PORT:3306} # ignored when using sqlite3 -github_token: !ENV ${TIRA_GITHUB_TOKEN} + +########################################################################################## +# Discourse # +########################################################################################## discourse_api_url: !ENV ${DISCOURSE_API_URL:https://www.tira.io} + +# [SECRET] discourse_api_key: !ENV ${DISCOURSE_API_KEY:""} + +########################################################################################## +# Django # +########################################################################################## +# A list of hostnames using which the backend may be addressed. The value "*" denotes any +# address. A value of ["tira.example.com", "example.com"] would only allow requests made +# addressing these hostnames. See +# https://docs.djangoproject.com/en/5.1/ref/settings/#allowed-hosts for more information. +allowed_hosts: + - "*" + +# [SECRET] See https://docs.djangoproject.com/en/5.1/ref/settings/#std-setting-SECRET_KEY +# for more information. +django_secret: !ENV ${DJANGO_SECRET:change-me!} + +########################################################################################## +# Deprecated and removed soon (we hope) # +########################################################################################## +# grpc_host can be local or remote. If local, it will call localhost (i.e., for testing). +# If remote, it will call the vm-host. When developing, set this option to local, +# otherwise you might accidentally remote-control the live-vms. +grpc_host: local +host_grpc_port: 50051 +application_grpc_port: 50052 diff --git a/documentation/development/backend/index.rst b/documentation/development/backend/index.rst index 946d1a9c..deb7b1ae 100644 --- a/documentation/development/backend/index.rst +++ b/documentation/development/backend/index.rst @@ -21,4 +21,37 @@ Linting FAQ --- -Yay, no questions yet. \ No newline at end of file +Yay, no questions yet. + + + + +.. This was copied over from the deployment documentation and should be worked into the development documentation instead + Development + ~~~~~~~~~~~ + Frequently used development commands are: + + - Start the application without any grpc server + + .. code:: bash + + application/src~$ python3 manage.py runserver 8080 + + - Start only the application's grpc server + + .. code:: bash + + application/src~$ python3 manage.py grpc_server + + - Start the application and the application's grpc server. This is used in make run-develop and the container + + .. code:: bash + + application/src~$ python3 manage.py run_develop + + - Start the application, the application's grpc server, and a mock host grpc server that will reply to the application + with fake commands. This is the simplest way to develop the application. + + .. code:: bash + + application/src~$ python3 manage.py run_mockup diff --git a/documentation/organizers/deployment/application.rst b/documentation/organizers/deployment/application.rst index 733f64b5..4580403e 100644 --- a/documentation/organizers/deployment/application.rst +++ b/documentation/organizers/deployment/application.rst @@ -1,190 +1,71 @@ -Deploying the Application Module -================================ +Deploying the Backend +===================== -Development Setup ------------------ -The following steps will setup a self-contained, local tira application and a mockup tira host. See `Development`_ for -more detailed options. +.. important:: TIRA makes greate use of the internet forum tool `Discourse `_. Before you + continue, please set up a Discourse instance and install the reverse proxy plugin + `Disraptor `_. -.. code:: bash +The TIRA backend is entirely contained within the official docker image, +`ghcr.io/tira-io/tira-backend:latest `_. To see, what a +deployment using this container can look like, please have a look at our +:bdg-ref-secondary:`demo deployment `. - # Install Python3, pip and virtualenv - sudo apt-get update && \ - sudo apt-get install python3 python3-pip python3-venv libmysqlclient-dev - # Setup the local environment - make setup # This creates the virtual environment and prepares Django's database - # Setup the local environment - make run-develop # This updates the config and runs the server within the venv. - -Docker ------- -You can run tira in a docker container for a simple deployment. - -You need to run two docker containers for a tira-application: - -- :code:`registry.webis.de/code-lib/public-images/tira-application` and -- :code:`registry.webis.de/code-lib/public-images/tira-application-grpc`. - -.. code:: bash - - ~$ docker run -d --rm --name=tira-application \ - -p 8080:80 \ - -v="":/mnt/ceph/tira \ - registry.webis.de/code-lib/public-images/tira-application:latest - - ~$ docker run -d --rm --name=tira-application-grpc \ - -p 50052:50052 \ - -v="":/mnt/ceph/tira \ - registry.webis.de/code-lib/public-images/tira-application-grpc:latest - -Use TIRA with Discourse via Disraptor -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. note:: This step is, in theory, optional but, if left out, user credentials will be transmitted and stored in - plaintext and can be read by anyone using the public REST-API. - -If you set up Discourse with `Disraptor `_, this will be useful information on how to -make TIRA work with it. - -(1) Since TIRA has a legacy and a Disraptor :code:`deployment` mode first change that to :code:`disraptor` in the - development config file :code:`application/config/tira-application-config.yml`. -(2) When you set up Disraptor you set a :code:`Disraptor App Secret Key` that allows Disraptor to communicate with your - web application. Since TIRA has to evaluate that this secret is correct we have to supply it to it. TIRA expects - that secret in an environment variable called :code:`DISRAPTOR_APP_SECRET_KEY`, so before starting your development - TIRA environment with :code:`make run-develop` or :code:`make run-git-develop` always remember to - :code:`export DISRAPTOR_APP_SECRET_KEY=`. -(3) Lastly, of course, Disraptor has to be supplied with the routes for TIRA at :code:`Settings > Plugins > Disraptor`. - Since routes may change in the future and depending on what you want to test you maybe do no need the full set of - routes arranged it is recommended to take a look at :code:`application/src/tira/urls.py`. Here is an example of - what can be used to have access to most of the **current** TIRA: - - ========================== =============================================== ======== - From To Method - ========================== =============================================== ======== - `/public/tira/*wildcard` `http://127.0.0.1:8080/public/tira/*wildcard` GET - `/task/*wildcard` `http://127.0.0.1:8080/task/*wildcard` GET - `/task/*wildcard` `http://127.0.0.1:8080/task/*wildcard` POST - `/api/*wildcard` `http://127.0.0.1:8080/api/*wildcard` GET - `/tira-admin` `http://127.0.0.1:8080/tira-admin` GET - `/tira-admin/*wildcard` `http://127.0.0.1:8080/tira-admin/*wildcard` GET - `/tira-admin/*wildcard` `http://127.0.0.1:8080/tira-admin/*wildcard` POST - `/static/*wildcard` `http://127.0.0.1:8080/static/*wildcard` GET - `/grpc/*wildcard` `http://127.0.0.1:8080/grpc/*wildcard` GET - `/grpc/*wildcard` `http://127.0.0.1:8080/grpc/*wildcard` POST - `/tasks` `http://127.0.0.1:8080/` GET - `/tira/static/*wildcard` `http://127.0.0.1:8080/tira/static/*wildcard` GET - ========================== =============================================== ======== - - .. note:: - Let's say route A gets added here first and then route B. Rails will evaluate them from latest to earliest. So - when checking for a destination Rails will first look if route B matches before it checks route A and so on. - This can lead to states where you might see unexpected behavior if your routes (namely these with wildcards) - have interseting "To"-pools. - .. note:: - Sometimes Rails does not instantly take in new routes so when adding new routes you might want to restart the - Rails server. - -Build and Deploy +Configuring TIRA ---------------- -Run the tests -~~~~~~~~~~~~~ -.. code:: bash - - # run all tests in application/src/tira/tests - application/src~$ python3 manage.py test test tira/tests/ - # run an individual test module - application/src~$ python3 manage.py test test tira/tests/tests.py - -Deploy on Kubernetes -~~~~~~~~~~~~~~~~~~~~ -.. todo:: This step is deprecated and the documentation must be updated - -Add the discourse secret in the namespace via: - -.. code:: bash - - ??? TODO? - -Re-build the docker images -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: bash - - # Build the protobuf libraries from source. - make build - # This creates the virtual environment and prepares Django's database - make setup - # Build the docker image (deploy mode with nginx) - make docker-build-tira-application - # Run the docker container with the make command (deploy mode) - make docker-run-tira-application - # (optional) Publish a new version - make docker-publish-tira-application - -These make targets from the deployment configuration: :code:`tira/application/config/settings-deploy.yml`. - -Development -~~~~~~~~~~~ -The settings used for the development setup are: :code:`tira/application/config/settings-dev.yml`. - -Frequently used development commands are: - -- Start the application without any grpc server - - .. code:: bash - - application/src~$ python3 manage.py runserver 8080 - -- Start only the application's grpc server - - .. code:: bash - - application/src~$ python3 manage.py grpc_server - -- Start the application and the application's grpc server. This is used in make run-develop and the container - - .. code:: bash - - application/src~$ python3 manage.py run_develop - -- Start the application, the application's grpc server, and a mock host grpc server that will reply to the application - with fake commands. This is the simplest way to develop the application. - - .. code:: bash - - application/src~$ python3 manage.py run_mockup - -Troubleshooting ---------------- -If there are problems with the precompiled protobuf parser, you can recompile them from the :code:`tira/protocol` -repository and copy them to :code:`tira/application/src/tira/proto`. - -Setup on MacOS (Monterey/M1) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. attention:: - This part of the documentation is experimental and may not work for everyone. -.. note:: - We will assume that `brew `_ is installed. - -(1) Install required software - - .. code:: bash - - brew install python@3.10 pipenv pyvenv mariadb uwsgi -(2) Inside :code:`tira/application/config/tira-application-config.yml` change :code:`tira_root` to the model you - want to use. -(3) From within :code:`tira/application` execute the makefile at least once. This copies the config and runs - :code:`manage.py index_model` once. - - .. code:: bash - - make setup - .. note:: - If above command did not work, you may try to build the venv and install the requirements manually by executing - the following commands within :code:`tira/application` - - .. code:: bash - - python3.10 -m venv venv - source venv/bin/activat - pip install -r requirements.txt +TIRA can be configured using two ways (non-exclusive): + +(1) Environment variables +(2) A configuration file + +To kill two birds with one stone (only figuratively speaking), we refer you to the fully documented default +configuration file (below). Any value of the form ``!ENV ${:}`` indicates that the value is read from the +environment variable ```` and, if that environment variable does not exist, the default value, ```` is +assigned. This means, that, to set TIRA into debug mode, for example, you have two options: + +(1) The line ``debug: !ENV ${TIRA_DEBUG:false}`` tells us that, per default, the debug mode is disabled but we can + enable it by setting the environment variable ``TIRA_DEBUG`` to ``true``. +(2) Copy the default configuration, replace ``debug: !ENV ${TIRA_DEBUG:false}`` with ``debug: true`` and map the new + configuration file into your container to ``/tira/config/tira-application-config.yml`` (this location can be + changed using the ``TIRA_CONFIG`` environment variable). + +.. literalinclude:: ../../../application/config/tira-application-config.yml + :language: yaml + :caption: TIRA's default configuration + +.. attention:: Some of these configuration parameters are **secrets** and should stay *secret*. Do not use their + default values for production and use `Docker secrets `_ + to set them. + + +Endpoints +--------- +Lastly, of course, Disraptor has to be supplied with the routes for TIRA at :code:`Settings > Plugins > Disraptor`. +Since routes may change in the future and depending on what you want to test you maybe do no need the full set of +routes arranged it is recommended to take a look at :code:`application/src/tira/urls.py`. Here is an example of +what can be used to have access to most of the **current** TIRA: + +========================== =============================================== ======== +From To Method +========================== =============================================== ======== +`/public/tira/*wildcard` `http://127.0.0.1:8080/public/tira/*wildcard` GET +`/task/*wildcard` `http://127.0.0.1:8080/task/*wildcard` GET +`/task/*wildcard` `http://127.0.0.1:8080/task/*wildcard` POST +`/api/*wildcard` `http://127.0.0.1:8080/api/*wildcard` GET +`/tira-admin` `http://127.0.0.1:8080/tira-admin` GET +`/tira-admin/*wildcard` `http://127.0.0.1:8080/tira-admin/*wildcard` GET +`/tira-admin/*wildcard` `http://127.0.0.1:8080/tira-admin/*wildcard` POST +`/static/*wildcard` `http://127.0.0.1:8080/static/*wildcard` GET +`/grpc/*wildcard` `http://127.0.0.1:8080/grpc/*wildcard` GET +`/grpc/*wildcard` `http://127.0.0.1:8080/grpc/*wildcard` POST +`/tasks` `http://127.0.0.1:8080/` GET +`/tira/static/*wildcard` `http://127.0.0.1:8080/tira/static/*wildcard` GET +`/health` `http://127.0.0.1:8080/health` GET +`/info` `http://127.0.0.1:8080/info` GET +`/v1/*wildcard` `http://127.0.0.1:8080/v1/*wildcard` GET +`/v1/*wildcard` `http://127.0.0.1:8080/v1/*wildcard` POST +`/v1/*wildcard` `http://127.0.0.1:8080/v1/*wildcard` DELETE +`/v1/*wildcard` `http://127.0.0.1:8080/v1/*wildcard` UPDATE +========================== =============================================== ======== + +.. important:: Routes that were added later take precedence over those added before when multiple rules match. +.. hint:: You may need to restart the Rails server to apply your changes.