Improve container dev environment

Closes #1859

Signed-off-by: Michal Konecny <[email protected]>

.gitignore

@@ -20,6 +20,7 @@ docs/docblocks/
 anitya.db
 *.sqlite
 *.cfg
+anitya.toml
 
 # Tests
 coverage.xml
@@ -45,4 +46,4 @@ htmlcov
 !.container/postgresql/.gitkeep
 
 # Javascript
-anitya/static/node_modules/
+anitya/static/node_modules/

Containerfile.dev

@@ -8,33 +8,47 @@ WORKDIR /app
 
 RUN dnf upgrade --refresh -y
 
-# Python3 and packages
-RUN dnf install -y python3 python3-devel python3-psycopg2 poetry
+# Install helpful development packages (for tests and setup)
+RUN dnf install -y git poetry python3-tox which python3.10 python3.11
 
-# Dependencies for tox
-RUN dnf install -y git python3-tox libpq libpq-devel gcc graphviz
+# System dependencies for Anitya
+RUN dnf install -y gcc graphviz libffi-devel python3-devel python3-psycopg2
 
 # Install npm to manage javascript dependencies
 RUN dnf install -y npm
 
-RUN dnf autoremove -y && dnf clean all -y
+# Dependencies for Anitya
+RUN dnf install -y \
+    fedora-messaging \
+    python3-alembic \
+    python3-arrow \
+    python3-authlib \
+    python3-beautifulsoup4 \
+    python3-dateutil \
+    python3-defusedxml \
+    python3-flask-login \
+    python3-flask-wtf \
+    python3-jinja2 \
+    python3-ordered-set \
+    python3-toml \
+    python3-semver \
+    python3-sqlalchemy \
+    python3-sqlalchemy_schemadisplay \
+    python3-sseclient \
+    python3-straight-plugin \
+    python3-wtforms
 
-COPY anitya ./anitya
-COPY ansible ./ansible
-COPY .container ./.container
-COPY mypy.cfg alembic.ini anitya.wsgi createdb.py pyproject.toml tox.ini README.rst poetry.lock .
+RUN dnf autoremove -y && dnf clean all -y
 
 # Copy over configuration files
-RUN mkdir -p /etc/anitya
-RUN cp /app/ansible/roles/anitya-dev/files/anitya.toml /etc/anitya
 RUN mkdir -p /etc/fedora-messaging
-RUN cp /app/.container/web/config.toml /etc/fedora-messaging
+COPY .container/web/config.toml /etc/fedora-messaging
 
-# Download javascript dependencies
-RUN pushd anitya/static && npm install && popd
+# Install javascript dependencies
+COPY anitya/static /app/static
+RUN pushd static && npm install && popd
 
-# Poetry installation
-RUN poetry build
-RUN pip install dist/*.whl
+# Set the poetry to use system packages
+RUN poetry config virtualenvs.options.system-site-packages true
 
-CMD ["sh","-c", "poetry build && pip install dist/*.whl && eval '$START_COMMAND'"]
+CMD ["sh", "-c", "poetry install && poetry run $START_COMMAND"]

Makefile

@@ -21,25 +21,29 @@ up:
 	$(call compose-tool) up -d
 # It takes some time before the postgres container is up, we need to wait before calling
 # the next step
-	sleep 10
+	sleep 15
 	$(MAKE) init-db
 	@echo "Empty database initialized. Run dump-restore to fill it by production dump."
 restart:
-	$(MAKE) halt && $(MAKE) up
+	$(MAKE) halt && $(call compose-tool) up -d
 halt:
 	$(call compose-tool) down -t1
 bash-web:
-	$(call container-tool) exec -it anitya-web bash -c "cat /app/.container/web/motd; bash;"
+	$(call container-tool) exec -it anitya-web bash -c "bash"
+bash-check:
+	$(call container-tool) exec -it anitya-check-service bash -c "bash"
 init-db:
-	$(call container-tool) exec -it anitya-web bash -c "python3 createdb.py"
+	$(call container-tool) exec -it anitya-web bash -c "poetry run python3 createdb.py"
 dump-restore: init-db
 	$(call download_dump)
 	$(call container-tool) exec -it postgres bash -c 'createuser anitya && xzcat /dump/anitya.dump.xz | psql anitya'
 	$(call remove_dump)
 logs:
-	$(call container-tool) logs -f anitya-web rabbitmq postgres
+	$(call container-tool) logs -f anitya-web anitya-check-service rabbitmq postgres
 clean: halt
 	$(call container-tool) rmi "localhost/anitya-base:latest" "docker.io/library/postgres:13.4" "docker.io/library/rabbitmq:3.8.16-management-alpine"
+tests:
+	$(call container-tool) exec -it anitya-web bash -c "tox $(PARAM)"
 
 .PHONY: up restart halt bash-web \
-	init-db dump-restore logs clean
+	init-db dump-restore logs clean tests

anitya.toml.example

@@ -0,0 +1,50 @@
+# This is a TOML-format file. For the spec, see https://github.com/toml-lang/toml#spec
+
+# URL to the database
+db_url = 'postgresql://postgres:anypasswordworkslocally@postgres/anitya'
+
+# List of web administrators. The values should be the value of the "id" column
+# for the user in the "users" table of the database. They need to log in before
+# this record is created. An example value would be
+# "65536ed7-bdd3-4a1e-8252-10d874fd706b"
+# You can also find this infromation in the settings page when logged in to Anitya
+anitya_web_admins = []
+
+preferred_url_scheme = "https"
+
+authlib_enabled_backends = ["Fedora"]
+
+fedora_client_id = "fedora"
+fedora_client_secret = "secret"
+fedora_server_metadata_url = "https://id.stg.fedoraproject.org/.well-known/openid-configuration"
+[fedora_client_kwargs]
+scope = "openid profile email"
+token_endpoint_auth_method = "client_secret_post"
+
+
+blacklisted_users = []
+
+# The logging configuration, in dictConfig format.
+[anitya_log_config]
+    version = 1
+    disable_existing_loggers = false
+
+    [anitya_log_config.formatters]
+        [anitya_log_config.formatters.simple]
+            format = "[%(name)s %(levelname)s] %(message)s"
+
+    [anitya_log_config.handlers]
+        [anitya_log_config.handlers.console]
+            class = "logging.StreamHandler"
+            formatter = "simple"
+            stream = "ext://sys.stdout"
+
+    [anitya_log_config.loggers]
+        [anitya_log_config.loggers.anitya]
+            level = "DEBUG"
+            propagate = false
+            handlers = ["console"]
+
+    [anitya_log_config.root]
+        level = "INFO"
+        handlers = ["console"]

anitya/app.py

@@ -21,7 +21,7 @@
 
 import anitya.lib
 import anitya.mail_logging
-from anitya import __version__, admin, api, api_v2, auth, authentication, ui
+from anitya import __version__, admin, api, api_v2, auth, authentication, debug, ui
 from anitya.config import config as anitya_config
 from anitya.db import Session
 from anitya.db import initialize as initialize_db
@@ -73,6 +73,11 @@ def create(config=None):
     app.register_blueprint(ui.ui_blueprint)
     app.register_blueprint(api.api_blueprint)
 
+    # Debug related initialization
+    # WARNING: For debug and development purpose only
+    if app.debug:  # pragma: no cover
+        app.register_blueprint(debug.debug_blueprint)
+
     oauth = OAuth(app)
     for auth_backend in app.config.get("AUTHLIB_ENABLED_BACKENDS", []):
         oauth.register(auth_backend.lower())

anitya/debug.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+"""
+This module contains blueprints that are only included
+ when application is running in debug mode.
+"""
+import flask
+import flask_login
+
+from anitya.db import Session, User
+
+debug_blueprint = flask.Blueprint(
+    "anitya_debug", __name__, static_folder="static", template_folder="templates"
+)
+
+
+@debug_blueprint.route("/login/debug/<name>")
+def login(name: str):
+    """
+    Debug login for prepared users.
+
+    Params:
+      name: User to log as in.
+    """
+    user = User.query.filter(User.username == name).first()
+
+    # Create user if it doesn't exist yet
+    if not user:
+        user = User(
+            username=name, email=(name + "@example.com"), admin=(name == "admin")
+        )
+        Session.add(user)
+        Session.commit()
+
+    flask_login.login_user(user)
+    if flask.session["next_url"]:
+        return flask.redirect(flask.session["next_url"])
+    return flask.redirect("/")

anitya/templates/login.html

@@ -11,4 +11,14 @@
     </a>
     {% endfor %}
 </div>
+{% if config["DEBUG"] %}
+<div class="d-flex align-itmes-start gap-3">
+    <a title="user" class="btn btn-outline-primary" role="button" href="{{ url_for('anitya_debug.login', name='user') }}">
+      <span>User</span>
+    </a>
+    <a title="admin" class="btn btn-outline-primary" role="button" href="{{ url_for('anitya_debug.login', name='admin') }}">
+      <span>Admin</span>
+    </a>
+</div>
+{% endif %}
 {% endblock %}

container-compose.yml

@@ -12,13 +12,33 @@ services:
     ports:
       - "127.0.0.1:5000:5000"
     volumes:
-      - ./anitya/:/app/anitya:z
-      - ./docs/:/app/docs:z
+      - .:/app:z
     restart: unless-stopped
     environment:
+      - ANITYA_WEB_CONFIG=anitya.toml
       - FLASK_APP=anitya.wsgi
       - FLASK_DEBUG=1
       - START_COMMAND=flask run -h 0.0.0.0
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "localhost", "5000" ]
+      interval: 3s
+      timeout: 3s
+      retries: 30
+    depends_on:
+      postgres:
+        condition: service_started
+      rabbitmq:
+        condition: service_started
+
+  anitya-check-service:
+    image: anitya-base
+    container_name: "anitya-check-service"
+    volumes:
+      - .:/app:z
+    restart: unless-stopped
+    environment:
+      - ANITYA_WEB_CONFIG=anitya.toml
+      - START_COMMAND=check_service
     depends_on:
       postgres:
         condition: service_started

docs/contributing.rst

@@ -15,7 +15,7 @@ will review your code. Please make sure you follow the guidelines below:
 Python Support
 --------------
 
-Anitya supports Python 3.8 or greater so please ensure the code
+Anitya supports Python 3.10 or greater so please ensure the code
 you submit works with these versions. The test suite will run against all supported
 Python versions to make this easier.
 
@@ -206,10 +206,11 @@ Requirements:
 * Docker / Podman
 * Docker Compose / Podman Compose
 
-Next, clone the repository and start containers::
+Next, clone the repository, prepare config and start containers::
 
     $ git clone https://github.com/fedora-infra/anitya
     $ cd anitya
+    $ cp anitya.toml.example anitya.toml
     $ make up
 
 .. list-table:: Container Service Informations:
@@ -235,25 +236,30 @@ Makefile scripts that provide easier container management:
 * ``make restart`` Restarts all the container services that are either stopped or running
 * ``make halt`` Stops and removes the containers
 * ``make bash-web`` Connects to anitya-web container
+* ``make bash-check-service`` Connects to anitya-check-service container
 * ``make init-db`` Creates database
 * ``make dump-restore`` Import production database
 * ``make logs`` Shows all logs of all containers
 * ``make clean`` Removes all images used by Anitya compose
+* ``make tests`` Run tests in Anitya container. You can provide parameters for
+  tox in ``PARAM`` variable. Few examples:
+
+  * ``PARAM="-e format" make tests`` - run only format test environment
+  * ``PARAM="-- anitya/tests/test_app.py" make tests`` - run tests only for ``test_app.py`` file
+  * ``PARAM="-e format -- anitya/tests/test_app.py" make tests`` - combine the two above
 
 Project files are bound to each other with host and container. Whenever you change any project file from the host or the container, the same change will happen on the opposite side as well.
 
 Anitya is accessible on http://localhost:5000
 
-Start the check service with::
+Check service for Anitya is running in separate container.
 
-    $ make bash-consumer or make-bash-web
-    $ check_service.py
+For testing purposes there are two predefined users.
+You can see them on login page.
 
-To apply changes run::
+To apply configuration changes::
     $ make restart
 
-This will restart the container, deploy the changes in code and start the development instance again.
-
 Python virtualenv
 -----------------
 
@@ -278,23 +284,6 @@ Create the database, by default it will be a sqlite database located at
 
     $ poetry run python createdb.py
 
-Configure social_sqlalchemy for Anitya if needed. This step is optional and depends on your use case: ::
-
-    #Example configuration for social_sqlalchemy in a Anitya
-    #in anitya/config.py
-    from flask import Flask
-    from social_flask_sqlalchemy.models import init_social
-
-    app = Flask(__name__)
-
-    #Configure SQLAlchemy database
-    app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///your_database.db'
-    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
-
-    #Initialize social_sqlalchemy 
-    init_social(app, app.config['SQLALCHEMY_ENGINE'])
-
-
 You can start the development web server included with Flask with::
 
     $ FLASK_APP=anitya.wsgi poetry run flask run

news/1859.dev

@@ -0,0 +1 @@
+Improve development container