diff --git a/.container/consumer/motd b/.container/consumer/motd deleted file mode 100644 index be9b2535..00000000 --- a/.container/consumer/motd +++ /dev/null @@ -1,12 +0,0 @@ - -Welcome to the Anitya Consumer development environment! - -Here are some tips: - -* The code is located at /app - -* To run check service simply run -check_service - -Happy hacking! - diff --git a/.container/web/motd b/.container/web/motd deleted file mode 100644 index 92968d0f..00000000 --- a/.container/web/motd +++ /dev/null @@ -1,17 +0,0 @@ - -Welcome to the Anitya Web development environment! - -Here are some tips: - -* The code is located at /app - -The development server is running on http://localhost:5000/. - -* To start check service simply run `check_service` - -* To start tests run `tox` - -* To apply changes run `make restart` outside the container - -Happy hacking! - diff --git a/.gitignore b/.gitignore index 7ba20f55..8b66bac0 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ docs/docblocks/ anitya.db *.sqlite *.cfg +anitya.toml # Tests coverage.xml @@ -45,4 +46,4 @@ htmlcov !.container/postgresql/.gitkeep # Javascript -anitya/static/node_modules/ \ No newline at end of file +anitya/static/node_modules/ diff --git a/Containerfile.dev b/Containerfile.dev index 051a7bab..ca625e17 100644 --- a/Containerfile.dev +++ b/Containerfile.dev @@ -8,33 +8,47 @@ WORKDIR /app RUN dnf upgrade --refresh -y -# Python3 and packages -RUN dnf install -y python3 python3-devel python3-psycopg2 poetry +# Install helpful development packages (for tests and setup) +RUN dnf install -y git poetry python3-tox which python3.10 python3.11 -# Dependencies for tox -RUN dnf install -y git python3-tox libpq libpq-devel gcc graphviz +# System dependencies for Anitya +RUN dnf install -y gcc graphviz libffi-devel python3-devel python3-psycopg2 # Install npm to manage javascript dependencies RUN dnf install -y npm -RUN dnf autoremove -y && dnf clean all -y +# Dependencies for Anitya +RUN dnf install -y \ + fedora-messaging \ + python3-alembic \ + python3-arrow \ + python3-authlib \ + python3-beautifulsoup4 \ + python3-dateutil \ + python3-defusedxml \ + python3-flask-login \ + python3-flask-wtf \ + python3-jinja2 \ + python3-ordered-set \ + python3-toml \ + python3-semver \ + python3-sqlalchemy \ + python3-sqlalchemy_schemadisplay \ + python3-sseclient \ + python3-straight-plugin \ + python3-wtforms -COPY anitya ./anitya -COPY ansible ./ansible -COPY .container ./.container -COPY mypy.cfg alembic.ini anitya.wsgi createdb.py pyproject.toml tox.ini README.rst poetry.lock . +RUN dnf autoremove -y && dnf clean all -y # Copy over configuration files -RUN mkdir -p /etc/anitya -RUN cp /app/ansible/roles/anitya-dev/files/anitya.toml /etc/anitya RUN mkdir -p /etc/fedora-messaging -RUN cp /app/.container/web/config.toml /etc/fedora-messaging +COPY .container/web/config.toml /etc/fedora-messaging -# Download javascript dependencies -RUN pushd anitya/static && npm install && popd +# Install javascript dependencies +COPY anitya/static /app/static +RUN pushd static && npm install && popd -# Poetry installation -RUN poetry build -RUN pip install dist/*.whl +# Set the poetry to use system packages +RUN poetry config virtualenvs.options.system-site-packages true -CMD ["sh","-c", "poetry build && pip install dist/*.whl && eval '$START_COMMAND'"] +CMD ["sh", "-c", "poetry install && poetry run $START_COMMAND"] diff --git a/Makefile b/Makefile index 3e916991..bc9fbf70 100644 --- a/Makefile +++ b/Makefile @@ -21,25 +21,29 @@ up: $(call compose-tool) up -d # It takes some time before the postgres container is up, we need to wait before calling # the next step - sleep 10 + sleep 15 $(MAKE) init-db @echo "Empty database initialized. Run dump-restore to fill it by production dump." restart: - $(MAKE) halt && $(MAKE) up + $(MAKE) halt && $(call compose-tool) up -d halt: $(call compose-tool) down -t1 bash-web: - $(call container-tool) exec -it anitya-web bash -c "cat /app/.container/web/motd; bash;" + $(call container-tool) exec -it anitya-web bash -c "bash" +bash-check: + $(call container-tool) exec -it anitya-check-service bash -c "bash" init-db: - $(call container-tool) exec -it anitya-web bash -c "python3 createdb.py" + $(call container-tool) exec -it anitya-web bash -c "poetry run python3 createdb.py" dump-restore: init-db $(call download_dump) $(call container-tool) exec -it postgres bash -c 'createuser anitya && xzcat /dump/anitya.dump.xz | psql anitya' $(call remove_dump) logs: - $(call container-tool) logs -f anitya-web rabbitmq postgres + $(call container-tool) logs -f anitya-web anitya-check-service rabbitmq postgres clean: halt $(call container-tool) rmi "localhost/anitya-base:latest" "docker.io/library/postgres:13.4" "docker.io/library/rabbitmq:3.8.16-management-alpine" +tests: + $(call container-tool) exec -it anitya-web bash -c "tox $(PARAM)" .PHONY: up restart halt bash-web \ - init-db dump-restore logs clean + init-db dump-restore logs clean tests diff --git a/anitya.toml.example b/anitya.toml.example new file mode 100644 index 00000000..31e48d63 --- /dev/null +++ b/anitya.toml.example @@ -0,0 +1,50 @@ +# This is a TOML-format file. For the spec, see https://github.com/toml-lang/toml#spec + +# URL to the database +db_url = 'postgresql://postgres:anypasswordworkslocally@postgres/anitya' + +# List of web administrators. The values should be the value of the "id" column +# for the user in the "users" table of the database. They need to log in before +# this record is created. An example value would be +# "65536ed7-bdd3-4a1e-8252-10d874fd706b" +# You can also find this infromation in the settings page when logged in to Anitya +anitya_web_admins = [] + +preferred_url_scheme = "https" + +authlib_enabled_backends = ["Fedora"] + +fedora_client_id = "fedora" +fedora_client_secret = "secret" +fedora_server_metadata_url = "https://id.stg.fedoraproject.org/.well-known/openid-configuration" +[fedora_client_kwargs] +scope = "openid profile email" +token_endpoint_auth_method = "client_secret_post" + + +blacklisted_users = [] + +# The logging configuration, in dictConfig format. +[anitya_log_config] + version = 1 + disable_existing_loggers = false + + [anitya_log_config.formatters] + [anitya_log_config.formatters.simple] + format = "[%(name)s %(levelname)s] %(message)s" + + [anitya_log_config.handlers] + [anitya_log_config.handlers.console] + class = "logging.StreamHandler" + formatter = "simple" + stream = "ext://sys.stdout" + + [anitya_log_config.loggers] + [anitya_log_config.loggers.anitya] + level = "DEBUG" + propagate = false + handlers = ["console"] + + [anitya_log_config.root] + level = "INFO" + handlers = ["console"] diff --git a/anitya/app.py b/anitya/app.py index 018bf0ee..89fdee8f 100644 --- a/anitya/app.py +++ b/anitya/app.py @@ -21,7 +21,7 @@ import anitya.lib import anitya.mail_logging -from anitya import __version__, admin, api, api_v2, auth, authentication, ui +from anitya import __version__, admin, api, api_v2, auth, authentication, debug, ui from anitya.config import config as anitya_config from anitya.db import Session from anitya.db import initialize as initialize_db @@ -73,6 +73,11 @@ def create(config=None): app.register_blueprint(ui.ui_blueprint) app.register_blueprint(api.api_blueprint) + # Debug related initialization + # WARNING: For debug and development purpose only + if app.debug: # pragma: no cover + app.register_blueprint(debug.debug_blueprint) + oauth = OAuth(app) for auth_backend in app.config.get("AUTHLIB_ENABLED_BACKENDS", []): oauth.register(auth_backend.lower()) diff --git a/anitya/debug.py b/anitya/debug.py new file mode 100644 index 00000000..9da77ca7 --- /dev/null +++ b/anitya/debug.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +""" +This module contains blueprints that are only included + when application is running in debug mode. +""" +import flask +import flask_login + +from anitya.db import Session, User + +debug_blueprint = flask.Blueprint( + "anitya_debug", __name__, static_folder="static", template_folder="templates" +) + + +@debug_blueprint.route("/login/debug/") +def login(name: str): + """ + Debug login for prepared users. + + Params: + name: User to log as in. + """ + user = User.query.filter(User.username == name).first() + + # Create user if it doesn't exist yet + if not user: + user = User( + username=name, email=(name + "@example.com"), admin=(name == "admin") + ) + Session.add(user) + Session.commit() + + flask_login.login_user(user) + if flask.session["next_url"]: + return flask.redirect(flask.session["next_url"]) + return flask.redirect("/") diff --git a/anitya/templates/login.html b/anitya/templates/login.html index fe93f388..2359cda3 100644 --- a/anitya/templates/login.html +++ b/anitya/templates/login.html @@ -11,4 +11,14 @@ {% endfor %} +{% if config["DEBUG"] %} +
+ + User + + + Admin + +
+{% endif %} {% endblock %} diff --git a/container-compose.yml b/container-compose.yml index 709b2e6b..f900f81b 100644 --- a/container-compose.yml +++ b/container-compose.yml @@ -12,13 +12,33 @@ services: ports: - "127.0.0.1:5000:5000" volumes: - - ./anitya/:/app/anitya:z - - ./docs/:/app/docs:z + - .:/app:z restart: unless-stopped environment: + - ANITYA_WEB_CONFIG=anitya.toml - FLASK_APP=anitya.wsgi - FLASK_DEBUG=1 - START_COMMAND=flask run -h 0.0.0.0 + healthcheck: + test: [ "CMD", "nc", "-z", "localhost", "5000" ] + interval: 3s + timeout: 3s + retries: 30 + depends_on: + postgres: + condition: service_started + rabbitmq: + condition: service_started + + anitya-check-service: + image: anitya-base + container_name: "anitya-check-service" + volumes: + - .:/app:z + restart: unless-stopped + environment: + - ANITYA_WEB_CONFIG=anitya.toml + - START_COMMAND=check_service depends_on: postgres: condition: service_started diff --git a/docs/contributing.rst b/docs/contributing.rst index d9276352..d56ca9fe 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -15,7 +15,7 @@ will review your code. Please make sure you follow the guidelines below: Python Support -------------- -Anitya supports Python 3.8 or greater so please ensure the code +Anitya supports Python 3.10 or greater so please ensure the code you submit works with these versions. The test suite will run against all supported Python versions to make this easier. @@ -206,10 +206,11 @@ Requirements: * Docker / Podman * Docker Compose / Podman Compose -Next, clone the repository and start containers:: +Next, clone the repository, prepare config and start containers:: $ git clone https://github.com/fedora-infra/anitya $ cd anitya + $ cp anitya.toml.example anitya.toml $ make up .. list-table:: Container Service Informations: @@ -235,25 +236,30 @@ Makefile scripts that provide easier container management: * ``make restart`` Restarts all the container services that are either stopped or running * ``make halt`` Stops and removes the containers * ``make bash-web`` Connects to anitya-web container +* ``make bash-check-service`` Connects to anitya-check-service container * ``make init-db`` Creates database * ``make dump-restore`` Import production database * ``make logs`` Shows all logs of all containers * ``make clean`` Removes all images used by Anitya compose +* ``make tests`` Run tests in Anitya container. You can provide parameters for + tox in ``PARAM`` variable. Few examples: + + * ``PARAM="-e format" make tests`` - run only format test environment + * ``PARAM="-- anitya/tests/test_app.py" make tests`` - run tests only for ``test_app.py`` file + * ``PARAM="-e format -- anitya/tests/test_app.py" make tests`` - combine the two above Project files are bound to each other with host and container. Whenever you change any project file from the host or the container, the same change will happen on the opposite side as well. Anitya is accessible on http://localhost:5000 -Start the check service with:: +Check service for Anitya is running in separate container. - $ make bash-consumer or make-bash-web - $ check_service.py +For testing purposes there are two predefined users. +You can see them on login page. -To apply changes run:: +To apply configuration changes:: $ make restart -This will restart the container, deploy the changes in code and start the development instance again. - Python virtualenv ----------------- @@ -278,23 +284,6 @@ Create the database, by default it will be a sqlite database located at $ poetry run python createdb.py -Configure social_sqlalchemy for Anitya if needed. This step is optional and depends on your use case: :: - - #Example configuration for social_sqlalchemy in a Anitya - #in anitya/config.py - from flask import Flask - from social_flask_sqlalchemy.models import init_social - - app = Flask(__name__) - - #Configure SQLAlchemy database - app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///your_database.db' - app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False - - #Initialize social_sqlalchemy - init_social(app, app.config['SQLALCHEMY_ENGINE']) - - You can start the development web server included with Flask with:: $ FLASK_APP=anitya.wsgi poetry run flask run diff --git a/news/1859.dev b/news/1859.dev new file mode 100644 index 00000000..92d2f72f --- /dev/null +++ b/news/1859.dev @@ -0,0 +1 @@ +Improve development container diff --git a/tox.ini b/tox.ini index 5fdd6869..abebb90c 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,8 @@ commands = [testenv:diff-cover] commands = poetry install - diff-cover coverage.xml --compare-branch=origin/master --fail-under=100 + diff-cover coverage.xml --compare-branch=origin/master \ + --exclude debug.py --fail-under=100 [testenv:docs] changedir = docs