From 334d6510a7a99084f7d84fd4696baa39db64426c Mon Sep 17 00:00:00 2001 From: Veelkoov Date: Tue, 2 May 2023 00:43:07 +0200 Subject: [PATCH] Fixer for Docker & sessions persistence (#193) --- Jenkinsfile | 2 +- README.md | 3 +- ansible/group_vars/public_envs.yaml | 2 + ansible/roles/environment/tasks/build.yaml | 12 +++-- .../roles/environment/tasks/clear_cache.yaml | 7 ++- .../tasks/setup_docker_container.yaml | 2 +- .../roles/environment/templates/update.sh.j2 | 7 ++- .../roles/public_host/tasks/setup_docker.yaml | 6 +++ docker/docker-compose.yml | 22 ++++---- docker/php-fpm/Dockerfile | 52 ++++++++++++++----- docker/php-fpm/entrypoint.sh | 24 +++++++++ docker/php-fpm/php.ini | 2 +- docker/php-fpm/php.ini-production.dist | 18 ++++++- toolbox | 12 ++--- 14 files changed, 120 insertions(+), 51 deletions(-) create mode 100644 ansible/group_vars/public_envs.yaml create mode 100755 docker/php-fpm/entrypoint.sh diff --git a/Jenkinsfile b/Jenkinsfile index 7db9214fb..946974515 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -63,7 +63,7 @@ pipeline { sh 'echo "GOOGLE_RECAPTCHA_SECRET=$GOOGLE_RECAPTCHA_SECRET" >> .env.test.local' sh './toolbox docker-up' - sh './toolbox composer install' + sh './toolbox composer install --no-progress' sh 'yarn install' sh './toolbox yep' sh './toolbox pu --version' diff --git a/README.md b/README.md index d7f26372b..3c432c2f8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See [getfursu.it/info](https://getfursu.it/info) ## Requirements * Docker w/Compose plugin -* sudo +* ACL-enabled filesystem * Yarn (to be able to do more than run tests in the `@small` group) @@ -39,4 +39,3 @@ To make the `dev` environment and tests in `@large` group work: * Yarn is not dockerized and automated * Tests in `@medium` group will not work without Yarn -* [Well, this](https://github.com/veelkoov/fuzzrake/issues/168) diff --git a/ansible/group_vars/public_envs.yaml b/ansible/group_vars/public_envs.yaml new file mode 100644 index 000000000..e8464fb39 --- /dev/null +++ b/ansible/group_vars/public_envs.yaml @@ -0,0 +1,2 @@ +--- +yarn_cache_dirpath: '{{ deployment_path }}/var/yarn-cache' diff --git a/ansible/roles/environment/tasks/build.yaml b/ansible/roles/environment/tasks/build.yaml index e7fde77a1..9fa9ee413 100644 --- a/ansible/roles/environment/tasks/build.yaml +++ b/ansible/roles/environment/tasks/build.yaml @@ -3,16 +3,18 @@ ansible.builtin.command: 'docker exec -ti {{ php_fpm_container_name | quote }} composer install' args: chdir: '{{ deployment_path }}' - # TODO: changed_when + register: 'cmd_out' + changed_when: '"Nothing to install, update or remove" not in cmd_out.stdout' - name: 'Execute: yarn install' - ansible.builtin.command: 'yarn --cache-folder {{ (deployment_path + "/var/yarn-cache") | quote }} install' # TODO: Var for the folder, deduplicate + ansible.builtin.command: 'yarn --cache-folder {{ yarn_cache_dirpath | quote }} install' args: chdir: '{{ deployment_path }}' - # TODO: changed_when + register: 'cmd_out' + changed_when: '"success Already up-to-date." not in cmd_out.stdout' - name: 'Execute: yarn encore production' - ansible.builtin.command: 'yarn --cache-folder {{ (deployment_path + "/var/yarn-cache") | quote }} encore production' + ansible.builtin.command: 'yarn --cache-folder {{ yarn_cache_dirpath | quote }} encore production' args: chdir: '{{ deployment_path }}' - # TODO: changed_when (if possible) + changed_when: 'not ansible_check_mode' # TODO: Make it real (if possible) diff --git a/ansible/roles/environment/tasks/clear_cache.yaml b/ansible/roles/environment/tasks/clear_cache.yaml index 423de2457..c932b16c1 100644 --- a/ansible/roles/environment/tasks/clear_cache.yaml +++ b/ansible/roles/environment/tasks/clear_cache.yaml @@ -1,5 +1,4 @@ --- -- name: 'Clear cache' # FIXME: This ruins all sessions - ansible.builtin.shell: 'rm -rf var/cache/*' - args: - chdir: '{{ deployment_path }}' +- name: 'Clear cache' + ansible.builtin.command: 'docker exec {{ php_fpm_container_name | quote }} bin/console cache:clear' + changed_when: 'not ansible_check_mode' diff --git a/ansible/roles/environment/tasks/setup_docker_container.yaml b/ansible/roles/environment/tasks/setup_docker_container.yaml index 4f97cbebf..939b90748 100644 --- a/ansible/roles/environment/tasks/setup_docker_container.yaml +++ b/ansible/roles/environment/tasks/setup_docker_container.yaml @@ -17,6 +17,6 @@ name: '{{ php_fpm_container_name }}' published_ports: '{{ php_fpm_container_port }}:9000' restart_policy: 'unless-stopped' - user: '1000:1000' + user: 'root:root' volumes: - '{{ deployment_path }}:/var/www/html' diff --git a/ansible/roles/environment/templates/update.sh.j2 b/ansible/roles/environment/templates/update.sh.j2 index 54581e206..842e0f506 100644 --- a/ansible/roles/environment/templates/update.sh.j2 +++ b/ansible/roles/environment/templates/update.sh.j2 @@ -4,10 +4,9 @@ set -euo pipefail date -u -# TODO: Path from group vars -docker exec fuzzrake-prod /var/www/html/bin/console app:status-tracker:run --refetch --commit -docker exec fuzzrake-prod /var/www/html/bin/console cache:clear -docker exec fuzzrake-prod /var/www/html/bin/console cache:warmup +docker exec fuzzrake-prod bin/console app:status-tracker:run --refetch --commit +docker exec fuzzrake-prod bin/console cache:clear +docker exec fuzzrake-prod bin/console cache:warmup curl -o /dev/null -s https://getfursu.it/ date -u diff --git a/ansible/roles/public_host/tasks/setup_docker.yaml b/ansible/roles/public_host/tasks/setup_docker.yaml index b795886b2..64e347969 100644 --- a/ansible/roles/public_host/tasks/setup_docker.yaml +++ b/ansible/roles/public_host/tasks/setup_docker.yaml @@ -34,3 +34,9 @@ enabled: true name: 'docker' state: 'started' + +- name: 'Add user to the "docker" group' + ansible.builtin.user: + append: true + name: '{{ ansible_ssh_user }}' + groups: 'docker' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 17ab35a70..9f1571500 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -6,21 +6,23 @@ services: web: - image: nginx:alpine - restart: unless-stopped - volumes: # FIXME: https://github.com/veelkoov/fuzzrake/issues/168 - - ..:/var/www/html - - ./dev-nginx.conf:/etc/nginx/conf.d/default.conf + image: 'nginx:alpine' + restart: 'unless-stopped' ports: - '${FUZZRAKE_DEV_WEBSRV_PORT:-8080}:80' + volumes: + - '..:/var/www/html' + - './dev-nginx.conf:/etc/nginx/conf.d/default.conf' php: build: - context: php-fpm + context: 'php-fpm' args: DEV_MACHINE: 'yes' - restart: unless-stopped - volumes: # FIXME: https://github.com/veelkoov/fuzzrake/issues/168 - - ..:/var/www/html + DOCKER_UID: '${FUZZRAKE_DOCKER_UID:-1000}' + restart: 'unless-stopped' + user: 'root:root' + volumes: + - '..:/var/www/html' - '${FUZZRAKE_COMPOSER_HOME:-../var/composer}:/composer' - - /dev/shm:/dev/shm + - '/dev/shm:/dev/shm' diff --git a/docker/php-fpm/Dockerfile b/docker/php-fpm/Dockerfile index 64da697ac..de291b5ad 100644 --- a/docker/php-fpm/Dockerfile +++ b/docker/php-fpm/Dockerfile @@ -1,10 +1,18 @@ -FROM php:8.2.1-fpm-alpine3.17 +FROM php:8.2.5-fpm-alpine3.17 ARG DEV_MACHINE=no +ENV DEV_MACHINE "$DEV_MACHINE" + +ARG DOCKER_UID=1000 +ENV DOCKER_UID "$DOCKER_UID" + +# +# Packages installation +# RUN apk add --no-cache --virtual .build-deps $PHPIZE_DEPS icu-dev linux-headers -RUN apk add --no-cache git icu php-intl python3 py3-pip util-linux zip libzip-dev libpng libpng-dev +RUN apk add --no-cache git icu php-intl python3 py3-pip util-linux zip libzip-dev libpng libpng-dev acl RUN wget https://get.symfony.com/cli/installer -O /tmp/symfony_installer \ && sed -ri 's/^binary_dest=.*$/binary_dest=\/usr\/local\/bin/g' /tmp/symfony_installer \ @@ -24,7 +32,6 @@ RUN if [ "$DEV_MACHINE" == "yes" ]; then \ echo "Setting up develop software" \ && pecl install xdebug \ && docker-php-ext-enable xdebug.so \ - && sed -r -i'' 's/set -e/set -e\ncd \/etc\nsed \/host.docker.internal\/d hosts > hosts.new\ncat hosts.new > hosts\nip r | grep "default via" | cut -f3 -d" " | tr -d "\\n" >> hosts\necho " host.docker.internal" >> hosts\ncd -\n/' /usr/local/bin/docker-php-entrypoint \ && apk add --no-cache chromium chromium-chromedriver; \ else \ echo "Skipping develop software"; \ @@ -32,6 +39,10 @@ RUN if [ "$DEV_MACHINE" == "yes" ]; then \ RUN apk del .build-deps +# +# Set up configuration files +# + COPY php.ini-production.dist php.ini docker-php-ext-xdebug.ini docker-php-ext-xdebug.ini.dist /tmp/ RUN if [ "$DEV_MACHINE" == "yes" ]; then \ @@ -44,19 +55,36 @@ RUN if [ "$DEV_MACHINE" == "yes" ]; then \ && rm "/tmp/docker-php-ext-xdebug.ini.dist" "/tmp/docker-php-ext-xdebug.ini"; \ fi -# It's OK for production image -ENV PANTHER_CHROME_DRIVER_BINARY /usr/lib/chromium/chromedriver -ENV PANTHER_NO_SANDBOX 1 - RUN diff "/tmp/php.ini-production.dist" "$PHP_INI_DIR/php.ini-production" \ && mv "/tmp/php.ini" "$PHP_INI_DIR/php.ini" \ && rm "/tmp/php.ini-production.dist" -# FIXME: https://github.com/veelkoov/fuzzrake/issues/168 -VOLUME /var/www/html -WORKDIR /var/www/html +# +# Set up entrypoint +# + +COPY entrypoint.sh /entrypoint.sh +CMD /entrypoint.sh + +# +# Set up working directory +# + RUN su www-data -s /bin/sh -c 'git config --global --add safe.directory /var/www/html' +WORKDIR /var/www/html +VOLUME /var/www/html -# FIXME: https://github.com/veelkoov/fuzzrake/issues/168 -VOLUME /composer +# +# Allow setting Composer home to a volume, to preserve cache e.g. between Jenkins jobs +# + +RUN mkdir /composer ENV COMPOSER_HOME /composer +VOLUME /composer + +# +# Development stuff, but OK to be included in production image +# + +ENV PANTHER_CHROME_DRIVER_BINARY /usr/lib/chromium/chromedriver +ENV PANTHER_NO_SANDBOX 1 diff --git a/docker/php-fpm/entrypoint.sh b/docker/php-fpm/entrypoint.sh new file mode 100755 index 000000000..27bb35959 --- /dev/null +++ b/docker/php-fpm/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +set -eux + +if [ "$DEV_MACHINE" = "yes" ]; then + sed '/host.docker.internal/d' /etc/hosts > /tmp/hosts.new && cat /tmp/hosts.new > /etc/hosts + ip r | grep "default via" | cut -f3 -d" " | tr -d "\n" >> /etc/hosts + echo " host.docker.internal" >> /etc/hosts +fi + +mkdir -p -m 700 ./var +mkdir -p -m 700 ./var/cache +mkdir -p -m 700 ./var/log +mkdir -p -m 700 ./var/sessions + +for TARGET in \ + /var/www/html/var \ + /composer \ +; do + setfacl -R -m u:www-data:rwX -m u:"$DOCKER_UID":rwX "$TARGET" + setfacl -dR -m u:www-data:rwX -m u:"$DOCKER_UID":rwX "$TARGET" +done + +exec php-fpm diff --git a/docker/php-fpm/php.ini b/docker/php-fpm/php.ini index 74613ecc0..c0bedf278 100644 --- a/docker/php-fpm/php.ini +++ b/docker/php-fpm/php.ini @@ -1365,7 +1365,7 @@ session.save_handler = files ; where MODE is the octal representation of the mode. Note that this ; does not overwrite the process's umask. ; https://php.net/session.save-path -;session.save_path = "/tmp" +session.save_path = "/var/www/html/var/sessions" ; Whether to use strict session mode. ; Strict session mode does not accept an uninitialized session ID, and diff --git a/docker/php-fpm/php.ini-production.dist b/docker/php-fpm/php.ini-production.dist index 21627c914..cb36654a7 100644 --- a/docker/php-fpm/php.ini-production.dist +++ b/docker/php-fpm/php.ini-production.dist @@ -425,6 +425,11 @@ max_input_time = 60 ; How many GET/POST/COOKIE input variables may be accepted ;max_input_vars = 1000 +; How many multipart body parts (combined input variable and file uploads) may +; be accepted. +; Default Value: -1 (Sum of max_input_vars and max_file_uploads) +;max_multipart_body_parts = 1500 + ; Maximum amount of memory a script may consume ; https://php.net/memory-limit memory_limit = 128M @@ -1097,6 +1102,10 @@ smtp_port = 25 ; Add X-PHP-Originating-Script: that will include uid of the script followed by the filename mail.add_x_header = Off +; Use mixed LF and CRLF line separators to keep compatibility with some +; RFC 2822 non conformant MTA. +mail.mixed_lf_and_crlf = Off + ; The path to a log file that will log all mail() calls. Log entries include ; the full path of the script, line number, To address and headers. ;mail.log = @@ -1898,8 +1907,13 @@ ldap.max_links = -1 ;opcache.file_cache_fallback=1 ; Enables or disables copying of PHP code (text segment) into HUGE PAGES. -; This should improve performance, but requires appropriate OS configuration. -;opcache.huge_code_pages=1 +; Under certain circumstances (if only a single global PHP process is +; started from which all others fork), this can increase performance +; by a tiny amount because TLB misses are reduced. On the other hand, this +; delays PHP startup, increases memory usage and degrades performance +; under memory pressure - use with care. +; Requires appropriate OS configuration. +;opcache.huge_code_pages=0 ; Validate cached file permissions. ;opcache.validate_permission=0 diff --git a/toolbox b/toolbox index c0fda8155..9c3aaaf4c 100755 --- a/toolbox +++ b/toolbox @@ -47,12 +47,6 @@ function assure_line_in_file() { } function action_run_setup() { - # FIXME: https://github.com/veelkoov/fuzzrake/issues/168 - run_command sudo mkdir -p ./var/cache - run_command sudo mkdir -p ./var/log - run_command sudo chmod -R a+w ./var - run_command touch ./var/db.sqlite - assure_line_in_file ./.env.local '^GOOGLE_RECAPTCHA_SITE_KEY=' 'GOOGLE_RECAPTCHA_SITE_KEY=__TODO_PROVIDE_THIS__' assure_line_in_file ./.env.local '^GOOGLE_RECAPTCHA_SECRET=' 'GOOGLE_RECAPTCHA_SECRET=__TODO_PROVIDE_THIS__' @@ -222,9 +216,9 @@ function action() { 'console') run_console "$@" ;; - # FIXME: https://github.com/veelkoov/fuzzrake/issues/168 - 'cc') run_command sudo rm -rf ./var/cache/* && sudo chmod a+w ./var/cache ;; - 'cc-prod') run_command ssh getfursu.it sudo docker exec fuzzrake-prod bin/console cache:clear ;; + 'cc') run_command run_console cache:clear ;; + 'cc-beta') run_command ssh getfursu.it docker exec fuzzrake-beta bin/console cache:clear ;; + 'cc-prod') run_command ssh getfursu.it docker exec fuzzrake-prod bin/console cache:clear ;; 'pu') run_docker_compose_exec ./bin/phpunit --testdox "$@" ;; 'pus') action pu --group small "$@" ;;