diff --git a/.github/ansible/production/nest.yaml b/.github/ansible/production/nest.yaml index 90f9594ca2..b727088a9e 100644 --- a/.github/ansible/production/nest.yaml +++ b/.github/ansible/production/nest.yaml @@ -1,76 +1,90 @@ - - name: Deploy Nest to Production - hosts: production_nest - tasks: - - name: Copy docker-compose.yaml - ansible.builtin.copy: - src: '{{ github_workspace }}/docker-compose/production/compose.yaml' - dest: ~/docker-compose.yaml - mode: '0644' +- name: Deploy Nest to Production + hosts: production_nest + tasks: + - name: Copy docker-compose.yaml + ansible.builtin.copy: + src: '{{ github_workspace }}/docker-compose/production/compose.yaml' + dest: ~/docker-compose.yaml + mode: '0644' - - name: Sync Makefile structure - ansible.builtin.synchronize: - src: '{{ github_workspace }}/' - dest: '~/' - recursive: yes - rsync_opts: - - '--include=*/' - - '--include=Makefile' - - '--include=*/Makefile' - - '--include=*/**/Makefile' - - '--include=*/**/**/Makefile' - - '--exclude=*' + - name: Sync Makefile structure + ansible.posix.synchronize: + src: '{{ github_workspace }}/' + dest: '~/' + recursive: true + rsync_opts: + - '--include=*/' + - '--include=Makefile' + - '--include=*/Makefile' + - '--include=*/**/Makefile' + - '--include=*/**/**/Makefile' + - '--exclude=*' - - name: Update Makefiles for production environment - shell: | - sed -i '/e2e-\|fuzz-/! s/\bnest-backend\b/production-nest-backend/g' ~/backend/Makefile - sed -i '/e2e-\|fuzz-/! s/\bnest-db\b/production-nest-db/g' ~/backend/Makefile - sed -i 's/\bnest-frontend\b/production-nest-frontend/g' ~/frontend/Makefile + - name: Update Makefiles for production environment + ansible.builtin.command: + argv: + - sed + - -i + - "{{ item.sed_expr }}" + - "{{ item.path }}" + loop: + - sed_expr: '/e2e-\|fuzz-/! s/\bnest-backend\b/production-nest-backend/g' + path: '{{ ansible_env.HOME }}/backend/Makefile' + - sed_expr: '/e2e-\|fuzz-/! s/\bnest-db\b/production-nest-db/g' + path: '{{ ansible_env.HOME }}/backend/Makefile' + - sed_expr: 's/\bnest-frontend\b/production-nest-frontend/g' + path: '{{ ansible_env.HOME }}/frontend/Makefile' + changed_when: false - - name: Copy secrets - copy: - src: '{{ github_workspace }}/{{ item }}' - dest: ~/ - mode: '0400' - loop: - - .env.backend - - .env.cache - - .env.db - - .env.frontend - - .github.pem + - name: Copy secrets + ansible.builtin.copy: + src: '{{ github_workspace }}/{{ item }}' + dest: ~/ + mode: '0400' + loop: + - .env.backend + - .env.cache + - .env.db + - .env.frontend + - .github.pem - - name: Clean up secrets - delegate_to: localhost - file: - path: '{{ github_workspace }}/{{ item }}' - state: absent - loop: - - .env.backend - - .env.cache - - .env.db - - .env.frontend - - .github.pem - run_once: true + - name: Clean up secrets + delegate_to: localhost + ansible.builtin.file: + path: '{{ github_workspace }}/{{ item }}' + state: absent + loop: + - .env.backend + - .env.cache + - .env.db + - .env.frontend + - .github.pem - - name: Copy crontab - copy: - src: '{{ github_workspace }}/cron/production' - dest: /tmp/production_crontab - mode: '0600' + - name: Copy crontab + ansible.builtin.copy: + src: '{{ github_workspace }}/cron/production' + dest: /tmp/production_crontab + mode: '0600' - - name: Install crontab - ansible.builtin.command: - cmd: crontab /tmp/production_crontab + - name: Install crontab + ansible.builtin.command: + cmd: crontab /tmp/production_crontab + changed_when: false - - name: Restart services - shell: - cmd: docker compose up -d --pull always + - name: Restart services + ansible.builtin.command: + cmd: docker compose up -d --pull always + changed_when: false - - name: Prune docker images - shell: - cmd: docker image prune -f + - name: Prune docker images + ansible.builtin.command: + cmd: docker image prune -f + changed_when: false - - name: Index data - async: 1800 # 30 minutes - poll: 0 - shell: | - make index-data > /var/log/nest/production/index-data.log 2>&1 + - name: Index data + async: 1800 # 30 minutes + poll: 0 + # Shell required for stdout/stderr redirect to log file. + ansible.builtin.shell: | + make index-data > /var/log/nest/production/index-data.log 2>&1 + changed_when: false diff --git a/.github/ansible/production/proxy.yaml b/.github/ansible/production/proxy.yaml index 0bf1bebccd..5c06a7200e 100644 --- a/.github/ansible/production/proxy.yaml +++ b/.github/ansible/production/proxy.yaml @@ -2,7 +2,7 @@ hosts: production_nest_proxy tasks: - name: Copy proxy configuration files - copy: + ansible.builtin.copy: src: '{{ github_workspace }}/proxy/{{ item }}' dest: ~/ mode: '0644' @@ -15,15 +15,26 @@ - redirects.conf - name: Copy docker compose file - copy: + ansible.builtin.copy: src: '{{ github_workspace }}/docker-compose/proxy/compose.yaml' dest: ~/docker-compose.yaml mode: '0644' + - name: Pull and start services + ansible.builtin.command: + cmd: docker compose up -d --pull always + args: + chdir: "{{ ansible_env.HOME }}" + changed_when: false + - name: Restart services - shell: - cmd: docker compose up -d --pull always && docker compose restart + ansible.builtin.command: + cmd: docker compose restart + args: + chdir: "{{ ansible_env.HOME }}" + changed_when: false - name: Prune docker images - shell: + ansible.builtin.command: cmd: docker image prune -f + changed_when: false diff --git a/.github/ansible/requirements.yaml b/.github/ansible/requirements.yaml new file mode 100644 index 0000000000..e3639a122c --- /dev/null +++ b/.github/ansible/requirements.yaml @@ -0,0 +1,2 @@ +collections: + - name: ansible.posix diff --git a/.github/ansible/staging/nest.yaml b/.github/ansible/staging/nest.yaml index 8986a6456c..d234e8130f 100644 --- a/.github/ansible/staging/nest.yaml +++ b/.github/ansible/staging/nest.yaml @@ -1,80 +1,94 @@ - - name: Deploy Nest to Staging - hosts: staging_nest - tasks: - - name: Copy docker-compose.yaml - ansible.builtin.copy: - src: '{{ github_workspace }}/docker-compose/staging/compose.yaml' - dest: ~/docker-compose.yaml - mode: '0644' +- name: Deploy Nest to Staging + hosts: staging_nest + tasks: + - name: Copy docker-compose.yaml + ansible.builtin.copy: + src: '{{ github_workspace }}/docker-compose/staging/compose.yaml' + dest: ~/docker-compose.yaml + mode: '0644' - - name: Sync Makefile structure - ansible.builtin.synchronize: - src: '{{ github_workspace }}/' - dest: '~/' - recursive: yes - rsync_opts: - - '--include=*/' - - '--include=Makefile' - - '--include=*/Makefile' - - '--include=*/**/Makefile' - - '--include=*/**/**/Makefile' - - '--exclude=*' + - name: Sync Makefile structure + ansible.posix.synchronize: + src: '{{ github_workspace }}/' + dest: '~/' + recursive: true + rsync_opts: + - '--include=*/' + - '--include=Makefile' + - '--include=*/Makefile' + - '--include=*/**/Makefile' + - '--include=*/**/**/Makefile' + - '--exclude=*' - - name: Update Makefiles for staging environment - shell: | - sed -i '/e2e-\|fuzz-/! s/\bnest-backend\b/staging-nest-backend/g' ~/backend/Makefile - sed -i '/e2e-\|fuzz-/! s/\bnest-db\b/staging-nest-db/g' ~/backend/Makefile - sed -i 's/\bnest-frontend\b/staging-nest-frontend/g' ~/frontend/Makefile + - name: Update Makefiles for staging environment + ansible.builtin.command: + argv: + - sed + - -i + - "{{ item.sed_expr }}" + - "{{ item.path }}" + loop: + - sed_expr: '/e2e-\|fuzz-/! s/\bnest-backend\b/staging-nest-backend/g' + path: '{{ ansible_env.HOME }}/backend/Makefile' + - sed_expr: '/e2e-\|fuzz-/! s/\bnest-db\b/staging-nest-db/g' + path: '{{ ansible_env.HOME }}/backend/Makefile' + - sed_expr: 's/\bnest-frontend\b/staging-nest-frontend/g' + path: '{{ ansible_env.HOME }}/frontend/Makefile' + changed_when: false - - name: Ensure data directory exists - ansible.builtin.file: - path: ~/backend/data - state: directory - mode: '0755' + - name: Ensure data directory exists + ansible.builtin.file: + path: ~/backend/data + state: directory + mode: '0755' - - name: Copy secrets - copy: - src: '{{ github_workspace }}/{{ item }}' - dest: ~/ - mode: '0400' - loop: - - .env.backend - - .env.cache - - .env.db - - .env.frontend + - name: Copy secrets + ansible.builtin.copy: + src: '{{ github_workspace }}/{{ item }}' + dest: ~/ + mode: '0400' + loop: + - .env.backend + - .env.cache + - .env.db + - .env.frontend - - name: Clean up secrets - delegate_to: localhost - file: - path: '{{ github_workspace }}/{{ item }}' - state: absent - loop: - - .env.backend - - .env.cache - - .env.db - - .env.frontend - run_once: true + - name: Clean up secrets + delegate_to: localhost + ansible.builtin.file: + path: '{{ github_workspace }}/{{ item }}' + state: absent + loop: + - .env.backend + - .env.cache + - .env.db + - .env.frontend - - name: Copy crontab - copy: - src: '{{ github_workspace }}/cron/staging' - dest: /tmp/staging_crontab - mode: '0600' + - name: Copy crontab + ansible.builtin.copy: + src: '{{ github_workspace }}/cron/staging' + dest: /tmp/staging_crontab + mode: '0600' - - name: Install crontab - ansible.builtin.command: - cmd: crontab /tmp/staging_crontab + - name: Install crontab + ansible.builtin.command: + cmd: crontab /tmp/staging_crontab + changed_when: false - - name: Restart services - shell: - cmd: docker compose up -d --pull always + - name: Restart services + ansible.builtin.command: + cmd: docker compose up -d --pull always + changed_when: false - - name: Prune docker images - shell: - cmd: docker image prune -f + - name: Prune docker images + ansible.builtin.command: + cmd: docker image prune -f + changed_when: false - - name: Index data - async: 1800 # 30 minutes - poll: 0 - shell: | - make index-data > /var/log/nest/staging/index-data.log 2>&1 + - name: Index data + async: 1800 # 30 minutes + poll: 0 + # Shell required for stdout/stderr redirect to log file. + ansible.builtin.shell: | + make index-data > /var/log/nest/staging/index-data.log 2>&1 + changed_when: false diff --git a/.github/ansible/staging/proxy.yaml b/.github/ansible/staging/proxy.yaml index dc75d8c107..7eb0981065 100644 --- a/.github/ansible/staging/proxy.yaml +++ b/.github/ansible/staging/proxy.yaml @@ -2,7 +2,7 @@ hosts: staging_nest_proxy tasks: - name: Copy proxy configuration files - copy: + ansible.builtin.copy: src: '{{ github_workspace }}/proxy/{{ item }}' dest: ~/ mode: '0644' @@ -15,15 +15,26 @@ - staging.conf - name: Copy docker compose file - copy: + ansible.builtin.copy: src: '{{ github_workspace }}/docker-compose/proxy/compose.yaml' dest: ~/docker-compose.yaml mode: '0644' + - name: Pull and start services + ansible.builtin.command: + cmd: docker compose up -d --pull always + args: + chdir: "{{ ansible_env.HOME }}" + changed_when: false + - name: Restart services - shell: - cmd: docker compose up -d --pull always && docker compose restart + ansible.builtin.command: + cmd: docker compose restart + args: + chdir: "{{ ansible_env.HOME }}" + changed_when: false - name: Prune docker images - shell: + ansible.builtin.command: cmd: docker image prune -f + changed_when: false diff --git a/.github/workflows/run-ci-cd.yaml b/.github/workflows/run-ci-cd.yaml index 3c4b1b8cbc..e90b5c8de1 100644 --- a/.github/workflows/run-ci-cd.yaml +++ b/.github/workflows/run-ci-cd.yaml @@ -52,6 +52,11 @@ jobs: restore-keys: | pre-commit-${{ runner.os }}- + - name: Install Ansible collections for ansible-lint + run: | + pip install ansible-core + ansible-galaxy collection install -r .github/ansible/requirements.yaml + - name: Run pre-commit uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd @@ -668,9 +673,13 @@ jobs: NEXTAUTH_URL=$NEXTAUTH_URL EOF - - name: Run Nest deploy + - name: Install Ansible collections + run: ansible-galaxy collection install -r requirements.yaml working-directory: .github/ansible + + - name: Run Nest deploy run: ansible-playbook -i inventory.yaml staging/nest.yaml -e "github_workspace=$GITHUB_WORKSPACE" + working-directory: .github/ansible timeout-minutes: 5 deploy-staging-nest-proxy: @@ -704,9 +713,13 @@ jobs: $PROXY_SSH_PRIVATE_KEY EOF - - name: Run proxy deploy + - name: Install Ansible collections + run: ansible-galaxy collection install -r requirements.yaml working-directory: .github/ansible + + - name: Run proxy deploy run: ansible-playbook -i inventory.yaml staging/proxy.yaml -e "github_workspace=$GITHUB_WORKSPACE" + working-directory: .github/ansible timeout-minutes: 5 run-staging-lighthouse-ci: @@ -1038,9 +1051,13 @@ jobs: "$NEST_GITHUB_APP_PRIVATE_KEY" EOF - - name: Run Nest deploy + - name: Install Ansible collections + run: ansible-galaxy collection install -r requirements.yaml working-directory: .github/ansible + + - name: Run Nest deploy run: ansible-playbook -i inventory.yaml production/nest.yaml -e "github_workspace=$GITHUB_WORKSPACE" + working-directory: .github/ansible timeout-minutes: 5 deploy-production-nest-proxy: @@ -1074,9 +1091,13 @@ jobs: $PROXY_SSH_PRIVATE_KEY EOF - - name: Run proxy deploy + - name: Install Ansible collections + run: ansible-galaxy collection install -r requirements.yaml working-directory: .github/ansible + + - name: Run proxy deploy run: ansible-playbook -i inventory.yaml production/proxy.yaml -e "github_workspace=$GITHUB_WORKSPACE" + working-directory: .github/ansible timeout-minutes: 5 run-production-zap-baseline-scan: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0fffc4946..acb5d8c91a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,16 @@ repos: - --strict exclude: (.github|pnpm-lock.yaml) + - repo: https://github.com/ansible/ansible-lint + rev: v26.1.1 + hooks: + - id: ansible-lint + args: + - .github/ansible + files: ^\.github/ansible/.*\.ya?ml$ + language_version: python3 + pass_filenames: true + - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.0 hooks: