OSAC-1568: Add local Vault infrastructure and e2e workflow

.github/workflows/e2e-test.yml

@@ -0,0 +1,23 @@
+---
+name: E2E Tests
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      test-suite:
+        description: "Test suite (vmaas, caas, or empty for all)"
+        required: false
+        default: "vmaas"
+      test-filter:
+        description: "pytest -k filter"
+        required: false
+        default: ""
+
+jobs:
+  e2e-vmaas:
+    uses: ./.github/workflows/e2e-vault-secrets.yml
+    with:
+      test-suite: ${{ github.event.inputs.test-suite || 'vmaas' }}
+      test-filter: ${{ github.event.inputs.test-filter || '' }}

.github/workflows/e2e-vault-secrets.yml

@@ -0,0 +1,108 @@
+---
+name: OSAC E2E — Vault Secrets
+
+on:
+  workflow_call:
+    inputs:
+      test-suite:
+        description: "Test suite to run: vmaas, caas, or empty string for all"
+        required: false
+        type: string
+        default: "vmaas"
+      test-filter:
+        description: "pytest -k filter expression (optional)"
+        required: false
+        type: string
+        default: ""
+      namespace:
+        description: "OSAC namespace on the cluster"
+        required: false
+        type: string
+        default: "osac-e2e-ci"
+      vm-template:
+        description: "VM template for compute instances"
+        required: false
+        type: string
+        default: "osac.templates.ocp_virt_vm"
+      test-image:
+        description: "Test container image override"
+        required: false
+        type: string
+        default: "ghcr.io/osac-project/osac-test-infra:latest"
+    outputs:
+      result:
+        description: "Test result: PASSED or FAILED"
+        value: ${{ jobs.e2e.outputs.result }}
+
+jobs:
+  e2e:
+    runs-on: osac-ci
+    environment: e2e-test
+    permissions:
+      contents: read
+      id-token: write
+    outputs:
+      result: ${{ steps.test.outcome == 'success' && 'PASSED' || 'FAILED' }}
+
+    steps:
+      - name: Retrieve secrets from Vault
+        id: vault
+        uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4
+        with:
+          url: http://127.0.0.1:8200
+          method: jwt
+          role: osac-e2e
+          jwtGithubAudience: https://github.com/osac-project
+          exportEnv: false
+          secrets: |
+            secret/data/osac/e2e/pull-secret dockerconfigjson | PULL_SECRET ;
+            secret/data/osac/e2e/aap-license license | AAP_LICENSE
+
+      - name: Write secrets to files
+        env:
+          PULL_SECRET: ${{ steps.vault.outputs.PULL_SECRET }}
+          AAP_LICENSE: ${{ steps.vault.outputs.AAP_LICENSE }}
+        run: |
+          printf '%s' "${PULL_SECRET}" > "$RUNNER_TEMP/pull-secret.json"
+          chmod 600 "$RUNNER_TEMP/pull-secret.json"
+
+          printf '%s' "${AAP_LICENSE}" | base64 -d > "$RUNNER_TEMP/aap-license.zip"
+          chmod 600 "$RUNNER_TEMP/aap-license.zip"
+
+      - name: Run E2E tests
+        id: test
+        env:
+          TEST_SUITE: ${{ inputs.test-suite }}
+          TEST_FILTER: ${{ inputs.test-filter }}
+          OSAC_NAMESPACE: ${{ inputs.namespace }}
+          OSAC_VM_TEMPLATE: ${{ inputs.vm-template }}
+          TEST_IMAGE: ${{ inputs.test-image }}
+        run: |
+          TEST_ARGS="tests/"
+          if [ -n "${TEST_SUITE}" ]; then
+            TEST_ARGS="tests/${TEST_SUITE}/"
+          fi
+
+          FILTER_ARGS=""
+          if [ -n "${TEST_FILTER}" ]; then
+            FILTER_ARGS="-k ${TEST_FILTER}"
+          fi
+
+          podman run --rm \
+            --network host \
+            -v "${HOME}/.kube/config:/root/.kube/config:ro,Z" \
+            -v "${RUNNER_TEMP}/pull-secret.json:/root/pull-secret:ro,Z" \
+            -v "${RUNNER_TEMP}/aap-license.zip:/root/aap-license.zip:ro,Z" \
+            -e KUBECONFIG=/root/.kube/config \
+            -e OSAC_VM_KUBECONFIG=/root/.kube/config \
+            -e OSAC_NAMESPACE="${OSAC_NAMESPACE}" \
+            -e OSAC_VM_TEMPLATE="${OSAC_VM_TEMPLATE}" \
+            -e OSAC_PULL_SECRET_PATH=/root/pull-secret \
+            "${TEST_IMAGE}" \
+            pytest ${TEST_ARGS} ${FILTER_ARGS} \
+              --junitxml=/tmp/junit.xml -v
+
+      - name: Clean up secrets
+        if: always()
+        run: |
+          rm -f "$RUNNER_TEMP/pull-secret.json" "$RUNNER_TEMP/aap-license.zip"

vault/config/vault.hcl

@@ -0,0 +1,20 @@
+# Vault server configuration for OSAC CI runner
+# Deployed by vault-setup.sh to ~/.vault-server/config/vault.hcl
+
+storage "file" {
+  path = "/vault/data"
+}
+
+listener "tcp" {
+  # Bind to all interfaces inside the container so Podman's port-forward
+  # works.  Host-side exposure is restricted to 127.0.0.1 by the Quadlet
+  # PublishPort directive.
+  address     = "0.0.0.0:8200"
+  tls_disable = 1
+}
+
+# Vault 2.0.2 removed IPC_LOCK from the official container image;
+# disabling mlock avoids the need for the capability in rootless Podman.
+disable_mlock = true
+
+ui = true

vault/quadlet/vault.container

@@ -0,0 +1,43 @@
+# Podman Quadlet unit file for HashiCorp Vault
+# Installed by vault-setup.sh to ~/.config/containers/systemd/vault.container
+#
+# Quadlet generates a systemd unit (vault.service) from this file.
+# See: man podman-systemd.unit(5)
+
+[Unit]
+Description=HashiCorp Vault (OSAC CI)
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+Image=docker.io/hashicorp/vault:2.0.3
+ContainerName=vault
+
+# Localhost-only binding -- no external exposure
+PublishPort=127.0.0.1:8200:8200
+
+# Run the vault server with our config
+Exec=server
+
+# Mount host directories into the container.
+# :Z applies the correct SELinux context for rootless Podman.
+Volume=%h/.vault-server/config:/vault/config:Z
+Volume=%h/.vault-server/data:/vault/data:Z
+Volume=%h/.vault-server/logs:/vault/logs:Z
+
+# Environment for rootless operation
+Environment=SKIP_CHOWN=true
+Environment=SKIP_SETCAP=true
+Environment=VAULT_ADDR=http://127.0.0.1:8200
+
+# Version pinned for CI stability -- update deliberately after testing
+
+[Service]
+Restart=always
+RestartSec=10
+TimeoutStartSec=120
+# Auto-unseal after the container starts
+ExecStartPost=%h/.vault-server/scripts/vault-unseal.sh
+
+[Install]
+WantedBy=default.target

vault/scripts/vault-backup.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# vault-backup.sh -- Backup Vault's encrypted data directory.
+# Called by vault-backup.service (daily via vault-backup.timer).
+set -euo pipefail
+
+VAULT_HOME="${HOME}/.vault-server"
+BACKUP_DIR="${VAULT_HOME}/backups"
+DATA_DIR="${VAULT_HOME}/data"
+RETENTION_DAYS=30
+
+mkdir -p "${BACKUP_DIR}"
+
+###############################################################################
+# Create backup
+###############################################################################
+timestamp=$(date +%Y%m%d-%H%M%S)
+backup_file="${BACKUP_DIR}/vault-data-${timestamp}.tar.gz"
+
+echo "[vault-backup] Backing up ${DATA_DIR} ..."
+tar -czf "${backup_file}" -C "${VAULT_HOME}" data
+chmod 0600 "${backup_file}"
+echo "[vault-backup] Backup created: ${backup_file}"
+
+###############################################################################
+# Prune old backups
+###############################################################################
+echo "[vault-backup] Pruning backups older than ${RETENTION_DAYS} days ..."
+pruned=$(find "${BACKUP_DIR}" -name "vault-data-*.tar.gz" -mtime "+${RETENTION_DAYS}" -delete -print | wc -l)
+echo "[vault-backup] Pruned ${pruned} old backup(s)."

vault/scripts/vault-health-check.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# vault-health-check.sh -- 10-point health check for the OSAC Vault instance.
+set -euo pipefail
+
+VAULT_ADDR="${VAULT_ADDR:-http://127.0.0.1:8200}"
+export VAULT_ADDR
+
+# Checks 5-8 query authenticated Vault endpoints; source root/admin token.
+INIT_JSON="${HOME}/.vault-server/.vault-init.json"
+if [[ -f "${INIT_JSON}" ]]; then
+    export VAULT_TOKEN
+    VAULT_TOKEN=$(jq -r '.root_token' "${INIT_JSON}")
+elif [[ -z "${VAULT_TOKEN:-}" ]]; then
+    echo "WARNING: No VAULT_TOKEN set and ${INIT_JSON} not found."
+    echo "         Authenticated checks (5-8) will fail."
+fi
+
+passed=0
+failed=0
+
+check() {
+    local num="$1" name="$2"
+    shift 2
+    if "$@" >/dev/null 2>&1; then
+        echo "  [PASS] ${num}. ${name}"
+        passed=$(( passed + 1 ))
+    else
+        echo "  [FAIL] ${num}. ${name}"
+        failed=$(( failed + 1 ))
+    fi
+}
+
+echo "=== Vault Health Check ==="
+echo ""
+
+# 1. Vault reachable
+check 1 "Vault is reachable" vault status -format=json
+
+# 2. Vault unsealed
+check 2 "Vault is unsealed" \
+    bash -c 'test "$(vault status -format=json | jq -r .sealed)" = "false"'
+
+# 3. Vault initialized
+check 3 "Vault is initialized" \
+    bash -c 'test "$(vault status -format=json | jq -r .initialized)" = "true"'
+
+# 4. Vault version reported
+check 4 "Vault version reported" \
+    bash -c 'vault status -format=json | jq -e .version'
+
+# 5. JWT auth enabled
+check 5 "JWT auth method enabled" \
+    bash -c 'vault auth list -format=json | jq -e ".\"jwt/\""'
+
+# 6. osac-e2e role exists
+check 6 "osac-e2e role exists" \
+    vault read auth/jwt/role/osac-e2e
+
+# 7. osac-e2e policy exists
+check 7 "osac-e2e policy exists" \
+    vault policy read osac-e2e
+
+# 8. KV v2 at secret/
+check 8 "KV v2 secrets engine at secret/" \
+    bash -c 'vault secrets list -format=json | jq -e ".\"secret/\""'
+
+# 9. systemd vault.service active
+check 9 "vault.service is active" \
+    systemctl --user is-active vault.service
+
+# 10. backup timer active
+check 10 "vault-backup.timer is active" \
+    systemctl --user is-active vault-backup.timer
+
+echo ""
+echo "=== Results: ${passed} passed, ${failed} failed ==="
+
+if (( failed > 0 )); then
+    exit 1
+fi

vault/scripts/vault-prerequisites.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# vault-prerequisites.sh -- Install required packages for the OSAC Vault instance.
+# Run as root (or with sudo) on a RHEL/CentOS/Fedora host.
+set -euo pipefail
+
+if (( EUID != 0 )); then
+    echo "ERROR: This script must be run as root (or via sudo)." >&2
+    exit 1
+fi
+
+###############################################################################
+# Detect package manager
+###############################################################################
+if command -v dnf &>/dev/null; then
+    PKG_MGR="dnf"
+elif command -v yum &>/dev/null; then
+    PKG_MGR="yum"
+else
+    echo "ERROR: Neither dnf nor yum found." >&2
+    exit 1
+fi
+
+###############################################################################
+# Install system packages
+###############################################################################
+if [[ "${PKG_MGR}" == "dnf" ]]; then
+    CONFIG_MGR_PKG="dnf-plugins-core"
+else
+    CONFIG_MGR_PKG="yum-utils"
+fi
+
+echo "==> Installing podman, jq, and ${CONFIG_MGR_PKG} ..."
+${PKG_MGR} install -y podman jq "${CONFIG_MGR_PKG}"
+
+###############################################################################
+# Add HashiCorp repo and install Vault CLI
+###############################################################################
+HASHICORP_REPO="/etc/yum.repos.d/hashicorp.repo"
+if [[ -f "${HASHICORP_REPO}" ]]; then
+    echo "==> HashiCorp repo already configured."
+else
+    echo "==> Adding HashiCorp RPM repository ..."
+    if [[ "${PKG_MGR}" == "dnf" ]]; then
+        dnf config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
+    else
+        yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
+    fi
+fi
+
+echo "==> Installing Vault CLI ..."
+${PKG_MGR} install -y vault
+
+###############################################################################
+# Verify
+###############################################################################
+echo ""
+echo "=== Installed versions ==="
+echo "  podman : $(podman --version)"
+echo "  vault  : $(vault --version)"
+echo "  jq     : $(jq --version)"
+echo ""
+echo "Prerequisites satisfied. Run vault-setup.sh next."