Add integration tests for force rotation and revocation

test/integration/common

@@ -107,6 +107,23 @@ check-server-started() {
   fail-now "timed out waiting for server to start"
 }
 
+check-log-line() {
+  # Check at most 30 times (with one second in between) that the agent has
+  # successfully synced down the workload entry.
+  MAXCHECKS=30
+  CHECKINTERVAL=1
+  for ((i=1;i<=MAXCHECKS;i++)); do
+      log-info "checking for log line ($i of $MAXCHECKS max)..."
+      if docker compose logs "$1" | grep -E "$2"; then
+          return 0
+      fi
+      sleep "${CHECKINTERVAL}"
+  done
+
+  docker compose logs "$1"
+  fail-now "timed out waiting for "$2" in log in container $1"
+}
+
 check-synced-entry() {
   # Check at most 30 times (with one second in between) that the agent has
   # successfully synced down the workload entry.

test/integration/suites/force-rotation-self-signed/00-setup

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# create shared folder for root agent socket
+mkdir -p -m 777 shared/rootSocket
+
+# create shared folder for intermediateA agent socket
+mkdir -p -m 777 shared/intermediateASocket
+
+# create shared folder for intermediateB agent socket
+mkdir -p -m 777 shared/intermediateBSocket
+
+# root certificates
+"${ROOTDIR}/setup/x509pop/setup.sh" root/server root/agent
+
+# intermediateA certificates
+"${ROOTDIR}/setup/x509pop/setup.sh" intermediateA/server intermediateA/agent
+
+# leafA certificates
+"${ROOTDIR}/setup/x509pop/setup.sh" leafA/server leafA/agent
+
+# intermediateB certificates
+"${ROOTDIR}/setup/x509pop/setup.sh" intermediateB/server intermediateB/agent
+
+# leafB certificates
+"${ROOTDIR}/setup/x509pop/setup.sh" leafB/server leafB/agent
+
+docker build --target nested-agent-alpine -t nested-agent-alpine .

test/integration/suites/force-rotation-self-signed/01-start-root

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+log-debug "Starting root-server..."
+docker-up root-server
+check-server-started "root-server"
+
+log-debug "bootstrapping root-agent..."
+docker compose exec -T root-server \
+    /opt/spire/bin/spire-server bundle show > root/agent/bootstrap.crt
+
+log-debug "Starting root-agent..."
+docker-up root-agent

test/integration/suites/force-rotation-self-signed/02-create-intermediate-downstream-entries

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+log-debug "creating intermediateA downstream registration entry..."
+docker compose exec -T root-server \
+    /opt/spire/bin/spire-server entry create \
+    -parentID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint root/agent/agent.crt.pem)" \
+    -spiffeID "spiffe://domain.test/intermediateA" \
+    -selector "docker:label:org.integration.name:intermediateA" \
+    -downstream
+check-synced-entry "root-agent" "spiffe://domain.test/intermediateA"
+
+log-debug "creating intermediateB downstream registration entry..."
+docker compose exec -T root-server \
+    /opt/spire/bin/spire-server entry create \
+    -parentID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint root/agent/agent.crt.pem)" \
+    -spiffeID "spiffe://domain.test/intermediateB" \
+    -selector "docker:label:org.integration.name:intermediateB" \
+    -downstream
+check-synced-entry "root-agent" "spiffe://domain.test/intermediateB"

test/integration/suites/force-rotation-self-signed/03-start-intermediateA

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+log-debug "Starting intermediateA-server.."
+docker-up intermediateA-server
+check-server-started "intermediateA-server"
+
+log-debug "bootstrapping intermediateA agent..."
+docker compose exec -T intermediateA-server \
+    /opt/spire/bin/spire-server bundle show > intermediateA/agent/bootstrap.crt
+
+log-debug "Starting intermediateA-agent..."
+docker-up intermediateA-agent

test/integration/suites/force-rotation-self-signed/04-create-leafA-downstream-entry

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+log-debug "creating leafA downstream registration entry..."
+# Create downstream registation entry on intermediateA-server for `leafA-server`
+docker compose exec -T intermediateA-server \
+    /opt/spire/bin/spire-server entry create \
+    -parentID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint intermediateA/agent/agent.crt.pem)" \
+    -spiffeID "spiffe://domain.test/leafA" \
+    -selector "docker:label:org.integration.name:leafA" \
+    -downstream
+
+check-synced-entry "intermediateA-agent" "spiffe://domain.test/leafA"

test/integration/suites/force-rotation-self-signed/05-start-leafA

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+log-debug "Starting leafA-server.."
+docker-up leafA-server
+check-server-started "leafA-server"
+
+log-debug "bootstrapping leafA agent..."
+docker compose exec -T leafA-server \
+    /opt/spire/bin/spire-server bundle show > leafA/agent/bootstrap.crt
+
+log-debug "Starting leafA-agent..."
+docker-up leafA-agent

test/integration/suites/force-rotation-self-signed/06-start-intermediateB

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+log-debug "Starting intermediateB-server.."
+docker-up intermediateB-server
+check-server-started "intermediateB-server"
+
+log-debug "bootstrapping intermediateB downstream agent..."
+docker compose exec -T intermediateB-server \
+    /opt/spire/bin/spire-server bundle show > intermediateB/agent/bootstrap.crt
+
+log-debug "Starting intermediateB-agent..."
+docker-up intermediateB-agent

test/integration/suites/force-rotation-self-signed/07-create-leafB-downstream-entry

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+log-debug "creating leafB downstream registration entry..."
+# Create downstream registration entry on itermediateB for leafB-server
+docker compose exec -T intermediateB-server \
+    /opt/spire/bin/spire-server entry create \
+    -parentID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint intermediateB/agent/agent.crt.pem)" \
+    -spiffeID "spiffe://domain.test/leafB" \
+    -selector "docker:label:org.integration.name:leafB" \
+    -downstream
+
+check-synced-entry "intermediateB-agent" "spiffe://domain.test/leafB"

test/integration/suites/force-rotation-self-signed/08-start-leafB

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+log-debug "Starting leafB-server.."
+docker-up leafB-server
+check-server-started "leafB-server"
+
+log-debug "bootstrapping leafB agent..."
+docker compose exec -T leafB-server \
+    /opt/spire/bin/spire-server bundle show > leafB/agent/bootstrap.crt
+
+log-debug "Starting leafB-agent..."
+docker-up leafB-agent

test/integration/suites/force-rotation-self-signed/09-create-workload-entries

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+log-debug "creating rootA workload registration entry..."
+docker compose exec -T root-server \
+    /opt/spire/bin/spire-server entry create \
+    -parentID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint root/agent/agent.crt.pem)" \
+    -spiffeID "spiffe://domain.test/root/workload" \
+    -selector "unix:uid:1001" \
+    -x509SVIDTTL 0
+check-synced-entry "root-agent" "spiffe://domain.test/root/workload"
+
+log-debug "creating intermediateA workload registration entry..."
+docker compose exec -T intermediateA-server \
+    /opt/spire/bin/spire-server entry create \
+    -parentID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint intermediateA/agent/agent.crt.pem)" \
+    -spiffeID "spiffe://domain.test/intermediateA/workload" \
+    -selector "unix:uid:1001" \
+    -x509SVIDTTL 0
+check-synced-entry "intermediateA-agent" "spiffe://domain.test/intermediateA/workload"
+
+log-debug "creating leafA workload registration entry..."
+docker compose exec -T leafA-server \
+    /opt/spire/bin/spire-server entry create \
+    -parentID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint leafA/agent/agent.crt.pem)" \
+    -spiffeID "spiffe://domain.test/leafA/workload" \
+    -selector "unix:uid:1001" \
+    -x509SVIDTTL 0
+check-synced-entry "leafA-agent" "spiffe://domain.test/leafA/workload"
+
+log-debug "creating intermediateB workload registration entry..."
+docker compose exec -T intermediateB-server \
+    /opt/spire/bin/spire-server entry create \
+    -parentID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint intermediateB/agent/agent.crt.pem)" \
+    -spiffeID "spiffe://domain.test/intermediateB/workload" \
+    -selector "unix:uid:1001" \
+    -x509SVIDTTL 0
+check-synced-entry "intermediateB-agent" "spiffe://domain.test/intermediateB/workload"
+
+log-debug "creating leafB workload registration entry..."
+docker compose exec -T leafB-server \
+    /opt/spire/bin/spire-server entry create \
+    -parentID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint leafB/agent/agent.crt.pem)" \
+    -spiffeID "spiffe://domain.test/leafB/workload" \
+    -selector "unix:uid:1001" \
+    -x509SVIDTTL 0
+check-synced-entry "leafB-agent" "spiffe://domain.test/leafB/workload"

test/integration/suites/force-rotation-self-signed/10-prepare-authority

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Constants
+MAXCHECKS=30
+RETRY_DELAY=1
+
+# Function to check x509 authorities propagation
+check-x509-authorities() {
+    local expected_bundle=$1
+    local container_name=$2
+    local retry_count=0
+
+    while [[ $retry_count -lt $MAXCHECKS ]]; do
+        log-info "Checking for x509 authorities propagation ($retry_count of $MAXCHECKS max)..."
+
+        x509_authorities=$(docker compose exec -T ${container_name} \
+            /opt/spire/bin/spire-server bundle show -output json | jq '.x509_authorities' -c)
+
+        if diff <(echo "$expected_bundle") <(echo "$x509_authorities") &>/dev/null; then
+            break
+        else
+            retry_count=$((retry_count + 1))
+            log-debug "x509 authorities not propagated on ${container_name}, retrying in $RETRY_DELAY seconds... ($retry_count/$MAXCHECKS)"
+            sleep "${RETRY_DELAY}"
+        fi
+
+        # Fail if retries exceed the maximum
+        if [[ $retry_count -eq $MAXCHECKS ]]; then
+            fail-now "Expected bundle: $expected_bundle got: $x509_authorities"
+        fi
+    done
+}
+
+# Initial check for x509 authorities in root-server
+x509_authorities=$(docker compose exec -T root-server \
+    /opt/spire/bin/spire-server bundle show -output json | jq '.x509_authorities' -c)
+
+amount_bundles=$(echo "$x509_authorities" | jq length)
+
+# Ensure only one bundle is present at the start
+if [[ $amount_bundles -ne 1 ]]; then
+    fail-now "Only one bundle expected at start"
+fi
+
+# Check x509 authorities propagation across all servers
+for server in intermediateA-server intermediateB-server leafA-server leafB-server; do
+    check-x509-authorities "$x509_authorities" "$server"
+done
+
+# Prepare authority
+prepared_authority_id=$(docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
+    /opt/spire/bin/spire-server localauthority x509 prepare -output json | jq -r .prepared_authority.authority_id)
+
+# Verify that the prepared authority is logged
+searching="X509 CA prepared.|local_authority_id=${prepared_authority_id}"
+check-log-line root-server "$searching"
+
+# Check for updated x509 authorities in root-server
+x509_authorities=$(docker compose exec -T root-server \
+    /opt/spire/bin/spire-server bundle show -output json | jq '.x509_authorities' -c)
+amount_bundles=$(echo "$x509_authorities" | jq length)
+
+# Ensure two bundles are present after preparation
+if [[ $amount_bundles -ne 2 ]]; then
+    fail-now "Two bundles expected after prepare"
+fi
+
+# Check x509 authorities propagation across all servers again
+for server in intermediateA-server intermediateB-server leafA-server leafB-server; do
+    check-x509-authorities "$x509_authorities" "$server"
+done

test/integration/suites/force-rotation-self-signed/11-activate-x509authority

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Fetch the prepared authority ID
+prepared_authority=$(docker compose exec -t -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
+    /opt/spire/bin/spire-server \
+    localauthority x509 show -output json | jq -r .prepared.authority_id) || fail-now "Failed to fetch prepared authority ID"
+
+# Activate the authority
+activated_authority=$(docker compose exec -t -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
+    /opt/spire/bin/spire-server \
+    localauthority x509 activate -authorityID "${prepared_authority}" \
+    -output json | jq -r .activated_authority.authority_id) || fail-now "Failed to activate authority"
+
+log-info "Activated authority: ${activated_authority}"
+
+# Check logs for specific lines
+check-log-line root-server "X509 CA activated|local_authority_id=${prepared_authority}"
+check-log-line root-server "Successfully rotated X\.509 CA"

test/integration/suites/force-rotation-self-signed/12-taint-x509authority

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+check-logs() {
+  local component=$1
+  shift
+  for log in "$@"; do
+        check-log-line "$component" "$log"
+  done
+}
+
+# Fetch old authority ID
+old_authority=$(docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
+      /opt/spire/bin/spire-server \
+      localauthority x509 show -output json | jq .old.authority_id -r) || fail-now "Failed to fetch old authority ID"
+
+# Taint the old authority
+docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
+      /opt/spire/bin/spire-server \
+      localauthority x509 taint -authorityID "${old_authority}" || fail-now "Failed to taint old authority"
+
+# Root server logs
+check-logs root-server \
+      "X\.509 authority tainted successfully|local_authority_id=${old_authority}" \
+      "Server SVID signed using a tainted authority, forcing rotation of the Server SVID"
+
+# Root agent logs
+check-logs root-agent \
+      "New tainted X.509 authorities found|subject_key_ids=${old_authority}" \
+      "Scheduled rotation for SVID entries due to tainted X\.509 authorities|count=3" \
+      "Agent SVID is tainted by a root authority, forcing rotation"
+
+# Verify workloads are rotated
+
+# Intermediate A server and agent logs
+check-logs intermediateA-server \
+      "Current root CA is signed by a tainted upstream authority, preparing rotation" \
+      "Server SVID signed using a tainted authority, forcing rotation of the Server SVID"
+check-logs intermediateA-agent \
+      "New tainted X\.509 authorities found|subject_key_ids=${old_authority}" \
+      "Scheduled rotation for SVID entries due to tainted X.509 authorities|count=2" \
+      "Agent SVID is tainted by a root authority, forcing rotation"
+
+# Intermediate B server and agent logs
+check-logs intermediateB-server \
+      "Current root CA is signed by a tainted upstream authority, preparing rotation" \
+      "Server SVID signed using a tainted authority, forcing rotation of the Server SVID"
+check-logs intermediateB-agent \
+      "New tainted X\.509 authorities found|subject_key_ids=${old_authority}" \
+      "Scheduled rotation for SVID entries due to tainted X\.509 authorities|count=2" \
+      "Agent SVID is tainted by a root authority, forcing rotation"
+
+# Leaf A server and agent logs
+check-logs leafA-server \
+      "Current root CA is signed by a tainted upstream authority, preparing rotation" \
+      "Server SVID signed using a tainted authority, forcing rotation of the Server SVID"
+check-logs leafA-agent \
+      "New tainted X.509 authorities found|subject_key_ids=${old_authority}" \
+      "Scheduled rotation for SVID entries due to tainted X\.509 authorities|count=1" \
+      "Agent SVID is tainted by a root authority, forcing rotation"
+
+# Leaf B server and agent logs
+check-logs leafB-server \
+      "Current root CA is signed by a tainted upstream authority, preparing rotation" \
+      "Server SVID signed using a tainted authority, forcing rotation of the Server SVID"
+check-logs leafB-agent \
+      "New tainted X.509 authorities found|subject_key_ids=${old_authority}" \
+      "Scheduled rotation for SVID entries due to tainted X\.509 authorities|count=1" \
+      "Agent SVID is tainted by a root authority, forcing rotation"