diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 06f0f5e0d0..174405fd19 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -180,6 +180,12 @@ Envoy Gateway is already installed by Skaffold (the `envoy-gateway` Helm release service for the proxy; klipper-lb binds it to hostPort 80, reachable via the `8080:80` load balancer port mapping. +To exercise the same path in the standard single-replica Kubernetes E2E suite, +run `mise run e2e:kubernetes:envoy`. The wrapper creates isolated Gateway API +resources, reuses an existing Envoy controller when present, and forwards the +test client through the generated Envoy proxy Service. It does not enable HA or +provision PostgreSQL. + ### Keycloak OIDC One-time setup — only needed once per cluster lifetime: diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 37154c75df..0a7aef248e 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -162,14 +162,22 @@ jobs: agent_sandbox_version: v0.5.0 topology: combined extra_helm_values: "" + use_envoy_gateway: false - agent_sandbox_api: v1alpha1 agent_sandbox_version: v0.4.6 topology: combined extra_helm_values: "" + use_envoy_gateway: false - agent_sandbox_api: v1beta1 agent_sandbox_version: v0.5.0 topology: sidecar extra_helm_values: deploy/helm/openshell/ci/values-sidecar.yaml + use_envoy_gateway: false + - agent_sandbox_api: v1beta1 + agent_sandbox_version: v0.5.0 + topology: envoy + extra_helm_values: "" + use_envoy_gateway: true permissions: actions: read contents: read @@ -180,6 +188,7 @@ jobs: job-name: Kubernetes E2E (Rust smoke, ${{ matrix.topology }}, Agent Sandbox ${{ matrix.agent_sandbox_api }}) agent-sandbox-version: ${{ matrix.agent_sandbox_version }} extra-helm-values: ${{ matrix.extra_helm_values }} + use-envoy-gateway: ${{ matrix.use_envoy_gateway }} cli-artifact-prefix: rust-binary-cli kubernetes-workspace-managed-e2e: diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index bf13ab7080..dafaf050e6 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -37,6 +37,11 @@ on: required: false type: string default: "v0.5.0" + use-envoy-gateway: + description: "Exercise Gateway API/Envoy routing for the standard single-replica e2e suite" + required: false + type: boolean + default: false e2e-task: description: "mise task to run for the Kubernetes e2e job" required: false @@ -141,6 +146,7 @@ jobs: OPENSHELL_E2E_KUBE_CONTEXT: kind-${{ env.KIND_CLUSTER_NAME }} OPENSHELL_E2E_KUBE_EXTRA_VALUES: ${{ inputs.extra-helm-values }} OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET: ${{ inputs.external-postgres-secret }} + OPENSHELL_E2E_KUBE_USE_ENVOY: ${{ inputs.use-envoy-gateway }} IMAGE_TAG: ${{ inputs.image-tag }} OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell E2E_TASK: ${{ inputs.e2e-task }} diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index f6d0efc4ec..0c630ab6d4 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -46,6 +46,11 @@ # `vault`; the Rust `credential_drivers` e2e test validates the active # backend. Vault mode installs a dev OpenBao fixture because it exposes the # Vault-compatible API used by the driver. +# +# Envoy Gateway path: +# Set OPENSHELL_E2E_KUBE_USE_ENVOY=1 to route the standard single-replica +# suite through the Gateway API GRPCRoute. The wrapper preserves an existing +# Envoy Gateway controller and cleans up only Envoy resources it created. set -euo pipefail @@ -94,6 +99,16 @@ VAULT_CHART_VERSION="${OPENSHELL_E2E_OPENBAO_CHART_VERSION:-0.28.3}" VAULT_DEV_ROOT_TOKEN="${OPENSHELL_E2E_VAULT_DEV_ROOT_TOKEN:-root}" CORPORATE_PROXY_FIXTURE_DEPLOYED=0 CORPORATE_PROXY_FIXTURE_SECRET="openshell-e2e-proxy-auth" +ENVOY_RUN_ID="$(date +%s)-$$" +ENVOY_RELEASE_NAME="openshell-e2e-envoy-${ENVOY_RUN_ID}" +ENVOY_NAMESPACE="openshell-e2e-envoy-${ENVOY_RUN_ID}" +ENVOY_CHART_VERSION="${OPENSHELL_E2E_ENVOY_VERSION:-v1.7.2}" +ENVOY_GATEWAY_CLASS="openshell-e2e-envoy-${ENVOY_RUN_ID}" +ENVOY_TRAFFIC_POLICY="openshell-e2e-envoy-${ENVOY_RUN_ID}" +ENVOY_HELM_INSTALLED=0 +ENVOY_NAMESPACE_CREATED=0 +ENVOY_GATEWAY_CLASS_CREATED=0 +ENVOY_TRAFFIC_POLICY_CREATED=0 # Isolate CLI/SDK gateway metadata from the developer's real config. export XDG_CONFIG_HOME="${WORKDIR}/config" @@ -184,6 +199,158 @@ cleanup_postgres_fixture() { EXTERNAL_PG_FIXTURE_SECRET="" } +use_envoy_gateway() { + case "${OPENSHELL_E2E_KUBE_USE_ENVOY:-0}" in + 1 | true | TRUE | yes | YES) return 0 ;; + *) return 1 ;; + esac +} + +envoy_controller_present() { + if kctl get gatewayclass \ + -o jsonpath='{range .items[*]}{.spec.controllerName}{"\n"}{end}' 2>/dev/null \ + | grep -qx 'gateway.envoyproxy.io/gatewayclass-controller'; then + return 0 + fi + [ -n "$(kctl get deployment -A -l app.kubernetes.io/name=gateway-helm \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" ] +} + +prepare_envoy_gateway() { + if envoy_controller_present; then + echo "Reusing an existing Envoy Gateway controller." + else + if kctl get namespace "${ENVOY_NAMESPACE}" >/dev/null 2>&1; then + echo "ERROR: refusing to reuse Envoy test namespace ${ENVOY_NAMESPACE}" >&2 + return 1 + fi + kctl create namespace "${ENVOY_NAMESPACE}" + ENVOY_NAMESPACE_CREATED=1 + + echo "Installing Envoy Gateway (${ENVOY_CHART_VERSION})..." + helmctl install "${ENVOY_RELEASE_NAME}" \ + oci://docker.io/envoyproxy/gateway-helm \ + --version "${ENVOY_CHART_VERSION}" \ + --namespace "${ENVOY_NAMESPACE}" --wait --timeout 5m + ENVOY_HELM_INSTALLED=1 + fi + + if kctl get gatewayclass "${ENVOY_GATEWAY_CLASS}" >/dev/null 2>&1; then + echo "ERROR: refusing to modify existing GatewayClass ${ENVOY_GATEWAY_CLASS}" >&2 + return 1 + fi + kctl create -f - </dev/null 2>&1; then + echo "ERROR: refusing to modify existing BackendTrafficPolicy ${ENVOY_TRAFFIC_POLICY}" >&2 + return 1 + fi + kctl -n "${NAMESPACE}" create -f - </dev/null || true)" + if [ -n "${svc_ref}" ]; then + svc_namespace="${svc_ref%%/*}" + if kctl -n "${svc_namespace}" wait --for=condition=Ready pod \ + -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME},gateway.envoyproxy.io/owning-gateway-namespace=${NAMESPACE}" \ + --timeout=5s >/dev/null 2>&1; then + printf '%s\n' "${svc_ref}" + return 0 + fi + fi + sleep 2 + done + + echo "ERROR: Envoy proxy Service for Gateway ${RELEASE_NAME} was not ready." >&2 + kctl -n "${NAMESPACE}" get gateway,grpcroute -o wide >&2 || true + kctl get svc -A \ + -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME},gateway.envoyproxy.io/owning-gateway-namespace=${NAMESPACE}" \ + -o wide >&2 || true + return 1 +} + +start_gateway_portforward() { + local elapsed=0 + local timeout=30 + local target_namespace="${NAMESPACE}" + local target_service="${RELEASE_NAME}" + local target_port=8080 + local target_service_ref="" + + LOCAL_PORT="$(e2e_pick_port)" + if use_envoy_gateway; then + target_service_ref="$(wait_for_envoy_service)" + target_namespace="${target_service_ref%%/*}" + target_service="${target_service_ref#*/}" + target_port=80 + echo "Starting kubectl port-forward -n ${target_namespace} svc/${target_service} ${LOCAL_PORT}:${target_port} (Envoy Gateway)..." + else + echo "Starting kubectl port-forward svc/${target_service} ${LOCAL_PORT}:${target_port}..." + fi + kctl -n "${target_namespace}" port-forward "svc/${target_service}" \ + "${LOCAL_PORT}:${target_port}" >"${PORTFORWARD_LOG}" 2>&1 & + PORTFORWARD_PID=$! + + while [ "${elapsed}" -lt "${timeout}" ]; do + if ! kill -0 "${PORTFORWARD_PID}" 2>/dev/null; then + echo "ERROR: kubectl port-forward exited before becoming reachable" >&2 + cat "${PORTFORWARD_LOG}" >&2 || true + return 1 + fi + if curl -s -o /dev/null --connect-timeout 1 "http://127.0.0.1:${LOCAL_PORT}"; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + + echo "ERROR: port-forward did not accept TCP within ${timeout}s" >&2 + cat "${PORTFORWARD_LOG}" >&2 || true + return 1 +} + deploy_vault_fixture() { echo "Deploying OpenBao fixture for Vault credential-driver validation..." @@ -272,6 +439,23 @@ cleanup() { cleanup_vault_fixture fi + if [ "${ENVOY_TRAFFIC_POLICY_CREATED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ]; then + kctl -n "${NAMESPACE}" delete backendtrafficpolicy.gateway.envoyproxy.io \ + "${ENVOY_TRAFFIC_POLICY}" --ignore-not-found --wait=false >/dev/null 2>&1 || true + fi + if [ "${ENVOY_GATEWAY_CLASS_CREATED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ]; then + kctl delete gatewayclass "${ENVOY_GATEWAY_CLASS}" --ignore-not-found \ + --wait=false >/dev/null 2>&1 || true + fi + if [ "${ENVOY_HELM_INSTALLED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ]; then + helmctl uninstall "${ENVOY_RELEASE_NAME}" --namespace "${ENVOY_NAMESPACE}" \ + --wait --timeout 60s >/dev/null 2>&1 || true + fi + if [ "${ENVOY_NAMESPACE_CREATED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ]; then + kctl delete namespace "${ENVOY_NAMESPACE}" --wait=true --timeout=60s \ + --ignore-not-found >/dev/null 2>&1 || true + fi + if [ "${CORPORATE_PROXY_FIXTURE_DEPLOYED}" = "1" ]; then kctl -n "${NAMESPACE}" delete secret "${CORPORATE_PROXY_FIXTURE_SECRET}" \ --ignore-not-found >/dev/null 2>&1 || true @@ -765,6 +949,16 @@ if [ -n "${OPENSHELL_E2E_KUBE_EXTRA_VALUES:-}" ]; then done fi +if use_envoy_gateway; then + if [ "${OPENSHELL_E2E_KUBE_DB_SCENARIOS:-0}" = "1" ]; then + echo "ERROR: Envoy Gateway mode supports only the standard single-replica Kubernetes e2e suite." >&2 + exit 2 + fi + helm_values_args+=(--values "${ROOT}/deploy/helm/openshell/ci/values-gateway.yaml") + helm_extra_args+=(--set "grpcRoute.gateway.className=${ENVOY_GATEWAY_CLASS}") + prepare_envoy_gateway +fi + if [ "${OPENSHELL_E2E_KUBE_DB_SCENARIOS:-0}" = "1" ]; then # --- Multi-scenario mode: test all database backends --- DB_PASSED=0 @@ -822,31 +1016,10 @@ else --docker-password=e2e-password fi - LOCAL_PORT="$(e2e_pick_port)" - echo "Starting kubectl port-forward svc/openshell ${LOCAL_PORT}:8080..." - kctl -n "${NAMESPACE}" port-forward "svc/openshell" \ - "${LOCAL_PORT}:8080" >"${PORTFORWARD_LOG}" 2>&1 & - PORTFORWARD_PID=$! - - elapsed=0 - timeout=30 - while [ "${elapsed}" -lt "${timeout}" ]; do - if ! kill -0 "${PORTFORWARD_PID}" 2>/dev/null; then - echo "ERROR: kubectl port-forward exited before becoming reachable" >&2 - cat "${PORTFORWARD_LOG}" >&2 || true - exit 1 - fi - if curl -s -o /dev/null --connect-timeout 1 "http://127.0.0.1:${LOCAL_PORT}"; then - break - fi - sleep 1 - elapsed=$((elapsed + 1)) - done - if [ "${elapsed}" -ge "${timeout}" ]; then - echo "ERROR: port-forward did not accept TCP within ${timeout}s" >&2 - cat "${PORTFORWARD_LOG}" >&2 || true - exit 1 + if use_envoy_gateway; then + create_envoy_traffic_policy fi + start_gateway_portforward HEALTH_LOCAL_PORT="$(e2e_pick_port)" WORKLOAD_REF="$(kube_workload_ref "${RELEASE_NAME}")" diff --git a/tasks/test.toml b/tasks/test.toml index bef8baf2b5..93708bd691 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -160,6 +160,11 @@ description = "Run Kubernetes e2e with the supervisor sidecar topology overlay" env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sidecar.yaml" } run = "e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:envoy"] +description = "Run the standard single-replica Kubernetes e2e suite through an Envoy Gateway GRPCRoute" +env = { OPENSHELL_E2E_KUBE_USE_ENVOY = "1" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:kubernetes:db"] description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)" env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" }