Rework & Simplify Kubeflow Auth

.github/workflows/kserve_m2m_test.yaml

@@ -1,5 +1,5 @@
 name: Deploy and test KServe with m2m auth in KinD
 on:
   pull_request:
     paths:
     - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
@@ -7,7 +7,8 @@
     - contrib/kserve/**
     - common/oauth2-proxy/**
     - common/istio*/**
-    - tests/gh-actions/install_istio_with_ext_auth.sh*
+    - tests/gh-actions/install_istio.sh
+    - tests/gh-actions/install_oauth2-proxy.sh
     - tests/gh-actions/install_cert_manager.sh
     - common/cert-manager/**
     - tests/gh-actions/install_knative.sh
@@ -30,8 +31,11 @@
     - name: Create kubeflow namespace
       run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
 
-    - name: Install Istio with external authentication
-      run: ./tests/gh-actions/install_istio_with_ext_auth.sh
+    - name: Install Istio
+      run: ./tests/gh-actions/install_istio.sh
+
+    - name: Install oauth2-proxy
+      run: ./tests/gh-actions/install_oauth2-proxy.sh
 
     - name: Install cert-manager
       run: ./tests/gh-actions/install_cert_manager.sh
@@ -59,10 +63,6 @@
         nohup kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8080:80 &
         while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
 
-    - name: Wait for the kubeflow-m2m-oidc-configurator Job
-      run: |
-        ./tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh
-
     - name: Run kserve tests with m2m token from SA default/default
       run: |
         export KSERVE_INGRESS_HOST_PORT=localhost:8080

.github/workflows/model_registry_test.yaml

@@ -1,7 +1,7 @@
 # If anyone changes or improve the following tests for Model Registry, please
 # consider reflecting the same changes on https://github.com/kubeflow/model-registry
 name: Deploy and test Kubeflow Model Registry
 on:
   pull_request:
     paths:
     - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
@@ -25,8 +25,11 @@
     - name: Create kubeflow namespace
       run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
 
-    - name: Install Istio with external authentication
-      run: ./tests/gh-actions/install_istio_with_ext_auth.sh*
+    - name: Install Istio
+      run: ./tests/gh-actions/install_istio.sh
+
+    - name: Install oauth2-proxy
+      run: ./tests/gh-actions/install_oauth2-proxy.sh
 
     - name: Install cert-manager
       run: ./tests/gh-actions/install_cert_manager.sh

.github/workflows/notebook_controller_m2m_test.yaml

@@ -1,5 +1,5 @@
 name: Test Notebook Controller with m2m auth manifests in KinD
 on:
   pull_request:
     paths:
     - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
@@ -7,7 +7,8 @@
     - apps/jupyter/**
     - common/oauth2-proxy/**
     - common/istio*/**
-    - tests/gh-actions/install_istio_with_ext_auth.sh*
+    - tests/gh-actions/install_istio.sh
+    - tests/gh-actions/install_oauth2-proxy.sh
     - tests/gh-actions/install_multi_tenancy.sh
 
 jobs:
@@ -26,8 +27,11 @@
     - name: Create kubeflow namespace
       run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
 
-    - name: Install Istio with external authentication
-      run: ./tests/gh-actions/install_istio_with_ext_auth.sh
+    - name: Install Istio
+      run: ./tests/gh-actions/install_istio.sh
+
+    - name: Install oauth2-proxy
+      run: ./tests/gh-actions/install_oauth2-proxy.sh
 
     - name: Install kubeflow-istio-resources
       run: kustomize build common/istio-1-22/kubeflow-istio-resources/base | kubectl apply -f -
@@ -51,10 +55,6 @@
         nohup kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8080:80 &
         while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
 
-    - name: Wait for the kubeflow-m2m-oidc-configurator Job
-      run: |
-        ./tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh
-
     - name: List notebooks over API with authorized SA Token
       run: |
         KF_PROFILE=kubeflow-user-example-com

.github/workflows/pipeline_run_from_notebook.yaml

@@ -1,5 +1,5 @@
 name: Create Pipeline Run from Kubeflow Notebook
 on:
   pull_request:
     paths:
     - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
@@ -24,8 +24,11 @@
     - name: Install KinD, Create KinD cluster and Install kustomize
       run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
 
-    - name: Install Istio with external authentication
-      run: ./tests/gh-actions/install_istio_with_ext_auth.sh
+    - name: Install Istio
+      run: ./tests/gh-actions/install_istio.sh
+
+    - name: Install oauth2-proxy
+      run: ./tests/gh-actions/install_oauth2-proxy.sh
 
     - name: Install cert-manager
       run: ./tests/gh-actions/install_cert_manager.sh
@@ -63,10 +66,6 @@
           -f tests/gh-actions/kf-objects/notebook.test.kubeflow-user-example.com.yaml \
           --timeout 600s
 
-    - name: Wait for the kubeflow-m2m-oidc-configurator Job
-      run: |
-        ./tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh
-
     - name: Copy and execute the pipeline run script in KF Notebook
       run: |
         kubectl -n kubeflow-user-example-com cp \

.github/workflows/pipeline_test.yaml

@@ -1,5 +1,5 @@
 name: Deploy and test Kubeflow Pipelines manifests with m2m auth in KinD
 on:
   pull_request:
     paths:
     - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
@@ -7,10 +7,10 @@
     - apps/pipeline/upstream/**
     - tests/gh-actions/install_istio.sh
     - tests/gh-actions/install_cert_manager.sh
+    - tests/gh-actions/install_oauth2-proxy.sh
     - common/cert-manager/**
     - common/oauth2-proxy/**
     - common/istio*/**
-    - tests/gh-actions/install_istio_with_ext_auth.sh
 
 jobs:
   build:
@@ -25,8 +25,11 @@
     - name: Install kubectl
       run: ./tests/gh-actions/install_kubectl.sh
 
-    - name: Install Istio with external authentication
-      run: ./tests/gh-actions/install_istio_with_ext_auth.sh
+    - name: Install Istio
+      run: ./tests/gh-actions/install_istio.sh
+
+    - name: Install oauth2-proxy
+      run: ./tests/gh-actions/install_oauth2-proxy.sh
 
     - name: Install cert-manager
       run: ./tests/gh-actions/install_cert_manager.sh
@@ -52,10 +55,6 @@
         nohup kubectl port-forward --namespace istio-system svc/${ingress_gateway_service} 8080:80 &
         while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
 
-    - name: Wait for the kubeflow-m2m-oidc-configurator Job
-      run: |
-        ./tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh
-
     - name: List and deploy test pipeline with authorized ServiceAccount Token
       run: |
         pip3 install kfp==2.4.0

.github/workflows/pss_test.yaml

@@ -1,5 +1,5 @@
 name: Apply PSS labels to namespaces
 on:
   pull_request:
     paths:
     - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
@@ -27,12 +27,11 @@
     - name: Install kubectl
       run: ./tests/gh-actions/install_kubectl.sh
 
-    - name: Install all istio-cni resources and kubeflow namespace
-      run: |
-        kustomize build common/kubeflow-namespace/base | kubectl apply -f -
-        ./tests/gh-actions/install_cert_manager.sh
-        ./tests/gh-actions/install_istio-cni.sh
-        kustomize build common/istio-cni-1-22/kubeflow-istio-resources/base | kubectl apply -f -
+    - name: Install cert-manager
+      run: ./tests/gh-actions/install_cert_manager.sh
+
+    - name: Install Istio CNI
+      run: ./tests/gh-actions/install_istio-cni.sh
 
     - name: Configure istio init container with seccompProfile attribute
       run: |
@@ -44,15 +43,25 @@
         kubectl apply -f temporary_patch.yaml
         rm temporary_patch.yaml
 
-    - name: Install all other deployments of static namespaces
+    - name: Install oauth2-proxy
+      run: ./tests/gh-actions/install_oauth2-proxy.sh
+
+    - name: Create kubeflow namespace
+      run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
+
+    - name: Install kubeflow-istio-resources
+      run: kustomize build common/istio-cni-1-22/kubeflow-istio-resources/base | kubectl apply -f -
+
+    - name: Install KF Multi Tenancy
+      run: ./tests/gh-actions/install_multi_tenancy.sh
+
+    - name: Install dex
       run: |
-        ./tests/gh-actions/install_multi_tenancy.sh
-        kustomize build ./common/oauth2-proxy/overlays/m2m-self-signed | kubectl apply -f -
-        echo "Waiting for all oauth2-proxy pods to become ready..."
-        kubectl wait --for=condition=ready pod -l 'app.kubernetes.io/name=oauth2-proxy' --timeout=180s -n oauth2-proxy
+        echo "Installing Dex..."
         kustomize build ./common/dex/overlays/oauth2-proxy | kubectl apply -f -
+
         echo "Waiting for pods in auth namespace to become ready..."
-        kubectl wait --for=condition=Ready pods --all --timeout=180s -n auth
+        kubectl wait --for=condition=ready pods --all --timeout=180s -n auth
 
     - name: Install KF Pipelines
       run: ./tests/gh-actions/install_pipelines.sh

.github/workflows/ray_test.yaml

@@ -1,5 +1,5 @@
 name: Build & Apply Ray manifest in KinD
 on:
   pull_request:
     paths:
     - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
@@ -16,8 +16,11 @@
     - name: Install KinD, Create KinD cluster and Install kustomize
       run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
 
-    - name: Install Istio with external authentication
-      run: ./tests/gh-actions/install_istio_with_ext_auth.sh
+    - name: Install Istio
+      run: ./tests/gh-actions/install_istio.sh
+
+    - name: Install oauth2-proxy
+      run: ./tests/gh-actions/install_oauth2-proxy.sh
 
     - name: Install cert-manager
       run: ./tests/gh-actions/install_cert_manager.sh

.github/workflows/training_operator_test.yaml

@@ -1,5 +1,5 @@
 name: Build & Apply Training Operator manifests in KinD
 on:
   pull_request:
     paths:
     - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
@@ -22,8 +22,11 @@
     - name: Install kubectl
       run: ./tests/gh-actions/install_kubectl.sh
 
-    - name: Install Istio with external authentication
-      run: ./tests/gh-actions/install_istio_with_ext_auth.sh
+    - name: Install Istio
+      run: ./tests/gh-actions/install_istio.sh
+
+    - name: Install oauth2-proxy
+      run: ./tests/gh-actions/install_oauth2-proxy.sh
 
     - name: Install cert-manager
       run: ./tests/gh-actions/install_cert_manager.sh

README.md

@@ -224,11 +224,25 @@ The oauth2-proxy extends your Istio Ingress-Gateway capabilities, to be able to
 
 ```sh
 echo "Installing oauth2-proxy..."
-kustomize build common/oauth2-proxy/overlays/m2m-self-signed/ | kubectl apply -f -
+
+# Only uncomment ONE of the following overlays, they are mutually exclusive,
+# see `common/oauth2-proxy/overlays/` for more options.
+
+# OPTION 1: works on most clusters, does NOT allow K8s service account 
+#           tokens to be used from outside the cluster via the Istio ingress-gateway.
+#
+kustomize build common/oauth2-proxy/overlays/m2m-dex-only/ | kubectl apply -f -
 kubectl wait --for=condition=ready pod -l 'app.kubernetes.io/name=oauth2-proxy' --timeout=180s -n oauth2-proxy
+
+# Option 2: works on Kind/K3D clusters, and allows K8s service account tokens to be used
+#           from outside the cluster via the Istio ingress-gateway.
+# 
+#kustomize build common/oauth2-proxy/overlays/m2m-dex-and-kind/ | kubectl apply -f -
+#kubectl wait --for=condition=ready pod -l 'app.kubernetes.io/name=oauth2-proxy' --timeout=180s -n oauth2-proxy
+#kubectl wait --for=condition=ready pod -l 'app.kubernetes.io/name=cluster-jwks-proxy' --timeout=180s -n istio-system
 ```
 
-It supports user sessions as well as proper token-based machine to machine atuhhentication.
+It supports user sessions as well as proper token-based machine to machine authentication.
 
 #### Dex
 
@@ -237,7 +251,9 @@ Dex is an OpenID Connect Identity (OIDC) with multiple authentication backends.
 Install Dex:
 
 ```sh
+echo "Installing Dex..."
 kustomize build common/dex/overlays/oauth2-proxy | kubectl apply -f -
+kubectl wait --for=condition=ready pods --all --timeout=180s -n auth
 ```
 
 #### Knative

apps/pipeline/upstream/base/installs/multi-user/istio-authorization-config.yaml

@@ -8,10 +8,21 @@ spec:
     matchLabels:
       app: ml-pipeline-ui
   rules:
+  # Allow all requests from the ingress gateway
   - from:
     - source:
-        namespaces:
-        - istio-system
+        principals:
+        - cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account
+
+  # Allow all requests with an `authorization` header but NOT a `kubeflow-userid` header.
+  # This is needed to allow Kubernetes JWTs to be passed to the KFP API.
+  - when:
+    - key: request.headers[authorization]
+      values:
+      - "*"
+    - key: request.headers[kubeflow-userid]
+      notValues:
+      - "*"
 ---
 apiVersion: security.istio.io/v1beta1
 kind: AuthorizationPolicy
@@ -32,11 +43,7 @@ spec:
         - cluster.local/ns/kubeflow/sa/ml-pipeline-scheduledworkflow
         - cluster.local/ns/kubeflow/sa/ml-pipeline-viewer-crd-service-account
         - cluster.local/ns/kubeflow/sa/kubeflow-pipelines-cache
-  # allow access by any trusted principal
-  - from:
-    - source:
-        requestPrincipals: ["*"]
-  # For user workloads, which cannot user http headers for authentication
+  # Allow all requests that dont have a `kubeflow-userid` header.
   - when:
     - key: request.headers[kubeflow-userid]
       notValues: ['*']

common/dex/overlays/istio/kustomization.yaml

@@ -6,9 +6,6 @@ resources:
 - ../../base
 - virtual-service.yaml
 
-configurations:
-- params.yaml
-
 replacements:
 - source:
     version: v1

common/dex/overlays/istio/virtual-service.yaml

@@ -1,7 +1,3 @@
-# This config is gated on kiali upgrade to 0.21 from 0.16 in istio 1.1.6:
-# https://github.com/kiali/kiali/issues/1154
-# https://github.com/istio/istio/issues/11131
-
 apiVersion: networking.istio.io/v1alpha3
 kind: VirtualService
 metadata:
@@ -17,6 +13,6 @@ spec:
         prefix: /dex/
     route:
     - destination:
-        host: $(DEX_SERVICE).$(DEX_NAMESPACE).svc.cluster.local
+        host: DEX_SERVICE.DEX_NAMESPACE.svc.cluster.local
         port:
           number: 5556

common/istio-1-22/README.md

@@ -84,7 +84,6 @@ The Istio kustomizations make the following changes:
 - Remove PodDisruptionBudget from `istio-install` and `cluster-local-gateway` kustomizations. See:
     - https://github.com/istio/istio/issues/12602
     - https://github.com/istio/istio/issues/24000
-- Add EnvoyFilter for adding an `X-Forwarded-For` header in requests passing through the Istio Ingressgateway, inside the `istio-install` kustomization.
 - Add Istio AuthorizationPolicy to allow all requests to the Istio Ingressgateway and the Istio cluster-local gateway.
 - Add Istio AuthorizationPolicy in Istio's root namespace, so that sidecars deny traffic by default (explicit deny-by-default authorization model).
 - Add Gateway CRs for the Istio Ingressgateway and the Istio cluster-local gateway, as `istioctl` stopped generating them in later versions.

common/istio-1-22/istio-install/base/kustomization.yaml

@@ -8,7 +8,6 @@ resources:
 - gateway_authorizationpolicy.yaml
 - deny_all_authorizationpolicy.yaml
 - gateway.yaml
-- x-forwarded-host.yaml
 
 patches:
 - path: patches/service.yaml