feat: add ambient mode support to kubeflow controllers

components/common/reconcilehelper/util.go

@@ -217,3 +217,4 @@ func CopyVirtualService(from, to *unstructured.Unstructured) bool {
 	}
 	return requiresUpdate
 }
+

components/notebook-controller/config/manager/manager.yaml

@@ -10,6 +10,10 @@ kind: Deployment
 metadata:
   name: deployment
 spec:
+  selector:
+    matchLabels:
+      app: notebook-controller
+      kustomize.component: notebook-controller
   template:
     metadata:
       labels:
@@ -21,42 +25,9 @@ spec:
         image: ghcr.io/kubeflow/kubeflow/notebook-controller
         command:
           - /manager
-        env:
-          - name: USE_ISTIO
-            valueFrom:
-              configMapKeyRef:
-                name: config
-                key: USE_ISTIO
-          - name: ISTIO_GATEWAY
-            valueFrom:
-              configMapKeyRef:
-                name: config
-                key: ISTIO_GATEWAY
-          - name: ISTIO_HOST
-            valueFrom:
-              configMapKeyRef:
-                name: config
-                key: ISTIO_HOST
-          - name: CLUSTER_DOMAIN
-            valueFrom:
-              configMapKeyRef:
-                name: config
-                key: CLUSTER_DOMAIN
-          - name: ENABLE_CULLING
-            valueFrom:
-              configMapKeyRef:
-                name: config
-                key: ENABLE_CULLING
-          - name: CULL_IDLE_TIME
-            valueFrom:
-              configMapKeyRef:
-                name: config
-                key: CULL_IDLE_TIME
-          - name: IDLENESS_CHECK_PERIOD
-            valueFrom:
-              configMapKeyRef:
-                name: config
-                key: IDLENESS_CHECK_PERIOD
+        envFrom:
+        - configMapRef:
+            name: config
         imagePullPolicy: IfNotPresent
         livenessProbe:
           httpGet:

components/notebook-controller/config/manager/params.env

@@ -1,5 +1,8 @@
 USE_ISTIO=true
+USE_GATEWAY_API=false
 ISTIO_GATEWAY=kubeflow/kubeflow-gateway
+K8S_GATEWAY_NAME=kubeflow-gateway
+K8S_GATEWAY_NAMESPACE=kubeflow
 ISTIO_HOST=*
 CLUSTER_DOMAIN=cluster.local
 ENABLE_CULLING=false

components/notebook-controller/config/rbac/role.yaml

@@ -35,6 +35,12 @@ rules:
   - services
   verbs:
   - '*'
+- apiGroups:
+  - gateway.networking.k8s.io
+  resources:
+  - httproutes
+  verbs:
+  - '*'
 - apiGroups:
   - kubeflow.org
   resources:

components/notebook-controller/controllers/notebook_controller.go

@@ -45,6 +45,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
+	gwapiv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 )
 
 const DefaultContainerPort = 8888
@@ -85,6 +86,7 @@ type NotebookReconciler struct {
 // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs="*"
 // +kubebuilder:rbac:groups=kubeflow.org,resources=notebooks;notebooks/status;notebooks/finalizers,verbs="*"
 // +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs="*"
+// +kubebuilder:rbac:groups="gateway.networking.k8s.io",resources=httproutes,verbs="*"
 
 func (r *NotebookReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := r.Log.WithValues("notebook", req.NamespacedName)
@@ -199,14 +201,22 @@ func (r *NotebookReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		}
 	}
 
-	// Reconcile virtual service if we use ISTIO.
+	// Reconcile VirtualService if we use ISTIO.
 	if os.Getenv("USE_ISTIO") == "true" {
 		err = r.reconcileVirtualService(instance)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
 	}
 
+	// Reconcile HTTPRoute if we use Gateway API.
+	if os.Getenv("USE_GATEWAY_API") == "true" {
+		err = r.reconcileHTTPRoute(instance)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
 	foundPod := &corev1.Pod{}
 	err = r.Get(ctx, types.NamespacedName{Name: ss.Name + "-0", Namespace: ss.Namespace}, foundPod)
 	if err != nil && apierrs.IsNotFound(err) {
@@ -567,7 +577,135 @@ func generateVirtualService(instance *v1beta1.Notebook) (*unstructured.Unstructu
 	}
 
 	return vsvc, nil
+}
+
+func generateHTTPRoute(instance *v1beta1.Notebook) (*gwapiv1beta1.HTTPRoute, error) {
+	name := instance.Name
+	namespace := instance.Namespace
+	pathPrefix := fmt.Sprintf("/notebook/%s/%s", namespace, name)
+
+	// unpack annotations from Notebook resource
+	annotations := make(map[string]string)
+	for k, v := range instance.ObjectMeta.Annotations {
+		annotations[k] = v
+	}
+
+	pathRewrite := fmt.Sprintf("/notebook/%s/%s", namespace, name)
+	// If AnnotationRewriteURI is present, use this value for path rewrite
+	if _, ok := annotations[AnnotationRewriteURI]; ok && len(annotations[AnnotationRewriteURI]) > 0 {
+		pathRewrite = annotations[AnnotationRewriteURI]
+	}
+
+	// Get gateway configuration
+	gatewayName := os.Getenv("K8S_GATEWAY_NAME")
+	if len(gatewayName) == 0 {
+		gatewayName = "kubeflow-gateway"
+	}
+	gatewayNamespace := os.Getenv("K8S_GATEWAY_NAMESPACE")
+	if len(gatewayNamespace) == 0 {
+		gatewayNamespace = "kubeflow"
+	}
+
+	pathMatchType := gwapiv1beta1.PathMatchPathPrefix
+	pathRewriteType := gwapiv1beta1.PrefixMatchHTTPPathModifier
+	urlRewriteType := gwapiv1beta1.HTTPRouteFilterURLRewrite
+	portNumber := gwapiv1beta1.PortNumber(DefaultServingPort)
+
+	httpRoute := &gwapiv1beta1.HTTPRoute{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      virtualServiceName(name, namespace),
+			Namespace: namespace,
+		},
+		Spec: gwapiv1beta1.HTTPRouteSpec{
+			CommonRouteSpec: gwapiv1beta1.CommonRouteSpec{
+				ParentRefs: []gwapiv1beta1.ParentReference{
+					{
+						Name:      gwapiv1beta1.ObjectName(gatewayName),
+						Namespace: (*gwapiv1beta1.Namespace)(&gatewayNamespace),
+					},
+				},
+			},
+			Rules: []gwapiv1beta1.HTTPRouteRule{
+				{
+					Matches: []gwapiv1beta1.HTTPRouteMatch{
+						{
+							Path: &gwapiv1beta1.HTTPPathMatch{
+								Type:  &pathMatchType,
+								Value: &pathPrefix,
+							},
+						},
+					},
+					Filters: []gwapiv1beta1.HTTPRouteFilter{
+						{
+							Type: urlRewriteType,
+							URLRewrite: &gwapiv1beta1.HTTPURLRewriteFilter{
+								Path: &gwapiv1beta1.HTTPPathModifier{
+									Type:               pathRewriteType,
+									ReplacePrefixMatch: &pathRewrite,
+								},
+							},
+						},
+					},
+					BackendRefs: []gwapiv1beta1.HTTPBackendRef{
+						{
+							BackendRef: gwapiv1beta1.BackendRef{
+								BackendObjectReference: gwapiv1beta1.BackendObjectReference{
+									Name: gwapiv1beta1.ObjectName(name),
+									Port: &portNumber,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	return httpRoute, nil
+}
+
+
+func (r *NotebookReconciler) reconcileHTTPRoute(instance *v1beta1.Notebook) error {
+	log := r.Log.WithValues("notebook", instance.Namespace)
+	httpRoute, err := generateHTTPRoute(instance)
+	if err != nil {
+		log.Info("Unable to generate HTTPRoute...", err)
+		return err
+	}
+	if err := ctrl.SetControllerReference(instance, httpRoute, r.Scheme); err != nil {
+		return err
+	}
+	// Check if the HTTPRoute already exists.
+	foundHTTPRoute := &gwapiv1beta1.HTTPRoute{}
+	justCreated := false
+	err = r.Get(context.TODO(), types.NamespacedName{Name: virtualServiceName(instance.Name,
+		instance.Namespace), Namespace: instance.Namespace}, foundHTTPRoute)
+	if err != nil && apierrs.IsNotFound(err) {
+		log.Info("Creating HTTPRoute", "namespace", instance.Namespace, "name",
+			virtualServiceName(instance.Name, instance.Namespace))
+		err = r.Create(context.TODO(), httpRoute)
+		justCreated = true
+		if err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
 
+	if !justCreated {
+		// Copy the spec from the desired HTTPRoute to the existing one
+		if !reflect.DeepEqual(httpRoute.Spec, foundHTTPRoute.Spec) {
+			log.Info("Updating HTTPRoute", "namespace", instance.Namespace, "name",
+				virtualServiceName(instance.Name, instance.Namespace))
+			foundHTTPRoute.Spec = httpRoute.Spec
+			err = r.Update(context.TODO(), foundHTTPRoute)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
 }
 
 func (r *NotebookReconciler) reconcileVirtualService(instance *v1beta1.Notebook) error {
@@ -730,6 +868,11 @@ func (r *NotebookReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		builder.Owns(virtualService)
 	}
 
+	// watch HTTPRoute
+	if os.Getenv("USE_GATEWAY_API") == "true" {
+		builder.Owns(&gwapiv1beta1.HTTPRoute{})
+	}
+
 	err := builder.Complete(r)
 	if err != nil {
 		return err

components/notebook-controller/go.mod

@@ -6,12 +6,13 @@ require (
 	github.com/go-logr/logr v1.2.0
 	github.com/kubeflow/kubeflow/components/common v0.0.0-20220218084159-4ad0158e955e
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.17.0
-	github.com/prometheus/client_golang v1.11.1
-	k8s.io/api v0.23.0
-	k8s.io/apimachinery v0.23.0
-	k8s.io/client-go v0.23.0
-	sigs.k8s.io/controller-runtime v0.11.0
+	github.com/onsi/gomega v1.18.1
+	github.com/prometheus/client_golang v1.12.1
+	k8s.io/api v0.24.1
+	k8s.io/apimachinery v0.24.1
+	k8s.io/client-go v0.24.1
+	sigs.k8s.io/controller-runtime v0.12.1
+	sigs.k8s.io/gateway-api v0.5.1
 )
 
 require (
@@ -22,54 +23,63 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-logr/zapr v1.2.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.1.2 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.28.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
-	golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect
-	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
+	golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0 // indirect
-	k8s.io/apiextensions-apiserver v0.23.0 // indirect
-	k8s.io/component-base v0.23.0 // indirect
-	k8s.io/klog/v2 v2.30.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
-	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
-	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect
+	k8s.io/apiextensions-apiserver v0.24.1 // indirect
+	k8s.io/component-base v0.24.1 // indirect
+	k8s.io/klog/v2 v2.60.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect
+	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
+	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )