fix(ci): update v2 deployment and ci tests (#1380)

* update deployment and ci tests * update test cases * update github action deps * update ci to build less images to avoid github rate limiting * update multi-user image name and patch * add debug statements * update script to make image build more efficient
kubeflow · Oct 18, 2023 · 5ac8032 · 5ac8032
1 parent add6aed
commit 5ac8032
Show file tree

Hide file tree

Showing 23 changed files with 109 additions and 61 deletions.
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -21,18 +21,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v4
       with:
         go-version: 1.20.x
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: "run go unit tests"
       run: go test -v -cover ./backend/...
   backend-integration:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Create k8s Kind Cluster
         uses: container-tools/kind-action@v2
         with:
@@ -43,7 +43,7 @@ jobs:
       - name: build images
         run: ./scripts/deploy/github/build-images.sh
       - name: Set up Python 3.10
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.10'
       - name: "deploy kfp-tekton"

diff --git a/backend/src/v2/compiler/tektoncompiler/container.go b/backend/src/v2/compiler/tektoncompiler/container.go
@@ -37,7 +37,7 @@ const (
 	MetadataGPRCServicePort = "8080"
 	MLPipelineServiceHost   = "ml-pipeline.kubeflow.svc.cluster.local"
 	MLPipelineServicePort   = "8887"
-	LauncherImage           = "gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4"
+	LauncherImage           = "gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7"
 )
 
 var (

diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml
@@ -99,7 +99,7 @@ spec:
                       - launcher-v2
                       - --copy
                       - /tekton/home/launch
-                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
                       imagePullPolicy: Always
                       name: kfp-launcher
                       computeResources: {}
@@ -276,7 +276,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}
@@ -434,7 +434,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}

diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml
@@ -81,7 +81,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}

diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/importer.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/importer.yaml
@@ -87,7 +87,7 @@ spec:
             value: metadata-grpc-service.kubeflow.svc.cluster.local
           - name: METADATA_GRPC_SERVICE_PORT
             value: "8080"
-          image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+          image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
           name: importer-main
           computeResources: {}
     - name: root-system-dag-driver

diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml
@@ -154,7 +154,7 @@ spec:
                       - launcher-v2
                       - --copy
                       - /tekton/home/launch
-                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
                       imagePullPolicy: Always
                       name: kfp-launcher
                       computeResources: {}
@@ -313,7 +313,7 @@ spec:
                       - launcher-v2
                       - --copy
                       - /tekton/home/launch
-                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
                       imagePullPolicy: Always
                       name: kfp-launcher
                       computeResources: {}
@@ -514,7 +514,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}

diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml
@@ -89,7 +89,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}
@@ -256,7 +256,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}
@@ -420,7 +420,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}
@@ -599,7 +599,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}
@@ -928,7 +928,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}
@@ -1119,7 +1119,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}
@@ -1284,7 +1284,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}
@@ -1457,7 +1457,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}
@@ -1630,7 +1630,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}

diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml
@@ -86,7 +86,7 @@ spec:
               - launcher-v2
               - --copy
               - /tekton/home/launch
-              image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+              image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
               imagePullPolicy: Always
               name: kfp-launcher
               computeResources: {}
@@ -326,7 +326,7 @@ spec:
                       - launcher-v2
                       - --copy
                       - /tekton/home/launch
-                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
                       imagePullPolicy: Always
                       name: kfp-launcher
                       computeResources: {}
@@ -538,7 +538,7 @@ spec:
                       - launcher-v2
                       - --copy
                       - /tekton/home/launch
-                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
                       imagePullPolicy: Always
                       name: kfp-launcher
                       computeResources: {}
@@ -751,7 +751,7 @@ spec:
                       - launcher-v2
                       - --copy
                       - /tekton/home/launch
-                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4
+                      image: gcr.io/ml-pipeline/kfp-launcher@sha256:6641bf94acaeec03ee7e231241800fce2f0ad92eee25371bd5248ca800a086d7
                       imagePullPolicy: Always
                       name: kfp-launcher
                       computeResources: {}

diff --git a/manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml b/manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml
@@ -0,0 +1,27 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kubeflow
+
+resources:
+- namespace.yaml
+bases:
+- ../third-party/application/cluster-scoped
+# - ../third-party/argo/installs/namespace/cluster-scoped
+- ../base/pipeline/cluster-scoped
+- ../base/cache-deployer/cluster-scoped
+vars:
+# NOTE: var name must be unique globally to allow composition of multiple kustomize
+# packages. Therefore, we added prefix `kfp-cluster-scoped-` to distinguish it from
+# others.
+- name: kfp-cluster-scoped-namespace
+  objref:
+    # cache deployer sa's metadata.namespace will be first transformed by namespace field in kustomization.yaml
+    # so that we only need to change kustomization.yaml's namespace field for namespace customization.
+    kind: ServiceAccount
+    name: kubeflow-pipelines-cache-deployer-sa
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.namespace
+configurations:
+- params.yaml
diff --git a/manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml b/manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: '$(kfp-cluster-scoped-namespace)'
diff --git a/manifests/kustomize/cluster-scoped-resources-tekton/params.yaml b/manifests/kustomize/cluster-scoped-resources-tekton/params.yaml
@@ -0,0 +1,4 @@
+# Allow Kustomize var to replace following fields.
+varReference:
+- path: metadata/name
+  kind: Namespace
diff --git a/manifests/kustomize/env/plain/kustomization.yaml b/manifests/kustomize/env/plain/kustomization.yaml
@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
+  - ../../cluster-scoped-resources-tekton
   - ../../base/installs/generic
   - ../../base/metadata/base
   - ../../third-party/minio/base

diff --git a/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml
@@ -15,13 +15,16 @@ commonLabels:
 images:
 - name: gcr.io/ml-pipeline/api-server
   newName: quay.io/aipipeline/apiserver
-  newTag: 2.0.0
+  newTag: 2.0.2
 - name: gcr.io/ml-pipeline/persistenceagent
   newName: quay.io/aipipeline/persistenceagent
-  newTag: 2.0.0
+  newTag: 2.0.2
 - name: gcr.io/ml-pipeline/scheduledworkflow
   newName: quay.io/aipipeline/scheduledworkflow
-  newTag: 2.0.0
+  newTag: 2.0.2
+
+patchesStrategicMerge:
+- tekton-config.yaml
 
 patches:
 - patch: |-

diff --git a/manifests/kustomize/env/platform-agnostic-tekton-multi-user/tekton-config.yaml b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/tekton-config.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: feature-flags
+  namespace: tekton-pipelines
+  labels:
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: tekton-pipelines
+data:
+  running-in-environment-with-injected-sidecars: "true"
diff --git a/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml
@@ -13,13 +13,13 @@ resources:
 images:
 - name: gcr.io/ml-pipeline/api-server
   newName: quay.io/aipipeline/apiserver
-  newTag: 2.0.0
+  newTag: 2.0.2
 - name: gcr.io/ml-pipeline/persistenceagent
   newName: quay.io/aipipeline/persistenceagent
-  newTag: 2.0.0
+  newTag: 2.0.2
 - name: gcr.io/ml-pipeline/scheduledworkflow
   newName: quay.io/aipipeline/scheduledworkflow
-  newTag: 2.0.0
+  newTag: 2.0.2
 
 labels:
 - includeSelectors: true

diff --git a/manifests/kustomize/third-party/tekton-custom-task/kustomization.yaml b/manifests/kustomize/third-party/tekton-custom-task/kustomization.yaml
@@ -16,16 +16,16 @@ images:
     newTag: 1.8.0
   - name: kfp-v2-dev-driver-controller
     newName: quay.io/aipipeline/tekton-driver
-    newTag: 2.0.0
+    newTag: 2.0.2
   - name: tekton-exithandler-controller
     newName: quay.io/aipipeline/tekton-exithandler-controller
-    newTag: 2.0.0
+    newTag: 2.0.2
   - name: tekton-exithandler-webhook
     newName: quay.io/aipipeline/tekton-exithandler-webhook
-    newTag: 2.0.0
+    newTag: 2.0.2
   - name: tekton-kfptask-controller
     newName: quay.io/aipipeline/tekton-kfptask-controller
-    newTag: 2.0.0
+    newTag: 2.0.2
   - name: tekton-kfptask-webhook
     newName: quay.io/aipipeline/tekton-kfptask-webhook
-    newTag: 2.0.0
+    newTag: 2.0.2
diff --git a/.../kustomize/third-party/tekton/upstream/manifests/base/tektoncd-install/tekton-config.yaml b/.../kustomize/third-party/tekton/upstream/manifests/base/tektoncd-install/tekton-config.yaml
@@ -7,5 +7,4 @@ metadata:
     app.kubernetes.io/instance: default
     app.kubernetes.io/part-of: tekton-pipelines
 data:
-  embedded-status: "full"
-  custom-task-version: "v1beta1"
+  running-in-environment-with-injected-sidecars: "false"
diff --git a/scripts/deploy/github/build-images.sh b/scripts/deploy/github/build-images.sh
@@ -23,14 +23,10 @@ TAG="${TAG:-latest}"
 
 docker system prune -a -f
 
-docker build -q -t "${REGISTRY}/apiserver:${TAG}" -f backend/Dockerfile . && docker push "${REGISTRY}/apiserver:${TAG}"
-docker build -q -t "${REGISTRY}/persistenceagent:${TAG}" -f backend/Dockerfile.persistenceagent . && docker push "${REGISTRY}/persistenceagent:${TAG}"
-docker build -q -t "${REGISTRY}/scheduledworkflow:${TAG}" -f backend/Dockerfile.scheduledworkflow . && docker push "${REGISTRY}/scheduledworkflow:${TAG}"
+docker build -q -t "${REGISTRY}/apiserver:${TAG}" -f backend/Dockerfile . && docker push "${REGISTRY}/apiserver:${TAG}" &
+docker build -q -t "${REGISTRY}/persistenceagent:${TAG}" -f backend/Dockerfile.persistenceagent . && docker push "${REGISTRY}/persistenceagent:${TAG}" &
+docker build -q -t "${REGISTRY}/scheduledworkflow:${TAG}" -f backend/Dockerfile.scheduledworkflow . && docker push "${REGISTRY}/scheduledworkflow:${TAG}" &
 docker build -q -t "${REGISTRY}/tekton-driver:${TAG}" -f backend/Dockerfile.tektondriver . && docker push "${REGISTRY}/tekton-driver:${TAG}" &
-docker build -q -t "${REGISTRY}/tekton-exithandler-controller:${TAG}" -f backend/Dockerfile.tekton-exithandler.controller . && docker push "${REGISTRY}/tekton-exithandler-controller:${TAG}" &
-docker build -q -t "${REGISTRY}/tekton-exithandler-webhook:${TAG}" -f backend/Dockerfile.tekton-exithandler.webhook . && docker push "${REGISTRY}/tekton-exithandler-webhook:${TAG}"
-docker build -q -t "${REGISTRY}/tekton-kfptask-controller:${TAG}" -f backend/Dockerfile.tekton-kfptask.controller . && docker push "${REGISTRY}/tekton-kfptask-controller:${TAG}" &
-docker build -q -t "${REGISTRY}/tekton-kfptask-webhook:${TAG}" -f backend/Dockerfile.tekton-kfptask.webhook . && docker push "${REGISTRY}/tekton-kfptask-webhook:${TAG}" &
 
 wait
 

diff --git a/scripts/deploy/github/deploy-kfp.sh b/scripts/deploy/github/deploy-kfp.sh
@@ -48,5 +48,8 @@ then
   exit 1
 fi
 
+echo "List Tekton control plane: "
+kubectl get pod -n tekton-pipelines
+
 echo "Finished kfp-tekton deployment."