From f825a037f66f20a8e11ba2c2c4c986cb6b6b8213 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Thu, 2 Nov 2023 15:10:51 -0700 Subject: [PATCH] feat(V2 backend): Update compiler to produce the merger task of driver + executor (#1412) * update ci and move manifests to 2.0.3 * update compiler and manifests with the new kfptask controller * remove comments' * update parameters * update tekton parameters to hyphen --- .../v2/compiler/tektoncompiler/container.go | 96 +--- backend/src/v2/compiler/tektoncompiler/dag.go | 36 +- .../v2/compiler/tektoncompiler/importer.go | 4 +- .../src/v2/compiler/tektoncompiler/tekton.go | 30 +- .../tektoncompiler/testdata/exit_handler.yaml | 213 +++----- .../tektoncompiler/testdata/hello_world.yaml | 83 ++- .../tektoncompiler/testdata/importer.yaml | 37 +- .../tektoncompiler/testdata/loop_static.yaml | 249 ++++----- .../testdata/mnist_pipeline.yaml | 515 ++++++------------ .../tektoncompiler/testdata/nestedloop.yaml | 411 ++++++-------- .../kfptask/201-clusterrole.yaml | 13 + .../tekton-custom-task/kfptask/201-role.yaml | 28 + .../tekton-custom-task/kustomization.yaml | 10 +- .../github/manifests/kustomization.yaml | 21 +- 14 files changed, 671 insertions(+), 1075 deletions(-) diff --git a/backend/src/v2/compiler/tektoncompiler/container.go b/backend/src/v2/compiler/tektoncompiler/container.go index 315fba00ad..ac89e0bcc7 100644 --- a/backend/src/v2/compiler/tektoncompiler/container.go +++ b/backend/src/v2/compiler/tektoncompiler/container.go @@ -237,39 +237,35 @@ func (i *containerDriverInputs) getParentDagCondition(isExitHandler bool) string func (c *pipelinerunCompiler) containerDriverTask(name string, inputs *containerDriverInputs) error { - containerDriverName := getContainerDriverTaskName(name) + t, err := c.containerExecutorTemplate(name, inputs.containerDef, c.spec.PipelineInfo.GetName()) + + if err != nil { + return err + } driverTask := &pipelineapi.PipelineTask{ - Name: containerDriverName, - TaskRef: &pipelineapi.TaskRef{ - APIVersion: "kfp-driver.tekton.dev/v1alpha1", - Kind: "KFPDriver", - }, + Name: name, + TaskSpec: t, Params: []pipelineapi.Param{ // "--type", "CONTAINER", { Name: paramNameType, Value: pipelineapi.ParamValue{Type: "string", StringVal: "CONTAINER"}, }, - // "--pipeline_name", c.spec.GetPipelineInfo().GetName(), + // "--pipeline-name", c.spec.GetPipelineInfo().GetName(), { Name: paramNamePipelineName, Value: pipelineapi.ParamValue{Type: "string", StringVal: c.spec.GetPipelineInfo().GetName()}, }, - // "--run_id", runID(), + // "--run-id", runID(), { - Name: paramNameRunId, + Name: paramRunId, Value: pipelineapi.ParamValue{Type: "string", StringVal: runID()}, }, - // "--dag_execution_id" + // "--dag-execution-id" { Name: paramNameDagExecutionId, Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.getParentDagID(c.ExitHandlerScope())}, }, - // "--component" - { - Name: paramComponent, - Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.component}, - }, // "--task" { Name: paramTask, @@ -280,30 +276,33 @@ func (c *pipelinerunCompiler) containerDriverTask(name string, inputs *container Name: paramContainer, Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.container}, }, - // "--iteration_index", inputValue(paramIterationIndex), + // "--iteration-index", inputValue(paramIterationIndex), { Name: paramNameIterationIndex, Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.iterationIndex}, }, - // "--kubernetes_config" + // "--kubernetes-config" { Name: paramKubernetesConfig, Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.kubernetesConfig}, }, - // "--mlmd_server_address" + // "--mlmd-server-address" { Name: paramNameMLMDServerHost, Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDHost()}, }, - // "--mlmd_server_port" + // "--mlmd-server-port" { Name: paramNameMLMDServerPort, Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDPort()}, }, + // "--component" + { + Name: paramComponent, + Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.component}, + }, // produce the following outputs: // - execution-id - // - executor-input - // - cached-decision // - condition }, } @@ -325,57 +324,6 @@ func (c *pipelinerunCompiler) containerDriverTask(name string, inputs *container c.addPipelineTask(driverTask) - // need container driver's output for executor - containerDriverOutputs := containerDriverOutputs{ - executionId: taskOutputParameter(containerDriverName, paramExecutionID), - condition: taskOutputParameter(containerDriverName, paramCondition), - executiorInput: taskOutputParameter(containerDriverName, paramExecutorInput), - cached: taskOutputParameter(containerDriverName, paramCachedDecision), - podSpecPatch: taskOutputParameter(containerDriverName, paramPodSpecPatch), - } - - t, err := c.containerExecutorTemplate(name, inputs.containerDef, c.spec.PipelineInfo.GetName()) - - if err != nil { - return err - } - - executorTask := &pipelineapi.PipelineTask{ - Name: name, - TaskSpec: t, - When: pipelineapi.WhenExpressions{ - { - Input: containerDriverOutputs.cached, - Operator: "in", - Values: []string{"false"}, - }, - }, - Params: []pipelineapi.Param{ - { - Name: paramExecutorInput, - Value: pipelineapi.ParamValue{Type: "string", StringVal: containerDriverOutputs.executiorInput}, - }, - { - Name: paramExecutionID, - Value: pipelineapi.ParamValue{Type: "string", StringVal: containerDriverOutputs.executionId}, - }, - { - Name: paramRunId, - Value: pipelineapi.ParamValue{Type: "string", StringVal: runID()}, - }, - { - Name: paramComponentSpec, - Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.component}, - }, - { - Name: paramPodSpecPatch, - Value: pipelineapi.ParamValue{Type: "string", StringVal: containerDriverOutputs.podSpecPatch}, - }, - }, - } - - c.addPipelineTask(executorTask) - return nil } @@ -393,7 +341,7 @@ func (c *pipelinerunCompiler) containerExecutorTemplate( "--run_id", inputValue(paramRunId), "--execution_id", inputValue(paramExecutionID), "--executor_input", inputValue(paramExecutorInput), - "--component_spec", inputValue(paramComponentSpec), + "--component_spec", inputValue(paramComponent), "--pod_name", "$(KFP_POD_NAME)", "--pod_uid", @@ -411,7 +359,7 @@ func (c *pipelinerunCompiler) containerExecutorTemplate( {Name: paramExecutorInput, Type: "string"}, // --executor_input {Name: paramExecutionID, Type: "string"}, // --execution_id {Name: paramRunId, Type: "string"}, // --run_id - {Name: paramComponentSpec, Type: "string"}, // --component_spec + {Name: paramComponent, Type: "string"}, // --component }, Steps: []pipelineapi.Step{ // step 1: copy launcher diff --git a/backend/src/v2/compiler/tektoncompiler/dag.go b/backend/src/v2/compiler/tektoncompiler/dag.go index 907bbc83e3..ec357a907d 100644 --- a/backend/src/v2/compiler/tektoncompiler/dag.go +++ b/backend/src/v2/compiler/tektoncompiler/dag.go @@ -233,8 +233,8 @@ func (c *pipelinerunCompiler) dagDriverTask( t := &pipelineapi.PipelineTask{ Name: name, TaskRef: &pipelineapi.TaskRef{ - APIVersion: "kfp-driver.tekton.dev/v1alpha1", - Kind: "KFPDriver", + APIVersion: "custom.tekton.dev/v1alpha1", + Kind: "KFPTask", }, Params: []pipelineapi.Param{ // "--type" @@ -242,17 +242,17 @@ func (c *pipelinerunCompiler) dagDriverTask( Name: paramNameType, Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.getDagType()}, }, - // "--pipeline_name" + // "--pipeline-name" { Name: paramNamePipelineName, Value: pipelineapi.ParamValue{Type: "string", StringVal: c.spec.GetPipelineInfo().GetName()}, }, - // "--run_id" + // "--run-id" { - Name: paramNameRunId, + Name: paramRunId, Value: pipelineapi.ParamValue{Type: "string", StringVal: runID()}, }, - // "--dag_execution_id" + // "--dag-execution-id" { Name: paramNameDagExecutionId, Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.getParentDagID(c.ExitHandlerScope())}, @@ -267,17 +267,17 @@ func (c *pipelinerunCompiler) dagDriverTask( Name: paramTask, Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.task}, }, - // "--runtime_config" + // "--runtime-config" { Name: paramNameRuntimeConfig, Value: pipelineapi.ParamValue{Type: "string", StringVal: runtimeConfigJson}, }, - // "--iteration_index" + // "--iteration-index" { Name: paramNameIterationIndex, Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.getIterationIndex()}, }, - // "--mlmd_server_address" + // "--mlmd-server-address" { Name: paramNameMLMDServerHost, Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDHost()}, @@ -309,8 +309,8 @@ func (c *pipelinerunCompiler) dagPubDriverTask( t := &pipelineapi.PipelineTask{ Name: name, TaskRef: &pipelineapi.TaskRef{ - APIVersion: "kfp-driver.tekton.dev/v1alpha1", - Kind: "KFPDriver", + APIVersion: "custom.tekton.dev/v1alpha1", + Kind: "KFPTask", }, Params: []pipelineapi.Param{ // "--type" @@ -318,27 +318,27 @@ func (c *pipelinerunCompiler) dagPubDriverTask( Name: paramNameType, Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.getDagType()}, }, - // "--pipeline_name" + // "--pipeline-name" { Name: paramNamePipelineName, Value: pipelineapi.ParamValue{Type: "string", StringVal: c.spec.GetPipelineInfo().GetName()}, }, - // "--run_id" + // "--run-id" { - Name: paramNameRunId, + Name: paramRunId, Value: pipelineapi.ParamValue{Type: "string", StringVal: runID()}, }, - // "--dag_execution_id" + // "--dag-execution-id" { Name: paramNameDagExecutionId, Value: pipelineapi.ParamValue{Type: "string", StringVal: inputs.getParentDagID(c.ExitHandlerScope() || rootDagPub)}, }, - // "--mlmd_server_address" + // "--mlmd-server-address" { Name: paramNameMLMDServerHost, Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDHost()}, }, - // "--mlmd_server_port" + // "--mlmd-server-port" { Name: paramNameMLMDServerPort, Value: pipelineapi.ParamValue{Type: "string", StringVal: GetMLMDPort()}, @@ -368,7 +368,7 @@ type pubDagDriverInputs struct { } func (i *pubDagDriverInputs) getDagType() string { - return "DAG-PUB" + return "DAG_PUB" } func (i *pubDagDriverInputs) getParentDagID(isExitHandler bool) string { diff --git a/backend/src/v2/compiler/tektoncompiler/importer.go b/backend/src/v2/compiler/tektoncompiler/importer.go index 7719133b12..d49518b4db 100644 --- a/backend/src/v2/compiler/tektoncompiler/importer.go +++ b/backend/src/v2/compiler/tektoncompiler/importer.go @@ -56,7 +56,7 @@ func (c *pipelinerunCompiler) Importer(name string, "--component_spec", inputValue(paramComponent), "--importer_spec", inputValue(paramImporter), "--pipeline_name", c.spec.PipelineInfo.GetName(), - "--run_id", inputValue(paramRunId), + "--run_id", inputValue(paramNameRunId), "--parent_dag_id", inputValue(paramParentDagID), "--pod_name", "$(KFP_POD_NAME)", @@ -140,7 +140,7 @@ func (c *pipelinerunCompiler) Importer(name string, Value: pipelineapi.ParamValue{Type: "string", StringVal: taskOutputParameter(getDAGDriverTaskName(c.CurrentDag()), paramExecutionID)}, }, { - Name: paramRunId, + Name: paramNameRunId, Value: pipelineapi.ParamValue{Type: "string", StringVal: runID()}, }, }, diff --git a/backend/src/v2/compiler/tektoncompiler/tekton.go b/backend/src/v2/compiler/tektoncompiler/tekton.go index 4757988485..a2946c7d9e 100644 --- a/backend/src/v2/compiler/tektoncompiler/tekton.go +++ b/backend/src/v2/compiler/tektoncompiler/tekton.go @@ -634,33 +634,33 @@ const ( paramImporter = "importer" // importer spec paramRuntimeConfig = "runtime-config" // job runtime config, pipeline level inputs paramParentDagID = "parent-dag-id" - paramExecutionID = "execution-id" paramIterationItem = "iteration-item" paramIterationCount = "iteration-count" paramIterationIndex = "iteration-index" - paramExecutorInput = "executor-input" paramDriverType = "driver-type" paramCachedDecision = "cached-decision" // indicate hit cache or not paramPodSpecPatch = "pod-spec-patch" // a strategic patch merged with the pod spec paramCondition = "condition" // condition = false -> skip the task paramRunId = "run-id" paramComponentSpec = "component-spec" + paramExecutionID = "execution-id" + paramExecutorInput = "executor-input" paramNameType = "type" - paramNamePipelineName = "pipeline_name" - paramNameRunId = "run_id" - paramNameDagExecutionId = "dag_execution_id" - paramNameRuntimeConfig = "runtime_config" - paramNameIterationIndex = "iteration_index" - paramNameExecutionId = "execution_id" - paramNameIterationCount = "iteration_count" + paramNamePipelineName = "pipeline-name" + paramNameRunId = "run-id" + paramNameDagExecutionId = "dag-execution-id" + paramNameRuntimeConfig = "runtime-config" + paramNameIterationIndex = "iteration-index" + paramNameExecutionId = "execution-id" + paramNameIterationCount = "iteration-count" paramNameCondition = "condition" - paramNameCachedDecision = "cached_decision" - paramNamePodSpecPatchPath = "pod_spec_patch_path" - paramNameExecutorInput = "executor_input" - paramNameMLMDServerHost = "mlmd_server_address" - paramNameMLMDServerPort = "mlmd_server_port" - paramKubernetesConfig = "kubernetes_config" // stores Kubernetes config + paramNameCachedDecision = "cached-decision" + paramNamePodSpecPatchPath = "pod-spec-patch-path" + paramNameExecutorInput = "executor-input" + paramNameMLMDServerHost = "mlmd-server-address" + paramNameMLMDServerPort = "mlmd-server-port" + paramKubernetesConfig = "kubernetes-config" // stores Kubernetes config kindPipelineLoop = "PipelineLoop" subfixPipelineLoop = "-pipelineloop" diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml index 9aef25b59a..df119bbe45 100755 --- a/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml @@ -27,18 +27,16 @@ spec: - name: parent-dag-id type: string tasks: - - name: print-op-driver + - name: print-op params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: pipeline-with-exit-handler - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(params.parent-dag-id) - - name: component - value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Exit handler has worked!"}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' @@ -52,29 +50,16 @@ spec: -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: print-op - params: - - name: executor-input - value: $(tasks.print-op-driver.results.executor-input) - - name: execution-id - value: $(tasks.print-op-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.print-op-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -92,17 +77,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -145,7 +130,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -155,6 +140,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -173,7 +159,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -182,44 +168,36 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.print-op-driver.results.cached-decision) - operator: in - values: - - "false" - name: root-system-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: pipeline-with-exit-handler - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(params.parent-dag-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - print-op taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask tasks: - - name: fail-op-driver + - name: fail-op params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: pipeline-with-exit-handler - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.exit-handler-1-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-fail-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task failed."}}}},"taskInfo":{"name":"fail-op"}}' @@ -233,29 +211,16 @@ spec: kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n print(message)\n sys.exit(1)\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: fail-op - params: - - name: executor-input - value: $(tasks.fail-op-driver.results.executor-input) - - name: execution-id - value: $(tasks.fail-op-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-fail-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.fail-op-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -273,17 +238,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -328,7 +293,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -338,6 +303,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -356,7 +322,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -365,24 +331,16 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.fail-op-driver.results.cached-decision) - operator: in - values: - - "false" - - name: print-op-2-driver + - name: print-op-2 params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: pipeline-with-exit-handler - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.exit-handler-1-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op-2"}}' - name: container @@ -395,29 +353,16 @@ spec: kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: print-op-2 - params: - - name: executor-input - value: $(tasks.print-op-2-driver.results.executor-input) - - name: execution-id - value: $(tasks.print-op-2-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.print-op-2-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -435,17 +380,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -488,7 +433,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -498,6 +443,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -516,7 +462,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -525,67 +471,61 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.print-op-2-driver.results.cached-decision) - operator: in - values: - - "false" - name: exit-handler-1-dag-driver params: - name: type value: DAG - - name: pipeline_name + - name: pipeline-name value: pipeline-with-exit-handler - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - name: component value: '{"dag":{"tasks":{"fail-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task failed."}}}},"taskInfo":{"name":"fail-op"}},"print-op-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--message":{"parameterType":"STRING"}}}}' - name: task value: '{"componentRef":{"name":"comp-exit-handler-1"},"inputs":{"parameters":{"pipelinechannel--message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"exit-handler-1"}}' - - name: runtime_config + - name: runtime-config value: "" - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: exit-handler-1-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: pipeline-with-exit-handler - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.exit-handler-1-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - fail-op - print-op-2 taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: root-system-dag-driver params: - name: type value: ROOT_DAG - - name: pipeline_name + - name: pipeline-name value: pipeline-with-exit-handler - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: "0" - name: component value: '{"dag":{"tasks":{"exit-handler-1":{"componentRef":{"name":"comp-exit-handler-1"},"inputs":{"parameters":{"pipelinechannel--message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"exit-handler-1"}},"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Exit @@ -593,15 +533,16 @@ spec: World!","parameterType":"STRING"}}}}' - name: task value: "" - - name: runtime_config + - name: runtime-config value: '{"parameterValues":{"message":"Hello World!"}}' - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + taskRunTemplate: {} status: {} diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml index 371be81b35..bb5490da0b 100644 --- a/backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml @@ -13,18 +13,16 @@ metadata: spec: pipelineSpec: tasks: - - name: hello-world-driver + - name: hello-world params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: namespace/n1/pipeline/hello-world - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container @@ -34,29 +32,16 @@ spec: = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: hello-world - params: - - name: executor-input - value: $(tasks.hello-world-driver.results.executor-input) - - name: execution-id - value: $(tasks.hello-world-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.hello-world-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -74,17 +59,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -ec @@ -116,7 +101,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -126,6 +111,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -144,7 +130,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -153,54 +139,49 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.hello-world-driver.results.cached-decision) - operator: in - values: - - "false" - name: root-system-dag-driver params: - name: type value: ROOT_DAG - - name: pipeline_name + - name: pipeline-name value: namespace/n1/pipeline/hello-world - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: "0" - name: component value: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - name: task value: "" - - name: runtime_config + - name: runtime-config value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: root-system-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: namespace/n1/pipeline/hello-world - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - hello-world taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + taskRunTemplate: {} status: {} diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/importer.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/importer.yaml index 81203269f2..9437e49630 100644 --- a/backend/src/v2/compiler/tektoncompiler/testdata/importer.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/importer.yaml @@ -74,6 +74,7 @@ spec: - $(METADATA_GRPC_SERVICE_PORT) command: - launcher-v2 + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -89,49 +90,49 @@ spec: value: "8080" image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e name: importer-main - computeResources: {} - name: root-system-dag-driver params: - name: type value: ROOT_DAG - - name: pipeline_name + - name: pipeline-name value: pipeline-with-importer - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: "0" - name: component value: '{"dag":{"tasks":{"importer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constantValue":{"stringValue":"gs://ml-pipeline-playground/shakespeare1.txt"}}}}},"taskInfo":{"name":"importer"}}}},"inputDefinitions":{"parameters":{"dataset2":{"type":"STRING"}}}}' - name: task value: "" - - name: runtime_config + - name: runtime-config value: '{}' - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: root-system-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: pipeline-with-importer - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - importer taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + taskRunTemplate: {} status: {} diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml index f185bb92f4..95591b1031 100644 --- a/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml @@ -41,59 +41,57 @@ spec: params: - name: type value: DAG - - name: pipeline_name + - name: pipeline-name value: pipeline-with-loop-static - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(params.parent-dag-id) - name: component value: '{"dag":{"tasks":{"concat-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-concat-op"},"inputs":{"parameters":{"a":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"},"b":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"b\"]"}}},"taskInfo":{"name":"concat-op"}},"print-op-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"dependentTasks":["concat-op"],"inputs":{"parameters":{"text":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"concat-op"}}}},"taskInfo":{"name":"print-op-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"STRUCT"}}}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[{\"a\": \"1\", \"b\": \"2\"}, {\"a\": \"10\", \"b\": \"20\"}]"}},"taskInfo":{"name":"for-loop-2"}}' - - name: runtime_config + - name: runtime-config value: "" - - name: iteration_index + - name: iteration-index value: $(params.iteration-index) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: for-loop-2-loop-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: pipeline-with-loop-static - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(params.parent-dag-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - print-op-2 taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: concat-op-driver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + - name: concat-op params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: pipeline-with-loop-static - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.for-loop-2-loop-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-concat-op","inputDefinitions":{"parameters":{"a":{"parameterType":"STRING"},"b":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-concat-op"},"inputs":{"parameters":{"a":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"},"b":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"b\"]"}}},"taskInfo":{"name":"concat-op"}}' - name: container @@ -107,29 +105,16 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef concat_op(a: str, b: str) -\u003e str:\n print(a + b)\n return a + b\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: concat-op - params: - - name: executor-input - value: $(tasks.concat-op-driver.results.executor-input) - - name: execution-id - value: $(tasks.concat-op-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-concat-op","inputDefinitions":{"parameters":{"a":{"parameterType":"STRING"},"b":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.concat-op-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -147,17 +132,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -200,7 +185,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -210,6 +195,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -228,7 +214,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -237,24 +223,16 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.concat-op-driver.results.cached-decision) - operator: in - values: - - "false" - - name: print-op-2-driver + - name: print-op-2 params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: pipeline-with-loop-static - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.for-loop-2-loop-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"dependentTasks":["concat-op"],"inputs":{"parameters":{"text":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"concat-op"}}}},"taskInfo":{"name":"print-op-2"}}' - name: container @@ -268,31 +246,18 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(text: str) -\u003e str:\n print(text)\n return text\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" + - name: component + value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' runAfter: - concat-op - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: print-op-2 - params: - - name: executor-input - value: $(tasks.print-op-2-driver.results.executor-input) - - name: execution-id - value: $(tasks.print-op-2-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec - value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.print-op-2-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -310,17 +275,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -363,7 +328,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -373,6 +338,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -391,7 +357,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -400,69 +366,61 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.print-op-2-driver.results.cached-decision) - operator: in - values: - - "false" - name: for-loop-2-dag-driver params: - name: type value: DAG - - name: pipeline_name + - name: pipeline-name value: pipeline-with-loop-static - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - name: component value: '{"dag":{"tasks":{"concat-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-concat-op"},"inputs":{"parameters":{"a":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"},"b":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"b\"]"}}},"taskInfo":{"name":"concat-op"}},"print-op-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"dependentTasks":["concat-op"],"inputs":{"parameters":{"text":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"concat-op"}}}},"taskInfo":{"name":"print-op-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"STRUCT"}}}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[{\"a\": \"1\", \"b\": \"2\"}, {\"a\": \"10\", \"b\": \"20\"}]"}},"taskInfo":{"name":"for-loop-2"}}' - - name: runtime_config + - name: runtime-config value: "" - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: for-loop-2-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: pipeline-with-loop-static - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.for-loop-2-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - for-loop-2-pipelineloop taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: print-op-driver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + - name: print-op params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: pipeline-with-loop-static - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"text":{"componentInputParameter":"greeting"}}},"taskInfo":{"name":"print-op"}}' - name: container @@ -475,29 +433,16 @@ spec: kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(text: str) -\u003e str:\n print(text)\n return text\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: print-op - params: - - name: executor-input - value: $(tasks.print-op-driver.results.executor-input) - - name: execution-id - value: $(tasks.print-op-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.print-op-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -515,17 +460,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -568,7 +513,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -578,6 +523,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -596,7 +542,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -605,21 +551,15 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.print-op-driver.results.cached-decision) - operator: in - values: - - "false" - name: root-system-dag-driver params: - name: type value: ROOT_DAG - - name: pipeline_name + - name: pipeline-name value: pipeline-with-loop-static - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: "0" - name: component value: '{"dag":{"tasks":{"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[{\"a\": @@ -627,36 +567,37 @@ spec: is a test for looping through parameters","isOptional":true,"parameterType":"STRING"}}}}' - name: task value: "" - - name: runtime_config + - name: runtime-config value: '{"parameterValues":{"greeting":"this is a test for looping through parameters"}}' - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: root-system-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: pipeline-with-loop-static - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - print-op - for-loop-2-pipelineloop taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + taskRunTemplate: {} status: {} diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml index 4bc897b71d..2204da7790 100755 --- a/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml @@ -13,18 +13,16 @@ metadata: spec: pipelineSpec: tasks: - - name: convert-experiment-spec-to-str-driver + - name: convert-experiment-spec-to-str params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-convert-experiment-spec-to-str","inputDefinitions":{"parameters":{"experiment_spec_json":{"parameterType":"STRUCT"}}},"outputDefinitions":{"parameters":{"experiment_spec_str_output":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-convert-experiment-spec-to-str"},"dependentTasks":["create-katib-experiment-task"],"inputs":{"parameters":{"experiment_spec_json":{"taskOutputParameter":{"outputParameterKey":"experiment_spec_json","producerTask":"create-katib-experiment-task"}}}},"taskInfo":{"name":"convert-experiment-spec-to-str"}}' - name: container @@ -40,31 +38,18 @@ spec: NamedTuple(''Outputs'', [(''experiment_spec_str_output'', str)]):\n import json\n output = NamedTuple(''Outputs'', [(''experiment_spec_str_output'', str)])\n return output(json.dumps(experiment_spec_json))\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" + - name: component + value: '{"executorLabel":"exec-convert-experiment-spec-to-str","inputDefinitions":{"parameters":{"experiment_spec_json":{"parameterType":"STRUCT"}}},"outputDefinitions":{"parameters":{"experiment_spec_str_output":{"parameterType":"STRING"}}}}' runAfter: - create-katib-experiment-task - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: convert-experiment-spec-to-str - params: - - name: executor-input - value: $(tasks.convert-experiment-spec-to-str-driver.results.executor-input) - - name: execution-id - value: $(tasks.convert-experiment-spec-to-str-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec - value: '{"executorLabel":"exec-convert-experiment-spec-to-str","inputDefinitions":{"parameters":{"experiment_spec_json":{"parameterType":"STRUCT"}}},"outputDefinitions":{"parameters":{"experiment_spec_str_output":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.convert-experiment-spec-to-str-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -82,17 +67,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -136,7 +121,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -146,6 +131,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -164,7 +150,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -173,24 +159,16 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.convert-experiment-spec-to-str-driver.results.cached-decision) - operator: in - values: - - "false" - - name: convert-katib-results-driver + - name: convert-katib-results params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-convert-katib-results","inputDefinitions":{"artifacts":{"katib_results":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-convert-katib-results"},"dependentTasks":["create-dataset"],"inputs":{"artifacts":{"katib_results":{"taskOutputArtifact":{"outputArtifactKey":"parameter_set","producerTask":"create-dataset"}}}},"taskInfo":{"name":"convert-katib-results"}}' - name: container @@ -211,31 +189,18 @@ spec: + pa[\"value\"])\n elif pa[\"name\"] == \"batch_size\":\n best_hps.append(\"--tf-batch-size=\" + pa[\"value\"])\n print(\"Best Hyperparameters: {}\".format(best_hps))\n return \" \".join(best_hps)\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" + - name: component + value: '{"executorLabel":"exec-convert-katib-results","inputDefinitions":{"artifacts":{"katib_results":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' runAfter: - create-dataset - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: convert-katib-results - params: - - name: executor-input - value: $(tasks.convert-katib-results-driver.results.executor-input) - - name: execution-id - value: $(tasks.convert-katib-results-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec - value: '{"executorLabel":"exec-convert-katib-results","inputDefinitions":{"artifacts":{"katib_results":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.convert-katib-results-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -253,17 +218,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -320,7 +285,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -330,6 +295,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -348,7 +314,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -357,53 +323,32 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.convert-katib-results-driver.results.cached-decision) - operator: in - values: - - "false" - - name: create-dataset-driver + - name: create-dataset params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-create-dataset","inputDefinitions":{"parameters":{"experiment_name":{"parameterType":"STRING"},"experiment_namespace":{"parameterType":"STRING"},"experiment_spec_json":{"parameterType":"STRING"},"experiment_timeout_minutes":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"parameter_set":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-dataset"},"dependentTasks":["convert-experiment-spec-to-str"],"inputs":{"parameters":{"experiment_name":{"componentInputParameter":"name"},"experiment_namespace":{"componentInputParameter":"namespace"},"experiment_spec_json":{"taskOutputParameter":{"outputParameterKey":"experiment_spec_str_output","producerTask":"convert-experiment-spec-to-str"}},"experiment_timeout_minutes":{"runtimeValue":{"constant":60}}}},"taskInfo":{"name":"create-dataset"}}' - name: container value: '{"args":["--experiment-name","{{$.inputs.parameters[''experiment_name'']}}","--experiment-namespace","{{$.inputs.parameters[''experiment_namespace'']}}","--experiment-spec","{{$.inputs.parameters[''experiment_spec_json'']}}","--experiment-timeout-minutes","{{$.inputs.parameters[''experiment_timeout_minutes'']}}","--delete-after-done","False","--output-file","{{$.outputs.artifacts[''parameter_set''].path}}"],"command":["python","src/launch_experiment.py"],"image":"docker.io/kubeflowkatib/kubeflow-pipelines-launcher"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" + - name: component + value: '{"executorLabel":"exec-create-dataset","inputDefinitions":{"parameters":{"experiment_name":{"parameterType":"STRING"},"experiment_namespace":{"parameterType":"STRING"},"experiment_spec_json":{"parameterType":"STRING"},"experiment_timeout_minutes":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"parameter_set":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' runAfter: - convert-experiment-spec-to-str - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: create-dataset - params: - - name: executor-input - value: $(tasks.create-dataset-driver.results.executor-input) - - name: execution-id - value: $(tasks.create-dataset-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec - value: '{"executorLabel":"exec-create-dataset","inputDefinitions":{"parameters":{"experiment_name":{"parameterType":"STRING"},"experiment_namespace":{"parameterType":"STRING"},"experiment_spec_json":{"parameterType":"STRING"},"experiment_timeout_minutes":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"parameter_set":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: pod-spec-patch - value: $(tasks.create-dataset-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -421,17 +366,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - python - src/launch_experiment.py @@ -458,7 +403,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -468,6 +413,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -486,7 +432,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -495,24 +441,16 @@ spec: optional: true image: docker.io/kubeflowkatib/kubeflow-pipelines-launcher name: user-main - computeResources: {} - when: - - input: $(tasks.create-dataset-driver.results.cached-decision) - operator: in - values: - - "false" - - name: create-katib-experiment-task-driver + - name: create-katib-experiment-task params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-create-katib-experiment-task","inputDefinitions":{"parameters":{"experiment_name":{"parameterType":"STRING"},"experiment_namespace":{"parameterType":"STRING"},"training_steps":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"experiment_spec_json":{"parameterType":"STRUCT"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-katib-experiment-task"},"inputs":{"parameters":{"experiment_name":{"componentInputParameter":"name"},"experiment_namespace":{"componentInputParameter":"namespace"},"training_steps":{"componentInputParameter":"training_steps"}}},"taskInfo":{"name":"create-katib-experiment-task"}}' - name: container @@ -564,29 +502,16 @@ spec: Convert experiment_spec to Dict type.\n experiment_spec_json = ApiClient().sanitize_for_serialization(experiment_spec)\n output = NamedTuple(''Outputs'', [(''experiment_spec_json'', Dict[str, str])])\n return output(experiment_spec_json)\n\n"],"image":"python:3.8"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: create-katib-experiment-task - params: - - name: executor-input - value: $(tasks.create-katib-experiment-task-driver.results.executor-input) - - name: execution-id - value: $(tasks.create-katib-experiment-task-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-create-katib-experiment-task","inputDefinitions":{"parameters":{"experiment_name":{"parameterType":"STRING"},"experiment_namespace":{"parameterType":"STRING"},"training_steps":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"experiment_spec_json":{"parameterType":"STRUCT"}}}}' - - name: pod-spec-patch - value: $(tasks.create-katib-experiment-task-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -604,17 +529,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -800,7 +725,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -810,6 +735,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -828,7 +754,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -837,24 +763,16 @@ spec: optional: true image: python:3.8 name: user-main - computeResources: {} - when: - - input: $(tasks.create-katib-experiment-task-driver.results.cached-decision) - operator: in - values: - - "false" - - name: create-tfjob-task-driver + - name: create-tfjob-task params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-create-tfjob-task","inputDefinitions":{"parameters":{"best_hps":{"parameterType":"STRING"},"model_volume_name":{"parameterType":"STRING"},"tfjob_name":{"parameterType":"STRING"},"tfjob_namespace":{"parameterType":"STRING"},"training_steps":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"chief_spec":{"parameterType":"STRUCT"},"worker_spec":{"parameterType":"STRUCT"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-tfjob-task"},"dependentTasks":["convert-katib-results"],"inputs":{"parameters":{"best_hps":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"convert-katib-results"}},"model_volume_name":{"componentInputParameter":"model_volume_name"},"tfjob_name":{"componentInputParameter":"name"},"tfjob_namespace":{"componentInputParameter":"namespace"},"training_steps":{"componentInputParameter":"training_steps"}}},"taskInfo":{"name":"create-tfjob-task"}}' - name: container @@ -895,31 +813,18 @@ spec: --tf-train-steps={} {}\".format(training_steps, best_hps) \n ],\n }\n ],\n }\n }\n }\n\n output = NamedTuple(''Outputs'', [(''chief_spec'', Dict[str, str]), (''worker_spec'', Dict[str, str])])\n return output(tfjob_chief_spec, tfjob_worker_spec)\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" + - name: component + value: '{"executorLabel":"exec-create-tfjob-task","inputDefinitions":{"parameters":{"best_hps":{"parameterType":"STRING"},"model_volume_name":{"parameterType":"STRING"},"tfjob_name":{"parameterType":"STRING"},"tfjob_namespace":{"parameterType":"STRING"},"training_steps":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"chief_spec":{"parameterType":"STRUCT"},"worker_spec":{"parameterType":"STRUCT"}}}}' runAfter: - convert-katib-results - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: create-tfjob-task - params: - - name: executor-input - value: $(tasks.create-tfjob-task-driver.results.executor-input) - - name: execution-id - value: $(tasks.create-tfjob-task-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec - value: '{"executorLabel":"exec-create-tfjob-task","inputDefinitions":{"parameters":{"best_hps":{"parameterType":"STRING"},"model_volume_name":{"parameterType":"STRING"},"tfjob_name":{"parameterType":"STRING"},"tfjob_namespace":{"parameterType":"STRING"},"training_steps":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"chief_spec":{"parameterType":"STRUCT"},"worker_spec":{"parameterType":"STRUCT"}}}}' - - name: pod-spec-patch - value: $(tasks.create-tfjob-task-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -937,17 +842,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -1021,7 +926,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -1031,6 +936,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -1049,7 +955,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -1058,24 +964,16 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.create-tfjob-task-driver.results.cached-decision) - operator: in - values: - - "false" - - name: convert-inference-service-to-artifact-driver + - name: convert-inference-service-to-artifact params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.serving-pipeline-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-convert-inference-service-to-artifact","inputDefinitions":{"parameters":{"inferenceservice_yaml":{"parameterType":"STRUCT"}}},"outputDefinitions":{"artifacts":{"inferenceservice_artifact":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-convert-inference-service-to-artifact"},"dependentTasks":["create-serving-task"],"inputs":{"parameters":{"inferenceservice_yaml":{"taskOutputParameter":{"outputParameterKey":"inferenceservice_yaml","producerTask":"create-serving-task"}}}},"taskInfo":{"name":"convert-inference-service-to-artifact"}}' - name: container @@ -1090,31 +988,18 @@ spec: convert_inference_service_to_artifact(inferenceservice_yaml: Dict[str, str], inferenceservice_artifact: Output[Artifact]):\n import json\n with open(inferenceservice_artifact.path, ''w'') as f:\n f.write(json.dumps(inferenceservice_yaml))\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" + - name: component + value: '{"executorLabel":"exec-convert-inference-service-to-artifact","inputDefinitions":{"parameters":{"inferenceservice_yaml":{"parameterType":"STRUCT"}}},"outputDefinitions":{"artifacts":{"inferenceservice_artifact":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' runAfter: - create-serving-task - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: convert-inference-service-to-artifact - params: - - name: executor-input - value: $(tasks.convert-inference-service-to-artifact-driver.results.executor-input) - - name: execution-id - value: $(tasks.convert-inference-service-to-artifact-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec - value: '{"executorLabel":"exec-convert-inference-service-to-artifact","inputDefinitions":{"parameters":{"inferenceservice_yaml":{"parameterType":"STRUCT"}}},"outputDefinitions":{"artifacts":{"inferenceservice_artifact":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: pod-spec-patch - value: $(tasks.convert-inference-service-to-artifact-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -1132,17 +1017,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -1186,7 +1071,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -1196,6 +1081,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -1214,7 +1100,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -1223,24 +1109,16 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.convert-inference-service-to-artifact-driver.results.cached-decision) - operator: in - values: - - "false" - - name: create-serving-task-driver + - name: create-serving-task params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.serving-pipeline-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-create-serving-task","inputDefinitions":{"parameters":{"model_name":{"parameterType":"STRING"},"model_namespace":{"parameterType":"STRING"},"model_volume_name":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"inferenceservice_yaml":{"parameterType":"STRUCT"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-serving-task"},"inputs":{"parameters":{"model_name":{"componentInputParameter":"model_name"},"model_namespace":{"componentInputParameter":"model_namespace"},"model_volume_name":{"componentInputParameter":"model_volume_name"}}},"taskInfo":{"name":"create-serving-task"}}' - name: container @@ -1261,29 +1139,16 @@ spec: {\n \"storageUri\": \"pvc://{}/\".format(model_volume_name)\n }\n }\n }\n }\n\n output = NamedTuple(''Outputs'', [(''inferenceservice_yaml'', Dict[str, str])])\n return output(inference_service)\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: create-serving-task - params: - - name: executor-input - value: $(tasks.create-serving-task-driver.results.executor-input) - - name: execution-id - value: $(tasks.create-serving-task-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-create-serving-task","inputDefinitions":{"parameters":{"model_name":{"parameterType":"STRING"},"model_namespace":{"parameterType":"STRING"},"model_volume_name":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"inferenceservice_yaml":{"parameterType":"STRUCT"}}}}' - - name: pod-spec-patch - value: $(tasks.create-serving-task-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -1301,17 +1166,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -1376,7 +1241,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -1386,6 +1251,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -1404,7 +1270,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -1413,54 +1279,33 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.create-serving-task-driver.results.cached-decision) - operator: in - values: - - "false" - - name: serving-launcher-driver + - name: serving-launcher params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.serving-pipeline-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-serving-launcher","inputDefinitions":{"parameters":{"action":{"parameterType":"STRING"},"inferenceservice_yaml":{"parameterType":"STRUCT"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-serving-launcher"},"dependentTasks":["convert-inference-service-to-artifact","create-serving-task"],"inputs":{"parameters":{"action":{"runtimeValue":{"constant":"apply"}},"inferenceservice_yaml":{"taskOutputParameter":{"outputParameterKey":"inferenceservice_yaml","producerTask":"create-serving-task"}}}},"taskInfo":{"name":"serving-launcher"}}' - name: container value: '{"args":["--action","{{$.inputs.parameters[''action'']}}","--inferenceservice-yaml","{{$.inputs.parameters[''inferenceservice_yaml'']}}"],"command":["python","kservedeployer.py"],"image":"quay.io/aipipeline/kserve-component:v0.7.0"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" + - name: component + value: '{"executorLabel":"exec-serving-launcher","inputDefinitions":{"parameters":{"action":{"parameterType":"STRING"},"inferenceservice_yaml":{"parameterType":"STRUCT"}}}}' runAfter: - convert-inference-service-to-artifact - create-serving-task - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: serving-launcher - params: - - name: executor-input - value: $(tasks.serving-launcher-driver.results.executor-input) - - name: execution-id - value: $(tasks.serving-launcher-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec - value: '{"executorLabel":"exec-serving-launcher","inputDefinitions":{"parameters":{"action":{"parameterType":"STRING"},"inferenceservice_yaml":{"parameterType":"STRUCT"}}}}' - - name: pod-spec-patch - value: $(tasks.serving-launcher-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -1478,17 +1323,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - python - kservedeployer.py @@ -1507,7 +1352,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -1517,6 +1362,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -1535,7 +1381,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -1544,100 +1390,79 @@ spec: optional: true image: quay.io/aipipeline/kserve-component:v0.7.0 name: user-main - computeResources: {} - when: - - input: $(tasks.serving-launcher-driver.results.cached-decision) - operator: in - values: - - "false" - name: serving-pipeline-dag-driver params: - name: type value: DAG - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - name: component value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"inferenceservice_artifact","producerSubtask":"convert-inference-service-to-artifact"}]}}},"tasks":{"convert-inference-service-to-artifact":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-convert-inference-service-to-artifact"},"dependentTasks":["create-serving-task"],"inputs":{"parameters":{"inferenceservice_yaml":{"taskOutputParameter":{"outputParameterKey":"inferenceservice_yaml","producerTask":"create-serving-task"}}}},"taskInfo":{"name":"convert-inference-service-to-artifact"}},"create-serving-task":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-serving-task"},"inputs":{"parameters":{"model_name":{"componentInputParameter":"model_name"},"model_namespace":{"componentInputParameter":"model_namespace"},"model_volume_name":{"componentInputParameter":"model_volume_name"}}},"taskInfo":{"name":"create-serving-task"}},"serving-launcher":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-serving-launcher"},"dependentTasks":["convert-inference-service-to-artifact","create-serving-task"],"inputs":{"parameters":{"action":{"runtimeValue":{"constant":"apply"}},"inferenceservice_yaml":{"taskOutputParameter":{"outputParameterKey":"inferenceservice_yaml","producerTask":"create-serving-task"}}}},"taskInfo":{"name":"serving-launcher"}}}},"inputDefinitions":{"parameters":{"model_name":{"parameterType":"STRING"},"model_namespace":{"parameterType":"STRING"},"model_volume_name":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-serving-pipeline"},"dependentTasks":["tfjob-launcher"],"inputs":{"parameters":{"model_name":{"componentInputParameter":"name"},"model_namespace":{"componentInputParameter":"namespace"},"model_volume_name":{"componentInputParameter":"model_volume_name"}}},"taskInfo":{"name":"serving-pipeline"}}' - - name: runtime_config + - name: runtime-config value: "" - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - tfjob-launcher taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: serving-pipeline-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.serving-pipeline-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - serving-launcher taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: tfjob-launcher-driver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + - name: tfjob-launcher params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-tfjob-launcher","inputDefinitions":{"parameters":{"chief_spec":{"parameterType":"STRUCT"},"tfjob_name":{"parameterType":"STRING"},"tfjob_namespace":{"parameterType":"STRING"},"tfjob_timeout_minutes":{"parameterType":"NUMBER_INTEGER"},"worker_spec":{"parameterType":"STRUCT"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-tfjob-launcher"},"dependentTasks":["convert-katib-results","create-tfjob-task"],"inputs":{"parameters":{"chief_spec":{"taskOutputParameter":{"outputParameterKey":"chief_spec","producerTask":"create-tfjob-task"}},"tfjob_name":{"componentInputParameter":"name"},"tfjob_namespace":{"componentInputParameter":"namespace"},"tfjob_timeout_minutes":{"runtimeValue":{"constant":60}},"worker_spec":{"taskOutputParameter":{"outputParameterKey":"worker_spec","producerTask":"create-tfjob-task"}}}},"taskInfo":{"name":"tfjob-launcher"}}' - name: container value: '{"args":["--name","{{$.inputs.parameters[''tfjob_name'']}}","--namespace","{{$.inputs.parameters[''tfjob_namespace'']}}","--workerSpec","{{$.inputs.parameters[''worker_spec'']}}","--chiefSpec","{{$.inputs.parameters[''chief_spec'']}}","--tfjobTimeoutMinutes","{{$.inputs.parameters[''tfjob_timeout_minutes'']}}","--deleteAfterDone","False"],"command":["python","/ml/launch_tfjob.py"],"image":"nikenano/launchernew:latest"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" + - name: component + value: '{"executorLabel":"exec-tfjob-launcher","inputDefinitions":{"parameters":{"chief_spec":{"parameterType":"STRUCT"},"tfjob_name":{"parameterType":"STRING"},"tfjob_namespace":{"parameterType":"STRING"},"tfjob_timeout_minutes":{"parameterType":"NUMBER_INTEGER"},"worker_spec":{"parameterType":"STRUCT"}}}}' runAfter: - convert-katib-results - create-tfjob-task - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: tfjob-launcher - params: - - name: executor-input - value: $(tasks.tfjob-launcher-driver.results.executor-input) - - name: execution-id - value: $(tasks.tfjob-launcher-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec - value: '{"executorLabel":"exec-tfjob-launcher","inputDefinitions":{"parameters":{"chief_spec":{"parameterType":"STRUCT"},"tfjob_name":{"parameterType":"STRING"},"tfjob_namespace":{"parameterType":"STRING"},"tfjob_timeout_minutes":{"parameterType":"NUMBER_INTEGER"},"worker_spec":{"parameterType":"STRUCT"}}}}' - - name: pod-spec-patch - value: $(tasks.tfjob-launcher-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -1655,17 +1480,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - python - /ml/launch_tfjob.py @@ -1692,7 +1517,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -1702,6 +1527,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -1720,7 +1546,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -1729,54 +1555,49 @@ spec: optional: true image: nikenano/launchernew:latest name: user-main - computeResources: {} - when: - - input: $(tasks.tfjob-launcher-driver.results.cached-decision) - operator: in - values: - - "false" - name: root-system-dag-driver params: - name: type value: ROOT_DAG - - name: pipeline_name + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: "0" - name: component value: '{"dag":{"tasks":{"convert-experiment-spec-to-str":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-convert-experiment-spec-to-str"},"dependentTasks":["create-katib-experiment-task"],"inputs":{"parameters":{"experiment_spec_json":{"taskOutputParameter":{"outputParameterKey":"experiment_spec_json","producerTask":"create-katib-experiment-task"}}}},"taskInfo":{"name":"convert-experiment-spec-to-str"}},"convert-katib-results":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-convert-katib-results"},"dependentTasks":["create-dataset"],"inputs":{"artifacts":{"katib_results":{"taskOutputArtifact":{"outputArtifactKey":"parameter_set","producerTask":"create-dataset"}}}},"taskInfo":{"name":"convert-katib-results"}},"create-dataset":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-dataset"},"dependentTasks":["convert-experiment-spec-to-str"],"inputs":{"parameters":{"experiment_name":{"componentInputParameter":"name"},"experiment_namespace":{"componentInputParameter":"namespace"},"experiment_spec_json":{"taskOutputParameter":{"outputParameterKey":"experiment_spec_str_output","producerTask":"convert-experiment-spec-to-str"}},"experiment_timeout_minutes":{"runtimeValue":{"constant":60}}}},"taskInfo":{"name":"create-dataset"}},"create-katib-experiment-task":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-katib-experiment-task"},"inputs":{"parameters":{"experiment_name":{"componentInputParameter":"name"},"experiment_namespace":{"componentInputParameter":"namespace"},"training_steps":{"componentInputParameter":"training_steps"}}},"taskInfo":{"name":"create-katib-experiment-task"}},"create-tfjob-task":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-create-tfjob-task"},"dependentTasks":["convert-katib-results"],"inputs":{"parameters":{"best_hps":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"convert-katib-results"}},"model_volume_name":{"componentInputParameter":"model_volume_name"},"tfjob_name":{"componentInputParameter":"name"},"tfjob_namespace":{"componentInputParameter":"namespace"},"training_steps":{"componentInputParameter":"training_steps"}}},"taskInfo":{"name":"create-tfjob-task"}},"serving-pipeline":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-serving-pipeline"},"dependentTasks":["tfjob-launcher"],"inputs":{"parameters":{"model_name":{"componentInputParameter":"name"},"model_namespace":{"componentInputParameter":"namespace"},"model_volume_name":{"componentInputParameter":"model_volume_name"}}},"taskInfo":{"name":"serving-pipeline"}},"tfjob-launcher":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-tfjob-launcher"},"dependentTasks":["convert-katib-results","create-tfjob-task"],"inputs":{"parameters":{"chief_spec":{"taskOutputParameter":{"outputParameterKey":"chief_spec","producerTask":"create-tfjob-task"}},"tfjob_name":{"componentInputParameter":"name"},"tfjob_namespace":{"componentInputParameter":"namespace"},"tfjob_timeout_minutes":{"runtimeValue":{"constant":60}},"worker_spec":{"taskOutputParameter":{"outputParameterKey":"worker_spec","producerTask":"create-tfjob-task"}}}},"taskInfo":{"name":"tfjob-launcher"}}}},"inputDefinitions":{"parameters":{"model_volume_name":{"defaultValue":"workflow1-model-volume","parameterType":"STRING"},"name":{"defaultValue":"mnist-e2e","parameterType":"STRING"},"namespace":{"defaultValue":"kubeflow-user-project","parameterType":"STRING"},"training_steps":{"defaultValue":"200","parameterType":"STRING"}}}}' - name: task value: "" - - name: runtime_config + - name: runtime-config value: '{"parameterValues":{"model_volume_name":"workflow1-model-volume","name":"mnist-e2e","namespace":"kubeflow-user-project","training_steps":"200"}}' - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: root-system-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: end-to-end-pipeline - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - serving-launcher taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + taskRunTemplate: {} status: {} diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml index 43d95a74f0..1955d6e64c 100644 --- a/backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml @@ -13,18 +13,16 @@ metadata: spec: pipelineSpec: tasks: - - name: flip-coin-op-driver + - name: flip-coin-op params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-flip-coin-op","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin-op"},"taskInfo":{"name":"flip-coin-op"}}' - name: container @@ -39,29 +37,16 @@ spec: flip_coin_op() -\u003e str:\n \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n import random\n result = random.choice([''heads'', ''tails''])\n print(result)\n return result\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: flip-coin-op - params: - - name: executor-input - value: $(tasks.flip-coin-op-driver.results.executor-input) - - name: execution-id - value: $(tasks.flip-coin-op-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-flip-coin-op","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.flip-coin-op-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -79,17 +64,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -135,7 +120,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -145,6 +130,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -163,7 +149,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -172,12 +158,6 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} - when: - - input: $(tasks.flip-coin-op-driver.results.cached-decision) - operator: in - values: - - "false" - name: for-loop-2-pipelineloop params: - name: parent-dag-id @@ -209,11 +189,11 @@ spec: params: - name: type value: DAG - - name: pipeline_name + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(params.parent-dag-id) - name: component value: '{"dag":{"tasks":{"condition-3":{"componentRef":{"name":"comp-condition-3"},"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop-item-param-1":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--loop-item-param-1''] @@ -221,49 +201,47 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["flip-coin-op"],"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op"}}}},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"heads\", \"tails\"]"}},"taskInfo":{"name":"for-loop-2"}}' - - name: runtime_config + - name: runtime-config value: "" - - name: iteration_index + - name: iteration-index value: $(params.iteration-index) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: for-loop-2-loop-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(params.parent-dag-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - - print-op - print-op-2 + - print-op taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: print-op-driver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + - name: print-op params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.condition-4-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--loop-item-param-1'']}} and {{$.inputs.parameters[''pipelinechannel--get-random-int-op-Output'']}} @@ -278,34 +256,16 @@ spec: -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(message: str):\n \"\"\"Print a message.\"\"\"\n print(message)\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - when: - - input: $(tasks.condition-4-dag-driver.results.condition) - operator: notin - values: - - "false" - - name: print-op - params: - - name: executor-input - value: $(tasks.print-op-driver.results.executor-input) - - name: execution-id - value: $(tasks.print-op-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.print-op-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -323,17 +283,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -376,7 +336,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -386,6 +346,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -404,7 +365,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -413,21 +374,20 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} when: - - input: $(tasks.print-op-driver.results.cached-decision) - operator: in + - input: $(tasks.condition-4-dag-driver.results.condition) + operator: notin values: - "false" - name: condition-4-dag-driver params: - name: type value: DAG - - name: pipeline_name + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.condition-3-dag-driver.results.execution-id) - name: component value: '{"dag":{"tasks":{"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--loop-item-param-1'']}} @@ -436,50 +396,48 @@ spec: - name: task value: '{"componentRef":{"name":"comp-condition-4"},"dependentTasks":["get-random-int-op"],"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--get-random-int-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"get-random-int-op"}},"pipelinechannel--loop-item-param-1":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"condition-4"},"triggerPolicy":{"condition":"int(inputs.parameter_values[''pipelinechannel--get-random-int-op-Output'']) \u003e 5"}}' - - name: runtime_config + - name: runtime-config value: "" - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - get-random-int-op taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: condition-4-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.condition-4-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - print-op taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: print-op-2-driver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + - name: print-op-2 params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.condition-5-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--loop-item-param-1'']}} and {{$.inputs.parameters[''pipelinechannel--get-random-int-op-Output'']}} @@ -494,34 +452,16 @@ spec: -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(message: str):\n \"\"\"Print a message.\"\"\"\n print(message)\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - when: - - input: $(tasks.condition-5-dag-driver.results.condition) - operator: notin - values: - - "false" - - name: print-op-2 - params: - - name: executor-input - value: $(tasks.print-op-2-driver.results.executor-input) - - name: execution-id - value: $(tasks.print-op-2-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: pod-spec-patch - value: $(tasks.print-op-2-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -539,17 +479,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -592,7 +532,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -602,6 +542,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -620,7 +561,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -629,21 +570,20 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} when: - - input: $(tasks.print-op-2-driver.results.cached-decision) - operator: in + - input: $(tasks.condition-5-dag-driver.results.condition) + operator: notin values: - "false" - name: condition-5-dag-driver params: - name: type value: DAG - - name: pipeline_name + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.condition-3-dag-driver.results.execution-id) - name: component value: '{"dag":{"tasks":{"print-op-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--loop-item-param-1'']}} @@ -652,50 +592,48 @@ spec: - name: task value: '{"componentRef":{"name":"comp-condition-5"},"dependentTasks":["get-random-int-op"],"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--get-random-int-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"get-random-int-op"}},"pipelinechannel--loop-item-param-1":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"condition-5"},"triggerPolicy":{"condition":"int(inputs.parameter_values[''pipelinechannel--get-random-int-op-Output'']) \u003c= 5"}}' - - name: runtime_config + - name: runtime-config value: "" - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - get-random-int-op taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: condition-5-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.condition-5-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - print-op-2 taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - - name: get-random-int-op-driver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + - name: get-random-int-op params: - name: type value: CONTAINER - - name: pipeline_name + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.condition-3-dag-driver.results.execution-id) - - name: component - value: '{"executorLabel":"exec-get-random-int-op","inputDefinitions":{"parameters":{"maximum":{"parameterType":"NUMBER_INTEGER"},"minimum":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-get-random-int-op"},"inputs":{"parameters":{"maximum":{"runtimeValue":{"constant":9}},"minimum":{"runtimeValue":{"constant":0}}}},"taskInfo":{"name":"get-random-int-op"}}' - name: container @@ -711,34 +649,16 @@ spec: a random number between minimum and maximum (inclusive).\"\"\"\n import random\n result = random.randint(minimum, maximum)\n print(result)\n return result\n\n"],"image":"python:3.7"}' - - name: iteration_index + - name: iteration-index value: "" - - name: kubernetes_config + - name: kubernetes-config value: "" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" - taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver - when: - - input: $(tasks.condition-3-dag-driver.results.condition) - operator: notin - values: - - "false" - - name: get-random-int-op - params: - - name: executor-input - value: $(tasks.get-random-int-op-driver.results.executor-input) - - name: execution-id - value: $(tasks.get-random-int-op-driver.results.execution-id) - - name: run-id - value: $(context.pipelineRun.uid) - - name: component-spec + - name: component value: '{"executorLabel":"exec-get-random-int-op","inputDefinitions":{"parameters":{"maximum":{"parameterType":"NUMBER_INTEGER"},"minimum":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: pod-spec-patch - value: $(tasks.get-random-int-op-driver.results.pod-spec-patch) taskSpec: apiVersion: custom.tekton.dev/v1alpha1 kind: KFPTask @@ -756,17 +676,17 @@ spec: type: string - name: run-id type: string - - name: component-spec + - name: component type: string steps: - command: - launcher-v2 - --copy - /tekton/home/launch + computeResources: {} image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher - computeResources: {} - args: - sh - -c @@ -812,7 +732,7 @@ spec: - --executor_input - $(params.executor-input) - --component_spec - - $(params.component-spec) + - $(params.component) - --pod_name - $(KFP_POD_NAME) - --pod_uid @@ -822,6 +742,7 @@ spec: - --mlmd_server_port - $(METADATA_GRPC_SERVICE_PORT) - -- + computeResources: {} env: - name: KFP_POD_NAME valueFrom: @@ -840,7 +761,7 @@ spec: - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" - name: MINIO_SERVICE_SERVICE_HOST - value: "minio-service.kubeflow.svc.cluster.local" + value: minio-service.kubeflow.svc.cluster.local - name: MINIO_SERVICE_SERVICE_PORT value: "9000" envFrom: @@ -849,21 +770,20 @@ spec: optional: true image: python:3.7 name: user-main - computeResources: {} when: - - input: $(tasks.get-random-int-op-driver.results.cached-decision) - operator: in + - input: $(tasks.condition-3-dag-driver.results.condition) + operator: notin values: - "false" - name: condition-3-dag-driver params: - name: type value: DAG - - name: pipeline_name + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.for-loop-2-loop-dag-driver.results.execution-id) - name: component value: '{"dag":{"tasks":{"condition-4":{"componentRef":{"name":"comp-condition-4"},"dependentTasks":["get-random-int-op"],"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--get-random-int-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"get-random-int-op"}},"pipelinechannel--loop-item-param-1":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"condition-4"},"triggerPolicy":{"condition":"int(inputs.parameter_values[''pipelinechannel--get-random-int-op-Output'']) @@ -872,46 +792,46 @@ spec: - name: task value: '{"componentRef":{"name":"comp-condition-3"},"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop-item-param-1":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--loop-item-param-1''] == inputs.parameter_values[''pipelinechannel--flip-coin-op-Output'']"}}' - - name: runtime_config + - name: runtime-config value: "" - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: condition-3-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.condition-3-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - print-op - print-op-2 taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: for-loop-2-dag-driver params: - name: type value: DAG - - name: pipeline_name + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - name: component value: '{"dag":{"tasks":{"condition-3":{"componentRef":{"name":"comp-condition-3"},"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop-item-param-1":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--loop-item-param-1''] @@ -919,81 +839,82 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["flip-coin-op"],"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op"}}}},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"heads\", \"tails\"]"}},"taskInfo":{"name":"for-loop-2"}}' - - name: runtime_config + - name: runtime-config value: "" - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - flip-coin-op taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: for-loop-2-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.for-loop-2-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - for-loop-2-pipelineloop taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: root-system-dag-driver params: - name: type value: ROOT_DAG - - name: pipeline_name + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: "0" - name: component value: '{"dag":{"tasks":{"flip-coin-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin-op"},"taskInfo":{"name":"flip-coin-op"}},"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["flip-coin-op"],"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op"}}}},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"heads\", \"tails\"]"}},"taskInfo":{"name":"for-loop-2"}}}}}' - name: task value: "" - - name: runtime_config + - name: runtime-config value: '{}' - - name: iteration_index + - name: iteration-index value: "-1" - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask - name: root-system-dag-pub-driver params: - name: type - value: DAG-PUB - - name: pipeline_name + value: DAG_PUB + - name: pipeline-name value: tutorial-control-flows - - name: run_id + - name: run-id value: $(context.pipelineRun.uid) - - name: dag_execution_id + - name: dag-execution-id value: $(tasks.root-system-dag-driver.results.execution-id) - - name: mlmd_server_address + - name: mlmd-server-address value: metadata-grpc-service.kubeflow.svc.cluster.local - - name: mlmd_server_port + - name: mlmd-server-port value: "8080" runAfter: - for-loop-2-pipelineloop taskRef: - apiVersion: kfp-driver.tekton.dev/v1alpha1 - kind: KFPDriver + apiVersion: custom.tekton.dev/v1alpha1 + kind: KFPTask + taskRunTemplate: {} status: {} diff --git a/manifests/kustomize/third-party/tekton-custom-task/kfptask/201-clusterrole.yaml b/manifests/kustomize/third-party/tekton-custom-task/kfptask/201-clusterrole.yaml index bf74ff2b6e..d8da4a5e61 100644 --- a/manifests/kustomize/third-party/tekton-custom-task/kfptask/201-clusterrole.yaml +++ b/manifests/kustomize/third-party/tekton-custom-task/kfptask/201-clusterrole.yaml @@ -34,6 +34,19 @@ rules: - apiGroups: ["apps"] resources: ["deployments", "deployments/finalizers"] verbs: ["get", "list", "create", "update", "delete", "patch", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims"] + verbs: ["*"] + + # Controller needs permission to emit events associated with Run CRs. + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "create", "update", "delete", "patch", "watch"] + + # driver needs to access configmaps to get configuration + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "watch", "list"] --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/kustomize/third-party/tekton-custom-task/kfptask/201-role.yaml b/manifests/kustomize/third-party/tekton-custom-task/kfptask/201-role.yaml index 055630eb1a..757ec7c4d6 100644 --- a/manifests/kustomize/third-party/tekton-custom-task/kfptask/201-role.yaml +++ b/manifests/kustomize/third-party/tekton-custom-task/kfptask/201-role.yaml @@ -33,6 +33,34 @@ rules: resources: ["podsecuritypolicies"] resourceNames: ["tekton-pipelines", "openshift-pipelines"] verbs: ["use"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["persistentvolumes", persistentvolumeclaims] + verbs: ["*"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["create", "delete", "get"] + - apiGroups: [""] + resources: ["pods", "pods/exec", "pods/log", "services"] + verbs: ["*"] + - apiGroups: ["", "apps", "extensions"] + resources: ["deployments", "replicasets"] + verbs: ["*"] + - apiGroups: ["kubeflow.org"] + resources: ["*"] + verbs: ["*"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["*"] + - apiGroups: ["machinelearning.seldon.io"] + resources: ["seldondeployments"] + verbs: ["*"] + - apiGroups: ["policy"] + resources: ["podsecuritypolicies"] + resourceNames: ["tekton-pipelines", "openshift-pipelines"] + verbs: ["use"] --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/kustomize/third-party/tekton-custom-task/kustomization.yaml b/manifests/kustomize/third-party/tekton-custom-task/kustomization.yaml index 125eb5701b..e2e80dd212 100644 --- a/manifests/kustomize/third-party/tekton-custom-task/kustomization.yaml +++ b/manifests/kustomize/third-party/tekton-custom-task/kustomization.yaml @@ -3,9 +3,10 @@ kind: Kustomization resources: - pipeline-loops -- driver-controller - exit-handler - kfptask +# Deprecated controller +# - driver-controller namespace: tekton-pipelines @@ -14,9 +15,6 @@ images: newTag: 1.8.1 - name: quay.io/aipipeline/pipelineloop-webhook newTag: 1.8.1 - - name: kfp-v2-dev-driver-controller - newName: quay.io/aipipeline/tekton-driver - newTag: 2.0.3 - name: tekton-exithandler-controller newName: quay.io/aipipeline/tekton-exithandler-controller newTag: 2.0.3 @@ -29,3 +27,7 @@ images: - name: tekton-kfptask-webhook newName: quay.io/aipipeline/tekton-kfptask-webhook newTag: 2.0.3 + # Deprecated controller + # - name: kfp-v2-dev-driver-controller + # newName: quay.io/aipipeline/tekton-driver + # newTag: 2.0.3 diff --git a/scripts/deploy/github/manifests/kustomization.yaml b/scripts/deploy/github/manifests/kustomization.yaml index 57e08a7c19..b4fc2b5817 100644 --- a/scripts/deploy/github/manifests/kustomization.yaml +++ b/scripts/deploy/github/manifests/kustomization.yaml @@ -23,18 +23,17 @@ images: - name: gcr.io/ml-pipeline/scheduledworkflow newName: kind-registry:5000/scheduledworkflow newTag: latest +- name: '*/aipipeline/tekton-exithandler-controller' + newTag: latest +- name: '*/aipipeline/tekton-exithandler-webhook' + newTag: latest +- name: '*/aipipeline/tekton-kfptask-controller' + newTag: latest +- name: '*/aipipeline/tekton-kfptask-webhook' + newTag: latest # For Staging image build only -# - name: '*/aipipeline/tekton-exithandler-controller' -# newName: kind-registry:5000/tekton-exithandler-controller -# newTag: latest -# - name: '*/aipipeline/tekton-exithandler-webhook' -# newName: kind-registry:5000/tekton-exithandler-webhook -# newTag: latest -# - name: '*/aipipeline/tekton-kfptask-controller' -# newName: kind-registry:5000/tekton-kfptask-controller -# newTag: latest -# - name: '*/aipipeline/tekton-kfptask-webhook' -# newName: kind-registry:5000/tekton-kfptask-webhook +# - name: '*/aipipeline/tekton-driver' +# newName: kind-registry:5000/tekton-driver # newTag: latest # - name: '*/aipipeline/tekton-driver' # newName: kind-registry:5000/tekton-driver