merge with kfp 2.0.5

kubeflow · Dec 8, 2023 · 75025c7 · 75025c7
2 parents 552c477 + c5658f0
commit 75025c7
Show file tree

Hide file tree

Showing 32 changed files with 117 additions and 65 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,22 @@
 # Changelog
 
+### [2.0.5](https://github.com/kubeflow/pipelines/compare/2.0.4...2.0.5) (2023-12-08)
+
+
+### Features
+
+* **backend:** Parameterizing v2 Launcher and Driver Images ([\#10269](https://github.com/kubeflow/pipelines/issues/10269)) ([23a2bc3](https://github.com/kubeflow/pipelines/commit/23a2bc3ba370e28a5ef5f0e0df1e40ec8f7888b9))
+
+
+### Bug Fixes
+
+* **components:** Append `tune-type` label when uploading models tuned by `preview.llm.rlhf_pipeline` ([708b8bd](https://github.com/kubeflow/pipelines/commit/708b8bd62357b1323e1a5127d5d914d7325bbdee))
+
+
+### Other Pull Requests
+
+* Update SECURITY.md ([\#10279](https://github.com/kubeflow/pipelines/issues/10279)) ([1499c4a](https://github.com/kubeflow/pipelines/commit/1499c4ae895613a7ff2458b2bd5ef98a548a33cb))
+
 ### [2.0.4](https://github.com/kubeflow/pipelines/compare/2.0.3...2.0.4) (2023-12-01)
 
 

diff --git a/SECURITY.md b/SECURITY.md
@@ -2,8 +2,4 @@
 
 When reporting a vulnerability, please include a description of the issue, the steps you took to create the issue, affected versions, and, if known, mitigations for the issue. If the issue is confirmed as a vulnerability, we will open a Security Advisory. This project follows a 90 day disclosure timeline.
 
-To report a security issue, please choose one of the options below:
-
-- Use [https://g.co/vulnz](https://g.co/vulnz). We use `g.co/vulnz` for our intake, and do coordination and disclosure here on GitHub (including using GitHub Security Advisory). The Google Security Team will respond within 5 working days of your report on `g.co/vulnz`.
-
-- Report a security vulnerability private via GitHub built-in function by following [these instructions](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability).
+To report a security issue, follow [these instructions](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability).
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.0.4
+2.0.5
diff --git a/backend/api/v1beta1/python_http_client/README.md b/backend/api/v1beta1/python_http_client/README.md
@@ -3,8 +3,8 @@ This file contains REST API specification for Kubeflow Pipelines. The file is au
 
 This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
-- API version: 2.0.4
-- Package version: 2.0.4
+- API version: 2.0.5
+- Package version: 2.0.5
 - Build package: org.openapitools.codegen.languages.PythonClientCodegen
 For more information, please visit [https://www.google.com](https://www.google.com)
 

diff --git a/backend/api/v1beta1/python_http_client/kfp_server_api/__init__.py b/backend/api/v1beta1/python_http_client/kfp_server_api/__init__.py
@@ -14,7 +14,7 @@
 
 from __future__ import absolute_import
 
-__version__ = "2.0.4"
+__version__ = "2.0.5"
 
 # import apis into sdk package
 from kfp_server_api.api.experiment_service_api import ExperimentServiceApi

diff --git a/backend/api/v1beta1/python_http_client/kfp_server_api/api_client.py b/backend/api/v1beta1/python_http_client/kfp_server_api/api_client.py
@@ -78,7 +78,7 @@ def __init__(self, configuration=None, header_name=None, header_value=None,
             self.default_headers[header_name] = header_value
         self.cookie = cookie
         # Set default User-Agent.
-        self.user_agent = 'OpenAPI-Generator/2.0.4/python'
+        self.user_agent = 'OpenAPI-Generator/2.0.5/python'
         self.client_side_validation = configuration.client_side_validation
 
     def __enter__(self):

diff --git a/backend/api/v1beta1/python_http_client/kfp_server_api/configuration.py b/backend/api/v1beta1/python_http_client/kfp_server_api/configuration.py
@@ -351,8 +351,8 @@ def to_debug_report(self):
         return "Python SDK Debug Report:\n"\
                "OS: {env}\n"\
                "Python Version: {pyversion}\n"\
-               "Version of the API: 2.0.4\n"\
-               "SDK Package Version: 2.0.4".\
+               "Version of the API: 2.0.5\n"\
+               "SDK Package Version: 2.0.5".\
                format(env=sys.platform, pyversion=sys.version)
 
     def get_host_settings(self):

diff --git a/backend/api/v1beta1/python_http_client/setup.py b/backend/api/v1beta1/python_http_client/setup.py
@@ -13,7 +13,7 @@
 from setuptools import setup, find_packages  # noqa: H301
 
 NAME = "kfp-server-api"
-VERSION = "2.0.4"
+VERSION = "2.0.5"
 # To install the library, run the following
 #
 # python setup.py install

diff --git a/backend/api/v1beta1/swagger/kfp_api_single_file.swagger.json b/backend/api/v1beta1/swagger/kfp_api_single_file.swagger.json
@@ -2,7 +2,7 @@
   "swagger": "2.0",
   "info": {
     "title": "Kubeflow Pipelines API",
-    "version": "2.0.4",
+    "version": "2.0.5",
     "description": "This file contains REST API specification for Kubeflow Pipelines. The file is autogenerated from the swagger definition.",
     "contact": {
       "name": "google",

diff --git a/backend/api/v2beta1/python_http_client/README.md b/backend/api/v2beta1/python_http_client/README.md
@@ -3,8 +3,8 @@ This file contains REST API specification for Kubeflow Pipelines. The file is au
 
 This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
-- API version: 2.0.4
-- Package version: 2.0.4
+- API version: 2.0.5
+- Package version: 2.0.5
 - Build package: org.openapitools.codegen.languages.PythonClientCodegen
 For more information, please visit [https://www.google.com](https://www.google.com)
 

diff --git a/backend/api/v2beta1/python_http_client/kfp_server_api/__init__.py b/backend/api/v2beta1/python_http_client/kfp_server_api/__init__.py
@@ -14,7 +14,7 @@
 
 from __future__ import absolute_import
 
-__version__ = "2.0.4"
+__version__ = "2.0.5"
 
 # import apis into sdk package
 from kfp_server_api.api.auth_service_api import AuthServiceApi

diff --git a/backend/api/v2beta1/python_http_client/kfp_server_api/api_client.py b/backend/api/v2beta1/python_http_client/kfp_server_api/api_client.py
@@ -78,7 +78,7 @@ def __init__(self, configuration=None, header_name=None, header_value=None,
             self.default_headers[header_name] = header_value
         self.cookie = cookie
         # Set default User-Agent.
-        self.user_agent = 'OpenAPI-Generator/2.0.4/python'
+        self.user_agent = 'OpenAPI-Generator/2.0.5/python'
         self.client_side_validation = configuration.client_side_validation
 
     def __enter__(self):

diff --git a/backend/api/v2beta1/python_http_client/kfp_server_api/configuration.py b/backend/api/v2beta1/python_http_client/kfp_server_api/configuration.py
@@ -351,8 +351,8 @@ def to_debug_report(self):
         return "Python SDK Debug Report:\n"\
                "OS: {env}\n"\
                "Python Version: {pyversion}\n"\
-               "Version of the API: 2.0.4\n"\
-               "SDK Package Version: 2.0.4".\
+               "Version of the API: 2.0.5\n"\
+               "SDK Package Version: 2.0.5".\
                format(env=sys.platform, pyversion=sys.version)
 
     def get_host_settings(self):

diff --git a/backend/api/v2beta1/python_http_client/setup.py b/backend/api/v2beta1/python_http_client/setup.py
@@ -13,7 +13,7 @@
 from setuptools import setup, find_packages  # noqa: H301
 
 NAME = "kfp-server-api"
-VERSION = "2.0.4"
+VERSION = "2.0.5"
 # To install the library, run the following
 #
 # python setup.py install

diff --git a/backend/api/v2beta1/swagger/kfp_api_single_file.swagger.json b/backend/api/v2beta1/swagger/kfp_api_single_file.swagger.json
@@ -2,7 +2,7 @@
   "swagger": "2.0",
   "info": {
     "title": "Kubeflow Pipelines API",
-    "version": "2.0.4",
+    "version": "2.0.5",
     "description": "This file contains REST API specification for Kubeflow Pipelines. The file is autogenerated from the swagger definition.",
     "contact": {
       "name": "google",

diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go
@@ -116,8 +116,8 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S
 		wf:        wf,
 		templates: make(map[string]*wfapi.Template),
 		// TODO(chensun): release process and update the images.
-		driverImage:   "gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707",
-		launcherImage: "gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e",
+		launcherImage: GetLauncherImage(),
+		driverImage:   GetDriverImage(),
 		job:           job,
 		spec:          spec,
 		executors:     deploy.GetExecutors(),

diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go
@@ -16,13 +16,18 @@ package argocompiler
 
 import (
 	wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
+	"os"
 	"github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec"
 	"github.com/kubeflow/pipelines/backend/src/v2/component"
 	k8score "k8s.io/api/core/v1"
 )
 
 const (
 	volumeNameKFPLauncher = "kfp-launcher"
+	DefaultLauncherImage = "gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4"
+	LauncherImageEnvVar   = "V2_LAUNCHER_IMAGE"
+	DefaultDriverImage = "gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707"
+	DriverImageEnvVar   = "V2_DRIVER_IMAGE"
 )
 
 func (c *workflowCompiler) Container(name string, component *pipelinespec.ComponentSpec, container *pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec) error {
@@ -52,6 +57,22 @@ type containerDriverInputs struct {
 	kubernetesConfig string // optional, used when Kubernetes config is not empty
 }
 
+func GetLauncherImage() string {
+    launcherImage := os.Getenv(LauncherImageEnvVar)
+    if launcherImage == "" {
+        launcherImage = DefaultLauncherImage
+    }
+    return launcherImage
+}
+
+func GetDriverImage() string {
+    driverImage := os.Getenv(DriverImageEnvVar)
+    if driverImage == "" {
+        driverImage = DefaultDriverImage
+    }
+    return driverImage
+}
+
 func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs) {
 	dagTask := &wfapi.DAGTask{
 		Name:     name,
@@ -111,7 +132,7 @@ func (c *workflowCompiler) addContainerDriverTemplate() string {
 			},
 		},
 		Container: &k8score.Container{
-			Image:   c.driverImage,
+			Image:   GetDriverImage(),
 			Command: []string{"driver"},
 			Args: []string{
 				"--type", "CONTAINER",
@@ -226,7 +247,7 @@ func (c *workflowCompiler) addContainerExecutorTemplate() string {
 		InitContainers: []wfapi.UserContainer{{
 			Container: k8score.Container{
 				Name:    "kfp-launcher",
-				Image:   c.launcherImage,
+				Image:   GetLauncherImage(),
 				Command: []string{"launcher-v2", "--copy", component.KFPLauncherPath},
 				VolumeMounts: []k8score.VolumeMount{{
 					Name:      volumeNameKFPLauncher,

diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md
@@ -1,5 +1,6 @@
 ## Upcoming release
 * Fix `v1.automl.training_job.AutoMLImageTrainingJobRunOp` `ModuleNotFoundError`
+* Append `tune-type` to existing labels when uploading models tuned by `preview.llm.rlhf_pipeline` instead of overriding them.
 
 
 ## Release 2.6.0

diff --git a/...nts/google-cloud/google_cloud_pipeline_components/_implementation/llm/deployment_graph.py b/...nts/google-cloud/google_cloud_pipeline_components/_implementation/llm/deployment_graph.py
@@ -75,22 +75,16 @@ def pipeline(
           'large_model_reference'
       ]
   ).set_display_name('Resolve Upload Model')
-  upload_task = (
-      upload_llm_model.upload_llm_model(
-          project=_placeholders.PROJECT_ID_PLACEHOLDER,
-          location=upload_location,
-          regional_endpoint=regional_endpoint.output,
-          artifact_uri=adapter_artifact.output,
-          model_display_name=display_name.output,
-          model_reference_name='text-bison@001',
-          upload_model=upload_model.output,
-      )
-      .set_env_variable(
-          name='VERTEX_AI_PIPELINES_RUN_LABELS',
-          value=json.dumps({'tune-type': 'rlhf'}),
-      )
-      .set_display_name('Upload Model')
-  )
+  upload_task = upload_llm_model.upload_llm_model(
+      project=_placeholders.PROJECT_ID_PLACEHOLDER,
+      location=upload_location,
+      regional_endpoint=regional_endpoint.output,
+      artifact_uri=adapter_artifact.output,
+      model_display_name=display_name.output,
+      model_reference_name='text-bison@001',
+      upload_model=upload_model.output,
+      tune_type='rlhf',
+  ).set_display_name('Upload Model')
   deploy_model = function_based.resolve_deploy_model(
       deploy_model=deploy_model,
       large_model_reference=reference_model_metadata.outputs[

diff --git a/...nts/google-cloud/google_cloud_pipeline_components/_implementation/llm/upload_llm_model.py b/...nts/google-cloud/google_cloud_pipeline_components/_implementation/llm/upload_llm_model.py
@@ -34,6 +34,7 @@ def upload_llm_model(
     gcp_resources: dsl.OutputPath(str),
     encryption_spec_key_name: str = '',
     upload_model: bool = True,
+    tune_type: str = '',
 ):
   """Uploads LLM model.
 
@@ -48,6 +49,8 @@ def upload_llm_model(
       upload_model: Whether to upload the model to the Model Registry. Default
         is ``True``. If ``False``, the model will not be uploaded and output
         artifacts will contain empty strings.
+      tune_type: Method used to tune the model, e.g. ``rlhf``. If present, this
+        value is used to set the ``tune-type`` run label during model upload.
 
   Returns:
       model_resource_name: Path to the created Model on Model Registry.
@@ -76,6 +79,8 @@ def upload_llm_model(
     labels['google-vertex-llm-tuning-base-model-id'] = (
         model_reference_name.replace('@', '-')
     )
+    if tune_type:
+      labels['tune-type'] = tune_type
 
     model_upload_payload = {
         'model': {

diff --git a/frontend/server/package-lock.json b/frontend/server/package-lock.json
diff --git a/frontend/server/package.json b/frontend/server/package.json
@@ -5,7 +5,7 @@
     "@aws-sdk/credential-providers": "^3.359.0",
     "@google-cloud/storage": "^2.5.0",
     "@kubernetes/client-node": "^0.8.2",
-    "axios": ">=0.21.1",
+    "axios": ">=1.6.0",
     "crypto-js": "^3.1.8",
     "express": "^4.17.3",
     "gunzip-maybe": "^1.4.1",

diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml
@@ -12,7 +12,7 @@ metadata:
 spec:
   descriptor:
     type: Kubeflow Pipelines
-    version: 2.0.4
+    version: 2.0.5
     description: |-
       Reusable end-to-end ML workflow
     maintainers:

diff --git a/manifests/gcp_marketplace/schema.yaml b/manifests/gcp_marketplace/schema.yaml
@@ -1,9 +1,9 @@
 x-google-marketplace:
   schemaVersion: v2
   applicationApiVersion: v1beta1
-  publishedVersion: 2.0.4
+  publishedVersion: 2.0.5
   publishedVersionMetadata:
-    releaseNote: Based on 2.0.4 version.
+    releaseNote: Based on 2.0.5 version.
     releaseTypes:
       - Feature
     recommended: false

diff --git a/manifests/kustomize/base/cache-deployer/kustomization.yaml b/manifests/kustomize/base/cache-deployer/kustomization.yaml
@@ -8,4 +8,4 @@ commonLabels:
   app: cache-deployer
 images:
   - name: gcr.io/ml-pipeline/cache-deployer
-    newTag: 2.0.4
+    newTag: 2.0.5
diff --git a/manifests/kustomize/base/cache/kustomization.yaml b/manifests/kustomize/base/cache/kustomization.yaml
@@ -10,4 +10,4 @@ commonLabels:
   app: cache-server
 images:
   - name: gcr.io/ml-pipeline/cache-server
-    newTag: 2.0.4
+    newTag: 2.0.5
diff --git a/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml b/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml
@@ -11,7 +11,7 @@ data:
     until the changes take effect. A quick way to restart all deployments in a
     namespace: `kubectl rollout restart deployment -n <your-namespace>`.
   appName: pipeline
-  appVersion: 2.0.4
+  appVersion: 2.0.5
   dbHost: mysql # relic to be removed after release
   dbPort: "3306" # relic to be removed after release
   dbType: mysql

diff --git a/manifests/kustomize/base/metadata/base/kustomization.yaml b/manifests/kustomize/base/metadata/base/kustomization.yaml
@@ -10,4 +10,4 @@ resources:
 namespace: kubeflow
 images:
   - name: gcr.io/ml-pipeline/metadata-envoy
-    newTag: 2.0.4
+    newTag: 2.0.5