Skip to content

Commit

Permalink
fix(components): Fix to model batch explanation component for Structu…
Browse files Browse the repository at this point in the history
…red Data pipelines

Signed-off-by: Googler <[email protected]>
PiperOrigin-RevId: 661475667
  • Loading branch information
Googler committed Aug 10, 2024
1 parent e8f0208 commit 289f64f
Show file tree
Hide file tree
Showing 40 changed files with 480 additions and 480 deletions.
2 changes: 1 addition & 1 deletion components/google-cloud/RELEASE.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
## Upcoming release
* Bump image for Structured Data pipelines.
* Fix to model batch explanation component for Structured Data pipelines; image bump.

## Release 2.16.0
* Updated the Starry Net pipeline's template gallery description, and added dataprep_nan_threshold and dataprep_zero_threshold args to the Starry Net pipeline.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def automl_forecasting_ensemble(
# fmt: on
job_id = dsl.PIPELINE_JOB_ID_PLACEHOLDER
task_id = dsl.PIPELINE_TASK_ID_PLACEHOLDER
image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625'
image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625'
display_name = f'automl-forecasting-ensemble-{job_id}-{task_id}'

error_file_path = f'{root_dir}/{job_id}/{task_id}/error.pb'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,14 +99,14 @@ def automl_forecasting_stage_1_tuner(
' 1, "machine_spec": {"machine_type": "n1-standard-8"},'
' "container_spec": {"image_uri":"'
),
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625',
'", "args": ["forecasting_mp_l2l_stage_1_tuner',
'", "--region=',
location,
'", "--transform_output_path=',
transform_output.uri,
'", "--training_docker_uri=',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625',
'", "--reduce_search_space_mode=',
reduce_search_space_mode,
f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,14 +97,14 @@ def automl_forecasting_stage_2_tuner(
' 1, "machine_spec": {"machine_type": "n1-standard-8"},'
' "container_spec": {"image_uri":"'
),
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625',
'", "args": ["forecasting_mp_l2l_stage_2_tuner',
'", "--region=',
location,
'", "--transform_output_path=',
transform_output.uri,
'", "--training_docker_uri=',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240804_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625',
f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}',
'", "--training_base_dir=',
root_dir,
Expand Down

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def automated_feature_engineering(
' 1, "machine_spec": {"machine_type": "n1-standard-16"},'
' "container_spec": {"image_uri":"'
),
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625',
'", "args": ["feature_engineering", "--project=', project,
'", "--location=', location, '", "--data_source_bigquery_table_path=',
data_source_bigquery_table_path,
Expand Down

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def distillation_stage_feature_transform_engine(
# fmt: on

return dsl.ContainerSpec(
image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625',
image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625',
command=[],
args=[
'distillation_stage_feature_transform_engine',
Expand Down Expand Up @@ -185,7 +185,7 @@ def distillation_stage_feature_transform_engine(
dataflow_machine_type,
]
),
'--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625',
'--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625',
dsl.ConcatPlaceholder(
items=[
'--dataflow_disk_size_gb=',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ def tabular_feature_ranking_and_selection(
' 1, "machine_spec": {"machine_type": "n1-standard-8"},'
' "container_spec": {"image_uri":"'
),
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625',
'", "args": ["feature_selection", "--data_source=',
data_source.uri,
'", "--target_column=',
Expand Down Expand Up @@ -137,7 +137,7 @@ def tabular_feature_ranking_and_selection(
),
dataflow_max_num_workers,
'", "--dataflow_worker_container_image=',
'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625',
'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625',
'", "--dataflow_machine_type=',
dataflow_machine_type,
'", "--dataflow_disk_size_gb=',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -983,8 +983,8 @@ deploymentSpec:
"/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}'
- '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}'
- '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}'
- --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625
- --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625
- --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625
- --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
- '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}'
- '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}'
- '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}'
Expand All @@ -1001,7 +1001,7 @@ deploymentSpec:
- '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat":
["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}'
- '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}'
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
resources:
cpuLimit: 8.0
memoryLimit: 30.0
Expand Down Expand Up @@ -1049,7 +1049,7 @@ deploymentSpec:
["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}'
- '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat":
["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}'
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
pipelineInfo:
description: Defines pipeline for feature transform engine component.
name: feature-selection
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ def feature_transform_engine(
# fmt: on

return dsl.ContainerSpec(
image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625',
image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625',
command=[],
args=[
'feature_transform_engine',
Expand Down Expand Up @@ -637,8 +637,8 @@ def feature_transform_engine(
dsl.ConcatPlaceholder(
items=['--dataflow_machine_type=', dataflow_machine_type]
),
'--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625',
'--feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625',
'--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625',
'--feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625',
dsl.ConcatPlaceholder(
items=['--dataflow_disk_size_gb=', dataflow_disk_size_gb]
),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,15 +158,15 @@ def tabnet_hyperparameter_tuning_job(
', "disk_spec": ',
training_disk_spec,
', "container_spec": {"image_uri":"',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240804_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240808_0625',
'", "args": ["--target_column=',
target_column,
'", "--weight_column=',
weight_column,
'", "--model_type=',
prediction_type,
'", "--prediction_docker_uri=',
'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625',
'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625',
'", "--prediction_docker_uri_artifact_path=',
prediction_docker_uri_output,
'", "--baseline_path=',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2826,7 +2826,7 @@ deploymentSpec:
\"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
"\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\":
{\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"",
"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240804_0625", "\",
"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\",
\"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}",
"/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=",
"{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}'
Expand All @@ -2841,7 +2841,7 @@ deploymentSpec:
args:
- --executor_input
- '{{$}}'
image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625
resources:
cpuLimit: 8.0
memoryLimit: 52.0
Expand All @@ -2866,7 +2866,7 @@ deploymentSpec:
\ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\
\ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\
\ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
exec-feature-transform-engine:
container:
args:
Expand Down Expand Up @@ -2951,8 +2951,8 @@ deploymentSpec:
"/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}'
- '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}'
- '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}'
- --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625
- --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625
- --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625
- --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
- '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}'
- '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}'
- '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}'
Expand All @@ -2969,7 +2969,7 @@ deploymentSpec:
- '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat":
["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}'
- '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}'
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
resources:
cpuLimit: 8.0
memoryLimit: 30.0
Expand Down Expand Up @@ -3037,7 +3037,7 @@ deploymentSpec:
\ = {\n 'instanceSchemaUri': instance_schema_uri,\n 'predictionSchemaUri':\
\ prediction_schema_uri,\n }\n unmanaged_container_model.uri = os.path.join(\n\
\ trials_dir, 'trial_{}'.format(best_trial['id']), 'model'\n )\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
exec-get-model-display-name:
container:
args:
Expand All @@ -3064,7 +3064,7 @@ deploymentSpec:
\n return collections.namedtuple(\n 'Outputs',\n [\n \
\ 'model_display_name',\n ],\n )(\n model_display_name,\n )\n\
\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
exec-get-tabnet-study-spec-parameters:
container:
args:
Expand Down Expand Up @@ -3580,7 +3580,7 @@ deploymentSpec:
\ = ', '.join(extra_overrides)\n warnings.warn(\n f'The overrides\
\ {extra_override_str} were not found in the params and '\n 'will\
\ be ignored.'\n )\n\n return study_spec_parameters\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
exec-model-batch-predict:
container:
args:
Expand Down Expand Up @@ -3821,7 +3821,7 @@ deploymentSpec:
\ 'training_disk_spec',\n 'eval_machine_spec',\n 'eval_replica_count',\n\
\ ],\n )(\n training_machine_spec,\n training_disk_spec,\n\
\ eval_machine_spec,\n eval_replica_count,\n )\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
exec-set-optional-inputs:
container:
args:
Expand Down Expand Up @@ -3869,7 +3869,7 @@ deploymentSpec:
\ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\
\ ],\n )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\
\ )\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
exec-split-materialized-data:
container:
args:
Expand Down Expand Up @@ -3915,7 +3915,7 @@ deploymentSpec:
\ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\
\ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\
\ 'w') as f:\n f.write(file_patterns[2])\n\n"
image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625
exec-tabnet-hyperparameter-tuning-job:
container:
args:
Expand Down Expand Up @@ -3943,11 +3943,11 @@ deploymentSpec:
", \"trial_job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"",
"1", "\", \"machine_spec\": ", "{{$.inputs.parameters[''training_machine_spec'']}}",
", \"disk_spec\": ", "{{$.inputs.parameters[''training_disk_spec'']}}",
", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240804_0625",
", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240808_0625",
"\", \"args\": [\"--target_column=", "{{$.inputs.parameters[''target_column'']}}",
"\", \"--weight_column=", "{{$.inputs.parameters[''weight_column'']}}",
"\", \"--model_type=", "{{$.inputs.parameters[''prediction_type'']}}", "\",
\"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625",
\"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625",
"\", \"--prediction_docker_uri_artifact_path=", "{{$.outputs.parameters[''prediction_docker_uri_output''].output_file}}",
"\", \"--baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}",
"\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\",
Expand Down Expand Up @@ -4016,7 +4016,7 @@ deploymentSpec:
["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}'
- '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat":
["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}'
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240804_0625
image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
pipelineInfo:
description: The TabNet built-in algorithm HyperparameterTuningJob pipeline.
name: automl-tabular-tabnet-hyperparameter-tuning-job
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,15 +165,15 @@ def tabnet_trainer(
', "disk_spec": ',
training_disk_spec,
', "container_spec": {"image_uri":"',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240804_0625',
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20240808_0625',
'", "args": ["--target_column=',
target_column,
'", "--weight_column=',
weight_column,
'", "--model_type=',
prediction_type,
'", "--prediction_docker_uri=',
'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240804_0625',
'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20240808_0625',
'", "--baseline_path=',
instance_baseline.uri,
'", "--metadata_path=',
Expand Down
Loading

0 comments on commit 289f64f

Please sign in to comment.