From 115e6bb6e029f08e32a21f701547ca373d314353 Mon Sep 17 00:00:00 2001 From: The Magician Date: Thu, 19 Sep 2024 14:05:28 -0700 Subject: [PATCH] Add new resource for creating Whistle Mapping, Reconciliation and Backfill Pipeline Jobs for Healthcare Data Engine (#11677) (#802) [upstream:1902bacf8e9c72befb6dfb29945bc5effb7355ab] Signed-off-by: Modular Magician --- .../backing_file.tf | 15 ++++ healthcare_pipeline_job_backfill/main.tf | 13 +++ healthcare_pipeline_job_backfill/motd | 7 ++ healthcare_pipeline_job_backfill/tutorial.md | 79 ++++++++++++++++++ .../backing_file.tf | 15 ++++ .../main.tf | 81 +++++++++++++++++++ .../motd | 7 ++ .../tutorial.md | 79 ++++++++++++++++++ .../backing_file.tf | 15 ++++ .../main.tf | 42 ++++++++++ healthcare_pipeline_job_reconciliation/motd | 7 ++ .../tutorial.md | 79 ++++++++++++++++++ .../backing_file.tf | 15 ++++ .../main.tf | 56 +++++++++++++ healthcare_pipeline_job_whistle_mapping/motd | 7 ++ .../tutorial.md | 79 ++++++++++++++++++ 16 files changed, 596 insertions(+) create mode 100644 healthcare_pipeline_job_backfill/backing_file.tf create mode 100644 healthcare_pipeline_job_backfill/main.tf create mode 100644 healthcare_pipeline_job_backfill/motd create mode 100644 healthcare_pipeline_job_backfill/tutorial.md create mode 100644 healthcare_pipeline_job_mapping_recon_dest/backing_file.tf create mode 100644 healthcare_pipeline_job_mapping_recon_dest/main.tf create mode 100644 healthcare_pipeline_job_mapping_recon_dest/motd create mode 100644 healthcare_pipeline_job_mapping_recon_dest/tutorial.md create mode 100644 healthcare_pipeline_job_reconciliation/backing_file.tf create mode 100644 healthcare_pipeline_job_reconciliation/main.tf create mode 100644 healthcare_pipeline_job_reconciliation/motd create mode 100644 healthcare_pipeline_job_reconciliation/tutorial.md create mode 100644 healthcare_pipeline_job_whistle_mapping/backing_file.tf create mode 100644 healthcare_pipeline_job_whistle_mapping/main.tf create mode 100644 healthcare_pipeline_job_whistle_mapping/motd create mode 100644 healthcare_pipeline_job_whistle_mapping/tutorial.md diff --git a/healthcare_pipeline_job_backfill/backing_file.tf b/healthcare_pipeline_job_backfill/backing_file.tf new file mode 100644 index 00000000..c60b1199 --- /dev/null +++ b/healthcare_pipeline_job_backfill/backing_file.tf @@ -0,0 +1,15 @@ +# This file has some scaffolding to make sure that names are unique and that +# a region and zone are selected when you try to create your Terraform resources. + +locals { + name_suffix = "${random_pet.suffix.id}" +} + +resource "random_pet" "suffix" { + length = 2 +} + +provider "google" { + region = "us-central1" + zone = "us-central1-c" +} diff --git a/healthcare_pipeline_job_backfill/main.tf b/healthcare_pipeline_job_backfill/main.tf new file mode 100644 index 00000000..621f2c6a --- /dev/null +++ b/healthcare_pipeline_job_backfill/main.tf @@ -0,0 +1,13 @@ +resource "google_healthcare_pipeline_job" "example-pipeline" { + name = "example_backfill_pipeline-${local.name_suffix}" + location = "us-central1" + dataset = google_healthcare_dataset.dataset.id + backfill_pipeline_job { + mapping_pipeline_job = "${google_healthcare_dataset.dataset.id}/pipelinejobs/example_mapping_pipeline-${local.name_suffix}" + } +} + +resource "google_healthcare_dataset" "dataset" { + name = "example_dataset-${local.name_suffix}" + location = "us-central1" +} diff --git a/healthcare_pipeline_job_backfill/motd b/healthcare_pipeline_job_backfill/motd new file mode 100644 index 00000000..45a906e8 --- /dev/null +++ b/healthcare_pipeline_job_backfill/motd @@ -0,0 +1,7 @@ +=== + +These examples use real resources that will be billed to the +Google Cloud Platform project you use - so make sure that you +run "terraform destroy" before quitting! + +=== diff --git a/healthcare_pipeline_job_backfill/tutorial.md b/healthcare_pipeline_job_backfill/tutorial.md new file mode 100644 index 00000000..688694ee --- /dev/null +++ b/healthcare_pipeline_job_backfill/tutorial.md @@ -0,0 +1,79 @@ +# Healthcare Pipeline Job Backfill - Terraform + +## Setup + + + +Welcome to Terraform in Google Cloud Shell! We need you to let us know what project you'd like to use with Terraform. + + + +Terraform provisions real GCP resources, so anything you create in this session will be billed against this project. + +## Terraforming! + +Let's use {{project-id}} with Terraform! Click the Cloud Shell icon below to copy the command +to your shell, and then run it from the shell by pressing Enter/Return. Terraform will pick up +the project name from the environment variable. + +```bash +export GOOGLE_CLOUD_PROJECT={{project-id}} +``` + +After that, let's get Terraform started. Run the following to pull in the providers. + +```bash +terraform init +``` + +With the providers downloaded and a project set, you're ready to use Terraform. Go ahead! + +```bash +terraform apply +``` + +Terraform will show you what it plans to do, and prompt you to accept. Type "yes" to accept the plan. + +```bash +yes +``` + + +## Post-Apply + +### Editing your config + +Now you've provisioned your resources in GCP! If you run a "plan", you should see no changes needed. + +```bash +terraform plan +``` + +So let's make a change! Try editing a number, or appending a value to the name in the editor. Then, +run a 'plan' again. + +```bash +terraform plan +``` + +Afterwards you can run an apply, which implicitly does a plan and shows you the intended changes +at the 'yes' prompt. + +```bash +terraform apply +``` + +```bash +yes +``` + +## Cleanup + +Run the following to remove the resources Terraform provisioned: + +```bash +terraform destroy +``` +```bash +yes +``` diff --git a/healthcare_pipeline_job_mapping_recon_dest/backing_file.tf b/healthcare_pipeline_job_mapping_recon_dest/backing_file.tf new file mode 100644 index 00000000..c60b1199 --- /dev/null +++ b/healthcare_pipeline_job_mapping_recon_dest/backing_file.tf @@ -0,0 +1,15 @@ +# This file has some scaffolding to make sure that names are unique and that +# a region and zone are selected when you try to create your Terraform resources. + +locals { + name_suffix = "${random_pet.suffix.id}" +} + +resource "random_pet" "suffix" { + length = 2 +} + +provider "google" { + region = "us-central1" + zone = "us-central1-c" +} diff --git a/healthcare_pipeline_job_mapping_recon_dest/main.tf b/healthcare_pipeline_job_mapping_recon_dest/main.tf new file mode 100644 index 00000000..9f0dcaf5 --- /dev/null +++ b/healthcare_pipeline_job_mapping_recon_dest/main.tf @@ -0,0 +1,81 @@ +resource "google_healthcare_pipeline_job" "recon" { + name = "example_recon_pipeline_job-${local.name_suffix}" + location = "us-central1" + dataset = google_healthcare_dataset.dataset.id + disable_lineage = true + reconciliation_pipeline_job { + merge_config { + description = "sample description for reconciliation rules" + whistle_config_source { + uri = "gs://${google_storage_bucket.bucket.name}/${google_storage_bucket_object.merge_file.name}" + import_uri_prefix = "gs://${google_storage_bucket.bucket.name}" + } + } + matching_uri_prefix = "gs://${google_storage_bucket.bucket.name}" + fhir_store_destination = "${google_healthcare_dataset.dataset.id}/fhirStores/${google_healthcare_fhir_store.dest_fhirstore.name}" + } +} + +resource "google_healthcare_pipeline_job" "example-mapping-pipeline" { + depends_on = [google_healthcare_pipeline_job.recon] + name = "example_mapping_pipeline_job-${local.name_suffix}" + location = "us-central1" + dataset = google_healthcare_dataset.dataset.id + disable_lineage = true + labels = { + example_label_key = "example_label_value" + } + mapping_pipeline_job { + mapping_config { + whistle_config_source { + uri = "gs://${google_storage_bucket.bucket.name}/${google_storage_bucket_object.mapping_file.name}" + import_uri_prefix = "gs://${google_storage_bucket.bucket.name}" + } + description = "example description for mapping configuration" + } + fhir_streaming_source { + fhir_store = "${google_healthcare_dataset.dataset.id}/fhirStores/${google_healthcare_fhir_store.source_fhirstore.name}" + description = "example description for streaming fhirstore" + } + reconciliation_destination = true + } +} + +resource "google_healthcare_dataset" "dataset" { + name = "example_dataset-${local.name_suffix}" + location = "us-central1" +} + +resource "google_healthcare_fhir_store" "source_fhirstore" { + name = "source_fhir_store-${local.name_suffix}" + dataset = google_healthcare_dataset.dataset.id + version = "R4" + enable_update_create = true + disable_referential_integrity = true +} + +resource "google_healthcare_fhir_store" "dest_fhirstore" { + name = "dest_fhir_store-${local.name_suffix}" + dataset = google_healthcare_dataset.dataset.id + version = "R4" + enable_update_create = true + disable_referential_integrity = true +} + +resource "google_storage_bucket" "bucket" { + name = "example_bucket_name-${local.name_suffix}" + location = "us-central1" + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_object" "mapping_file" { + name = "mapping.wstl" + content = " " + bucket = google_storage_bucket.bucket.name +} + +resource "google_storage_bucket_object" "merge_file" { + name = "merge.wstl" + content = " " + bucket = google_storage_bucket.bucket.name +} diff --git a/healthcare_pipeline_job_mapping_recon_dest/motd b/healthcare_pipeline_job_mapping_recon_dest/motd new file mode 100644 index 00000000..45a906e8 --- /dev/null +++ b/healthcare_pipeline_job_mapping_recon_dest/motd @@ -0,0 +1,7 @@ +=== + +These examples use real resources that will be billed to the +Google Cloud Platform project you use - so make sure that you +run "terraform destroy" before quitting! + +=== diff --git a/healthcare_pipeline_job_mapping_recon_dest/tutorial.md b/healthcare_pipeline_job_mapping_recon_dest/tutorial.md new file mode 100644 index 00000000..b5e2e5dd --- /dev/null +++ b/healthcare_pipeline_job_mapping_recon_dest/tutorial.md @@ -0,0 +1,79 @@ +# Healthcare Pipeline Job Mapping Recon Dest - Terraform + +## Setup + + + +Welcome to Terraform in Google Cloud Shell! We need you to let us know what project you'd like to use with Terraform. + + + +Terraform provisions real GCP resources, so anything you create in this session will be billed against this project. + +## Terraforming! + +Let's use {{project-id}} with Terraform! Click the Cloud Shell icon below to copy the command +to your shell, and then run it from the shell by pressing Enter/Return. Terraform will pick up +the project name from the environment variable. + +```bash +export GOOGLE_CLOUD_PROJECT={{project-id}} +``` + +After that, let's get Terraform started. Run the following to pull in the providers. + +```bash +terraform init +``` + +With the providers downloaded and a project set, you're ready to use Terraform. Go ahead! + +```bash +terraform apply +``` + +Terraform will show you what it plans to do, and prompt you to accept. Type "yes" to accept the plan. + +```bash +yes +``` + + +## Post-Apply + +### Editing your config + +Now you've provisioned your resources in GCP! If you run a "plan", you should see no changes needed. + +```bash +terraform plan +``` + +So let's make a change! Try editing a number, or appending a value to the name in the editor. Then, +run a 'plan' again. + +```bash +terraform plan +``` + +Afterwards you can run an apply, which implicitly does a plan and shows you the intended changes +at the 'yes' prompt. + +```bash +terraform apply +``` + +```bash +yes +``` + +## Cleanup + +Run the following to remove the resources Terraform provisioned: + +```bash +terraform destroy +``` +```bash +yes +``` diff --git a/healthcare_pipeline_job_reconciliation/backing_file.tf b/healthcare_pipeline_job_reconciliation/backing_file.tf new file mode 100644 index 00000000..c60b1199 --- /dev/null +++ b/healthcare_pipeline_job_reconciliation/backing_file.tf @@ -0,0 +1,15 @@ +# This file has some scaffolding to make sure that names are unique and that +# a region and zone are selected when you try to create your Terraform resources. + +locals { + name_suffix = "${random_pet.suffix.id}" +} + +resource "random_pet" "suffix" { + length = 2 +} + +provider "google" { + region = "us-central1" + zone = "us-central1-c" +} diff --git a/healthcare_pipeline_job_reconciliation/main.tf b/healthcare_pipeline_job_reconciliation/main.tf new file mode 100644 index 00000000..515af8a7 --- /dev/null +++ b/healthcare_pipeline_job_reconciliation/main.tf @@ -0,0 +1,42 @@ +resource "google_healthcare_pipeline_job" "example-pipeline" { + name = "example_pipeline_job-${local.name_suffix}" + location = "us-central1" + dataset = google_healthcare_dataset.dataset.id + disable_lineage = true + reconciliation_pipeline_job { + merge_config { + description = "sample description for reconciliation rules" + whistle_config_source { + uri = "gs://${google_storage_bucket.bucket.name}/${google_storage_bucket_object.merge_file.name}" + import_uri_prefix = "gs://${google_storage_bucket.bucket.name}" + } + } + matching_uri_prefix = "gs://${google_storage_bucket.bucket.name}" + fhir_store_destination = "${google_healthcare_dataset.dataset.id}/fhirStores/${google_healthcare_fhir_store.fhirstore.name}" + } +} + +resource "google_healthcare_dataset" "dataset" { + name = "example_dataset-${local.name_suffix}" + location = "us-central1" +} + +resource "google_healthcare_fhir_store" "fhirstore" { + name = "fhir_store-${local.name_suffix}" + dataset = google_healthcare_dataset.dataset.id + version = "R4" + enable_update_create = true + disable_referential_integrity = true +} + +resource "google_storage_bucket" "bucket" { + name = "example_bucket_name-${local.name_suffix}" + location = "us-central1" + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_object" "merge_file" { + name = "merge.wstl" + content = " " + bucket = google_storage_bucket.bucket.name +} diff --git a/healthcare_pipeline_job_reconciliation/motd b/healthcare_pipeline_job_reconciliation/motd new file mode 100644 index 00000000..45a906e8 --- /dev/null +++ b/healthcare_pipeline_job_reconciliation/motd @@ -0,0 +1,7 @@ +=== + +These examples use real resources that will be billed to the +Google Cloud Platform project you use - so make sure that you +run "terraform destroy" before quitting! + +=== diff --git a/healthcare_pipeline_job_reconciliation/tutorial.md b/healthcare_pipeline_job_reconciliation/tutorial.md new file mode 100644 index 00000000..2945935e --- /dev/null +++ b/healthcare_pipeline_job_reconciliation/tutorial.md @@ -0,0 +1,79 @@ +# Healthcare Pipeline Job Reconciliation - Terraform + +## Setup + + + +Welcome to Terraform in Google Cloud Shell! We need you to let us know what project you'd like to use with Terraform. + + + +Terraform provisions real GCP resources, so anything you create in this session will be billed against this project. + +## Terraforming! + +Let's use {{project-id}} with Terraform! Click the Cloud Shell icon below to copy the command +to your shell, and then run it from the shell by pressing Enter/Return. Terraform will pick up +the project name from the environment variable. + +```bash +export GOOGLE_CLOUD_PROJECT={{project-id}} +``` + +After that, let's get Terraform started. Run the following to pull in the providers. + +```bash +terraform init +``` + +With the providers downloaded and a project set, you're ready to use Terraform. Go ahead! + +```bash +terraform apply +``` + +Terraform will show you what it plans to do, and prompt you to accept. Type "yes" to accept the plan. + +```bash +yes +``` + + +## Post-Apply + +### Editing your config + +Now you've provisioned your resources in GCP! If you run a "plan", you should see no changes needed. + +```bash +terraform plan +``` + +So let's make a change! Try editing a number, or appending a value to the name in the editor. Then, +run a 'plan' again. + +```bash +terraform plan +``` + +Afterwards you can run an apply, which implicitly does a plan and shows you the intended changes +at the 'yes' prompt. + +```bash +terraform apply +``` + +```bash +yes +``` + +## Cleanup + +Run the following to remove the resources Terraform provisioned: + +```bash +terraform destroy +``` +```bash +yes +``` diff --git a/healthcare_pipeline_job_whistle_mapping/backing_file.tf b/healthcare_pipeline_job_whistle_mapping/backing_file.tf new file mode 100644 index 00000000..c60b1199 --- /dev/null +++ b/healthcare_pipeline_job_whistle_mapping/backing_file.tf @@ -0,0 +1,15 @@ +# This file has some scaffolding to make sure that names are unique and that +# a region and zone are selected when you try to create your Terraform resources. + +locals { + name_suffix = "${random_pet.suffix.id}" +} + +resource "random_pet" "suffix" { + length = 2 +} + +provider "google" { + region = "us-central1" + zone = "us-central1-c" +} diff --git a/healthcare_pipeline_job_whistle_mapping/main.tf b/healthcare_pipeline_job_whistle_mapping/main.tf new file mode 100644 index 00000000..e57a15ed --- /dev/null +++ b/healthcare_pipeline_job_whistle_mapping/main.tf @@ -0,0 +1,56 @@ +resource "google_healthcare_pipeline_job" "example-mapping-pipeline" { + name = "example_mapping_pipeline_job-${local.name_suffix}" + location = "us-central1" + dataset = google_healthcare_dataset.dataset.id + disable_lineage = true + labels = { + example_label_key = "example_label_value" + } + mapping_pipeline_job { + mapping_config { + whistle_config_source { + uri = "gs://${google_storage_bucket.bucket.name}/${google_storage_bucket_object.mapping_file.name}" + import_uri_prefix = "gs://${google_storage_bucket.bucket.name}" + } + description = "example description for mapping configuration" + } + fhir_streaming_source { + fhir_store = "${google_healthcare_dataset.dataset.id}/fhirStores/${google_healthcare_fhir_store.source_fhirstore.name}" + description = "example description for streaming fhirstore" + } + fhir_store_destination = "${google_healthcare_dataset.dataset.id}/fhirStores/${google_healthcare_fhir_store.dest_fhirstore.name}" + } +} + +resource "google_healthcare_dataset" "dataset" { + name = "example_dataset-${local.name_suffix}" + location = "us-central1" +} + +resource "google_healthcare_fhir_store" "source_fhirstore" { + name = "source_fhir_store-${local.name_suffix}" + dataset = google_healthcare_dataset.dataset.id + version = "R4" + enable_update_create = true + disable_referential_integrity = true +} + +resource "google_healthcare_fhir_store" "dest_fhirstore" { + name = "dest_fhir_store-${local.name_suffix}" + dataset = google_healthcare_dataset.dataset.id + version = "R4" + enable_update_create = true + disable_referential_integrity = true +} + +resource "google_storage_bucket" "bucket" { + name = "example_bucket_name-${local.name_suffix}" + location = "us-central1" + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_object" "mapping_file" { + name = "mapping.wstl" + content = " " + bucket = google_storage_bucket.bucket.name +} diff --git a/healthcare_pipeline_job_whistle_mapping/motd b/healthcare_pipeline_job_whistle_mapping/motd new file mode 100644 index 00000000..45a906e8 --- /dev/null +++ b/healthcare_pipeline_job_whistle_mapping/motd @@ -0,0 +1,7 @@ +=== + +These examples use real resources that will be billed to the +Google Cloud Platform project you use - so make sure that you +run "terraform destroy" before quitting! + +=== diff --git a/healthcare_pipeline_job_whistle_mapping/tutorial.md b/healthcare_pipeline_job_whistle_mapping/tutorial.md new file mode 100644 index 00000000..4eeb3de2 --- /dev/null +++ b/healthcare_pipeline_job_whistle_mapping/tutorial.md @@ -0,0 +1,79 @@ +# Healthcare Pipeline Job Whistle Mapping - Terraform + +## Setup + + + +Welcome to Terraform in Google Cloud Shell! We need you to let us know what project you'd like to use with Terraform. + + + +Terraform provisions real GCP resources, so anything you create in this session will be billed against this project. + +## Terraforming! + +Let's use {{project-id}} with Terraform! Click the Cloud Shell icon below to copy the command +to your shell, and then run it from the shell by pressing Enter/Return. Terraform will pick up +the project name from the environment variable. + +```bash +export GOOGLE_CLOUD_PROJECT={{project-id}} +``` + +After that, let's get Terraform started. Run the following to pull in the providers. + +```bash +terraform init +``` + +With the providers downloaded and a project set, you're ready to use Terraform. Go ahead! + +```bash +terraform apply +``` + +Terraform will show you what it plans to do, and prompt you to accept. Type "yes" to accept the plan. + +```bash +yes +``` + + +## Post-Apply + +### Editing your config + +Now you've provisioned your resources in GCP! If you run a "plan", you should see no changes needed. + +```bash +terraform plan +``` + +So let's make a change! Try editing a number, or appending a value to the name in the editor. Then, +run a 'plan' again. + +```bash +terraform plan +``` + +Afterwards you can run an apply, which implicitly does a plan and shows you the intended changes +at the 'yes' prompt. + +```bash +terraform apply +``` + +```bash +yes +``` + +## Cleanup + +Run the following to remove the resources Terraform provisioned: + +```bash +terraform destroy +``` +```bash +yes +```