From 9cb02e92862f29f93f57d55659584b24e91f4012 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Mon, 19 Sep 2022 15:27:51 -0700 Subject: [PATCH 01/42] Added initial datastream Stream structure --- mmv1/products/datastream/api.yaml | 858 +++++++++++++++++++++++- mmv1/products/datastream/terraform.yaml | 10 + 2 files changed, 864 insertions(+), 4 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 21e6f107a540..4d4497bc2b1b 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -65,7 +65,7 @@ objects: - !ruby/object:Api::Type::String name: 'location' description: | - The name of the location this repository is located in. + The name of the location this connection profile is located in. required: true input: true url_param_only: true @@ -321,8 +321,6 @@ objects: api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.privateConnections' description: | The PrivateConnection resource is used to establish private connectivity between Datastream and a customer's network. - - input: true parameters: - !ruby/object:Api::Type::String @@ -335,7 +333,7 @@ objects: - !ruby/object:Api::Type::String name: 'location' description: | - The name of the location this repository is located in. + The name of the location this connection is located in. required: true input: true url_param_only: true @@ -394,3 +392,855 @@ objects: required: true description: | A free subnet for peering. (CIDR of /29) + - !ruby/object:Api::Resource + name: 'Stream' + base_url: "projects/{{project}}/locations/{{location}}/streams" + create_url: "projects/{{project}}/locations/{{location}}/streams?streamId={{stream_id}}" + self_link: "projects/{{project}}/locations/{{location}}/streams/{{stream_id}}" + references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-a-stream' + api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.streams' + description: | + A resource representing streaming data from a source to a destination. + input: true + parameters: + - !ruby/object:Api::Type::String + name: streamId + description: |- + The stream identifier. + required: true + input: true + url_param_only: true + - !ruby/object:Api::Type::String + name: 'location' + description: | + The name of the location this stream is located in. + required: true + input: true + url_param_only: true + properties: + - !ruby/object:Api::Type::String + name: 'name' + output: true + description: The stream's name. + - !ruby/object:Api::Type::KeyValuePairs + name: 'labels' + description: Labels. + - !ruby/object:Api::Type::String + name: 'displayName' + required: true + description: Display name. + - !ruby/object:Api::Type::NestedObject + name: 'sourceConfig' + required: true + description: | + Source connection profile configuration. + properties: + - !ruby/object:Api::Type::String + name: 'sourceConnectionProfile' + required: true + description: | + Source connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} + - !ruby/object:Api::Type::NestedObject + name: 'oracleSourceConfig' + conflicts: + - source_config.mysql_source_config + - source_config.postgresql_source_config + description: | + Oracle data source configuration. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'includeObjects' + description: | + Oracle objects to include in the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 + description: | + Oracle schemas/databases in the database server. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + required: true + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' + required: true + min_size: 1 + description: | + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'excludeObjects' + description: | + Oracle objects to exclude from the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 + description: | + Oracle schemas/databases in the database server. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + required: true + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' + required: true + min_size: 1 + description: | + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentCdcTasks' + description: | + Maximum number of concurrent CDC tasks. The number should be non negative. + If not set (or set to 0), the system's default value will be used. + - !ruby/object:Api::Type::NestedObject + name: 'mysqlSourceConfig' + conflicts: + - source_config.oracle_source_config + - source_config.postgresql_source_config + description: | + MySQL data source configuration. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'includeObjects' + description: | + MySQL objects to retrieve from the source. + properties: + - !ruby/object:Api::Type::Array + name: 'mysqlDatabases' + required: true + min_size: 1 + description: | + MySQL databases on the server + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'database' + required: true + description: | + Database name. + - !ruby/object:Api::Type::Array + name: 'mysqlTables' + required: true + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'mysqlColumns' + required: true + min_size: 1 + description: | + MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The MySQL data type. Full data types list can be found here: + https://dev.mysql.com/doc/refman/8.0/en/data-types.html + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::String + name: 'collation' + description: | + Column collation. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'excludeObjects' + description: | + MySQL objects to exclude from the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'mysqlDatabases' + required: true + min_size: 1 + description: | + MySQL databases on the server + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'database' + required: true + description: | + Database name. + - !ruby/object:Api::Type::Array + name: 'mysqlTables' + required: true + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'mysqlColumns' + required: true + min_size: 1 + description: | + MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The MySQL data type. Full data types list can be found here: + https://dev.mysql.com/doc/refman/8.0/en/data-types.html + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::String + name: 'collation' + description: | + Column collation. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentCdcTasks' + description: | + Maximum number of concurrent CDC tasks. The number should be non negative. + If not set (or set to 0), the system's default value will be used. + - !ruby/object:Api::Type::NestedObject + name: 'postgresqlSourceConfig' + conflicts: + - source_config.oracle_source_config + - source_config.mysql_source_config + description: | + PostgreSQL data source configuration. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'includeObjects' + description: | + PostgreSQL objects to include in the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'postgresqlSchemas' + required: true + min_size: 1 + description: | + PostgreSQL schemas in the database server. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'postgresqlTables' + required: true + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'postgresqlColumns' + required: true + min_size: 1 + description: | + PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The PostgreSQL data type. + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + description: | + Column scale. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'excludeObjects' + description: | + PostgreSQL objects to exclude from the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'postgresqlSchemas' + required: true + min_size: 1 + description: | + PostgreSQL schemas in the database server. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'postgresqlTables' + required: true + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'postgresqlColumns' + required: true + min_size: 1 + description: | + PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The PostgreSQL data type. + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + description: | + Column scale. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::String + name: 'replicationSlot' + required: true + description: | + The name of the logical replication slot that's configured with the pgoutput plugin. + - !ruby/object:Api::Type::String + name: 'publication' + required: true + description: | + The name of the publication that includes the set of all tables that are defined in the stream's includeObjects. + - !ruby/object:Api::Type::NestedObject + name: 'destinationConfig' + required: true + description: | + Destination connection profile configuration. + properties: + - !ruby/object:Api::Type::String + name: 'destinationConnectionProfile' + required: true + description: | + Destination connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} + - !ruby/object:Api::Type::NestedObject + name: 'gcsDestinationConfig' + description: | + A configuration for how data should be loaded to Cloud Storage. + properties: + - !ruby/object:Api::Type::String + name: 'path' + description: | + Path inside the Cloud Storage bucket to write data to. + - !ruby/object:Api::Type::String + name: 'fileRotationMb' + description: | + The maximum file size to be saved in the bucket. + - !ruby/object:Api::Type::String + name: 'fileRotationInterval' + description: | + The maximum duration for which new events are added before a file is closed and a new file is created. + A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. + - !ruby/object:Api::Type::NestedObject + name: 'jsonFileFormat' + description: | + JSON file format configuration. + properties: + - !ruby/object:Api::Type::Enum + name: 'schemaFileFormat' + description: | + The schema file format along JSON data files. + values: + - NO_SCHEMA_FILE + - AVRO_SCHEMA_FILE + - !ruby/object:Api::Type::Enum + name: 'compression' + description: | + Compression of the loaded JSON file. + values: + - NO_COMPRESSION + - GZIP + - !ruby/object:Api::Type::NestedObject + name: 'backfillAll' + description: | + Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'oracleExcludedObjects' + conflicts: + - backfill_all.mysql_excluded_objects + - backfill_all.postgresql_excluded_objects + description: | + Oracle data source objects to avoid backfilling. + properties: + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 + description: | + Oracle schemas/databases in the database server. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + required: true + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' + required: true + min_size: 1 + description: | + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'mysqlExcludedObjects' + description: | + MySQL data source objects to avoid backfilling. + conflicts: + - backfill_all.oracle_excluded_objects + - backfill_all.postgresql_excluded_objects + properties: + - !ruby/object:Api::Type::Array + name: 'mysqlDatabases' + required: true + min_size: 1 + description: | + MySQL databases on the server + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL database. + properties: + - !ruby/object:Api::Type::String + name: 'database' + required: true + description: | + Database name. + - !ruby/object:Api::Type::Array + name: 'mysqlTables' + required: true + min_size: 1 + description: | + Tables in the database. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'mysqlColumns' + required: true + min_size: 1 + description: | + MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + MySQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The MySQL data type. Full data types list can be found here: + https://dev.mysql.com/doc/refman/8.0/en/data-types.html + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::String + name: 'collation' + description: | + Column collation. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'postgresqlExcludedObjects' + conflicts: + - backfill_all.oracle_excluded_objects + - backfill_all.mysql_excluded_objects + description: | + PostgreSQL data source objects to avoid backfilling. + properties: + - !ruby/object:Api::Type::Array + name: 'postgresqlSchemas' + required: true + min_size: 1 + description: | + PostgreSQL schemas in the database server. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'postgresqlTables' + required: true + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'postgresqlColumns' + required: true + min_size: 1 + description: | + PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + PostgreSQL Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The PostgreSQL data type. + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + description: | + Column scale. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index bb6e5708c703..c1a41c4e099a 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -93,6 +93,16 @@ overrides: !ruby/object:Overrides::ResourceOverrides constants: templates/terraform/constants/private_connection.go.erb post_create: templates/terraform/post_create/private_connection.go.erb post_import: templates/terraform/post_import/private_connection.go.erb + Stream: !ruby/object:Overrides::Terraform::ResourceOverride + id_format: projects/{{project}}/locations/{{location}}/streams/{{stream_id}} + import_format: ["projects/{{project}}/locations/{{location}}/streams/{{stream_id}}"] + properties: + sourceConfig.oracleSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride + validation: !ruby/object:Provider::Terraform::Validation + function: 'validation.IntAtLeast(0)' + sourceConfig.mysqlSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride + validation: !ruby/object:Provider::Terraform::Validation + function: 'validation.IntAtLeast(0)' # This is for copying files over files: !ruby/object:Provider::Config::Files From 940de80765e31e259cc692522e8f3274e26a9adc Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 28 Sep 2022 11:56:18 -0700 Subject: [PATCH 02/42] Added basic test for datastream --- mmv1/products/datastream/api.yaml | 46 ++++--- mmv1/products/datastream/terraform.yaml | 13 ++ .../examples/datastream_stream_basic.tf.erb | 120 ++++++++++++++++++ 3 files changed, 164 insertions(+), 15 deletions(-) create mode 100644 mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 4d4497bc2b1b..d899e85840c4 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -444,9 +444,12 @@ objects: Source connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} - !ruby/object:Api::Type::NestedObject name: 'oracleSourceConfig' - conflicts: - - source_config.mysql_source_config - - source_config.postgresql_source_config + allow_empty_object: true + send_empty_value: true + exactly_one_of: + - source_config.0.oracle_source_config + - source_config.0.mysql_source_config + - source_config.0.postgresql_source_config description: | Oracle data source configuration. properties: @@ -614,14 +617,18 @@ objects: The ordinal position of the column in the table. - !ruby/object:Api::Type::Integer name: 'maxConcurrentCdcTasks' + send_empty_value: true description: | Maximum number of concurrent CDC tasks. The number should be non negative. If not set (or set to 0), the system's default value will be used. - !ruby/object:Api::Type::NestedObject name: 'mysqlSourceConfig' - conflicts: - - source_config.oracle_source_config - - source_config.postgresql_source_config + allow_empty_object: true + send_empty_value: true + exactly_one_of: + - source_config.0.oracle_source_config + - source_config.0.mysql_source_config + - source_config.0.postgresql_source_config description: | MySQL data source configuration. properties: @@ -775,14 +782,16 @@ objects: The ordinal position of the column in the table. - !ruby/object:Api::Type::Integer name: 'maxConcurrentCdcTasks' + send_empty_value: true description: | Maximum number of concurrent CDC tasks. The number should be non negative. If not set (or set to 0), the system's default value will be used. - !ruby/object:Api::Type::NestedObject name: 'postgresqlSourceConfig' - conflicts: - - source_config.oracle_source_config - - source_config.mysql_source_config + exactly_one_of: + - source_config.0.oracle_source_config + - source_config.0.mysql_source_config + - source_config.0.postgresql_source_config description: | PostgreSQL data source configuration. properties: @@ -963,6 +972,7 @@ objects: Destination connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} - !ruby/object:Api::Type::NestedObject name: 'gcsDestinationConfig' + required: true description: | A configuration for how data should be loaded to Cloud Storage. properties: @@ -981,6 +991,9 @@ objects: A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. - !ruby/object:Api::Type::NestedObject name: 'jsonFileFormat' + required: true + allow_empty_object: true + send_empty_value: true description: | JSON file format configuration. properties: @@ -1000,14 +1013,17 @@ objects: - GZIP - !ruby/object:Api::Type::NestedObject name: 'backfillAll' + required: true + allow_empty_object: true + send_empty_value: true description: | Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. properties: - !ruby/object:Api::Type::NestedObject name: 'oracleExcludedObjects' conflicts: - - backfill_all.mysql_excluded_objects - - backfill_all.postgresql_excluded_objects + - backfill_all.0.mysql_excluded_objects + - backfill_all.0.postgresql_excluded_objects description: | Oracle data source objects to avoid backfilling. properties: @@ -1092,8 +1108,8 @@ objects: description: | MySQL data source objects to avoid backfilling. conflicts: - - backfill_all.oracle_excluded_objects - - backfill_all.postgresql_excluded_objects + - backfill_all.0.oracle_excluded_objects + - backfill_all.0.postgresql_excluded_objects properties: - !ruby/object:Api::Type::Array name: 'mysqlDatabases' @@ -1167,8 +1183,8 @@ objects: - !ruby/object:Api::Type::NestedObject name: 'postgresqlExcludedObjects' conflicts: - - backfill_all.oracle_excluded_objects - - backfill_all.mysql_excluded_objects + - backfill_all.0.oracle_excluded_objects + - backfill_all.0.mysql_excluded_objects description: | PostgreSQL data source objects to avoid backfilling. properties: diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index c1a41c4e099a..b3edb88c6ea5 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -103,6 +103,19 @@ overrides: !ruby/object:Overrides::ResourceOverrides sourceConfig.mysqlSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride validation: !ruby/object:Provider::Terraform::Validation function: 'validation.IntAtLeast(0)' + examples: + - !ruby/object:Provider::Terraform::Examples + name: "datastream_stream_basic" + primary_resource_id: "default" + vars: + stream_id: "my-stream" + private_connection_id: "my-connection" + network_name: "my-network" + source_connection_profile_id: "source-profile" + database_instance_name: "my-instance" + deletion_protection: "true" + bucket_id: "my-bucket" + destination_connection_profile_id: "destination-profile" # This is for copying files over files: !ruby/object:Provider::Config::Files diff --git a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb new file mode 100644 index 000000000000..40414a843058 --- /dev/null +++ b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb @@ -0,0 +1,120 @@ +data "google_project" "project" { +} + +resource "google_sql_database_instance" "instance" { + name = "<%= ctx[:vars]['database_instance_name'] %>" + database_version = "MYSQL_8_0" + region = "us-central1" + settings { + tier = "db-f1-micro" + backup_configuration { + enabled = true + binary_log_enabled = true + } + + ip_configuration { + + // Datastream IPs will vary by region. + authorized_networks { + value = "34.71.242.81" + } + + authorized_networks { + value = "34.72.28.29" + } + + authorized_networks { + value = "34.67.6.157" + } + + authorized_networks { + value = "34.67.234.134" + } + + authorized_networks { + value = "34.72.239.218" + } + } + } + + deletion_protection = "<%= ctx[:vars]['deletion_protection'] %>" +} + +resource "google_sql_database" "db" { + instance = google_sql_database_instance.instance.name + name = "db" +} + +resource "random_password" "pwd" { + length = 16 + special = false +} + +resource "google_sql_user" "user" { + name = "user" + instance = google_sql_database_instance.instance.name + host = "%" + password = random_password.pwd.result +} + +resource "google_datastream_connection_profile" "source_connection_profile" { + display_name = "Source connection profile" + location = "us-central1" + connection_profile_id = "<%= ctx[:vars]['source_connection_profile_id'] %>" + + mysql_profile { + hostname = google_sql_database_instance.instance.public_ip_address + username = google_sql_user.user.name + password = google_sql_user.user.password + } +} + +resource "google_storage_bucket" "bucket" { + name = "<%= ctx[:vars]['bucket_name'] %>" + location = "US" + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_iam_member" "viewer" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.objectViewer" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + +resource "google_storage_bucket_iam_member" "creator" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.objectCreator" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + +resource "google_storage_bucket_iam_member" "reader" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.legacyBucketReader" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + +resource "google_datastream_connection_profile" "destination_connection_profile" { + display_name = "Connection profile" + location = "us-central1" + connection_profile_id = "<%= ctx[:vars]['destination_connection_profile_id'] %>" + + gcs_profile { + bucket = "<%= ctx[:vars]['bucket_name'] %>" + root_path = "/path" + } +} + +resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { + stream_id = "<%= ctx[:vars]['stream_id'] %>" + location = "us-central1" + display_name = "my stream" + source_config { + source_connection_profile = google_datastream_connection_profile.source_connection_profile.id + } + destination_config { + destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id + } + + backfill_all { + } +} From 848267b6f38960617394c61353189554ad63c671 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 9 Nov 2022 15:22:01 -0800 Subject: [PATCH 03/42] Removed oracle support since it's hard to test --- mmv1/products/datastream/api.yaml | 181 ------------------------------ 1 file changed, 181 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index d899e85840c4..e23c612b1490 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -442,191 +442,11 @@ objects: required: true description: | Source connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} - - !ruby/object:Api::Type::NestedObject - name: 'oracleSourceConfig' - allow_empty_object: true - send_empty_value: true - exactly_one_of: - - source_config.0.oracle_source_config - - source_config.0.mysql_source_config - - source_config.0.postgresql_source_config - description: | - Oracle data source configuration. - properties: - - !ruby/object:Api::Type::NestedObject - name: 'includeObjects' - description: | - Oracle objects to include in the stream. - properties: - - !ruby/object:Api::Type::Array - name: 'oracleSchemas' - required: true - min_size: 1 - description: | - Oracle schemas/databases in the database server. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle schema. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'oracleTables' - required: true - min_size: 1 - description: | - Tables in the schema. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'oracleColumns' - required: true - min_size: 1 - description: | - Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The Oracle data type. - - !ruby/object:Api::Type::Integer - name: 'length' - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - description: | - Column scale. - - !ruby/object:Api::Type::String - name: 'encoding' - description: | - Column encoding. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::NestedObject - name: 'excludeObjects' - description: | - Oracle objects to exclude from the stream. - properties: - - !ruby/object:Api::Type::Array - name: 'oracleSchemas' - required: true - min_size: 1 - description: | - Oracle schemas/databases in the database server. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle schema. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'oracleTables' - required: true - min_size: 1 - description: | - Tables in the schema. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'oracleColumns' - required: true - min_size: 1 - description: | - Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The Oracle data type. - - !ruby/object:Api::Type::Integer - name: 'length' - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - description: | - Column scale. - - !ruby/object:Api::Type::String - name: 'encoding' - description: | - Column encoding. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::Integer - name: 'maxConcurrentCdcTasks' - send_empty_value: true - description: | - Maximum number of concurrent CDC tasks. The number should be non negative. - If not set (or set to 0), the system's default value will be used. - !ruby/object:Api::Type::NestedObject name: 'mysqlSourceConfig' allow_empty_object: true send_empty_value: true exactly_one_of: - - source_config.0.oracle_source_config - source_config.0.mysql_source_config - source_config.0.postgresql_source_config description: | @@ -789,7 +609,6 @@ objects: - !ruby/object:Api::Type::NestedObject name: 'postgresqlSourceConfig' exactly_one_of: - - source_config.0.oracle_source_config - source_config.0.mysql_source_config - source_config.0.postgresql_source_config description: | From 0e83051b4d3a3b17cbc499b42233fd8d836fd49e Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 9 Nov 2022 15:22:51 -0800 Subject: [PATCH 04/42] Added some additional fields that could not previously be handled --- mmv1/products/datastream/api.yaml | 92 ++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 3 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index e23c612b1490..84f80ad84317 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -791,7 +791,9 @@ objects: Destination connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} - !ruby/object:Api::Type::NestedObject name: 'gcsDestinationConfig' - required: true + exactly_one_of: + - destination_config.0.gcs_destination_config + - destination_config.0.bigquery_destination_config description: | A configuration for how data should be loaded to Cloud Storage. properties: @@ -808,9 +810,21 @@ objects: description: | The maximum duration for which new events are added before a file is closed and a new file is created. A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. + - !ruby/object:Api::Type::NestedObject + name: 'avroFileFormat' + exactly_one_of: + - destination_config.0.gcs_destination_config.0.avro_file_format + - destination_config.0.gcs_destination_config.0.json_file_format + allow_empty_object: true + send_empty_value: true + description: | + AVRO file format configuration. + properties: [] - !ruby/object:Api::Type::NestedObject name: 'jsonFileFormat' - required: true + exactly_one_of: + - destination_config.0.gcs_destination_config.0.avro_file_format + - destination_config.0.gcs_destination_config.0.json_file_format allow_empty_object: true send_empty_value: true description: | @@ -830,9 +844,71 @@ objects: values: - NO_COMPRESSION - GZIP + - !ruby/object:Api::Type::NestedObject + name: 'bigqueryDestinationConfig' + allow_empty_object: true + send_empty_value: true + exactly_one_of: + - destination_config.0.gcs_destination_config + - destination_config.0.bigquery_destination_config + description: | + A configuration for how data should be loaded to Cloud Storage. + properties: + - !ruby/object:Api::Type::String + name: 'dataFreshness' + description: | + The guaranteed data freshness (in seconds) when querying tables created by the stream. + Editing this field will only affect new tables created in the future, but existing tables + will not be impacted. Lower values mean that queries will return fresher data, but may result in higher cost. + A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s. + - !ruby/object:Api::Type::NestedObject + name: 'singleTargetDataset' + exactly_one_of: + - destination_config.0.bigquery_destination_config.0.single_target_dataset + - destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets + description: | + A single target dataset to which all data will be streamed. + properties: + - !ruby/object:Api::Type::String + name: 'datasetId' + required: true + description: | + Dataset ID in the format projects/{project}/datasets/{dataset_id} + - !ruby/object:Api::Type::NestedObject + name: 'sourceHierarchyDatasets' + exactly_one_of: + - destination_config.0.bigquery_destination_config.0.single_target_dataset + - destination_config.0.bigquery_destination_config.0.source_hierarchy_datasets + description: | + Destination datasets are created so that hierarchy of the destination data objects matches the source hierarchy. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'datasetTemplate' + required: true + description: | + Dataset template used for dynamic dataset creation. + properties: + - !ruby/object:Api::Type::String + name: 'location' + required: true + description: | + The geographic location where the dataset should reside. + See https://cloud.google.com/bigquery/docs/locations for supported locations. + - !ruby/object:Api::Type::String + name: 'datasetIdPrefix' + description: | + If supplied, every created dataset will have its name prefixed by the provided value. + The prefix and name will be separated by an underscore. i.e. _. + - !ruby/object:Api::Type::String + name: 'location' + description: | + The geographic location where the dataset should reside. + See https://cloud.google.com/bigquery/docs/locations for supported locations. - !ruby/object:Api::Type::NestedObject name: 'backfillAll' - required: true + exactly_one_of: + - backfill_all + - backfill_none allow_empty_object: true send_empty_value: true description: | @@ -1079,3 +1155,13 @@ objects: name: 'ordinalPosition' description: | The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'backfillNone' + exactly_one_of: + - backfill_all + - backfill_none + allow_empty_object: true + send_empty_value: true + description: | + Backfill strategy to disable automatic backfill for the Stream's objects. + properties: [] From 434463e215129d9f4d76d5864b925eb63fa63212 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 9 Nov 2022 15:42:06 -0800 Subject: [PATCH 05/42] Added full example for datastream stream with MySQL -> GCS --- mmv1/products/datastream/terraform.yaml | 13 ++ .../examples/datastream_stream_basic.tf.erb | 2 +- .../examples/datastream_stream_full.tf.erb | 186 ++++++++++++++++++ 3 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 mmv1/templates/terraform/examples/datastream_stream_full.tf.erb diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index b3edb88c6ea5..6f028d2157cf 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -116,6 +116,19 @@ overrides: !ruby/object:Overrides::ResourceOverrides deletion_protection: "true" bucket_id: "my-bucket" destination_connection_profile_id: "destination-profile" + skip_docs: true + - !ruby/object:Provider::Terraform::Examples + name: "datastream_stream_full" + primary_resource_id: "default" + vars: + stream_id: "my-stream" + private_connection_id: "my-connection" + network_name: "my-network" + source_connection_profile_id: "source-profile" + database_instance_name: "my-instance" + deletion_protection: "true" + bucket_id: "my-bucket" + destination_connection_profile_id: "destination-profile" # This is for copying files over files: !ruby/object:Provider::Config::Files diff --git a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb index 40414a843058..bcbefcf47130 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb @@ -115,6 +115,6 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id } - backfill_all { + backfill_none { } } diff --git a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb new file mode 100644 index 000000000000..c8219d042620 --- /dev/null +++ b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb @@ -0,0 +1,186 @@ +data "google_project" "project" { +} + +resource "google_sql_database_instance" "instance" { + name = "<%= ctx[:vars]['database_instance_name'] %>" + database_version = "MYSQL_8_0" + region = "us-central1" + settings { + tier = "db-f1-micro" + backup_configuration { + enabled = true + binary_log_enabled = true + } + + ip_configuration { + + // Datastream IPs will vary by region. + authorized_networks { + value = "34.71.242.81" + } + + authorized_networks { + value = "34.72.28.29" + } + + authorized_networks { + value = "34.67.6.157" + } + + authorized_networks { + value = "34.67.234.134" + } + + authorized_networks { + value = "34.72.239.218" + } + } + } + + deletion_protection = "<%= ctx[:vars]['deletion_protection'] %>" +} + +resource "google_sql_database" "db" { + instance = google_sql_database_instance.instance.name + name = "db" +} + +resource "random_password" "pwd" { + length = 16 + special = false +} + +resource "google_sql_user" "user" { + name = "user" + instance = google_sql_database_instance.instance.name + host = "%" + password = random_password.pwd.result +} + +resource "google_datastream_connection_profile" "source_connection_profile" { + display_name = "Source connection profile" + location = "us-central1" + connection_profile_id = "<%= ctx[:vars]['source_connection_profile_id'] %>" + + mysql_profile { + hostname = google_sql_database_instance.instance.public_ip_address + username = google_sql_user.user.name + password = google_sql_user.user.password + } +} + +resource "google_storage_bucket" "bucket" { + name = "<%= ctx[:vars]['bucket_name'] %>" + location = "US" + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_iam_member" "viewer" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.objectViewer" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + +resource "google_storage_bucket_iam_member" "creator" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.objectCreator" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + +resource "google_storage_bucket_iam_member" "reader" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.legacyBucketReader" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + +resource "google_datastream_connection_profile" "destination_connection_profile" { + display_name = "Connection profile" + location = "us-central1" + connection_profile_id = "<%= ctx[:vars]['destination_connection_profile_id'] %>" + + gcs_profile { + bucket = "<%= ctx[:vars]['bucket_name'] %>" + root_path = "/path" + } +} + +resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { + stream_id = "<%= ctx[:vars]['stream_id'] %>" + location = "us-central1" + display_name = "my stream" + source_config { + source_connection_profile = google_datastream_connection_profile.source_connection_profile.id + mysql_source_config { + include_objects { + mysql_databases { + database { + mysql_tables { + table = "includedTable" + mysql_columns { + column = "includedColumn" + data_type = "VARCHAR" + length = 20 + collation = "utf8mb4" + primary_key = false + nullable = false + ordinal_position = false + } + } + } + } + } + exclude_objects { + mysql_databases { + database { + mysql_tables { + table = "excludedTable" + mysql_columns { + column = "excludedColumn" + data_type = "VARCHAR" + length = 20 + collation = "utf8mb4" + primary_key = false + nullable = false + ordinal_position = false + } + } + } + } + } + max_concurrent_cdc_tasks = 5 + } + } + destination_config { + destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id + gcs_destination_config { + path = "mydata" + file_rotation_mb = 200 + file_rotation_interval = "900s" + json_file_format { + schema_file_format = "NO_SCHEMA_FILE" + compression = "GZIP" + } + } + } + + backfill_all { + mysql_excluded_objects { + mysql_databases { + database { + mysql_tables { + table = "excludedTable" + mysql_columns { + column = "excludedColumn" + data_type = "VARCHAR" + length = 20 + collation = "utf8mb4" + primary_key = false + nullable = false + ordinal_position = false + } + } + } + } + } + } +} From 05729da68723a3bfa43f104ba565ea17ac7dad24 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 9 Nov 2022 16:01:35 -0800 Subject: [PATCH 06/42] Removed additional references to oracle on datastream stream --- mmv1/products/datastream/api.yaml | 86 ------------------------- mmv1/products/datastream/terraform.yaml | 3 - 2 files changed, 89 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 84f80ad84317..5d3ce082a53d 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -914,96 +914,11 @@ objects: description: | Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. properties: - - !ruby/object:Api::Type::NestedObject - name: 'oracleExcludedObjects' - conflicts: - - backfill_all.0.mysql_excluded_objects - - backfill_all.0.postgresql_excluded_objects - description: | - Oracle data source objects to avoid backfilling. - properties: - - !ruby/object:Api::Type::Array - name: 'oracleSchemas' - required: true - min_size: 1 - description: | - Oracle schemas/databases in the database server. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle schema. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'oracleTables' - required: true - min_size: 1 - description: | - Tables in the schema. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'oracleColumns' - required: true - min_size: 1 - description: | - Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The Oracle data type. - - !ruby/object:Api::Type::Integer - name: 'length' - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - description: | - Column scale. - - !ruby/object:Api::Type::String - name: 'encoding' - description: | - Column encoding. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - !ruby/object:Api::Type::NestedObject name: 'mysqlExcludedObjects' description: | MySQL data source objects to avoid backfilling. conflicts: - - backfill_all.0.oracle_excluded_objects - backfill_all.0.postgresql_excluded_objects properties: - !ruby/object:Api::Type::Array @@ -1078,7 +993,6 @@ objects: - !ruby/object:Api::Type::NestedObject name: 'postgresqlExcludedObjects' conflicts: - - backfill_all.0.oracle_excluded_objects - backfill_all.0.mysql_excluded_objects description: | PostgreSQL data source objects to avoid backfilling. diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 6f028d2157cf..86b33f2c9fd3 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -97,9 +97,6 @@ overrides: !ruby/object:Overrides::ResourceOverrides id_format: projects/{{project}}/locations/{{location}}/streams/{{stream_id}} import_format: ["projects/{{project}}/locations/{{location}}/streams/{{stream_id}}"] properties: - sourceConfig.oracleSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride - validation: !ruby/object:Provider::Terraform::Validation - function: 'validation.IntAtLeast(0)' sourceConfig.mysqlSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride validation: !ruby/object:Provider::Terraform::Validation function: 'validation.IntAtLeast(0)' From 14b7832883c68ab04f8f6839f56642e1c15a612a Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 9 Nov 2022 16:02:04 -0800 Subject: [PATCH 07/42] cleaned up api.yaml and terraform.yaml --- mmv1/products/datastream/api.yaml | 5 ----- mmv1/products/datastream/terraform.yaml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 5d3ce082a53d..942c26f4b132 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -899,11 +899,6 @@ objects: description: | If supplied, every created dataset will have its name prefixed by the provided value. The prefix and name will be separated by an underscore. i.e. _. - - !ruby/object:Api::Type::String - name: 'location' - description: | - The geographic location where the dataset should reside. - See https://cloud.google.com/bigquery/docs/locations for supported locations. - !ruby/object:Api::Type::NestedObject name: 'backfillAll' exactly_one_of: diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 86b33f2c9fd3..1e00cfb0116f 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -104,6 +104,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides - !ruby/object:Provider::Terraform::Examples name: "datastream_stream_basic" primary_resource_id: "default" + skip_docs: true vars: stream_id: "my-stream" private_connection_id: "my-connection" @@ -113,7 +114,6 @@ overrides: !ruby/object:Overrides::ResourceOverrides deletion_protection: "true" bucket_id: "my-bucket" destination_connection_profile_id: "destination-profile" - skip_docs: true - !ruby/object:Provider::Terraform::Examples name: "datastream_stream_full" primary_resource_id: "default" From 8e5b56dc6cfbab95fac6ec959f7b72f83195a473 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 10 Nov 2022 09:12:33 -0800 Subject: [PATCH 08/42] Revert "Removed additional references to oracle on datastream stream" This reverts commit 6ce0d705e309f79f8372e9f9706c9c21d34a1415. --- mmv1/products/datastream/api.yaml | 86 +++++++++++++++++++++++++ mmv1/products/datastream/terraform.yaml | 3 + 2 files changed, 89 insertions(+) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 942c26f4b132..7246e0238312 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -909,11 +909,96 @@ objects: description: | Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. properties: + - !ruby/object:Api::Type::NestedObject + name: 'oracleExcludedObjects' + conflicts: + - backfill_all.0.mysql_excluded_objects + - backfill_all.0.postgresql_excluded_objects + description: | + Oracle data source objects to avoid backfilling. + properties: + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 + description: | + Oracle schemas/databases in the database server. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + required: true + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' + required: true + min_size: 1 + description: | + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. - !ruby/object:Api::Type::NestedObject name: 'mysqlExcludedObjects' description: | MySQL data source objects to avoid backfilling. conflicts: + - backfill_all.0.oracle_excluded_objects - backfill_all.0.postgresql_excluded_objects properties: - !ruby/object:Api::Type::Array @@ -988,6 +1073,7 @@ objects: - !ruby/object:Api::Type::NestedObject name: 'postgresqlExcludedObjects' conflicts: + - backfill_all.0.oracle_excluded_objects - backfill_all.0.mysql_excluded_objects description: | PostgreSQL data source objects to avoid backfilling. diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 1e00cfb0116f..1260a8dee845 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -97,6 +97,9 @@ overrides: !ruby/object:Overrides::ResourceOverrides id_format: projects/{{project}}/locations/{{location}}/streams/{{stream_id}} import_format: ["projects/{{project}}/locations/{{location}}/streams/{{stream_id}}"] properties: + sourceConfig.oracleSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride + validation: !ruby/object:Provider::Terraform::Validation + function: 'validation.IntAtLeast(0)' sourceConfig.mysqlSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride validation: !ruby/object:Provider::Terraform::Validation function: 'validation.IntAtLeast(0)' From d6579e3cd368ff367a96eca7366efb0af16327d7 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 10 Nov 2022 09:20:59 -0800 Subject: [PATCH 09/42] Revert "Removed oracle support since it's hard to test" This reverts commit 732cf3036db07192046cda17c85fcec3247b8588. --- mmv1/products/datastream/api.yaml | 181 ++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 7246e0238312..df3691b605ba 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -442,11 +442,191 @@ objects: required: true description: | Source connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} + - !ruby/object:Api::Type::NestedObject + name: 'oracleSourceConfig' + allow_empty_object: true + send_empty_value: true + exactly_one_of: + - source_config.0.oracle_source_config + - source_config.0.mysql_source_config + - source_config.0.postgresql_source_config + description: | + Oracle data source configuration. + properties: + - !ruby/object:Api::Type::NestedObject + name: 'includeObjects' + description: | + Oracle objects to include in the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 + description: | + Oracle schemas/databases in the database server. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + required: true + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' + required: true + min_size: 1 + description: | + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::NestedObject + name: 'excludeObjects' + description: | + Oracle objects to exclude from the stream. + properties: + - !ruby/object:Api::Type::Array + name: 'oracleSchemas' + required: true + min_size: 1 + description: | + Oracle schemas/databases in the database server. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle schema. + properties: + - !ruby/object:Api::Type::String + name: 'schema' + required: true + description: | + Schema name. + - !ruby/object:Api::Type::Array + name: 'oracleTables' + required: true + min_size: 1 + description: | + Tables in the schema. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle table. + properties: + - !ruby/object:Api::Type::String + name: 'table' + required: true + description: | + Table name. + - !ruby/object:Api::Type::Array + name: 'oracleColumns' + required: true + min_size: 1 + description: | + Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. + item_type: !ruby/object:Api::Type::NestedObject + description: | + Oracle Column. + properties: + - !ruby/object:Api::Type::String + name: 'column' + description: | + Column name. + - !ruby/object:Api::Type::String + name: 'dataType' + description: | + The Oracle data type. + - !ruby/object:Api::Type::Integer + name: 'length' + description: | + Column length. + - !ruby/object:Api::Type::Integer + name: 'precision' + description: | + Column precision. + - !ruby/object:Api::Type::Integer + name: 'scale' + description: | + Column scale. + - !ruby/object:Api::Type::String + name: 'encoding' + description: | + Column encoding. + - !ruby/object:Api::Type::Boolean + name: 'primaryKey' + description: | + Whether or not the column represents a primary key. + - !ruby/object:Api::Type::Boolean + name: 'nullable' + description: | + Whether or not the column can accept a null value. + - !ruby/object:Api::Type::Integer + name: 'ordinalPosition' + description: | + The ordinal position of the column in the table. + - !ruby/object:Api::Type::Integer + name: 'maxConcurrentCdcTasks' + send_empty_value: true + description: | + Maximum number of concurrent CDC tasks. The number should be non negative. + If not set (or set to 0), the system's default value will be used. - !ruby/object:Api::Type::NestedObject name: 'mysqlSourceConfig' allow_empty_object: true send_empty_value: true exactly_one_of: + - source_config.0.oracle_source_config - source_config.0.mysql_source_config - source_config.0.postgresql_source_config description: | @@ -609,6 +789,7 @@ objects: - !ruby/object:Api::Type::NestedObject name: 'postgresqlSourceConfig' exactly_one_of: + - source_config.0.oracle_source_config - source_config.0.mysql_source_config - source_config.0.postgresql_source_config description: | From a8881fe82cc3f31ffbefc8cc00b8f485220e37fe Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 10 Nov 2022 09:25:33 -0800 Subject: [PATCH 10/42] Added oracleSourceConfig.dropLargeObjects and oracleSourceConfig.streamLargeObjects --- mmv1/products/datastream/api.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index df3691b605ba..5aa770b31e5b 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -621,6 +621,25 @@ objects: description: | Maximum number of concurrent CDC tasks. The number should be non negative. If not set (or set to 0), the system's default value will be used. + - !ruby/object:Api::Type::NestedObject + name: 'dropLargeObjects' + allow_empty_object: true + send_empty_value: true + conflicts: + - oracle_source_config.0.stream_large_objects + description: | + Drop large object values. + properties: [] + - !ruby/object:Api::Type::NestedObject + name: 'streamLargeObjects' + allow_empty_object: true + send_empty_value: true + conflicts: + - oracle_source_config.0.drop_large_objects + description: | + Stream large object values. NOTE: This feature is currently experimental. + properties: [] + - !ruby/object:Api::Type::NestedObject name: 'mysqlSourceConfig' allow_empty_object: true From b0601ed0bbf5006085e8b029e98c05a6f02ce41c Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 10 Nov 2022 09:26:52 -0800 Subject: [PATCH 11/42] Require a value inside jsonFileFormat --- mmv1/products/datastream/api.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 5aa770b31e5b..62de50a4297b 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -1025,8 +1025,6 @@ objects: exactly_one_of: - destination_config.0.gcs_destination_config.0.avro_file_format - destination_config.0.gcs_destination_config.0.json_file_format - allow_empty_object: true - send_empty_value: true description: | JSON file format configuration. properties: From 24df2f094ea175dfd8799769f1f72548fc2cae74 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 10 Nov 2022 09:27:29 -0800 Subject: [PATCH 12/42] Require a value inside bigqueryDestinationConfig --- mmv1/products/datastream/api.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 62de50a4297b..24772deaaf44 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -1044,8 +1044,6 @@ objects: - GZIP - !ruby/object:Api::Type::NestedObject name: 'bigqueryDestinationConfig' - allow_empty_object: true - send_empty_value: true exactly_one_of: - destination_config.0.gcs_destination_config - destination_config.0.bigquery_destination_config From 3f2f3a9f89b7a3c74a9d78bc2db73dc492fd3d84 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Mon, 14 Nov 2022 08:53:56 -0800 Subject: [PATCH 13/42] Corrected type of fileRotationMb --- mmv1/products/datastream/api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 24772deaaf44..e0a96b532d7f 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -1001,7 +1001,7 @@ objects: name: 'path' description: | Path inside the Cloud Storage bucket to write data to. - - !ruby/object:Api::Type::String + - !ruby/object:Api::Type::Integer name: 'fileRotationMb' description: | The maximum file size to be saved in the bucket. From 8ee1290918176e3816114c7e2aa37fca754254b4 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Mon, 14 Nov 2022 08:54:10 -0800 Subject: [PATCH 14/42] Added state field and added default_from_api to a few fields --- mmv1/products/datastream/api.yaml | 13 +++++++++++++ mmv1/products/datastream/terraform.yaml | 4 ++++ .../examples/datastream_stream_full.tf.erb | 1 + 3 files changed, 18 insertions(+) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index e0a96b532d7f..77afcd74ed76 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -1095,6 +1095,19 @@ objects: description: | If supplied, every created dataset will have its name prefixed by the provided value. The prefix and name will be separated by an underscore. i.e. _. + - !ruby/object::Api::Type::Enum + name: 'state' + description: The state of the stream. + values: + - :NOT_STARTED + - :RUNNING + - :PAUSED + - :MAINTENANCE + - :FAILED + - :FAILED_PERMANENTLY + - :STARTING + - :DRAINING + default: :NOT_STARTED - !ruby/object:Api::Type::NestedObject name: 'backfillAll' exactly_one_of: diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 1260a8dee845..649e922dcced 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -97,10 +97,14 @@ overrides: !ruby/object:Overrides::ResourceOverrides id_format: projects/{{project}}/locations/{{location}}/streams/{{stream_id}} import_format: ["projects/{{project}}/locations/{{location}}/streams/{{stream_id}}"] properties: + destinationConfig.gcsDestinationConfig.fileRotationInterval: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true sourceConfig.oracleSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true validation: !ruby/object:Provider::Terraform::Validation function: 'validation.IntAtLeast(0)' sourceConfig.mysqlSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true validation: !ruby/object:Provider::Terraform::Validation function: 'validation.IntAtLeast(0)' examples: diff --git a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb index c8219d042620..2245291f4f54 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb @@ -106,6 +106,7 @@ resource "google_datastream_connection_profile" "destination_connection_profile" resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { stream_id = "<%= ctx[:vars]['stream_id'] %>" + state = "NOT_STARTED" location = "us-central1" display_name = "my stream" source_config { From 165e9855a838c64c59270eb559b70de9a215d557 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Mon, 14 Nov 2022 08:59:38 -0800 Subject: [PATCH 15/42] Fixed typo --- mmv1/products/datastream/api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 77afcd74ed76..f85a232a8e9c 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -1095,7 +1095,7 @@ objects: description: | If supplied, every created dataset will have its name prefixed by the provided value. The prefix and name will be separated by an underscore. i.e. _. - - !ruby/object::Api::Type::Enum + - !ruby/object:Api::Type::Enum name: 'state' description: The state of the stream. values: From aae92f7bd57c60ecb099bc996ff267a6f8f55e1f Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Mon, 14 Nov 2022 09:00:42 -0800 Subject: [PATCH 16/42] Corrected name of default_value property --- mmv1/products/datastream/api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index f85a232a8e9c..47260c798471 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -1107,7 +1107,7 @@ objects: - :FAILED_PERMANENTLY - :STARTING - :DRAINING - default: :NOT_STARTED + default_value: :NOT_STARTED - !ruby/object:Api::Type::NestedObject name: 'backfillAll' exactly_one_of: From cd31c2130c14af604449f3f6daa6a759daa9d4d5 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 22 Nov 2022 11:04:22 -0800 Subject: [PATCH 17/42] Removed support for oracle and postgres from Datastream Stream This simplifies the PR; also they are not automatically testable --- mmv1/products/datastream/api.yaml | 543 +----------------------- mmv1/products/datastream/terraform.yaml | 4 - 2 files changed, 1 insertion(+), 546 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 47260c798471..845644b3fb94 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -442,212 +442,11 @@ objects: required: true description: | Source connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} - - !ruby/object:Api::Type::NestedObject - name: 'oracleSourceConfig' - allow_empty_object: true - send_empty_value: true - exactly_one_of: - - source_config.0.oracle_source_config - - source_config.0.mysql_source_config - - source_config.0.postgresql_source_config - description: | - Oracle data source configuration. - properties: - - !ruby/object:Api::Type::NestedObject - name: 'includeObjects' - description: | - Oracle objects to include in the stream. - properties: - - !ruby/object:Api::Type::Array - name: 'oracleSchemas' - required: true - min_size: 1 - description: | - Oracle schemas/databases in the database server. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle schema. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'oracleTables' - required: true - min_size: 1 - description: | - Tables in the schema. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'oracleColumns' - required: true - min_size: 1 - description: | - Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The Oracle data type. - - !ruby/object:Api::Type::Integer - name: 'length' - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - description: | - Column scale. - - !ruby/object:Api::Type::String - name: 'encoding' - description: | - Column encoding. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::NestedObject - name: 'excludeObjects' - description: | - Oracle objects to exclude from the stream. - properties: - - !ruby/object:Api::Type::Array - name: 'oracleSchemas' - required: true - min_size: 1 - description: | - Oracle schemas/databases in the database server. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle schema. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'oracleTables' - required: true - min_size: 1 - description: | - Tables in the schema. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'oracleColumns' - required: true - min_size: 1 - description: | - Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The Oracle data type. - - !ruby/object:Api::Type::Integer - name: 'length' - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - description: | - Column scale. - - !ruby/object:Api::Type::String - name: 'encoding' - description: | - Column encoding. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::Integer - name: 'maxConcurrentCdcTasks' - send_empty_value: true - description: | - Maximum number of concurrent CDC tasks. The number should be non negative. - If not set (or set to 0), the system's default value will be used. - - !ruby/object:Api::Type::NestedObject - name: 'dropLargeObjects' - allow_empty_object: true - send_empty_value: true - conflicts: - - oracle_source_config.0.stream_large_objects - description: | - Drop large object values. - properties: [] - - !ruby/object:Api::Type::NestedObject - name: 'streamLargeObjects' - allow_empty_object: true - send_empty_value: true - conflicts: - - oracle_source_config.0.drop_large_objects - description: | - Stream large object values. NOTE: This feature is currently experimental. - properties: [] - - !ruby/object:Api::Type::NestedObject name: 'mysqlSourceConfig' allow_empty_object: true send_empty_value: true - exactly_one_of: - - source_config.0.oracle_source_config - - source_config.0.mysql_source_config - - source_config.0.postgresql_source_config + required: true description: | MySQL data source configuration. properties: @@ -805,179 +604,6 @@ objects: description: | Maximum number of concurrent CDC tasks. The number should be non negative. If not set (or set to 0), the system's default value will be used. - - !ruby/object:Api::Type::NestedObject - name: 'postgresqlSourceConfig' - exactly_one_of: - - source_config.0.oracle_source_config - - source_config.0.mysql_source_config - - source_config.0.postgresql_source_config - description: | - PostgreSQL data source configuration. - properties: - - !ruby/object:Api::Type::NestedObject - name: 'includeObjects' - description: | - PostgreSQL objects to include in the stream. - properties: - - !ruby/object:Api::Type::Array - name: 'postgresqlSchemas' - required: true - min_size: 1 - description: | - PostgreSQL schemas in the database server. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL schema. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'postgresqlTables' - required: true - min_size: 1 - description: | - Tables in the schema. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'postgresqlColumns' - required: true - min_size: 1 - description: | - PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The PostgreSQL data type. - - !ruby/object:Api::Type::Integer - name: 'length' - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - description: | - Column scale. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::NestedObject - name: 'excludeObjects' - description: | - PostgreSQL objects to exclude from the stream. - properties: - - !ruby/object:Api::Type::Array - name: 'postgresqlSchemas' - required: true - min_size: 1 - description: | - PostgreSQL schemas in the database server. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL schema. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'postgresqlTables' - required: true - min_size: 1 - description: | - Tables in the schema. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'postgresqlColumns' - required: true - min_size: 1 - description: | - PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The PostgreSQL data type. - - !ruby/object:Api::Type::Integer - name: 'length' - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - description: | - Column scale. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - - !ruby/object:Api::Type::String - name: 'replicationSlot' - required: true - description: | - The name of the logical replication slot that's configured with the pgoutput plugin. - - !ruby/object:Api::Type::String - name: 'publication' - required: true - description: | - The name of the publication that includes the set of all tables that are defined in the stream's includeObjects. - !ruby/object:Api::Type::NestedObject name: 'destinationConfig' required: true @@ -1118,97 +744,10 @@ objects: description: | Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. properties: - - !ruby/object:Api::Type::NestedObject - name: 'oracleExcludedObjects' - conflicts: - - backfill_all.0.mysql_excluded_objects - - backfill_all.0.postgresql_excluded_objects - description: | - Oracle data source objects to avoid backfilling. - properties: - - !ruby/object:Api::Type::Array - name: 'oracleSchemas' - required: true - min_size: 1 - description: | - Oracle schemas/databases in the database server. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle schema. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'oracleTables' - required: true - min_size: 1 - description: | - Tables in the schema. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'oracleColumns' - required: true - min_size: 1 - description: | - Oracle columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - Oracle Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The Oracle data type. - - !ruby/object:Api::Type::Integer - name: 'length' - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - description: | - Column scale. - - !ruby/object:Api::Type::String - name: 'encoding' - description: | - Column encoding. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - !ruby/object:Api::Type::NestedObject name: 'mysqlExcludedObjects' description: | MySQL data source objects to avoid backfilling. - conflicts: - - backfill_all.0.oracle_excluded_objects - - backfill_all.0.postgresql_excluded_objects properties: - !ruby/object:Api::Type::Array name: 'mysqlDatabases' @@ -1279,86 +818,6 @@ objects: name: 'ordinalPosition' description: | The ordinal position of the column in the table. - - !ruby/object:Api::Type::NestedObject - name: 'postgresqlExcludedObjects' - conflicts: - - backfill_all.0.oracle_excluded_objects - - backfill_all.0.mysql_excluded_objects - description: | - PostgreSQL data source objects to avoid backfilling. - properties: - - !ruby/object:Api::Type::Array - name: 'postgresqlSchemas' - required: true - min_size: 1 - description: | - PostgreSQL schemas in the database server. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL schema. - properties: - - !ruby/object:Api::Type::String - name: 'schema' - required: true - description: | - Schema name. - - !ruby/object:Api::Type::Array - name: 'postgresqlTables' - required: true - min_size: 1 - description: | - Tables in the schema. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL table. - properties: - - !ruby/object:Api::Type::String - name: 'table' - required: true - description: | - Table name. - - !ruby/object:Api::Type::Array - name: 'postgresqlColumns' - required: true - min_size: 1 - description: | - PostgreSQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. - item_type: !ruby/object:Api::Type::NestedObject - description: | - PostgreSQL Column. - properties: - - !ruby/object:Api::Type::String - name: 'column' - description: | - Column name. - - !ruby/object:Api::Type::String - name: 'dataType' - description: | - The PostgreSQL data type. - - !ruby/object:Api::Type::Integer - name: 'length' - description: | - Column length. - - !ruby/object:Api::Type::Integer - name: 'precision' - description: | - Column precision. - - !ruby/object:Api::Type::Integer - name: 'scale' - description: | - Column scale. - - !ruby/object:Api::Type::Boolean - name: 'primaryKey' - description: | - Whether or not the column represents a primary key. - - !ruby/object:Api::Type::Boolean - name: 'nullable' - description: | - Whether or not the column can accept a null value. - - !ruby/object:Api::Type::Integer - name: 'ordinalPosition' - description: | - The ordinal position of the column in the table. - !ruby/object:Api::Type::NestedObject name: 'backfillNone' exactly_one_of: diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 649e922dcced..59bdad66466d 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -99,10 +99,6 @@ overrides: !ruby/object:Overrides::ResourceOverrides properties: destinationConfig.gcsDestinationConfig.fileRotationInterval: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true - sourceConfig.oracleSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride - default_from_api: true - validation: !ruby/object:Provider::Terraform::Validation - function: 'validation.IntAtLeast(0)' sourceConfig.mysqlSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true validation: !ruby/object:Provider::Terraform::Validation From bb6a18f5e8f79d86046a20ab39ead64a99e8cc09 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 22 Nov 2022 11:05:57 -0800 Subject: [PATCH 18/42] Converted status field to be computed We will likely want to use either default_from_api or use a desired_state field because this field is only partially specified --- mmv1/products/datastream/api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 845644b3fb94..7b3ad94f11e5 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -733,7 +733,7 @@ objects: - :FAILED_PERMANENTLY - :STARTING - :DRAINING - default_value: :NOT_STARTED + output: true - !ruby/object:Api::Type::NestedObject name: 'backfillAll' exactly_one_of: From 6d760d6cf53de3fc587a35085009f79afbf041a8 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 22 Nov 2022 12:05:01 -0800 Subject: [PATCH 19/42] Added desired_state field --- mmv1/products/datastream/api.yaml | 11 +--- mmv1/products/datastream/terraform.yaml | 17 ++++++ .../constants/datastream_stream.go.erb | 56 +++++++++++++++++++ .../encoders/datastream_stream.go.erb | 18 ++++++ .../post_create/datastream_stream.go.erb | 17 ++++++ .../post_import/datastream_stream.go.erb | 17 ++++++ .../post_update/datastream_stream.go.erb | 17 ++++++ .../pre_update/datastream_stream.go.erb | 17 ++++++ 8 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 mmv1/templates/terraform/constants/datastream_stream.go.erb create mode 100644 mmv1/templates/terraform/encoders/datastream_stream.go.erb create mode 100644 mmv1/templates/terraform/post_create/datastream_stream.go.erb create mode 100644 mmv1/templates/terraform/post_import/datastream_stream.go.erb create mode 100644 mmv1/templates/terraform/post_update/datastream_stream.go.erb create mode 100644 mmv1/templates/terraform/pre_update/datastream_stream.go.erb diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 7b3ad94f11e5..d857c520aafe 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -721,18 +721,9 @@ objects: description: | If supplied, every created dataset will have its name prefixed by the provided value. The prefix and name will be separated by an underscore. i.e. _. - - !ruby/object:Api::Type::Enum + - !ruby/object:Api::Type::String name: 'state' description: The state of the stream. - values: - - :NOT_STARTED - - :RUNNING - - :PAUSED - - :MAINTENANCE - - :FAILED - - :FAILED_PERMANENTLY - - :STARTING - - :DRAINING output: true - !ruby/object:Api::Type::NestedObject name: 'backfillAll' diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 59bdad66466d..c024aa2cd5cd 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -103,6 +103,23 @@ overrides: !ruby/object:Overrides::ResourceOverrides default_from_api: true validation: !ruby/object:Provider::Terraform::Validation function: 'validation.IntAtLeast(0)' + virtual_fields: + - !ruby/object:Api::Type::Enum + name: 'desired_state' + description: | + Desired state of the Stream. Set this field to `RUNNING` to start the stream, and `PAUSED` to pause the stream. + values: + - :NOT_STARTED + - :RUNNING + - :PAUSED + default_value: :NOT_STARTED + custom_code: !ruby/object:Provider::Terraform::CustomCode + constants: 'templates/terraform/constants/datastream_stream.go.erb' + post_create: 'templates/terraform/post_create/datastream_stream.go.erb' + post_import: 'templates/terraform/post_import/datastream_stream.go.erb' + pre_update: 'templates/terraform/pre_update/datastream_stream.go.erb' + post_update: 'templates/terraform/post_update/datastream_stream.go.erb' + encoder: 'templates/terraform/encoders/datastream_stream.go.erb' examples: - !ruby/object:Provider::Terraform::Examples name: "datastream_stream_basic" diff --git a/mmv1/templates/terraform/constants/datastream_stream.go.erb b/mmv1/templates/terraform/constants/datastream_stream.go.erb new file mode 100644 index 000000000000..d4e29613f254 --- /dev/null +++ b/mmv1/templates/terraform/constants/datastream_stream.go.erb @@ -0,0 +1,56 @@ +<%- # the license inside this block applies to this file + # Copyright 2022 Google Inc. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +-%> +func resourceDatastreamStreamCustomDiff(_ context.Context, diff *schema.ResourceDiff, meta interface{}) error { + if diff.HasChange("desired_state") { + oldState, newState := diff.GetChange("desired_state") + + if isNewResource(diff) { + if newState.(string) != "NOT_STARTED" && newState.(string) != "RUNNING" { + return fmt.Errorf("`desired_state` can only be set to `NOT_STARTED` or `RUNNING` when creating a new Stream") + } + } else { + if newState.(string) == "NOT_STARTED" && oldState.(string) != newState.(string) { + return fmt.Errorf("Field `desired_state` can only be set to `NOT_STARTED` when creating a new Stream") + } + + if newState.(string) != "RUNNING" && newState.(string) != "PAUSED" { + return fmt.Errorf("`desired_state` can only be set to `RUNNING` or `PAUSED` when updating a Stream") + } + } + } + return nil +} + +<% unless compiler == "terraformvalidator-codegen" -%> +// waitForDatastreamStreamReady waits for an agent pool to reach a stable state to indicate that it's ready. +func waitForDatastreamStreamReady(d *schema.ResourceData, config *Config, timeout time.Duration) error { + return resource.Retry(timeout, func() *resource.RetryError { + if err := resourceDatastreamStreamRead(d, config); err != nil { + return resource.NonRetryableError(err) + } + + name := d.Get("name").(string) + state := d.Get("state").(string) + if state == "STARTING" || state == "DRAINING" { + return resource.RetryableError(fmt.Errorf("Stream %q has state %q.", name, state)) + } else if state == "NOT_STARTED" || state == "RUNNING" || state == "PAUSED" { + log.Printf("[DEBUG] Stream %q has state %q.", name, state) + return nil + } else { + return resource.NonRetryableError(fmt.Errorf("Stream %q has state %q.", name, state)) + } + }) +} +<% end -%> diff --git a/mmv1/templates/terraform/encoders/datastream_stream.go.erb b/mmv1/templates/terraform/encoders/datastream_stream.go.erb new file mode 100644 index 000000000000..beeb16c84801 --- /dev/null +++ b/mmv1/templates/terraform/encoders/datastream_stream.go.erb @@ -0,0 +1,18 @@ +<%- # the license inside this block applies to this file + # Copyright 2022 Google Inc. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +-%> +if d.HasChange("desired_state") { + obj["state"] = d.Get("desired_state") +} +return obj, nil \ No newline at end of file diff --git a/mmv1/templates/terraform/post_create/datastream_stream.go.erb b/mmv1/templates/terraform/post_create/datastream_stream.go.erb new file mode 100644 index 000000000000..e45d6054f670 --- /dev/null +++ b/mmv1/templates/terraform/post_create/datastream_stream.go.erb @@ -0,0 +1,17 @@ +<%- # the license inside this block applies to this file + # Copyright 2022 Google Inc. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +-%> +if err := waitForDatastreamStreamReady(d, config, d.Timeout(schema.TimeoutCreate) - time.Minute); err != nil { + return fmt.Errorf("Error waiting for Stream %q to be NOT_STARTED or RUNNING during creation: %q", d.Get("name").(string), err) +} \ No newline at end of file diff --git a/mmv1/templates/terraform/post_import/datastream_stream.go.erb b/mmv1/templates/terraform/post_import/datastream_stream.go.erb new file mode 100644 index 000000000000..7e1d083b2528 --- /dev/null +++ b/mmv1/templates/terraform/post_import/datastream_stream.go.erb @@ -0,0 +1,17 @@ +<%- # the license inside this block applies to this file + # Copyright 2022 Google Inc. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +-%> +if err := waitForDatastreamStreamReady(d, config, d.Timeout(schema.TimeoutCreate) - time.Minute); err != nil { + return nil, fmt.Errorf("Error waiting for Stream %q to be NOT_STARTED, RUNNING, or PAUSED during import: %q", d.Get("name").(string), err) +} \ No newline at end of file diff --git a/mmv1/templates/terraform/post_update/datastream_stream.go.erb b/mmv1/templates/terraform/post_update/datastream_stream.go.erb new file mode 100644 index 000000000000..c56e295a62bc --- /dev/null +++ b/mmv1/templates/terraform/post_update/datastream_stream.go.erb @@ -0,0 +1,17 @@ +<%- # the license inside this block applies to this file + # Copyright 2022 Google Inc. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +-%> +if err := waitForDatastreamStreamReady(d, config, d.Timeout(schema.TimeoutCreate) - time.Minute); err != nil { + return fmt.Errorf("Error waiting for Stream %q to be NOT_STARTED, RUNNING, or PAUSED during update: %q", d.Get("name").(string), err) +} \ No newline at end of file diff --git a/mmv1/templates/terraform/pre_update/datastream_stream.go.erb b/mmv1/templates/terraform/pre_update/datastream_stream.go.erb new file mode 100644 index 000000000000..be70550c7468 --- /dev/null +++ b/mmv1/templates/terraform/pre_update/datastream_stream.go.erb @@ -0,0 +1,17 @@ +<%- # the license inside this block applies to this file + # Copyright 2022 Google Inc. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +-%> +if err := waitForDatastreamStreamReady(d, config, d.Timeout(schema.TimeoutCreate) - time.Minute); err != nil { + return fmt.Errorf("Error waiting for Stream %q to be NOT_STARTED, RUNNING, or PAUSED before updating: %q", d.Get("name").(string), err) +} \ No newline at end of file From f11c78b80d5a6a1f7fa8d102463421f500493747 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 22 Nov 2022 15:24:22 -0800 Subject: [PATCH 20/42] removed input: true from stream --- mmv1/products/datastream/api.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index d857c520aafe..a0a23193a804 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -403,7 +403,6 @@ objects: api: 'https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.streams' description: | A resource representing streaming data from a source to a destination. - input: true parameters: - !ruby/object:Api::Type::String name: streamId From a8fb6229ed2326e602f62d4c24f5c4dda3cc3c87 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 29 Nov 2022 09:21:02 -0800 Subject: [PATCH 21/42] ignore read on stream connection profiles --- mmv1/products/datastream/terraform.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index c024aa2cd5cd..14d242a3765d 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -99,6 +99,12 @@ overrides: !ruby/object:Overrides::ResourceOverrides properties: destinationConfig.gcsDestinationConfig.fileRotationInterval: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true + destinationConfig.destinationConnectionProfile: !ruby/object:Overrides::Terraform::PropertyOverride + # API currently returns an id that uses a number instead of a project id + ignore_read: true + sourceConfig.sourceConnectionProfile: !ruby/object:Overrides::Terraform::PropertyOverride + # API currently returns an id that uses a number instead of a project id + ignore_read: true sourceConfig.mysqlSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true validation: !ruby/object:Provider::Terraform::Validation From c3ede7f6c97c0da0ee18abb5f2fed4c7994a9f7f Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 29 Nov 2022 09:32:42 -0800 Subject: [PATCH 22/42] Added update test --- mmv1/products/datastream/api.yaml | 1 - .../tests/resource_datastream_stream_test.go | 214 ++++++++++++++++++ 2 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 mmv1/third_party/terraform/tests/resource_datastream_stream_test.go diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index a0a23193a804..46fbd02b4e63 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -445,7 +445,6 @@ objects: name: 'mysqlSourceConfig' allow_empty_object: true send_empty_value: true - required: true description: | MySQL data source configuration. properties: diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go new file mode 100644 index 000000000000..17e0e00aa350 --- /dev/null +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -0,0 +1,214 @@ +package google + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestAccDatastreamStream_update(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": randString(t, 10), + } + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckDatastreamStreamDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDatastreamStream_datastreamStreamBasicExample(context), + }, + { + ResourceName: "google_datastream_stream.default", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"stream_id", "location"}, + }, + { + Config: testAccDatastreamStream_datastreamStreamBasicUpdate(context, "RUNNING", true), + }, + { + ResourceName: "google_datastream_stream.default", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"stream_id", "location"}, + }, + { + Config: testAccDatastreamStream_datastreamStreamBasicUpdate(context, "PAUSED", true), + }, + { + ResourceName: "google_datastream_stream.default", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"stream_id", "location"}, + }, + { + Config: testAccDatastreamStream_datastreamStreamBasicUpdate(context, "RUNNING", true), + }, + { + ResourceName: "google_datastream_stream.default", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"stream_id", "location"}, + }, + { + // Disable prevent_destroy + Config: testAccDatastreamStream_datastreamStreamBasicUpdate(context, "RUNNING", false), + }, + }, + }) +} + +func testAccDatastreamStream_datastreamStreamBasicUpdate(context map[string]interface{}, desiredState string, preventDestroy bool) string { + context["lifecycle_block"] = "" + if preventDestroy { + context["lifecycle_block"] = ` + lifecycle { + prevent_destroy = true + }` + } + context["desired_state"] = desiredState + return Nprintf(` +data "google_project" "project" { +} + +resource "google_sql_database_instance" "instance" { + name = "tf-test-my-instance%{random_suffix}" + database_version = "MYSQL_8_0" + region = "us-central1" + settings { + tier = "db-f1-micro" + backup_configuration { + enabled = true + binary_log_enabled = true + } + + ip_configuration { + + // Datastream IPs will vary by region. + authorized_networks { + value = "34.71.242.81" + } + + authorized_networks { + value = "34.72.28.29" + } + + authorized_networks { + value = "34.67.6.157" + } + + authorized_networks { + value = "34.67.234.134" + } + + authorized_networks { + value = "34.72.239.218" + } + } + } + + deletion_protection = "true%{random_suffix}" +} + +resource "google_sql_database" "db" { + instance = google_sql_database_instance.instance.name + name = "db" +} + +resource "random_password" "pwd" { + length = 16 + special = false +} + +resource "google_sql_user" "user" { + name = "user" + instance = google_sql_database_instance.instance.name + host = "%" + password = random_password.pwd.result +} + +resource "google_datastream_connection_profile" "source_connection_profile_updated" { + display_name = "Source connection profile" + location = "us-central1" + connection_profile_id = "tf-test-source-profile-update%{random_suffix}" + + mysql_profile { + hostname = google_sql_database_instance.instance.public_ip_address + username = google_sql_user.user.name + password = google_sql_user.user.password + } +} + +resource "google_storage_bucket" "bucket" { + name = "" + location = "US" + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_iam_member" "viewer" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.objectViewer" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + +resource "google_storage_bucket_iam_member" "creator" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.objectCreator" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + +resource "google_storage_bucket_iam_member" "reader" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.legacyBucketReader" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" +} + +resource "google_datastream_connection_profile" "destination_connection_profile_updated" { + display_name = "Connection profile" + location = "us-central1" + connection_profile_id = "tf-test-destination-profile-update%{random_suffix}" + + gcs_profile { + bucket = "" + root_path = "/path" + } +} + +resource "google_datastream_stream" "default" { + stream_id = "tf-test-my-stream%{random_suffix}" + location = "us-central1" + display_name = "my stream update" + desired_state = "%{desired_state}" + + labels { + key = "updated" + } + + source_config { + source_connection_profile = google_datastream_connection_profile.source_connection_profile_updated.id + + mysql_source_config {} + } + destination_config { + destination_connection_profile = google_datastream_connection_profile.destination_connection_profile_updated.id + gcs_destination_config { + path = "mydata" + file_rotation_mb = 200 + file_rotation_interval = "900s" + json_file_format { + schema_file_format = "NO_SCHEMA_FILE" + compression = "GZIP" + } + } + } + + backfill_all { + } + %{lifecycle_block} +} +`, context) +} \ No newline at end of file From 67be969da0cc679d2b1070741b654e1f73d6476f Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 29 Nov 2022 09:51:35 -0800 Subject: [PATCH 23/42] Added unit test for customizediff --- .../tests/resource_datastream_stream_test.go | 104 +++++++++++++++++- 1 file changed, 98 insertions(+), 6 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index 17e0e00aa350..ee2b9f8c421f 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -6,6 +6,98 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" ) +func TestDatastreamStreamCustomDiff(t *testing.T) { + t.Parallel() + + cases := []struct { + isNew bool + old string + new string + wantError bool + }{ + { + isNew: true, + new: "NOT_STARTED", + wantError: false, + }, + { + isNew: true, + new: "RUNNING", + wantError: false, + }, + { + isNew: true, + new: "PAUSED", + wantError: true, + }, + { + isNew: true, + new: "MAINTENANCE", + wantError: true, + }, + { + old: "NOT_STARTED", + new: "RUNNING", + wantError: false, + }, + { + old: "NOT_STARTED", + new: "MAINTENANCE", + wantError: true, + }, + { + old: "NOT_STARTED", + new: "PAUSED", + wantError: true, + }, + { + old: "NOT_STARTED", + new: "NOT_STARTED", + wantError: false, + }, + { + old: "RUNNING", + new: "PAUSED", + wantError: false, + }, + { + old: "RUNNING", + new: "NOT_STARTED", + wantError: true, + }, + { + old: "RUNNING", + new: "RUNNING", + wantError: false, + }, + { + old: "RUNNING", + new: "MAINTENANCE", + wantError: true, + }, + { + old: "PAUSED", + new: "PAUSED", + wantError: false, + }, + { + old: "PAUSED", + new: "NOT_STARTED", + wantError: true, + }, + { + old: "PAUSED", + new: "RUNNING", + wantError: false, + }, + { + old: "PAUSED", + new: "MAINTENANCE", + wantError: true, + }, + } +} + func TestAccDatastreamStream_update(t *testing.T) { t.Parallel() @@ -63,14 +155,14 @@ func TestAccDatastreamStream_update(t *testing.T) { } func testAccDatastreamStream_datastreamStreamBasicUpdate(context map[string]interface{}, desiredState string, preventDestroy bool) string { - context["lifecycle_block"] = "" - if preventDestroy { - context["lifecycle_block"] = ` + context["lifecycle_block"] = "" + if preventDestroy { + context["lifecycle_block"] = ` lifecycle { prevent_destroy = true }` - } - context["desired_state"] = desiredState + } + context["desired_state"] = desiredState return Nprintf(` data "google_project" "project" { } @@ -211,4 +303,4 @@ resource "google_datastream_stream" "default" { %{lifecycle_block} } `, context) -} \ No newline at end of file +} From b7a5f79e25604baa55493373f0e94043ff4c57b9 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 29 Nov 2022 11:25:25 -0800 Subject: [PATCH 24/42] Added unit tests for customize diff --- mmv1/products/datastream/terraform.yaml | 1 + .../constants/datastream_stream.go.erb | 20 ++++++++--- .../privateca_certificate_authority.go.erb | 2 +- .../datastream_stream.go.erb | 15 ++++++++ .../tests/resource_datastream_stream_test.go | 35 +++++++++++++++++++ 5 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 mmv1/templates/terraform/resource_definition/datastream_stream.go.erb diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 14d242a3765d..5accffa611c6 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -126,6 +126,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides pre_update: 'templates/terraform/pre_update/datastream_stream.go.erb' post_update: 'templates/terraform/post_update/datastream_stream.go.erb' encoder: 'templates/terraform/encoders/datastream_stream.go.erb' + resource_definition: 'templates/terraform/resource_definition/datastream_stream.go.erb' examples: - !ruby/object:Provider::Terraform::Examples name: "datastream_stream_basic" diff --git a/mmv1/templates/terraform/constants/datastream_stream.go.erb b/mmv1/templates/terraform/constants/datastream_stream.go.erb index d4e29613f254..bbde7af65b51 100644 --- a/mmv1/templates/terraform/constants/datastream_stream.go.erb +++ b/mmv1/templates/terraform/constants/datastream_stream.go.erb @@ -12,26 +12,36 @@ # See the License for the specific language governing permissions and # limitations under the License. -%> -func resourceDatastreamStreamCustomDiff(_ context.Context, diff *schema.ResourceDiff, meta interface{}) error { +func resourceDatastreamStreamCustomDiffFunc(diff TerraformResourceDiff) error { if diff.HasChange("desired_state") { - oldState, newState := diff.GetChange("desired_state") + old, new := diff.GetChange("desired_state") + oldState := old.(string) + newState := new.(string) if isNewResource(diff) { - if newState.(string) != "NOT_STARTED" && newState.(string) != "RUNNING" { + if newState != "NOT_STARTED" && newState != "RUNNING" { return fmt.Errorf("`desired_state` can only be set to `NOT_STARTED` or `RUNNING` when creating a new Stream") } } else { - if newState.(string) == "NOT_STARTED" && oldState.(string) != newState.(string) { + if newState == "NOT_STARTED" && oldState != newState { return fmt.Errorf("Field `desired_state` can only be set to `NOT_STARTED` when creating a new Stream") } - if newState.(string) != "RUNNING" && newState.(string) != "PAUSED" { + if oldState == "NOT_STARTED" && newState != "RUNNING" { + return fmt.Errorf("Field `desired_state` can only be set to `RUNNING` from `NOT_STARTED`") + } + + if newState != "RUNNING" && newState != "PAUSED" { return fmt.Errorf("`desired_state` can only be set to `RUNNING` or `PAUSED` when updating a Stream") } } } return nil } +func resourceDatastreamStreamCustomDiff(_ context.Context, diff *schema.ResourceDiff, meta interface{}) error { + // separate func to allow unit testing + return resourceDatastreamStreamCustomDiffFunc(diff) +} <% unless compiler == "terraformvalidator-codegen" -%> // waitForDatastreamStreamReady waits for an agent pool to reach a stable state to indicate that it's ready. diff --git a/mmv1/templates/terraform/constants/privateca_certificate_authority.go.erb b/mmv1/templates/terraform/constants/privateca_certificate_authority.go.erb index 4493f64e36b4..f3cf0318af03 100644 --- a/mmv1/templates/terraform/constants/privateca_certificate_authority.go.erb +++ b/mmv1/templates/terraform/constants/privateca_certificate_authority.go.erb @@ -18,7 +18,7 @@ func resourcePrivateCaCACustomDiff(_ context.Context, diff *schema.ResourceDiff, return nil } -func isNewResource(diff *schema.ResourceDiff) bool { +func isNewResource(diff TerraformResourceDiff) bool { name := diff.Get("name") return name.(string) == "" } diff --git a/mmv1/templates/terraform/resource_definition/datastream_stream.go.erb b/mmv1/templates/terraform/resource_definition/datastream_stream.go.erb new file mode 100644 index 000000000000..186ad8e571f3 --- /dev/null +++ b/mmv1/templates/terraform/resource_definition/datastream_stream.go.erb @@ -0,0 +1,15 @@ +<%# The license inside this block applies to this file. + # Copyright 2022 Google Inc. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +-%> +CustomizeDiff: resourceDatastreamStreamCustomDiff, diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index ee2b9f8c421f..eef488e41e8f 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -1,6 +1,7 @@ package google import ( + "fmt" "testing" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" @@ -35,6 +36,14 @@ func TestDatastreamStreamCustomDiff(t *testing.T) { new: "MAINTENANCE", wantError: true, }, + { + // Normally this transition is okay, but if the resource is "new" + // (for example being recreated) it's not. + isNew: true, + old: "RUNNING", + new: "PAUSED", + wantError: true, + }, { old: "NOT_STARTED", new: "RUNNING", @@ -96,6 +105,32 @@ func TestDatastreamStreamCustomDiff(t *testing.T) { wantError: true, }, } + for _, tc := range cases { + name := "whatever" + tn := fmt.Sprintf("%s => %s", tc.old, tc.new) + if tc.isNew { + name = "" + tn = fmt.Sprintf("(new) %s => %s", tc.old, tc.new) + } + t.Run(tn, func(t *testing.T) { + diff := &ResourceDiffMock{ + Before: map[string]interface{}{ + "desired_state": tc.old, + }, + After: map[string]interface{}{ + "name": name, + "desired_state": tc.new, + }, + } + err := resourceDatastreamStreamCustomDiffFunc(diff) + if tc.wantError && err == nil { + t.Fatalf("want error, got nil") + } + if !tc.wantError && err != nil { + t.Fatalf("got unexpected error: %v", err) + } + }) + } } func TestAccDatastreamStream_update(t *testing.T) { From c05993ed8de744225ba76a21679d3eaf47ae5a11 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 29 Nov 2022 13:13:30 -0800 Subject: [PATCH 25/42] Added external providers --- mmv1/products/datastream/terraform.yaml | 2 ++ .../terraform/tests/resource_datastream_stream_test.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 5accffa611c6..59aea40b0e71 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -130,6 +130,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides examples: - !ruby/object:Provider::Terraform::Examples name: "datastream_stream_basic" + pull_external: true primary_resource_id: "default" skip_docs: true vars: @@ -143,6 +144,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides destination_connection_profile_id: "destination-profile" - !ruby/object:Provider::Terraform::Examples name: "datastream_stream_full" + pull_external: true primary_resource_id: "default" vars: stream_id: "my-stream" diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index eef488e41e8f..d8d24d4ed3b3 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -143,6 +143,9 @@ func TestAccDatastreamStream_update(t *testing.T) { vcrTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, Providers: testAccProviders, + ExternalProviders: map[string]resource.ExternalProvider{ + "random": {}, + }, CheckDestroy: testAccCheckDatastreamStreamDestroyProducer(t), Steps: []resource.TestStep{ { From c5ca4e56a7cc68d52a2189128910b12f803e690f Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 29 Nov 2022 13:34:03 -0800 Subject: [PATCH 26/42] gofmt --- .../terraform/tests/resource_datastream_stream_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index d8d24d4ed3b3..2d798fdce70e 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -141,8 +141,8 @@ func TestAccDatastreamStream_update(t *testing.T) { } vcrTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, ExternalProviders: map[string]resource.ExternalProvider{ "random": {}, }, From 722c17e73e0ff4cb256323521d98eaa0299144a1 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 29 Nov 2022 14:49:25 -0800 Subject: [PATCH 27/42] Corrected deletion_protection usage --- mmv1/products/datastream/terraform.yaml | 4 ++++ .../terraform/examples/datastream_stream_basic.tf.erb | 2 +- .../terraform/examples/datastream_stream_full.tf.erb | 2 +- .../terraform/tests/resource_datastream_stream_test.go | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 59aea40b0e71..c0b05f8837a2 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -142,6 +142,8 @@ overrides: !ruby/object:Overrides::ResourceOverrides deletion_protection: "true" bucket_id: "my-bucket" destination_connection_profile_id: "destination-profile" + test_vars_overrides: + deletion_protection: "false" - !ruby/object:Provider::Terraform::Examples name: "datastream_stream_full" pull_external: true @@ -155,6 +157,8 @@ overrides: !ruby/object:Overrides::ResourceOverrides deletion_protection: "true" bucket_id: "my-bucket" destination_connection_profile_id: "destination-profile" + test_vars_overrides: + deletion_protection: "false" # This is for copying files over files: !ruby/object:Provider::Config::Files diff --git a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb index bcbefcf47130..00215525e64f 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb @@ -37,7 +37,7 @@ resource "google_sql_database_instance" "instance" { } } - deletion_protection = "<%= ctx[:vars]['deletion_protection'] %>" + deletion_protection = <%= ctx[:vars]['deletion_protection'] %> } resource "google_sql_database" "db" { diff --git a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb index 2245291f4f54..e7dfa0f947e6 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb @@ -37,7 +37,7 @@ resource "google_sql_database_instance" "instance" { } } - deletion_protection = "<%= ctx[:vars]['deletion_protection'] %>" + deletion_protection = <%= ctx[:vars]['deletion_protection'] %> } resource "google_sql_database" "db" { diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index 2d798fdce70e..c684a882c7ad 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -241,7 +241,7 @@ resource "google_sql_database_instance" "instance" { } } - deletion_protection = "true%{random_suffix}" + deletion_protection = false } resource "google_sql_database" "db" { From bd636d543866aa99749b19c5da9719ecc4461516 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 30 Nov 2022 08:49:24 -0800 Subject: [PATCH 28/42] Updates due to code review --- mmv1/products/datastream/api.yaml | 9 ++------- .../terraform/constants/datastream_stream.go.erb | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 46fbd02b4e63..dda23543c861 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -333,7 +333,7 @@ objects: - !ruby/object:Api::Type::String name: 'location' description: | - The name of the location this connection is located in. + The name of the location this private connection is located in. required: true input: true url_param_only: true @@ -445,6 +445,7 @@ objects: name: 'mysqlSourceConfig' allow_empty_object: true send_empty_value: true + required: true description: | MySQL data source configuration. properties: @@ -470,7 +471,6 @@ objects: Database name. - !ruby/object:Api::Type::Array name: 'mysqlTables' - required: true min_size: 1 description: | Tables in the database. @@ -485,7 +485,6 @@ objects: Table name. - !ruby/object:Api::Type::Array name: 'mysqlColumns' - required: true min_size: 1 description: | MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. @@ -544,7 +543,6 @@ objects: Database name. - !ruby/object:Api::Type::Array name: 'mysqlTables' - required: true min_size: 1 description: | Tables in the database. @@ -559,7 +557,6 @@ objects: Table name. - !ruby/object:Api::Type::Array name: 'mysqlColumns' - required: true min_size: 1 description: | MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. @@ -755,7 +752,6 @@ objects: Database name. - !ruby/object:Api::Type::Array name: 'mysqlTables' - required: true min_size: 1 description: | Tables in the database. @@ -770,7 +766,6 @@ objects: Table name. - !ruby/object:Api::Type::Array name: 'mysqlColumns' - required: true min_size: 1 description: | MySQL columns in the schema. When unspecified as part of include/exclude objects, includes/excludes everything. diff --git a/mmv1/templates/terraform/constants/datastream_stream.go.erb b/mmv1/templates/terraform/constants/datastream_stream.go.erb index bbde7af65b51..7f4cebc715dc 100644 --- a/mmv1/templates/terraform/constants/datastream_stream.go.erb +++ b/mmv1/templates/terraform/constants/datastream_stream.go.erb @@ -24,7 +24,7 @@ func resourceDatastreamStreamCustomDiffFunc(diff TerraformResourceDiff) error { } } else { if newState == "NOT_STARTED" && oldState != newState { - return fmt.Errorf("Field `desired_state` can only be set to `NOT_STARTED` when creating a new Stream") + return fmt.Errorf("Field `desired_state` cannot be changed to `NOT_STARTED` for an already-created Stream") } if oldState == "NOT_STARTED" && newState != "RUNNING" { From 10695bb738f90ad18f5417368c7984c207719c2f Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 30 Nov 2022 08:59:31 -0800 Subject: [PATCH 29/42] Cleaned up test issues --- mmv1/products/datastream/terraform.yaml | 4 +- .../examples/datastream_stream_basic.tf.erb | 1 + .../examples/datastream_stream_full.tf.erb | 69 +++++++++---------- .../tests/resource_datastream_stream_test.go | 2 +- 4 files changed, 37 insertions(+), 39 deletions(-) diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index c0b05f8837a2..d012a3bbdd71 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -140,7 +140,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides source_connection_profile_id: "source-profile" database_instance_name: "my-instance" deletion_protection: "true" - bucket_id: "my-bucket" + bucket_name: "my-bucket" destination_connection_profile_id: "destination-profile" test_vars_overrides: deletion_protection: "false" @@ -155,7 +155,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides source_connection_profile_id: "source-profile" database_instance_name: "my-instance" deletion_protection: "true" - bucket_id: "my-bucket" + bucket_name: "my-bucket" destination_connection_profile_id: "destination-profile" test_vars_overrides: deletion_protection: "false" diff --git a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb index 00215525e64f..402a749c0ab7 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb @@ -110,6 +110,7 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { display_name = "my stream" source_config { source_connection_profile = google_datastream_connection_profile.source_connection_profile.id + mysql_source_config {} } destination_config { destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id diff --git a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb index e7dfa0f947e6..1a051e08f782 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb @@ -114,36 +114,34 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { mysql_source_config { include_objects { mysql_databases { - database { - mysql_tables { - table = "includedTable" - mysql_columns { - column = "includedColumn" - data_type = "VARCHAR" - length = 20 - collation = "utf8mb4" - primary_key = false - nullable = false - ordinal_position = false - } + database = "my-database" + mysql_tables { + table = "includedTable" + mysql_columns { + column = "includedColumn" + data_type = "VARCHAR" + length = 20 + collation = "utf8mb4" + primary_key = false + nullable = false + ordinal_position = false } } } } exclude_objects { mysql_databases { - database { - mysql_tables { - table = "excludedTable" - mysql_columns { - column = "excludedColumn" - data_type = "VARCHAR" - length = 20 - collation = "utf8mb4" - primary_key = false - nullable = false - ordinal_position = false - } + database = "my-database" + mysql_tables { + table = "excludedTable" + mysql_columns { + column = "excludedColumn" + data_type = "VARCHAR" + length = 20 + collation = "utf8mb4" + primary_key = false + nullable = false + ordinal_position = false } } } @@ -167,18 +165,17 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { backfill_all { mysql_excluded_objects { mysql_databases { - database { - mysql_tables { - table = "excludedTable" - mysql_columns { - column = "excludedColumn" - data_type = "VARCHAR" - length = 20 - collation = "utf8mb4" - primary_key = false - nullable = false - ordinal_position = false - } + database = "my-database" + mysql_tables { + table = "excludedTable" + mysql_columns { + column = "excludedColumn" + data_type = "VARCHAR" + length = 20 + collation = "utf8mb4" + primary_key = false + nullable = false + ordinal_position = false } } } diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index c684a882c7ad..0cbbc65062a4 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -274,7 +274,7 @@ resource "google_datastream_connection_profile" "source_connection_profile_updat } resource "google_storage_bucket" "bucket" { - name = "" + name = "tf-test-my-bucket%{random_suffix}" location = "US" uniform_bucket_level_access = true } From 5a54936d65d1b74f053098ddb3b8bdb1a82a63fb Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 30 Nov 2022 13:51:56 -0800 Subject: [PATCH 30/42] Corrections to tests --- .../terraform/examples/datastream_stream_full.tf.erb | 8 ++++---- .../terraform/tests/resource_datastream_stream_test.go | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb index 1a051e08f782..6bdd208b4908 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb @@ -106,7 +106,7 @@ resource "google_datastream_connection_profile" "destination_connection_profile" resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { stream_id = "<%= ctx[:vars]['stream_id'] %>" - state = "NOT_STARTED" + desired_state = "NOT_STARTED" location = "us-central1" display_name = "my stream" source_config { @@ -124,7 +124,7 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { collation = "utf8mb4" primary_key = false nullable = false - ordinal_position = false + ordinal_position = 0 } } } @@ -141,7 +141,7 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { collation = "utf8mb4" primary_key = false nullable = false - ordinal_position = false + ordinal_position = 0 } } } @@ -175,7 +175,7 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { collation = "utf8mb4" primary_key = false nullable = false - ordinal_position = false + ordinal_position = 0 } } } diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index 0cbbc65062a4..3d99b4583958 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -138,6 +138,7 @@ func TestAccDatastreamStream_update(t *testing.T) { context := map[string]interface{}{ "random_suffix": randString(t, 10), + "deletion_protection": false, } vcrTest(t, resource.TestCase{ @@ -241,7 +242,7 @@ resource "google_sql_database_instance" "instance" { } } - deletion_protection = false + deletion_protection = %{deletion_protection} } resource "google_sql_database" "db" { From 375774954b7bab20afdd9a28a36c0a685c3ced15 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 1 Dec 2022 09:10:15 -0800 Subject: [PATCH 31/42] Fixed test issues --- .../templates/terraform/examples/datastream_stream_basic.tf.erb | 1 + .../terraform/tests/resource_datastream_stream_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb index 402a749c0ab7..dd81906e7af7 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb @@ -114,6 +114,7 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { } destination_config { destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id + gcs_destination_config {} } backfill_none { diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index 3d99b4583958..f06a93e73ea9 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -137,7 +137,7 @@ func TestAccDatastreamStream_update(t *testing.T) { t.Parallel() context := map[string]interface{}{ - "random_suffix": randString(t, 10), + "random_suffix": randString(t, 10), "deletion_protection": false, } From 583406c0a6f79a7fab083cb2378b2e2a53e3f0fc Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 1 Dec 2022 09:32:31 -0800 Subject: [PATCH 32/42] Added gcs file format to fix tests --- .../terraform/examples/datastream_stream_basic.tf.erb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb index dd81906e7af7..87e1bfa0683d 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb @@ -114,7 +114,9 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { } destination_config { destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id - gcs_destination_config {} + gcs_destination_config { + avro_file_format {} + } } backfill_none { From b81ca7da9fbf4fdac1cfe82c551cc981a3d0e259 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 8 Dec 2022 15:55:08 -0800 Subject: [PATCH 33/42] Added projectNumberDiffSuppress --- mmv1/products/datastream/terraform.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index d012a3bbdd71..4f9d270f9efd 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -100,11 +100,9 @@ overrides: !ruby/object:Overrides::ResourceOverrides destinationConfig.gcsDestinationConfig.fileRotationInterval: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true destinationConfig.destinationConnectionProfile: !ruby/object:Overrides::Terraform::PropertyOverride - # API currently returns an id that uses a number instead of a project id - ignore_read: true + diff_suppress_func: "projectNumberDiffSuppress" sourceConfig.sourceConnectionProfile: !ruby/object:Overrides::Terraform::PropertyOverride - # API currently returns an id that uses a number instead of a project id - ignore_read: true + diff_suppress_func: "projectNumberDiffSuppress" sourceConfig.mysqlSourceConfig.maxConcurrentCdcTasks: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true validation: !ruby/object:Provider::Terraform::Validation From d65050fdc5ab7aa5f95a29dd1fa2657ea9ac6ee1 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 8 Dec 2022 16:04:03 -0800 Subject: [PATCH 34/42] Added default_from_api for fileRotationMb --- mmv1/products/datastream/terraform.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mmv1/products/datastream/terraform.yaml b/mmv1/products/datastream/terraform.yaml index 4f9d270f9efd..c0510efbc6d3 100644 --- a/mmv1/products/datastream/terraform.yaml +++ b/mmv1/products/datastream/terraform.yaml @@ -97,6 +97,8 @@ overrides: !ruby/object:Overrides::ResourceOverrides id_format: projects/{{project}}/locations/{{location}}/streams/{{stream_id}} import_format: ["projects/{{project}}/locations/{{location}}/streams/{{stream_id}}"] properties: + destinationConfig.gcsDestinationConfig.fileRotationMb: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true destinationConfig.gcsDestinationConfig.fileRotationInterval: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true destinationConfig.destinationConnectionProfile: !ruby/object:Overrides::Terraform::PropertyOverride From d1b99a8153a251a1d5454ec29b604fd7392b5898 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Fri, 9 Dec 2022 08:12:42 -0800 Subject: [PATCH 35/42] Corrected labels usage in tests --- .../examples/datastream_connection_profile_full.tf.erb | 3 +++ .../templates/terraform/examples/datastream_stream_full.tf.erb | 3 +++ .../terraform/tests/resource_datastream_stream_test.go | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/mmv1/templates/terraform/examples/datastream_connection_profile_full.tf.erb b/mmv1/templates/terraform/examples/datastream_connection_profile_full.tf.erb index 4446b36eb872..a06c613daeef 100644 --- a/mmv1/templates/terraform/examples/datastream_connection_profile_full.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_connection_profile_full.tf.erb @@ -14,4 +14,7 @@ resource "google_datastream_connection_profile" "<%= ctx[:primary_resource_id] % port = 8022 password = "swordfish" } + labels = { + key = "value" + } } \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb index 6bdd208b4908..088f35961cd5 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb @@ -109,6 +109,9 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { desired_state = "NOT_STARTED" location = "us-central1" display_name = "my stream" + labels = { + key = "value" + } source_config { source_connection_profile = google_datastream_connection_profile.source_connection_profile.id mysql_source_config { diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index f06a93e73ea9..be68f1a52cf7 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -315,7 +315,7 @@ resource "google_datastream_stream" "default" { display_name = "my stream update" desired_state = "%{desired_state}" - labels { + labels = { key = "updated" } From d471ab6f962e82856e7037c2e74b728bc13e249e Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Fri, 9 Dec 2022 11:35:23 -0800 Subject: [PATCH 36/42] Corrected stream update verb --- mmv1/products/datastream/api.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index dda23543c861..da6e02eb4042 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -397,6 +397,8 @@ objects: base_url: "projects/{{project}}/locations/{{location}}/streams" create_url: "projects/{{project}}/locations/{{location}}/streams?streamId={{stream_id}}" self_link: "projects/{{project}}/locations/{{location}}/streams/{{stream_id}}" + update_verb: :PATCH + update_mask: true references: !ruby/object:Api::Resource::ReferenceLinks guides: 'Official Documentation': 'https://cloud.google.com/datastream/docs/create-a-stream' From 43a05bb3b7b762a01f7e93b8ae5593bf9f3d85cc Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Fri, 9 Dec 2022 13:14:11 -0800 Subject: [PATCH 37/42] Cleaned up references to google_storage_bucket.bucket.name --- .../templates/terraform/examples/datastream_stream_basic.tf.erb | 2 +- mmv1/templates/terraform/examples/datastream_stream_full.tf.erb | 2 +- .../terraform/tests/resource_datastream_stream_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb index 87e1bfa0683d..142af2597a7b 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_basic.tf.erb @@ -99,7 +99,7 @@ resource "google_datastream_connection_profile" "destination_connection_profile" connection_profile_id = "<%= ctx[:vars]['destination_connection_profile_id'] %>" gcs_profile { - bucket = "<%= ctx[:vars]['bucket_name'] %>" + bucket = google_storage_bucket.bucket.name root_path = "/path" } } diff --git a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb index 088f35961cd5..60420e8ac697 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb @@ -99,7 +99,7 @@ resource "google_datastream_connection_profile" "destination_connection_profile" connection_profile_id = "<%= ctx[:vars]['destination_connection_profile_id'] %>" gcs_profile { - bucket = "<%= ctx[:vars]['bucket_name'] %>" + bucket = google_storage_bucket.bucket.name root_path = "/path" } } diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index be68f1a52cf7..edfcadb90bac 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -304,7 +304,7 @@ resource "google_datastream_connection_profile" "destination_connection_profile_ connection_profile_id = "tf-test-destination-profile-update%{random_suffix}" gcs_profile { - bucket = "" + bucket = google_storage_bucket.bucket.name root_path = "/path" } } From 374fc584d73962c08ab470b21da7055d2fdb17e2 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Mon, 12 Dec 2022 08:27:58 -0800 Subject: [PATCH 38/42] Source / destination profile are not updatable --- mmv1/products/datastream/api.yaml | 2 ++ .../tests/resource_datastream_stream_test.go | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index da6e02eb4042..1ea2e4d3a871 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -440,6 +440,7 @@ objects: properties: - !ruby/object:Api::Type::String name: 'sourceConnectionProfile' + input: true required: true description: | Source connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} @@ -609,6 +610,7 @@ objects: properties: - !ruby/object:Api::Type::String name: 'destinationConnectionProfile' + input: true required: true description: | Destination connection profile resource. Format: projects/{project}/locations/{location}/connectionProfiles/{name} diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index edfcadb90bac..eb233c15add8 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -262,10 +262,10 @@ resource "google_sql_user" "user" { password = random_password.pwd.result } -resource "google_datastream_connection_profile" "source_connection_profile_updated" { +resource "google_datastream_connection_profile" "source_connection_profile" { display_name = "Source connection profile" location = "us-central1" - connection_profile_id = "tf-test-source-profile-update%{random_suffix}" + connection_profile_id = "tf-test-source-profile%{random_suffix}" mysql_profile { hostname = google_sql_database_instance.instance.public_ip_address @@ -298,10 +298,10 @@ resource "google_storage_bucket_iam_member" "reader" { member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com" } -resource "google_datastream_connection_profile" "destination_connection_profile_updated" { +resource "google_datastream_connection_profile" "destination_connection_profile" { display_name = "Connection profile" location = "us-central1" - connection_profile_id = "tf-test-destination-profile-update%{random_suffix}" + connection_profile_id = "tf-test-destination-profile%{random_suffix}" gcs_profile { bucket = google_storage_bucket.bucket.name @@ -320,12 +320,12 @@ resource "google_datastream_stream" "default" { } source_config { - source_connection_profile = google_datastream_connection_profile.source_connection_profile_updated.id + source_connection_profile = google_datastream_connection_profile.source_connection_profile.id mysql_source_config {} } destination_config { - destination_connection_profile = google_datastream_connection_profile.destination_connection_profile_updated.id + destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id gcs_destination_config { path = "mydata" file_rotation_mb = 200 From 8867dd8ba785c6b7c6dcfc566032836c4a54a01b Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 20 Dec 2022 09:16:43 -0800 Subject: [PATCH 39/42] Marked column length fields as output-only --- mmv1/products/datastream/api.yaml | 3 +++ .../templates/terraform/examples/datastream_stream_full.tf.erb | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmv1/products/datastream/api.yaml b/mmv1/products/datastream/api.yaml index 1ea2e4d3a871..cd10ca264b18 100644 --- a/mmv1/products/datastream/api.yaml +++ b/mmv1/products/datastream/api.yaml @@ -506,6 +506,7 @@ objects: https://dev.mysql.com/doc/refman/8.0/en/data-types.html - !ruby/object:Api::Type::Integer name: 'length' + output: true description: | Column length. - !ruby/object:Api::Type::String @@ -578,6 +579,7 @@ objects: https://dev.mysql.com/doc/refman/8.0/en/data-types.html - !ruby/object:Api::Type::Integer name: 'length' + output: true description: | Column length. - !ruby/object:Api::Type::String @@ -788,6 +790,7 @@ objects: https://dev.mysql.com/doc/refman/8.0/en/data-types.html - !ruby/object:Api::Type::Integer name: 'length' + output: true description: | Column length. - !ruby/object:Api::Type::String diff --git a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb index 60420e8ac697..25c664cbdcf0 100644 --- a/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb +++ b/mmv1/templates/terraform/examples/datastream_stream_full.tf.erb @@ -123,7 +123,6 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { mysql_columns { column = "includedColumn" data_type = "VARCHAR" - length = 20 collation = "utf8mb4" primary_key = false nullable = false @@ -140,7 +139,6 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { mysql_columns { column = "excludedColumn" data_type = "VARCHAR" - length = 20 collation = "utf8mb4" primary_key = false nullable = false @@ -174,7 +172,6 @@ resource "google_datastream_stream" "<%= ctx[:primary_resource_id] %>" { mysql_columns { column = "excludedColumn" data_type = "VARCHAR" - length = 20 collation = "utf8mb4" primary_key = false nullable = false From 95e9175690c615107122c16e0815bcbe948b11a4 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 20 Dec 2022 09:36:37 -0800 Subject: [PATCH 40/42] Made update test use checks instead of importstateverify for state / desired_state --- .../tests/resource_datastream_stream_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index eb233c15add8..f23139f1445b 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -151,39 +151,43 @@ func TestAccDatastreamStream_update(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccDatastreamStream_datastreamStreamBasicExample(context), + Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "NOT_STARTED"), }, { ResourceName: "google_datastream_stream.default", ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"stream_id", "location"}, + ImportStateVerifyIgnore: []string{"stream_id", "location", "desired_state"}, }, { Config: testAccDatastreamStream_datastreamStreamBasicUpdate(context, "RUNNING", true), + Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "RUNNING"), }, { ResourceName: "google_datastream_stream.default", ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"stream_id", "location"}, + ImportStateVerifyIgnore: []string{"stream_id", "location", "desired_state"}, }, { Config: testAccDatastreamStream_datastreamStreamBasicUpdate(context, "PAUSED", true), + Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "PAUSED"), }, { ResourceName: "google_datastream_stream.default", ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"stream_id", "location"}, + ImportStateVerifyIgnore: []string{"stream_id", "location", "desired_state"}, }, { Config: testAccDatastreamStream_datastreamStreamBasicUpdate(context, "RUNNING", true), + Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "RUNNING"), }, { ResourceName: "google_datastream_stream.default", ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"stream_id", "location"}, + ImportStateVerifyIgnore: []string{"stream_id", "location", "desired_state"}, }, { // Disable prevent_destroy From 75ae319fa306b621aa5211216267f48c12bc143f Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 20 Dec 2022 09:58:57 -0800 Subject: [PATCH 41/42] gofmt --- .../terraform/tests/resource_datastream_stream_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go index f23139f1445b..bbf29c033a8e 100644 --- a/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go +++ b/mmv1/third_party/terraform/tests/resource_datastream_stream_test.go @@ -151,7 +151,7 @@ func TestAccDatastreamStream_update(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccDatastreamStream_datastreamStreamBasicExample(context), - Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "NOT_STARTED"), + Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "NOT_STARTED"), }, { ResourceName: "google_datastream_stream.default", @@ -161,7 +161,7 @@ func TestAccDatastreamStream_update(t *testing.T) { }, { Config: testAccDatastreamStream_datastreamStreamBasicUpdate(context, "RUNNING", true), - Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "RUNNING"), + Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "RUNNING"), }, { ResourceName: "google_datastream_stream.default", @@ -171,7 +171,7 @@ func TestAccDatastreamStream_update(t *testing.T) { }, { Config: testAccDatastreamStream_datastreamStreamBasicUpdate(context, "PAUSED", true), - Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "PAUSED"), + Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "PAUSED"), }, { ResourceName: "google_datastream_stream.default", @@ -181,7 +181,7 @@ func TestAccDatastreamStream_update(t *testing.T) { }, { Config: testAccDatastreamStream_datastreamStreamBasicUpdate(context, "RUNNING", true), - Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "RUNNING"), + Check: resource.TestCheckResourceAttr("google_datastream_stream.default", "state", "RUNNING"), }, { ResourceName: "google_datastream_stream.default", From b2719d82a46293604da993be231a5541500aeaea Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 4 Jan 2023 09:21:31 -0800 Subject: [PATCH 42/42] Add state to update mask if desired_state is changed --- .../terraform/pre_update/datastream_stream.go.erb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mmv1/templates/terraform/pre_update/datastream_stream.go.erb b/mmv1/templates/terraform/pre_update/datastream_stream.go.erb index be70550c7468..600a88abfa69 100644 --- a/mmv1/templates/terraform/pre_update/datastream_stream.go.erb +++ b/mmv1/templates/terraform/pre_update/datastream_stream.go.erb @@ -12,6 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -%> + +if d.HasChange("desired_state") { + updateMask = append(updateMask, "state") +} + +// Override the previous setting of updateMask to include state. +// updateMask is a URL parameter but not present in the schema, so replaceVars +// won't set it +url, err = addQueryParams(url, map[string]string{"updateMask": strings.Join(updateMask, ",")}) +if err != nil { + return err +} + if err := waitForDatastreamStreamReady(d, config, d.Timeout(schema.TimeoutCreate) - time.Minute); err != nil { return fmt.Errorf("Error waiting for Stream %q to be NOT_STARTED, RUNNING, or PAUSED before updating: %q", d.Get("name").(string), err) } \ No newline at end of file