feat: Add subscription module that detects if topic exists (#69)

Author: tembleking

examples/organization/README.md

@@ -76,6 +76,7 @@ module "secure-for-cloud_example_organization" {
 | <a name="module_cloud_connector"></a> [cloud\_connector](#module\_cloud\_connector) | ../../modules/services/cloud-connector |  |
 | <a name="module_cloud_scanning"></a> [cloud\_scanning](#module\_cloud\_scanning) | ../../modules/services/cloud-scanning |  |
 | <a name="module_connector_organization_sink"></a> [connector\_organization\_sink](#module\_connector\_organization\_sink) | ../../modules/infrastructure/organization_sink |  |
+| <a name="module_pubsub_http_subscription"></a> [pubsub\_http\_subscription](#module\_pubsub\_http\_subscription) | ../../modules/infrastructure/pubsub_push_http_subscription |  |
 | <a name="module_scanning_organization_sink"></a> [scanning\_organization\_sink](#module\_scanning\_organization\_sink) | ../../modules/infrastructure/organization_sink |  |
 | <a name="module_secure_secrets"></a> [secure\_secrets](#module\_secure\_secrets) | ../../modules/infrastructure/secrets |  |
 
@@ -100,7 +101,6 @@ module "secure-for-cloud_example_organization" {
 | <a name="input_benchmark_project_ids"></a> [benchmark\_project\_ids](#input\_benchmark\_project\_ids) | Google cloud project IDs to run Benchmarks on. If empty, all organization projects will be defaulted. | `list(string)` | `[]` | no |
 | <a name="input_benchmark_regions"></a> [benchmark\_regions](#input\_benchmark\_regions) | List of regions in which to run the benchmark. If empty, the task will contain all regions by default. | `list(string)` | `[]` | no |
 | <a name="input_benchmark_role_name"></a> [benchmark\_role\_name](#input\_benchmark\_role\_name) | The name of the Service Account that will be created. | `string` | `"sysdigcloudbench"` | no |
-| <a name="input_create_gcr_topic"></a> [create\_gcr\_topic](#input\_create\_gcr\_topic) | Deploys a PubSub topic called `gcr` as part of this stack, which is needed for GCR scanning. Set to `true` if it doesn't exist yet. If this is not deployed, and no existing `gcr` topic is found, the GCR scanning is ommited and won't be deployed. For more info see [GCR PubSub topic](https://cloud.google.com/container-registry/docs/configuring-notifications#create_a_topic). | `bool` | `true` | no |
 | <a name="input_deploy_bench"></a> [deploy\_bench](#input\_deploy\_bench) | whether benchmark module is to be deployed | `bool` | `true` | no |
 | <a name="input_location"></a> [location](#input\_location) | Zone where the stack will be deployed | `string` | `"us-central1"` | no |
 | <a name="input_max_instances"></a> [max\_instances](#input\_max\_instances) | Max number of instances for the workloads | `number` | `1` | no |

examples/organization/main.tf

@@ -127,9 +127,7 @@ module "secure_secrets" {
 #--------------------
 # scanning
 #--------------------
-locals {
-  repository_project_ids = length(var.repository_project_ids) == 0 ? [for p in data.google_projects.all_projects.projects : p.project_id] : var.repository_project_ids
-}
+
 
 
 module "cloud_scanning" {
@@ -144,12 +142,27 @@ module "cloud_scanning" {
   cloud_scanning_sa_email  = google_service_account.scanning_sa.email
   scanning_pubsub_topic_id = module.connector_organization_sink.pubsub_topic_id
   project_id               = var.project_id
-  create_gcr_topic         = var.create_gcr_topic
-  repository_project_ids   = local.repository_project_ids
 
   max_instances = var.max_instances
 }
 
+locals {
+  repository_project_ids = length(var.repository_project_ids) == 0 ? [for p in data.google_projects.all_projects.projects : p.project_id] : var.repository_project_ids
+}
+
+module "pubsub_http_subscription" {
+  for_each = toset(local.repository_project_ids)
+  source   = "../../modules/infrastructure/pubsub_push_http_subscription"
+
+  topic_project_id        = each.key
+  subscription_project_id = var.project_id
+  topic_name              = "gcr"
+  name                    = "${var.name}-gcr"
+  service_account_email   = google_service_account.scanning_sa.email
+
+  push_http_endpoint = "${module.cloud_scanning.cloud_run_service_url}/gcr_scanning"
+}
+
 #--------------------
 # benchmark
 #--------------------

examples/organization/variables.tf

@@ -55,13 +55,6 @@ variable "repository_project_ids" {
   description = "Projects were a `gcr`-named topic will be to subscribe to its repository events. If empty, all organization projects will be defaulted."
 }
 
-variable "create_gcr_topic" {
-  type        = bool
-  description = "Deploys a PubSub topic called `gcr` as part of this stack, which is needed for GCR scanning. Set to `true` if it doesn't exist yet. If this is not deployed, and no existing `gcr` topic is found, the GCR scanning is ommited and won't be deployed. For more info see [GCR PubSub topic](https://cloud.google.com/container-registry/docs/configuring-notifications#create_a_topic)."
-  default     = true
-}
-
-
 # benchmark
 
 variable "deploy_bench" {

examples/single-project/README.md

@@ -70,6 +70,7 @@ module "secure-for-cloud_example_single-project" {
 | <a name="module_cloud_connector"></a> [cloud\_connector](#module\_cloud\_connector) | ../../modules/services/cloud-connector |  |
 | <a name="module_cloud_scanning"></a> [cloud\_scanning](#module\_cloud\_scanning) | ../../modules/services/cloud-scanning |  |
 | <a name="module_connector_project_sink"></a> [connector\_project\_sink](#module\_connector\_project\_sink) | ../../modules/infrastructure/project_sink |  |
+| <a name="module_pubsub_http_subscription"></a> [pubsub\_http\_subscription](#module\_pubsub\_http\_subscription) | ../../modules/infrastructure/pubsub_push_http_subscription |  |
 | <a name="module_scanning_project_sink"></a> [scanning\_project\_sink](#module\_scanning\_project\_sink) | ../../modules/infrastructure/project_sink |  |
 | <a name="module_secure_secrets"></a> [secure\_secrets](#module\_secure\_secrets) | ../../modules/infrastructure/secrets |  |
 
@@ -88,7 +89,6 @@ module "secure-for-cloud_example_single-project" {
 | <a name="input_sysdig_secure_api_token"></a> [sysdig\_secure\_api\_token](#input\_sysdig\_secure\_api\_token) | Sysdig's Secure API Token | `string` | n/a | yes |
 | <a name="input_benchmark_regions"></a> [benchmark\_regions](#input\_benchmark\_regions) | List of regions in which to run the benchmark. If empty, the task will contain all regions by default. | `list(string)` | `[]` | no |
 | <a name="input_benchmark_role_name"></a> [benchmark\_role\_name](#input\_benchmark\_role\_name) | The name of the Service Account that will be created. | `string` | `"sysdigcloudbench"` | no |
-| <a name="input_create_gcr_topic"></a> [create\_gcr\_topic](#input\_create\_gcr\_topic) | Deploys a PubSub topic called `gcr` as part of this stack, which is needed for GCR scanning. Set to `true` if it doesn't exist yet. If this is not deployed, and no existing `gcr` topic is found, the GCR scanning is ommited and won't be deployed. For more info see [GCR PubSub topic](https://cloud.google.com/container-registry/docs/configuring-notifications#create_a_topic). | `bool` | `true` | no |
 | <a name="input_deploy_bench"></a> [deploy\_bench](#input\_deploy\_bench) | whether benchmark module is to be deployed | `bool` | `true` | no |
 | <a name="input_location"></a> [location](#input\_location) | Zone where the stack will be deployed | `string` | `"us-central1"` | no |
 | <a name="input_name"></a> [name](#input\_name) | Name to be assigned to all child resources. A suffix may be added internally when required. Use default value unless you need to install multiple instances | `string` | `"sfc"` | no |

examples/single-project/main.tf

@@ -85,9 +85,7 @@ module "cloud_scanning" {
 
   cloud_scanning_sa_email  = google_service_account.scanning_sa.email
   scanning_pubsub_topic_id = module.scanning_project_sink.pubsub_topic_id
-  create_gcr_topic         = var.create_gcr_topic
   project_id               = var.project_id
-  repository_project_ids   = [var.project_id]
 
   secure_api_token_secret_id = module.secure_secrets.secure_api_token_secret_name
   sysdig_secure_api_token    = var.sysdig_secure_api_token
@@ -97,6 +95,18 @@ module "cloud_scanning" {
   verify_ssl = local.verify_ssl
 }
 
+module "pubsub_http_subscription" {
+  source = "../../modules/infrastructure/pubsub_push_http_subscription"
+
+  topic_project_id        = var.project_id
+  subscription_project_id = var.project_id
+  topic_name              = "gcr"
+  name                    = "${var.name}-gcr"
+  service_account_email   = google_service_account.scanning_sa.email
+
+  push_http_endpoint = "${module.cloud_scanning.cloud_run_service_url}/gcr_scanning"
+}
+
 
 #######################
 #      BENCHMARKS     #

examples/single-project/variables.tf

@@ -35,12 +35,6 @@ variable "name" {
   }
 }
 
-variable "create_gcr_topic" {
-  type        = bool
-  description = "Deploys a PubSub topic called `gcr` as part of this stack, which is needed for GCR scanning. Set to `true` if it doesn't exist yet. If this is not deployed, and no existing `gcr` topic is found, the GCR scanning is ommited and won't be deployed. For more info see [GCR PubSub topic](https://cloud.google.com/container-registry/docs/configuring-notifications#create_a_topic)."
-  default     = true
-}
-
 # benchmark
 
 variable "deploy_bench" {

modules/infrastructure/pubsub_push_http_subscription/README.md

@@ -0,0 +1,54 @@
+# PubSub Subscription module
+
+Creates a PubSub Push subscription that sends the events to an HTTP endpoint. It will reuse the specified topic if it
+already exists in the project. It will create the topic if it doesn't exist.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.14.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | >= 3.67.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_google"></a> [google](#provider\_google) | >= 3.67.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [google_pubsub_subscription.subscription](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource |
+| [google_pubsub_topic.topic](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource |
+| [google_pubsub_topic.topic](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/pubsub_topic) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_push_http_endpoint"></a> [push\_http\_endpoint](#input\_push\_http\_endpoint) | HTTP endpoint to push the events to | `string` | n/a | yes |
+| <a name="input_service_account_email"></a> [service\_account\_email](#input\_service\_account\_email) | Service account email to use | `string` | n/a | yes |
+| <a name="input_subscription_project_id"></a> [subscription\_project\_id](#input\_subscription\_project\_id) | Project ID where the subscription must be created | `string` | n/a | yes |
+| <a name="input_topic_name"></a> [topic\_name](#input\_topic\_name) | Topic to create a subscription | `string` | n/a | yes |
+| <a name="input_topic_project_id"></a> [topic\_project\_id](#input\_topic\_project\_id) | Project ID where the topic exists / must be created | `string` | n/a | yes |
+| <a name="input_name"></a> [name](#input\_name) | Name to be assigned to all child resources. A suffix may be added internally when required. Use default value unless you need to install multiple instances | `string` | `"sfc"` | no |
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+## Authors
+
+Module is maintained and supported by [Sysdig](https://github.com/sysdiglabs/terraform-google-cloudvision).
+
+## License
+
+Apache 2 Licensed. See LICENSE for full details.

modules/infrastructure/pubsub_push_http_subscription/main.tf

@@ -0,0 +1,37 @@
+data "google_pubsub_topic" "topic" {
+  name    = var.topic_name
+  project = var.topic_project_id
+}
+
+locals {
+  create_topic = (data.google_pubsub_topic.topic.name == null || lookup(coalesce(data.google_pubsub_topic.topic.labels, {}), "sysdig-managed", "false") == "true")
+}
+
+resource "google_pubsub_topic" "topic" {
+  count   = local.create_topic ? 1 : 0
+  name    = var.topic_name
+  project = var.topic_project_id
+  labels = {
+    sysdig-managed = "true"
+  }
+}
+
+resource "google_pubsub_subscription" "subscription" {
+  name    = "${var.name}-${var.topic_project_id}"
+  topic   = "projects/${var.topic_project_id}/topics/${var.topic_name}"
+  project = var.subscription_project_id
+
+  ack_deadline_seconds = 10
+
+  push_config {
+    push_endpoint = var.push_http_endpoint
+    oidc_token {
+      service_account_email = var.service_account_email
+    }
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "300s"
+  }
+}

modules/infrastructure/pubsub_push_http_subscription/outputs.tf

@@ -0,0 +1,30 @@
+variable "topic_name" {
+  type        = string
+  description = "Topic to create a subscription"
+}
+
+variable "push_http_endpoint" {
+  type        = string
+  description = "HTTP endpoint to push the events to"
+}
+
+variable "subscription_project_id" {
+  type        = string
+  description = "Project ID where the subscription must be created"
+}
+
+variable "service_account_email" {
+  type        = string
+  description = "Service account email to use"
+}
+
+variable "topic_project_id" {
+  type        = string
+  description = "Project ID where the topic exists / must be created"
+}
+
+variable "name" {
+  type        = string
+  description = "Name to be assigned to all child resources. A suffix may be added internally when required. Use default value unless you need to install multiple instances"
+  default     = "sfc"
+}