diff --git a/terraform/modules/scheduled-job/README.md b/terraform/modules/scheduled-job/README.md index 2cc49ca..172699f 100644 --- a/terraform/modules/scheduled-job/README.md +++ b/terraform/modules/scheduled-job/README.md @@ -261,6 +261,7 @@ module "data_processor" { ### Alerting (optional) - `enable_alerting` - Whether to enable alerting for job failures (true) +- `slack_token_rotation` - Increment after rotating the alertlib Slack token to rewrite the channel's write-only token from the latest secret version (1) - `slack_channel` - Slack channel to send notifications to (e.g., "#1s-and-0s") (required when alerting enabled) - `slack_mention_users` - List of Slack users or groups to mention in alerts (e.g., ["@user", "@group"]) ([]) - `alert_project_id` - GCP project ID where monitoring and alerting resources will be created (defaults to project_id) (null) @@ -426,9 +427,13 @@ gcloud builds submit --tag gcr.io/YOUR_PROJECT_ID/YOUR_JOB_NAME:latest ./jobs/yo The module supports optional Slack alerting for job failures. When enabled, it creates: - **Monitoring policies**: Cloud Monitoring alert policies for different failure scenarios -- **Slack notification channel**: Direct integration with Slack using the Slack API token from Secret Manager +- **Slack notification channel**: fully Terraform-managed, with the token kept out of state -**Note**: The module automatically fetches the Slack API token from Secret Manager in the `khan-academy` project (secret: `Slack__API_token_for_alertlib`). Ensure your Terraform service account has access to read this secret. +**How the token is handled**: the module reads `Slack__API_token_for_alertlib` (Secret Manager, `khan-academy` project) with an ephemeral resource and writes it to the channel via the write-only `sensitive_labels.auth_token_wo` argument. Ephemeral values and write-only arguments are never persisted to Terraform state or saved plan files, so the token value cannot leak through state or plan artifacts (versions of this module before v0.4.0 used a data source, which persisted the token in both). Ensure your Terraform service account can read the secret. + +**Requirements**: Terraform >= 1.11 and hashicorp/google >= 7.19.0 (write-only `sensitive_labels` support). + +**Rotation**: after adding a new secret version, bump `slack_token_rotation`; the next apply re-reads the latest version and rewrites the channel token. ### Enabling Alerting diff --git a/terraform/modules/scheduled-job/main.tf b/terraform/modules/scheduled-job/main.tf index 8229dba..1665e8c 100644 --- a/terraform/modules/scheduled-job/main.tf +++ b/terraform/modules/scheduled-job/main.tf @@ -3,12 +3,15 @@ # Required providers terraform { - required_version = ">= 1.3.0" + # 1.11+ for write-only arguments (1.10 introduced ephemeral resources). + required_version = ">= 1.11.0" required_providers { google = { + # 7.19.0 added the write-only sensitive_labels variants + # (auth_token_wo) on google_monitoring_notification_channel. source = "hashicorp/google" - version = ">= 6.0.0" + version = ">= 7.19.0" } archive = { source = "hashicorp/archive" @@ -288,8 +291,12 @@ resource "google_cloud_scheduler_job" "job_scheduler" { # Alerting resources (only created when enable_alerting is true) -# Fetch Slack API token from Secret Manager -data "google_secret_manager_secret_version" "slack_token" { +# Read the Slack API token ephemerally: the value is available to this run at +# plan/apply time but is never persisted to Terraform state or saved plan +# files. A regular data source would store its full response, including +# secret_data, in both; that is how this token was exposed by the +# committed-tfplan incident. +ephemeral "google_secret_manager_secret_version" "slack_token" { count = var.enable_alerting ? 1 : 0 project = "khan-academy" @@ -298,7 +305,6 @@ data "google_secret_manager_secret_version" "slack_token" { locals { alert_project_id = var.alert_project_id != null ? var.alert_project_id : var.project_id - slack_auth_token = var.enable_alerting ? data.google_secret_manager_secret_version.slack_token[0].secret_data : null slack_cc_mention = length(var.slack_mention_users) > 0 ? "\n\nCC: ${join(" ", var.slack_mention_users)}" : "" # Console URLs for functions and jobs @@ -319,7 +325,13 @@ resource "google_monitoring_notification_channel" "slack_channel" { } sensitive_labels { - auth_token = local.slack_auth_token + # Write-only: the token is sent to the Monitoring API but never stored in + # Terraform state or plan files. The ephemeral read above re-resolves the + # latest secret version on each run; the _wo_version counter controls + # when the API value is actually rewritten, so bump slack_token_rotation + # after rotating the secret. + auth_token_wo = ephemeral.google_secret_manager_secret_version.slack_token[0].secret_data + auth_token_wo_version = var.slack_token_rotation } } diff --git a/terraform/modules/scheduled-job/variables.tf b/terraform/modules/scheduled-job/variables.tf index 191ee90..fe0aa26 100644 --- a/terraform/modules/scheduled-job/variables.tf +++ b/terraform/modules/scheduled-job/variables.tf @@ -202,6 +202,12 @@ variable "enable_alerting" { default = true } +variable "slack_token_rotation" { + description = "Increment after rotating Slack__API_token_for_alertlib. The token is read ephemerally (never persisted to state or plans) and written to the notification channel via a write-only argument; bumping this counter is what triggers rewriting the channel's token from the latest secret version." + type = number + default = 1 +} + variable "slack_channel" { description = "Slack channel to send notifications to (e.g., '#1s-and-0s')" type = string