scheduled-job: keep the Slack channel token out of state via ephemeral + write-only

terraform/modules/scheduled-job/README.md

@@ -261,6 +261,7 @@ module "data_processor" {
 ### Alerting (optional)
 
 - `enable_alerting` - Whether to enable alerting for job failures (true)
+- `slack_token_rotation` - Increment after rotating the alertlib Slack token to rewrite the channel's write-only token from the latest secret version (1)
 - `slack_channel` - Slack channel to send notifications to (e.g., "#1s-and-0s") (required when alerting enabled)
 - `slack_mention_users` - List of Slack users or groups to mention in alerts (e.g., ["@user", "@group"]) ([])
 - `alert_project_id` - GCP project ID where monitoring and alerting resources will be created (defaults to project_id) (null)
@@ -426,9 +427,13 @@ gcloud builds submit --tag gcr.io/YOUR_PROJECT_ID/YOUR_JOB_NAME:latest ./jobs/yo
 The module supports optional Slack alerting for job failures. When enabled, it creates:
 
 - **Monitoring policies**: Cloud Monitoring alert policies for different failure scenarios
-- **Slack notification channel**: Direct integration with Slack using the Slack API token from Secret Manager
+- **Slack notification channel**: fully Terraform-managed, with the token kept out of state
 
-**Note**: The module automatically fetches the Slack API token from Secret Manager in the `khan-academy` project (secret: `Slack__API_token_for_alertlib`). Ensure your Terraform service account has access to read this secret.
+**How the token is handled**: the module reads `Slack__API_token_for_alertlib` (Secret Manager, `khan-academy` project) with an ephemeral resource and writes it to the channel via the write-only `sensitive_labels.auth_token_wo` argument. Ephemeral values and write-only arguments are never persisted to Terraform state or saved plan files, so the token value cannot leak through state or plan artifacts (versions of this module before v0.4.0 used a data source, which persisted the token in both). Ensure your Terraform service account can read the secret.
+
+**Requirements**: Terraform >= 1.11 and hashicorp/google >= 7.19.0 (write-only `sensitive_labels` support).
+
+**Rotation**: after adding a new secret version, bump `slack_token_rotation`; the next apply re-reads the latest version and rewrites the channel token.
 
 ### Enabling Alerting
 

terraform/modules/scheduled-job/main.tf

@@ -3,12 +3,15 @@
 
 # Required providers
 terraform {
-  required_version = ">= 1.3.0"
+  # 1.11+ for write-only arguments (1.10 introduced ephemeral resources).
+  required_version = ">= 1.11.0"
 
   required_providers {
     google = {
+      # 7.19.0 added the write-only sensitive_labels variants
+      # (auth_token_wo) on google_monitoring_notification_channel.
       source  = "hashicorp/google"
-      version = ">= 6.0.0"
+      version = ">= 7.19.0"
     }
     archive = {
       source  = "hashicorp/archive"
@@ -288,8 +291,12 @@ resource "google_cloud_scheduler_job" "job_scheduler" {
 
 # Alerting resources (only created when enable_alerting is true)
 
-# Fetch Slack API token from Secret Manager
-data "google_secret_manager_secret_version" "slack_token" {
+# Read the Slack API token ephemerally: the value is available to this run at
+# plan/apply time but is never persisted to Terraform state or saved plan
+# files. A regular data source would store its full response, including
+# secret_data, in both; that is how this token was exposed by the
+# committed-tfplan incident.
+ephemeral "google_secret_manager_secret_version" "slack_token" {
   count = var.enable_alerting ? 1 : 0
 
   project = "khan-academy"
@@ -298,7 +305,6 @@ data "google_secret_manager_secret_version" "slack_token" {
 
 locals {
   alert_project_id = var.alert_project_id != null ? var.alert_project_id : var.project_id
-  slack_auth_token = var.enable_alerting ? data.google_secret_manager_secret_version.slack_token[0].secret_data : null
   slack_cc_mention = length(var.slack_mention_users) > 0 ? "\n\nCC: ${join(" ", var.slack_mention_users)}" : ""
 
   # Console URLs for functions and jobs
@@ -319,7 +325,13 @@ resource "google_monitoring_notification_channel" "slack_channel" {
   }
 
   sensitive_labels {
-    auth_token = local.slack_auth_token
+    # Write-only: the token is sent to the Monitoring API but never stored in
+    # Terraform state or plan files. The ephemeral read above re-resolves the
+    # latest secret version on each run; the _wo_version counter controls
+    # when the API value is actually rewritten, so bump slack_token_rotation
+    # after rotating the secret.
+    auth_token_wo         = ephemeral.google_secret_manager_secret_version.slack_token[0].secret_data
+    auth_token_wo_version = var.slack_token_rotation
   }
 }
 

terraform/modules/scheduled-job/variables.tf

@@ -202,6 +202,12 @@ variable "enable_alerting" {
   default     = true
 }
 
+variable "slack_token_rotation" {
+  description = "Increment after rotating Slack__API_token_for_alertlib. The token is read ephemerally (never persisted to state or plans) and written to the notification channel via a write-only argument; bumping this counter is what triggers rewriting the channel's token from the latest secret version."
+  type        = number
+  default     = 1
+}
+
 variable "slack_channel" {
   description = "Slack channel to send notifications to (e.g., '#1s-and-0s')"
   type        = string