From be1787d412805e5f941c1a269f3d393847b7ab4c Mon Sep 17 00:00:00 2001
From: James Wiesebron <jameswiesebron@khanacademy.org>
Date: Mon, 20 Jul 2026 21:25:54 -0700
Subject: [PATCH 1/2] [jwies/scheduled-job-channel-ids] scheduled-job: accept
 pre-created notification channels

Reading Slack__API_token_for_alertlib with a data source persists the
token value into Terraform state (state stores the full data-source
response) and from there into every saved plan file; this is how the
token was exposed by the committed-tfplan incident.

New notification_channel_ids input: pass full resource names of
pre-created Cloud Monitoring channels and the alert policies use them
directly; the data source and module-managed channel are skipped
entirely, so Terraform never touches the token. The token-based path
remains as a deprecated default for backward compatibility and should
be removed in the next major version.

Suggest releasing as scheduled-job-v0.4.0.
---
 terraform/modules/scheduled-job/README.md    |  9 ++++---
 terraform/modules/scheduled-job/main.tf      | 25 ++++++++++++++------
 terraform/modules/scheduled-job/variables.tf |  6 +++++
 3 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/terraform/modules/scheduled-job/README.md b/terraform/modules/scheduled-job/README.md
index 2cc49ca..5c8f786 100644
--- a/terraform/modules/scheduled-job/README.md
+++ b/terraform/modules/scheduled-job/README.md
@@ -261,7 +261,8 @@ module "data_processor" {
 ### Alerting (optional)
 
 - `enable_alerting` - Whether to enable alerting for job failures (true)
-- `slack_channel` - Slack channel to send notifications to (e.g., "#1s-and-0s") (required when alerting enabled)
+- `notification_channel_ids` - Full resource names of pre-created notification channels for failure alerts; strongly preferred over the deprecated module-managed Slack channel, which puts the alertlib token value into Terraform state ([])
+- `slack_channel` - Slack channel to send notifications to (e.g., "#1s-and-0s") (required when alerting enabled and notification_channel_ids is empty)
 - `slack_mention_users` - List of Slack users or groups to mention in alerts (e.g., ["@user", "@group"]) ([])
 - `alert_project_id` - GCP project ID where monitoring and alerting resources will be created (defaults to project_id) (null)
 
@@ -426,9 +427,11 @@ gcloud builds submit --tag gcr.io/YOUR_PROJECT_ID/YOUR_JOB_NAME:latest ./jobs/yo
 The module supports optional Slack alerting for job failures. When enabled, it creates:
 
 - **Monitoring policies**: Cloud Monitoring alert policies for different failure scenarios
-- **Slack notification channel**: Direct integration with Slack using the Slack API token from Secret Manager
+- **Notification channel wiring**: either pre-created channels you pass in, or (deprecated) a module-managed Slack channel
 
-**Note**: The module automatically fetches the Slack API token from Secret Manager in the `khan-academy` project (secret: `Slack__API_token_for_alertlib`). Ensure your Terraform service account has access to read this secret.
+**Pass `notification_channel_ids` (strongly preferred).** Provide the full resource names (`projects/PROJECT/notificationChannels/ID`) of one or more pre-created Cloud Monitoring notification channels, and the alert policies use them directly. Terraform then never touches the Slack token. Create the channel once per project, ideally via the Cloud Console's Slack integration (an OAuth flow, so no token handling at all), and share it across jobs.
+
+**Deprecated default (empty `notification_channel_ids`)**: the module creates a Slack channel itself by fetching `Slack__API_token_for_alertlib` from Secret Manager in `khan-academy` with a data source. Terraform state stores the full data-source response, so the token VALUE ends up in state and in any saved plan file; this is how the token was exposed by the committed-tfplan incident. This path remains only for backward compatibility and will be removed in a future major version. If you must use it, ensure your Terraform service account can read the secret, and treat your state and plan artifacts as containing the token.
 
 ### Enabling Alerting
 
diff --git a/terraform/modules/scheduled-job/main.tf b/terraform/modules/scheduled-job/main.tf
index 8229dba..a598348 100644
--- a/terraform/modules/scheduled-job/main.tf
+++ b/terraform/modules/scheduled-job/main.tf
@@ -288,9 +288,15 @@ resource "google_cloud_scheduler_job" "job_scheduler" {
 
 # Alerting resources (only created when enable_alerting is true)
 
-# Fetch Slack API token from Secret Manager
+# DEPRECATED token-based channel creation, used only when
+# notification_channel_ids is empty. Reading the Slack token with a data
+# source persists the token VALUE into Terraform state (state stores the full
+# data-source response, including secret_data), and from there into every
+# saved plan file. Prefer passing pre-created channel IDs via
+# notification_channel_ids; this path exists for backward compatibility and
+# will be removed in a future major version.
 data "google_secret_manager_secret_version" "slack_token" {
-  count = var.enable_alerting ? 1 : 0
+  count = var.enable_alerting && length(var.notification_channel_ids) == 0 ? 1 : 0
 
   project = "khan-academy"
   secret  = "Slack__API_token_for_alertlib"
@@ -298,17 +304,22 @@ data "google_secret_manager_secret_version" "slack_token" {
 
 locals {
   alert_project_id = var.alert_project_id != null ? var.alert_project_id : var.project_id
-  slack_auth_token = var.enable_alerting ? data.google_secret_manager_secret_version.slack_token[0].secret_data : null
+  slack_auth_token = var.enable_alerting && length(var.notification_channel_ids) == 0 ? data.google_secret_manager_secret_version.slack_token[0].secret_data : null
   slack_cc_mention = length(var.slack_mention_users) > 0 ? "\n\nCC: ${join(" ", var.slack_mention_users)}" : ""
   
+  # Channels the alert policies notify: pre-created channels when provided,
+  # otherwise the deprecated module-managed channel.
+  alert_notification_channels = length(var.notification_channel_ids) > 0 ? var.notification_channel_ids : [google_monitoring_notification_channel.slack_channel[0].name]
+
   # Console URLs for functions and jobs
   function_console_url = "https://console.cloud.google.com/run/detail/${var.region}/${var.job_name}/observability/logs?project=${var.project_id}"
   job_console_url      = "https://console.cloud.google.com/run/jobs/detail/${var.region}/${var.job_name}/observability/logs?project=${var.project_id}"
 }
 
-# Monitoring notification channel for Slack
+# Monitoring notification channel for Slack (DEPRECATED path, see above;
+# skipped entirely when notification_channel_ids is provided)
 resource "google_monitoring_notification_channel" "slack_channel" {
-  count = var.enable_alerting ? 1 : 0
+  count = var.enable_alerting && length(var.notification_channel_ids) == 0 ? 1 : 0
 
   project      = local.alert_project_id
   display_name = "${var.job_name} Slack Alerts"
@@ -359,7 +370,7 @@ resource "google_monitoring_alert_policy" "function_failure" {
     }
   }
 
-  notification_channels = [google_monitoring_notification_channel.slack_channel[0].name]
+  notification_channels = local.alert_notification_channels
 
   documentation {
     content   = "The Cloud Function ${var.job_name} has failed to execute. Check the function logs for more details.\n\n[View Function in Console](${local.function_console_url})${local.slack_cc_mention}"
@@ -403,7 +414,7 @@ resource "google_monitoring_alert_policy" "job_failure" {
     }
   }
 
-  notification_channels = [google_monitoring_notification_channel.slack_channel[0].name]
+  notification_channels = local.alert_notification_channels
 
   documentation {
     content   = "The Cloud Run Job ${var.job_name} has failed to execute or complete successfully. Check the job logs for more details.\n\n[View Job in Console](${local.job_console_url})${local.slack_cc_mention}"
diff --git a/terraform/modules/scheduled-job/variables.tf b/terraform/modules/scheduled-job/variables.tf
index 191ee90..732fd5f 100644
--- a/terraform/modules/scheduled-job/variables.tf
+++ b/terraform/modules/scheduled-job/variables.tf
@@ -202,6 +202,12 @@ variable "enable_alerting" {
   default     = true
 }
 
+variable "notification_channel_ids" {
+  description = "Full resource names of pre-created Cloud Monitoring notification channels (projects/PROJECT/notificationChannels/ID) for failure alerts. Strongly preferred over the default: when empty, the module creates a Slack channel itself by reading the alertlib token with a data source, which persists the token value into Terraform state and any saved plan files. Pre-create the channel once per project (e.g. via the console's Slack OAuth flow, which never handles the token in Terraform) and pass its name here."
+  type        = list(string)
+  default     = []
+}
+
 variable "slack_channel" {
   description = "Slack channel to send notifications to (e.g., '#1s-and-0s')"
   type        = string

From a2e8dd324540d03e60ebb6a45465c98c9740df03 Mon Sep 17 00:00:00 2001
From: James Wiesebron <jameswiesebron@khanacademy.org>
Date: Mon, 20 Jul 2026 21:32:28 -0700
Subject: [PATCH 2/2] [jwies/scheduled-job-channel-ids] scheduled-job: keep the
 channel Terraform-managed via ephemeral + write-only

Reworks the previous commit per review: instead of offloading channel
creation to pre-created channels, keep the Slack notification channel
fully module-managed and fix the leak with the mechanism built for it.
The token is read with an ephemeral resource and written via the
write-only sensitive_labels.auth_token_wo argument (google provider
7.19.0+), so the value never touches Terraform state or plan files.
Rotation is an explicit slack_token_rotation counter bump.

Requires Terraform >= 1.11 and hashicorp/google >= 7.19.0; consumers
currently pin 7.8.0 and bump the provider when they adopt v0.4.0.
---
 terraform/modules/scheduled-job/README.md    | 12 +++---
 terraform/modules/scheduled-job/main.tf      | 45 ++++++++++----------
 terraform/modules/scheduled-job/variables.tf |  8 ++--
 3 files changed, 34 insertions(+), 31 deletions(-)

diff --git a/terraform/modules/scheduled-job/README.md b/terraform/modules/scheduled-job/README.md
index 5c8f786..172699f 100644
--- a/terraform/modules/scheduled-job/README.md
+++ b/terraform/modules/scheduled-job/README.md
@@ -261,8 +261,8 @@ module "data_processor" {
 ### Alerting (optional)
 
 - `enable_alerting` - Whether to enable alerting for job failures (true)
-- `notification_channel_ids` - Full resource names of pre-created notification channels for failure alerts; strongly preferred over the deprecated module-managed Slack channel, which puts the alertlib token value into Terraform state ([])
-- `slack_channel` - Slack channel to send notifications to (e.g., "#1s-and-0s") (required when alerting enabled and notification_channel_ids is empty)
+- `slack_token_rotation` - Increment after rotating the alertlib Slack token to rewrite the channel's write-only token from the latest secret version (1)
+- `slack_channel` - Slack channel to send notifications to (e.g., "#1s-and-0s") (required when alerting enabled)
 - `slack_mention_users` - List of Slack users or groups to mention in alerts (e.g., ["@user", "@group"]) ([])
 - `alert_project_id` - GCP project ID where monitoring and alerting resources will be created (defaults to project_id) (null)
 
@@ -427,11 +427,13 @@ gcloud builds submit --tag gcr.io/YOUR_PROJECT_ID/YOUR_JOB_NAME:latest ./jobs/yo
 The module supports optional Slack alerting for job failures. When enabled, it creates:
 
 - **Monitoring policies**: Cloud Monitoring alert policies for different failure scenarios
-- **Notification channel wiring**: either pre-created channels you pass in, or (deprecated) a module-managed Slack channel
+- **Slack notification channel**: fully Terraform-managed, with the token kept out of state
 
-**Pass `notification_channel_ids` (strongly preferred).** Provide the full resource names (`projects/PROJECT/notificationChannels/ID`) of one or more pre-created Cloud Monitoring notification channels, and the alert policies use them directly. Terraform then never touches the Slack token. Create the channel once per project, ideally via the Cloud Console's Slack integration (an OAuth flow, so no token handling at all), and share it across jobs.
+**How the token is handled**: the module reads `Slack__API_token_for_alertlib` (Secret Manager, `khan-academy` project) with an ephemeral resource and writes it to the channel via the write-only `sensitive_labels.auth_token_wo` argument. Ephemeral values and write-only arguments are never persisted to Terraform state or saved plan files, so the token value cannot leak through state or plan artifacts (versions of this module before v0.4.0 used a data source, which persisted the token in both). Ensure your Terraform service account can read the secret.
 
-**Deprecated default (empty `notification_channel_ids`)**: the module creates a Slack channel itself by fetching `Slack__API_token_for_alertlib` from Secret Manager in `khan-academy` with a data source. Terraform state stores the full data-source response, so the token VALUE ends up in state and in any saved plan file; this is how the token was exposed by the committed-tfplan incident. This path remains only for backward compatibility and will be removed in a future major version. If you must use it, ensure your Terraform service account can read the secret, and treat your state and plan artifacts as containing the token.
+**Requirements**: Terraform >= 1.11 and hashicorp/google >= 7.19.0 (write-only `sensitive_labels` support).
+
+**Rotation**: after adding a new secret version, bump `slack_token_rotation`; the next apply re-reads the latest version and rewrites the channel token.
 
 ### Enabling Alerting
 
diff --git a/terraform/modules/scheduled-job/main.tf b/terraform/modules/scheduled-job/main.tf
index a598348..1665e8c 100644
--- a/terraform/modules/scheduled-job/main.tf
+++ b/terraform/modules/scheduled-job/main.tf
@@ -3,12 +3,15 @@
 
 # Required providers
 terraform {
-  required_version = ">= 1.3.0"
+  # 1.11+ for write-only arguments (1.10 introduced ephemeral resources).
+  required_version = ">= 1.11.0"
 
   required_providers {
     google = {
+      # 7.19.0 added the write-only sensitive_labels variants
+      # (auth_token_wo) on google_monitoring_notification_channel.
       source  = "hashicorp/google"
-      version = ">= 6.0.0"
+      version = ">= 7.19.0"
     }
     archive = {
       source  = "hashicorp/archive"
@@ -288,15 +291,13 @@ resource "google_cloud_scheduler_job" "job_scheduler" {
 
 # Alerting resources (only created when enable_alerting is true)
 
-# DEPRECATED token-based channel creation, used only when
-# notification_channel_ids is empty. Reading the Slack token with a data
-# source persists the token VALUE into Terraform state (state stores the full
-# data-source response, including secret_data), and from there into every
-# saved plan file. Prefer passing pre-created channel IDs via
-# notification_channel_ids; this path exists for backward compatibility and
-# will be removed in a future major version.
-data "google_secret_manager_secret_version" "slack_token" {
-  count = var.enable_alerting && length(var.notification_channel_ids) == 0 ? 1 : 0
+# Read the Slack API token ephemerally: the value is available to this run at
+# plan/apply time but is never persisted to Terraform state or saved plan
+# files. A regular data source would store its full response, including
+# secret_data, in both; that is how this token was exposed by the
+# committed-tfplan incident.
+ephemeral "google_secret_manager_secret_version" "slack_token" {
+  count = var.enable_alerting ? 1 : 0
 
   project = "khan-academy"
   secret  = "Slack__API_token_for_alertlib"
@@ -304,22 +305,16 @@ data "google_secret_manager_secret_version" "slack_token" {
 
 locals {
   alert_project_id = var.alert_project_id != null ? var.alert_project_id : var.project_id
-  slack_auth_token = var.enable_alerting && length(var.notification_channel_ids) == 0 ? data.google_secret_manager_secret_version.slack_token[0].secret_data : null
   slack_cc_mention = length(var.slack_mention_users) > 0 ? "\n\nCC: ${join(" ", var.slack_mention_users)}" : ""
   
-  # Channels the alert policies notify: pre-created channels when provided,
-  # otherwise the deprecated module-managed channel.
-  alert_notification_channels = length(var.notification_channel_ids) > 0 ? var.notification_channel_ids : [google_monitoring_notification_channel.slack_channel[0].name]
-
   # Console URLs for functions and jobs
   function_console_url = "https://console.cloud.google.com/run/detail/${var.region}/${var.job_name}/observability/logs?project=${var.project_id}"
   job_console_url      = "https://console.cloud.google.com/run/jobs/detail/${var.region}/${var.job_name}/observability/logs?project=${var.project_id}"
 }
 
-# Monitoring notification channel for Slack (DEPRECATED path, see above;
-# skipped entirely when notification_channel_ids is provided)
+# Monitoring notification channel for Slack
 resource "google_monitoring_notification_channel" "slack_channel" {
-  count = var.enable_alerting && length(var.notification_channel_ids) == 0 ? 1 : 0
+  count = var.enable_alerting ? 1 : 0
 
   project      = local.alert_project_id
   display_name = "${var.job_name} Slack Alerts"
@@ -330,7 +325,13 @@ resource "google_monitoring_notification_channel" "slack_channel" {
   }
 
   sensitive_labels {
-    auth_token = local.slack_auth_token
+    # Write-only: the token is sent to the Monitoring API but never stored in
+    # Terraform state or plan files. The ephemeral read above re-resolves the
+    # latest secret version on each run; the _wo_version counter controls
+    # when the API value is actually rewritten, so bump slack_token_rotation
+    # after rotating the secret.
+    auth_token_wo         = ephemeral.google_secret_manager_secret_version.slack_token[0].secret_data
+    auth_token_wo_version = var.slack_token_rotation
   }
 }
 
@@ -370,7 +371,7 @@ resource "google_monitoring_alert_policy" "function_failure" {
     }
   }
 
-  notification_channels = local.alert_notification_channels
+  notification_channels = [google_monitoring_notification_channel.slack_channel[0].name]
 
   documentation {
     content   = "The Cloud Function ${var.job_name} has failed to execute. Check the function logs for more details.\n\n[View Function in Console](${local.function_console_url})${local.slack_cc_mention}"
@@ -414,7 +415,7 @@ resource "google_monitoring_alert_policy" "job_failure" {
     }
   }
 
-  notification_channels = local.alert_notification_channels
+  notification_channels = [google_monitoring_notification_channel.slack_channel[0].name]
 
   documentation {
     content   = "The Cloud Run Job ${var.job_name} has failed to execute or complete successfully. Check the job logs for more details.\n\n[View Job in Console](${local.job_console_url})${local.slack_cc_mention}"
diff --git a/terraform/modules/scheduled-job/variables.tf b/terraform/modules/scheduled-job/variables.tf
index 732fd5f..fe0aa26 100644
--- a/terraform/modules/scheduled-job/variables.tf
+++ b/terraform/modules/scheduled-job/variables.tf
@@ -202,10 +202,10 @@ variable "enable_alerting" {
   default     = true
 }
 
-variable "notification_channel_ids" {
-  description = "Full resource names of pre-created Cloud Monitoring notification channels (projects/PROJECT/notificationChannels/ID) for failure alerts. Strongly preferred over the default: when empty, the module creates a Slack channel itself by reading the alertlib token with a data source, which persists the token value into Terraform state and any saved plan files. Pre-create the channel once per project (e.g. via the console's Slack OAuth flow, which never handles the token in Terraform) and pass its name here."
-  type        = list(string)
-  default     = []
+variable "slack_token_rotation" {
+  description = "Increment after rotating Slack__API_token_for_alertlib. The token is read ephemerally (never persisted to state or plans) and written to the notification channel via a write-only argument; bumping this counter is what triggers rewriting the channel's token from the latest secret version."
+  type        = number
+  default     = 1
 }
 
 variable "slack_channel" {