feat: alarms module

Add API alarms that trigger when errors are logged or a
secret is detected.

.github/workflows/tf_apply_production.yml

@@ -15,6 +15,7 @@ env:
   TF_VAR_api_config: ${{ secrets.API_CONFIG }}
   TF_VAR_log_analytics_workspace_id: ${{ secrets.LOG_ANALYTICS_WORKSPACE_ID }}
   TF_VAR_log_analytics_workspace_key: ${{ secrets.LOG_ANALYTICS_WORKSPACE_KEY }}
+  TF_VAR_slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
 
 permissions:
   id-token: write
@@ -51,3 +52,7 @@ jobs:
       - name: Apply cloudfront
         working-directory: terragrunt/env/production/cloudfront
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
+
+      - name: Apply alarms
+        working-directory: terragrunt/env/production/alarms
+        run: terragrunt apply --terragrunt-non-interactive -auto-approve

.github/workflows/tf_plan_production.yml

@@ -15,6 +15,7 @@ env:
   TF_VAR_api_config: ${{ secrets.API_CONFIG }}
   TF_VAR_log_analytics_workspace_id: ${{ secrets.LOG_ANALYTICS_WORKSPACE_ID }}
   TF_VAR_log_analytics_workspace_key: ${{ secrets.LOG_ANALYTICS_WORKSPACE_KEY }}
+  TF_VAR_slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
 
 permissions:
   id-token: write
@@ -29,7 +30,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:          
+        include:
+          - module: alarms
           - module: api
           - module: cloudfront
           - module: hosted_zone

terragrunt/aws/alarms/alarms.tf

@@ -0,0 +1,57 @@
+resource "aws_cloudwatch_log_metric_filter" "api_error" {
+  name           = local.error_logged_api
+  pattern        = "?ERROR ?Error ?error ?failed"
+  log_group_name = local.api_cloudwatch_log_group
+
+  metric_transformation {
+    name      = local.error_logged_api
+    namespace = local.metric_namespace
+    value     = "1"
+  }
+}
+
+resource "aws_cloudwatch_metric_alarm" "api_error" {
+  alarm_name          = local.error_logged_api
+  alarm_description   = "Errors logged by the API lambda function"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+
+  metric_name        = aws_cloudwatch_log_metric_filter.api_error.metric_transformation[0].name
+  namespace          = aws_cloudwatch_log_metric_filter.api_error.metric_transformation[0].namespace
+  period             = "60"
+  evaluation_periods = "1"
+  statistic          = "Sum"
+  threshold          = "1"
+  treat_missing_data = "notBreaching"
+
+  alarm_actions = [local.sns_topic_arn]
+  ok_actions    = [local.sns_topic_arn]
+}
+
+resource "aws_cloudwatch_log_metric_filter" "api_secret_detected" {
+  name           = local.secret_detected_api
+  pattern        = "Secret detected"
+  log_group_name = local.api_cloudwatch_log_group
+
+  metric_transformation {
+    name      = local.secret_detected_api
+    namespace = local.metric_namespace
+    value     = "1"
+  }
+}
+
+resource "aws_cloudwatch_metric_alarm" "api_secret_detected" {
+  alarm_name          = local.secret_detected_api
+  alarm_description   = "GitHub alert that a secret has been detected"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+
+  metric_name        = aws_cloudwatch_log_metric_filter.api_secret_detected.metric_transformation[0].name
+  namespace          = aws_cloudwatch_log_metric_filter.api_secret_detected.metric_transformation[0].namespace
+  period             = "60"
+  evaluation_periods = "1"
+  statistic          = "Sum"
+  threshold          = "1"
+  treat_missing_data = "notBreaching"
+
+  alarm_actions = [local.sns_topic_arn]
+  ok_actions    = [local.sns_topic_arn]
+}

terragrunt/aws/alarms/inputs.tf

@@ -0,0 +1,10 @@
+variable "api_function_name" {
+  description = "The name of the API function."
+  type        = string
+}
+
+variable "slack_webhook_url" {
+  description = "The URL of the Slack webhook."
+  type        = string
+  sensitive   = true
+}

terragrunt/aws/alarms/locals.tf

@@ -0,0 +1,7 @@
+locals {
+  api_cloudwatch_log_group = "/aws/lambda/${var.api_function_name}"
+  error_logged_api         = "ErrorLoggedAPI"
+  metric_namespace         = "GitHubSecretScanning"
+  secret_detected_api      = "SecretDetectedAPI"
+  sns_topic_arn            = "arn:aws:sns:${var.region}:${var.account_id}:internal-sre-alert"
+}

terragrunt/aws/alarms/slack.tf

@@ -0,0 +1,12 @@
+module "cloudwatch_alarms_slack" {
+  source = "github.com/cds-snc/terraform-modules?ref=v5.0.2//notify_slack"
+
+  function_name     = var.product_name
+  project_name      = var.product_name
+  slack_webhook_url = var.slack_webhook_url
+  sns_topic_arns = [
+    local.sns_topic_arn,
+  ]
+
+  billing_tag_value = var.billing_code
+}

terragrunt/env/production/alarms/terragrunt.hcl

@@ -0,0 +1,25 @@
+terraform {
+  source = "../../../aws//alarms"
+}
+
+dependencies {
+  paths = ["../api"]
+}
+
+dependency "api" {
+  config_path = "../api"
+
+  mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"]
+  mock_outputs_merge_with_state           = true
+  mock_outputs = {
+    api_function_name = "github-secret-scanning-api"
+  }
+}
+
+inputs = {
+  api_function_name = dependency.api.outputs.function_name
+}  
+
+include {
+  path = find_in_parent_folders()
+}