Skip to content

Commit

Permalink
Revert "feat: use Lambda and Cloudwatch Logs to send reviewed alarms …
Browse files Browse the repository at this point in the history 
…to Slack channels (#421)" (#426)

This reverts commit 7f502df.
  • Loading branch information
@craigzour
craigzour authored Jul 11, 2023
1 parent 7f502df commit 063e411
Show file tree
Hide file tree
Showing 34 changed files with 842 additions and 735 deletions.
252 changes: 186 additions & 66 deletions aws/alarms/cloudwatch.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,51 @@ resource "aws_cloudwatch_metric_alarm" "forms_memory_utilization_high_warn" {
#
# Error alarms
#
resource "aws_cloudwatch_log_metric_filter" "five_hundred_response" {
name = "500Response"
pattern = "\"HTTP/1.1 5\""
log_group_name = var.ecs_cloudwatch_log_group_name

metric_transformation {
name = "500Response"
namespace = "forms"
value = "1"
default_value = "0"
}
}

resource "aws_cloudwatch_metric_alarm" "five_hundred_response_warn" {
alarm_name = "500ResponseWarn"
comparison_operator = "GreaterThanThreshold"
evaluation_periods = "1"
metric_name = aws_cloudwatch_log_metric_filter.five_hundred_response.name
namespace = "forms"
period = "60"
statistic = "Sum"
threshold = "0"
treat_missing_data = "notBreaching"
alarm_description = "End User Forms Warning - A 5xx HTML error was detected coming from the Forms."

alarm_actions = [var.sns_topic_alert_warning_arn]

tags = {
(var.billing_tag_key) = var.billing_tag_value
Terraform = true
}
}

resource "aws_cloudwatch_log_metric_filter" "application_error" {
name = "ApplicationError"
pattern = "Error"
log_group_name = var.ecs_cloudwatch_log_group_name

metric_transformation {
name = "ApplicationError"
namespace = "forms"
value = "1"
default_value = "0"
}
}

resource "aws_cloudwatch_metric_alarm" "ELB_5xx_error_warn" {
alarm_name = "HTTPCode_ELB_5XX_Count"
Expand All @@ -72,7 +117,25 @@ resource "aws_cloudwatch_metric_alarm" "ELB_5xx_error_warn" {
}
}

### Lambdas (to be replaced with subscription)
resource "aws_cloudwatch_metric_alarm" "application_error_warn" {
alarm_name = "ApplicationErrorWarn"
comparison_operator = "GreaterThanThreshold"
evaluation_periods = "1"
metric_name = aws_cloudwatch_log_metric_filter.application_error.name
namespace = "forms"
period = "60"
statistic = "Sum"
threshold = "0"
treat_missing_data = "notBreaching"
alarm_description = "End User Forms Warning - An error message was detected in the ECS logs"

alarm_actions = [var.sns_topic_alert_warning_arn]

tags = {
(var.billing_tag_key) = var.billing_tag_value
Terraform = true
}
}

resource "aws_cloudwatch_log_metric_filter" "reliability_error" {
name = "ReliabilityQueueError"
Expand Down Expand Up @@ -393,6 +456,128 @@ resource "aws_cloudwatch_metric_alarm" "route53_ddos" {
# Monitoring alarms
#

resource "aws_cloudwatch_metric_alarm" "temporary_token_generated_outside_canada_warn" {
alarm_name = "TemporaryTokenGeneratedOutsideCanadaWarn"
namespace = "AWS/WAFV2"
metric_name = "CountedRequests"
statistic = "SampleCount"
period = "300"
comparison_operator = "GreaterThanThreshold"
threshold = "0"
evaluation_periods = "1"
treat_missing_data = "notBreaching"
dimensions = {
Region = "ca-central-1"
Rule = "TemporaryTokenGeneratedOutsideCanada"
WebACL = "GCForms"
}

alarm_description = "End User Forms Warning - A temporary token has been generated from outside Canada"
alarm_actions = [var.sns_topic_alert_warning_arn]
ok_actions = [var.sns_topic_alert_ok_arn]

tags = {
(var.billing_tag_key) = var.billing_tag_value
Terraform = true
}
}

resource "aws_cloudwatch_log_metric_filter" "expired_bearer_token" {
name = "ExpiredBearerToken"
pattern = "expired bearer token"
log_group_name = var.ecs_cloudwatch_log_group_name
metric_transformation {
name = "ExpiredBearerToken"
namespace = "forms"
value = "1"
}
}

resource "aws_cloudwatch_metric_alarm" "expired_bearer_token" {
alarm_name = "ExpiredBearerToken"
namespace = "forms"
metric_name = aws_cloudwatch_log_metric_filter.expired_bearer_token.metric_transformation[0].name
statistic = "SampleCount"
period = "60"
comparison_operator = "GreaterThanThreshold"
threshold = "0"
evaluation_periods = "1"
treat_missing_data = "notBreaching"

alarm_description = "End User Forms Warning - An expired bearer token has been used"
alarm_actions = [var.sns_topic_alert_warning_arn]
ok_actions = [var.sns_topic_alert_ok_arn]

tags = {
(var.billing_tag_key) = var.billing_tag_value
Terraform = true
}
}

resource "aws_cloudwatch_log_metric_filter" "generate_temporary_token_api_failure" {
name = "GenerateTemporaryTokenApiFailure"
pattern = "Failed to generate temporary token"
log_group_name = var.ecs_cloudwatch_log_group_name
metric_transformation {
name = "GenerateTemporaryTokenApiFailure"
namespace = "forms"
value = "1"
}
}

resource "aws_cloudwatch_metric_alarm" "generate_temporary_token_api_failure" {
alarm_name = "GenerateTemporaryTokenApiFailure"
namespace = "forms"
metric_name = aws_cloudwatch_log_metric_filter.generate_temporary_token_api_failure.metric_transformation[0].name
statistic = "SampleCount"
period = "300"
comparison_operator = "GreaterThanThreshold"
threshold = "5"
evaluation_periods = "1"
treat_missing_data = "notBreaching"

alarm_description = "End User Forms Warning - Failed to generate temporary token too many times"
alarm_actions = [var.sns_topic_alert_warning_arn]
ok_actions = [var.sns_topic_alert_ok_arn]

tags = {
(var.billing_tag_key) = var.billing_tag_value
Terraform = true
}
}

resource "aws_cloudwatch_log_metric_filter" "request_temporary_token_api_using_unauthorized_email_address" {
name = "RequestTemporaryTokenApiUsingUnauthorizedEmailAddress"
pattern = "\"An email address with no access to any form has been locked out\""
log_group_name = var.ecs_cloudwatch_log_group_name
metric_transformation {
name = "RequestTemporaryTokenApiUsingUnauthorizedEmailAddress"
namespace = "forms"
value = "1"
}
}

resource "aws_cloudwatch_metric_alarm" "request_temporary_token_api_using_unauthorized_email_address" {
alarm_name = "RequestTemporaryTokenApiUsingUnauthorizedEmailAddress"
namespace = "forms"
metric_name = aws_cloudwatch_log_metric_filter.request_temporary_token_api_using_unauthorized_email_address.metric_transformation[0].name
statistic = "SampleCount"
period = "60"
comparison_operator = "GreaterThanThreshold"
threshold = "0"
evaluation_periods = "1"
treat_missing_data = "notBreaching"

alarm_description = "End User Forms Warning - Someone tried to request a temporary token using an unauthorized email address"
alarm_actions = [var.sns_topic_alert_warning_arn]
ok_actions = [var.sns_topic_alert_ok_arn]

tags = {
(var.billing_tag_key) = var.billing_tag_value
Terraform = true
}
}

resource "aws_cloudwatch_log_metric_filter" "cognito_signin_exceeded" {
name = "CognitoSigninExceeded"
pattern = "\"Cognito Lockout: Password attempts exceeded\""
Expand Down Expand Up @@ -457,69 +642,4 @@ resource "aws_cloudwatch_metric_alarm" "twoFa_verification_exceeded" {
(var.billing_tag_key) = var.billing_tag_value
Terraform = true
}
}

// Dynamic Stream to Lambda

resource "aws_cloudwatch_log_subscription_filter" "forms_unhandled_error_steam" {
depends_on = [aws_lambda_permission.allow_cloudwatch_to_run_lambda]
name = "forms_unhandled_error_stream"
log_group_name = var.ecs_cloudwatch_log_group_name
filter_pattern = "Error -level" # Do not catch JSON formatted logs with `level` property because those will be handled by the filters below
destination_arn = aws_lambda_function.notify_slack.arn
}

resource "aws_cloudwatch_log_subscription_filter" "forms_app_log_stream" {
name = "forms_app_log_stream"
log_group_name = var.ecs_cloudwatch_log_group_name
filter_pattern = "{($.level = \"warn\") || ($.level = \"error\")}"
destination_arn = aws_lambda_function.notify_slack.arn
}

resource "aws_cloudwatch_log_subscription_filter" "reliability_log_stream" {
name = "reliability_log_stream"
log_group_name = var.lambda_reliability_log_group_name
filter_pattern = "{($.level = \"warn\") || ($.level = \"error\")}"
destination_arn = aws_lambda_function.notify_slack.arn
}

resource "aws_cloudwatch_log_subscription_filter" "archiver_log_stream" {
name = "archiver_log_stream"
log_group_name = var.lambda_archiver_log_group_name
filter_pattern = "{($.level = \"warn\") || ($.level = \"error\")}"
destination_arn = aws_lambda_function.notify_slack.arn
}
resource "aws_cloudwatch_log_subscription_filter" "dlq_consumer_log_stream" {
name = "dql_consumer_log_stream"
log_group_name = var.lambda_dlq_consumer_log_group_name
filter_pattern = "{($.level = \"warn\") || ($.level = \"error\")}"
destination_arn = aws_lambda_function.notify_slack.arn
}
resource "aws_cloudwatch_log_subscription_filter" "template_archiver_log_stream" {
name = "template_archiver_log_stream"
log_group_name = var.lambda_template_archiver_log_group_name
filter_pattern = "{($.level = \"warn\") || ($.level = \"error\")}"
destination_arn = aws_lambda_function.notify_slack.arn
}
resource "aws_cloudwatch_log_subscription_filter" "audit_log_stream" {
name = "audit_log_stream"
log_group_name = var.lambda_audit_log_group_name
filter_pattern = "{($.level = \"warn\") || ($.level = \"error\")}"
destination_arn = aws_lambda_function.notify_slack.arn
}
resource "aws_cloudwatch_log_subscription_filter" "nagware_log_stream" {
name = "nagware_log_stream"
log_group_name = var.lambda_nagware_log_group_name
filter_pattern = "{($.level = \"warn\") || ($.level = \"error\")}"
destination_arn = aws_lambda_function.notify_slack.arn
}

// Allow Cloudwatch filters to trigger Lambda

resource "aws_lambda_permission" "allow_cloudwatch_to_run_lambda" {
statement_id = "AllowExecutionFromCloudWatch"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.notify_slack.function_name
principal = "events.amazonaws.com"
source_arn = "arn:aws:ssm:ca-central-1:${var.account_id}:log-group:*:*"
}
37 changes: 1 addition & 36 deletions aws/alarms/inputs.tf
View file
Original file line number Diff line number Diff line change
@@ -1,45 +1,10 @@
variable "kms_key_cloudwatch_arn" {
description = "CloudWatch KMS key ARN used to encrypt the logs"
type = string
}

variable "ecs_cloudwatch_log_group_name" {
description = "ECS Forms CloudWatch log group name, used by app error metric alarms"
type = string
}

variable "lambda_reliability_log_group_name" {
description = "Reliability Queues CloudWatch log group name"
type = string
}

variable "lambda_submission_log_group_name" {
description = "Submission Lambda CloudWatch log group name"
type = string
}

variable "lambda_archiver_log_group_name" {
description = "Response Archiver Lambda CloudWatch log group name"
type = string
}

variable "lambda_dlq_consumer_log_group_name" {
description = "DQL Consumer CloudWatch log group name"
type = string
}

variable "lambda_template_archiver_log_group_name" {
description = "Template Archiver Lambda CloudWatch log group name"
type = string
}

variable "lambda_audit_log_group_name" {
description = "Audit Log Lambda CloudWatch log group name"
type = string
}

variable "lambda_nagware_log_group_name" {
description = "Nagware Lambda CloudWatch log group name"
description = "Lambda Reliability Queue CloudWatch log group name, used by lambda error metric alarms"
type = string
}

Expand Down
18 changes: 6 additions & 12 deletions aws/alarms/lambda.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ data "archive_file" "notify_slack" {
}

#tfsec:ignore:aws-lambda-enable-tracing
resource "aws_lambda_function" "notify_slack" {
resource "aws_lambda_function" "notify_slack_sns" {
filename = "/tmp/notify_slack.zip"
function_name = "NotifySlackSNS"
role = aws_iam_role.notify_slack_lambda.arn
Expand Down Expand Up @@ -41,39 +41,39 @@ resource "aws_lambda_function" "notify_slack" {
resource "aws_lambda_permission" "notify_slack_critical" {
statement_id = "AllowExecutionFromSNSCriticalAlert"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.notify_slack.function_name
function_name = aws_lambda_function.notify_slack_sns.function_name
principal = "sns.amazonaws.com"
source_arn = var.sns_topic_alert_critical_arn
}

resource "aws_lambda_permission" "notify_slack_warning" {
statement_id = "AllowExecutionFromSNSWarningAlert"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.notify_slack.function_name
function_name = aws_lambda_function.notify_slack_sns.function_name
principal = "sns.amazonaws.com"
source_arn = var.sns_topic_alert_warning_arn
}

resource "aws_lambda_permission" "notify_slack_ok" {
statement_id = "AllowExecutionFromSNSOkAlert"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.notify_slack.function_name
function_name = aws_lambda_function.notify_slack_sns.function_name
principal = "sns.amazonaws.com"
source_arn = var.sns_topic_alert_ok_arn
}

resource "aws_lambda_permission" "notify_slack_warning_us_east" {
statement_id = "AllowExecutionFromSNSWarningAlertUSEast"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.notify_slack.function_name
function_name = aws_lambda_function.notify_slack_sns.function_name
principal = "sns.amazonaws.com"
source_arn = var.sns_topic_alert_warning_us_east_arn
}

resource "aws_lambda_permission" "notify_slack_ok_us_east" {
statement_id = "AllowExecutionFromSNSOkAlertUSEast"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.notify_slack.function_name
function_name = aws_lambda_function.notify_slack_sns.function_name
principal = "sns.amazonaws.com"
source_arn = var.sns_topic_alert_ok_us_east_arn
}
Expand Down Expand Up @@ -108,9 +108,3 @@ resource "aws_iam_role_policy_attachment" "notify_slack_lambda_basic_access" {
role = aws_iam_role.notify_slack_lambda.name
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}

resource "aws_cloudwatch_log_group" "notify_slack" {
name = "/aws/lambda/${aws_lambda_function.notify_slack.function_name}"
kms_key_id = var.kms_key_cloudwatch_arn
retention_in_days = 90
}
Loading

0 comments on commit 063e411

Please sign in to comment.