test: rework smoketests and improve usability (#508)

* test: rework smoketests and improve usability

always tee to tf.log
improve folder name and move smoketest under testing
remove smokedir, we only have one smoketest
move function under smoketest

* feat: support multiple function runtime and fix cloudwatch logs

.github/workflows/smoke-tests.yml

@@ -50,10 +50,10 @@ jobs:
           export_to_environment: true
           secrets: |-
             EC_API_KEY:elastic-observability/elastic-cloud-observability-team-pro-api-key
-      - run: make smoketest/run TEST_DIR=./tf
+      - run: make smoketest/run
       - if: always()
         name: Tear down
-        run: make smoketest/all/cleanup TEST_DIR=./tf
+        run: make smoketest/cleanup
 
       - if: always()
         uses: elastic/oblt-actions/slack/[email protected]

.gitignore

@@ -31,6 +31,4 @@ dist/
 .aws-*/
 *arn-file.md
 
-tf/*.zip
-
 build

Makefile

@@ -62,29 +62,13 @@ check-notice:
 ##############################################################################
 
 SMOKETEST_VERSIONS ?= latest
-SMOKETEST_DIRS = $$(find ./tf -mindepth 0 -maxdepth 0 -type d)
-
-.PHONY: smoketest/discover
-smoketest/discover:
-	@echo "$(SMOKETEST_DIRS)"
 
 .PHONY: smoketest/run
 smoketest/run: zip
-	@ for version in $(shell echo $(SMOKETEST_VERSIONS) | tr ',' ' '); do \
-		echo "-> Running $(TEST_DIR) smoke tests for version $${version}..."; \
-		cd $(TEST_DIR) && ./test.sh $${version}; \
-	done
+	@echo "-> Running smoke tests for version $${version}..."
+	cd testing/smoketest && ./test.sh $${version}
 
 .PHONY: smoketest/cleanup
-smoketest/cleanup:
-	@ cd $(TEST_DIR); \
-	if [ -f "./cleanup.sh" ]; then \
-		./cleanup.sh; \
-	fi
-
-.PHONY: smoketest/all
-smoketest/all/cleanup:
-	@ for test_dir in $(SMOKETEST_DIRS); do \
-		echo "-> Cleanup $${test_dir} smoke tests..."; \
-		$(MAKE) smoketest/cleanup TEST_DIR=$${test_dir}; \
-	done
+smoketest/cleanup: zip
+	@echo "-> Running cleanup"
+	cd testing/smoketest && ./cleanup.sh

tf/.gitignore → testing/smoketest/.gitignore

@@ -9,3 +9,4 @@ response.json
 tf.log
 
 builds/
+*.zip

tf/cleanup.sh → testing/smoketest/cleanup.sh

@@ -6,4 +6,4 @@ export TF_IN_AUTOMATION=1
 export TF_CLI_ARGS=-no-color
 
 echo "-> Tearing down the underlying infrastructure..."
-terraform destroy -auto-approve >> tf.log
+terraform destroy -auto-approve | tee -a tf.log

testing/smoketest/function/main.py

@@ -0,0 +1,16 @@
+import json
+
+coldstart = "true"
+def handler(event, context):
+    global coldstart
+    print("Example function log", context.aws_request_id)
+    resp = {
+        "statusCode": 200,
+        "body": json.dumps("Hello from Lambda!"+context.aws_request_id),
+        "headers": {
+            "coldstart": coldstart,
+        }
+    }
+    coldstart = "false"
+    return resp
+

tf/main.tf → testing/smoketest/main.tf

@@ -24,6 +24,30 @@ module "ec_deployment" {
   tags                     = module.tags.tags
 }
 
+locals {
+  runtimeVars = {
+    "nodejs" = {
+      "source_file" = "./function/index.js"
+      "handler"     = "index.handler"
+      "runtime"     = "nodejs18.x"
+      "agent_layer" = "arn:aws:lambda:${var.aws_region}:267093732750:layer:elastic-apm-node-ver-4-3-0:1"
+      "envvars" = {
+        "NODE_OPTIONS" = "-r elastic-apm-node/start"
+      }
+    }
+    "python" = {
+      "source_file" = "./function/main.py"
+      "handler"     = "main.handler"
+      "runtime"     = "python3.9"
+      "agent_layer" = "arn:aws:lambda:${var.aws_region}:267093732750:layer:elastic-apm-python-ver-6-22-3:1"
+      "envvars" = {
+        "AWS_LAMBDA_EXEC_WRAPPER" = "/opt/python/bin/elasticapm-lambda"
+      }
+    }
+  }
+}
+
+
 data "aws_iam_policy_document" "assume_role" {
   statement {
     effect = "Allow"
@@ -44,35 +68,71 @@ resource "aws_iam_role" "lambda" {
 
 data "archive_file" "lambda" {
   type        = "zip"
-  source_file = "../testdata/function/index.js"
+  source_file = local.runtimeVars[var.function_runtime]["source_file"]
   output_path = "lambda_function_payload.zip"
 }
 
 resource "aws_lambda_function" "test_lambda" {
   filename      = "lambda_function_payload.zip"
   function_name = "${var.user_name}-smoke-testing-test"
   role          = aws_iam_role.lambda.arn
-  handler       = "index.handler"
+  handler       = local.runtimeVars[var.function_runtime]["handler"]
 
-  source_code_hash = data.archive_file.lambda.output_base64sha256
+  runtime = local.runtimeVars[var.function_runtime]["runtime"]
 
-  runtime = "nodejs16.x"
+  source_code_hash = data.archive_file.lambda.output_base64sha256
 
   layers = [
     aws_lambda_layer_version.lambda_layer.arn,
-    "arn:aws:lambda:${var.aws_region}:267093732750:layer:elastic-apm-node-ver-4-3-0:1",
+    local.runtimeVars[var.function_runtime]["agent_layer"]
   ]
 
   environment {
-    variables = {
-      NODE_OPTIONS                                = "-r elastic-apm-node/start"
+    variables = merge({
       ELASTIC_APM_LOG_LEVEL                       = var.log_level
       ELASTIC_APM_LAMBDA_APM_SERVER               = module.ec_deployment.apm_url
       ELASTIC_APM_SECRETS_MANAGER_SECRET_TOKEN_ID = aws_secretsmanager_secret.apm_secret_token.id
-    }
+    }, local.runtimeVars[var.function_runtime]["envvars"])
+  }
+
+  depends_on = [
+    aws_iam_role_policy_attachment.lambda_logs,
+    aws_iam_role_policy_attachment.secrets_manager_elastic_apm_policy_attach,
+    aws_cloudwatch_log_group.example,
+  ]
+}
+
+resource "aws_cloudwatch_log_group" "example" {
+  name              = "/aws/lambda/${var.user_name}-smoke-testing-test"
+  retention_in_days = 1
+}
+
+data "aws_iam_policy_document" "lambda_logging" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+    ]
+
+    resources = ["arn:aws:logs:*:*:*"]
   }
 }
 
+resource "aws_iam_policy" "lambda_logging" {
+  name        = "smoketest_extension_lambda_logging"
+  path        = "/"
+  description = "IAM policy for logging during smoketest for apm aws lambda extension"
+  policy      = data.aws_iam_policy_document.lambda_logging.json
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_logs" {
+  role       = aws_iam_role.lambda.name
+  policy_arn = aws_iam_policy.lambda_logging.arn
+}
+
 resource "aws_secretsmanager_secret" "apm_secret_token" {
   name_prefix             = "apm-aws-lambda-smoke-testing-secret"
   recovery_window_in_days = 0
@@ -103,13 +163,12 @@ resource "aws_iam_role_policy_attachment" "secrets_manager_elastic_apm_policy_at
 }
 
 locals {
-  zip_files = tolist(fileset("../dist/", "*-linux-amd64.zip"))
+  zip_files = tolist(fileset("../../dist/", "*-linux-amd64.zip"))
 }
 
 resource "aws_lambda_layer_version" "lambda_layer" {
-  filename   = "../dist/${local.zip_files[0]}"
+  filename   = "../../dist/${local.zip_files[0]}"
   layer_name = "apm-aws-lambda-smoke-testing-lambda_layer_name"
 
-  description         = "AWS Lambda Extension Layer for Elastic APM - smoke testing"
-  compatible_runtimes = ["nodejs16.x"]
+  description = "AWS Lambda Extension Layer for Elastic APM - smoke testing"
 }

tf/output.tf → testing/smoketest/output.tf

@@ -23,3 +23,7 @@ output "aws_region" {
 output "user_name" {
   value = var.user_name
 }
+
+output "agent_name" {
+  value = var.function_runtime
+}

tf/test.sh → testing/smoketest/test.sh

@@ -8,15 +8,17 @@ export TF_CLI_ARGS=-no-color
 cleanup() {
   if [ "$SKIP_DESTROY" != "1" ]; then
     echo "-> Tearing down the underlying infrastructure..."
-    terraform destroy -auto-approve >> tf.log
+    terraform destroy -auto-approve | tee -a tf.log
   fi
 }
 
 trap "cleanup" EXIT
 
+
+
 echo "-> Creating the underlying infrastructure..."
 terraform init | tee tf.log
-terraform apply -auto-approve | tee -a tf.log
+terraform apply -auto-approve -var "function_runtime=${FUNCTION_RUNTIME:-nodejs}" | tee -a tf.log
 
 # https://github.com/hashicorp/setup-terraform/issues/167#issuecomment-1090760365
 TERRAFORM_WRAPPER=terraform-bin
@@ -36,21 +38,22 @@ echo "-> Waiting for the agent documents to be indexed in Elasticsearch..."
 ES_HOST=$($TERRAFORM_WRAPPER output -raw elasticsearch_url)
 ES_USER=$($TERRAFORM_WRAPPER output -raw elasticsearch_username)
 ES_PASS=$($TERRAFORM_WRAPPER output -raw elasticsearch_password)
+AGENT_NAME=$($TERRAFORM_WRAPPER output -raw agent_name)
 
 hits=0
 attempts=0
 while [ ${hits} -eq 0 ]; do
     # Check that ES has transaction documents on the GET endpoint.
-    resp=$(curl -s -H 'Content-Type: application/json' -u ${ES_USER}:${ES_PASS} "${ES_HOST}/traces-apm-*/_search" -d '{
-    "query": {
-        "match": {
-        "agent.name": "nodejs"
+    resp=$(curl -s -H 'Content-Type: application/json' -u ${ES_USER}:${ES_PASS} "${ES_HOST}/traces-apm-*/_search" -d "{
+    \"query\": {
+        \"match\": {
+        \"agent.name\": \"${AGENT_NAME}\"
         }
     }
-    }')
+    }")
     hits=$(echo ${resp} | jq '.hits.total.value')
     if [[ ${attempts} -ge 30 ]]; then
-        echo "-> Didn't find any traces for the NodeJS agents after ${attempts} attempts"
+        echo "-> Didn't find any traces for the ${AGENT_NAME} agents after ${attempts} attempts"
         exit 1
     fi
     let "attempts+=1"

tf/variables.tf → testing/smoketest/variables.tf

@@ -9,6 +9,12 @@ variable "user_name" {
   type        = string
 }
 
+variable "function_runtime" {
+  description = "function runtime and apm agent "
+  type        = string
+  default     = "nodejs"
+}
+
 variable "log_level" {
   type        = string
   description = "lambda extension log level"
@@ -24,7 +30,7 @@ variable "ess_region" {
 variable "ess_deployment_template" {
   type        = string
   description = "Elastic Cloud deployment template"
-  default     = "gcp-compute-optimized-v3"
+  default     = "gcp-vector-search-optimized"
 }
 
 variable "ess_version" {