add singer log dir

Author: AJ Steers

Diff for: catalog/aws/singer-taps/main.tf

@@ -62,22 +62,24 @@ module "ecs_cluster" {
 }
 
 module "ecs_tap_sync_task" {
-  count               = length(local.taps_specs)
-  source              = "../../../components/aws/ecs-task"
-  name_prefix         = "${local.name_prefix}task${count.index}-"
-  environment         = var.environment
-  resource_tags       = var.resource_tags
-  ecs_cluster_name    = module.ecs_cluster.ecs_cluster_name
-  container_image     = local.taps_specs[count.index].image
-  container_command   = local.taps_specs[count.index].sync_command
-  container_ram_gb    = var.container_ram_gb
-  container_num_cores = var.container_num_cores
-  use_private_subnet  = var.use_private_subnet
-  use_fargate         = true
+  count                = length(local.taps_specs)
+  source               = "../../../components/aws/ecs-task"
+  name_prefix          = "${local.name_prefix}task${count.index}-"
+  environment          = var.environment
+  resource_tags        = var.resource_tags
+  ecs_cluster_name     = module.ecs_cluster.ecs_cluster_name
+  container_image      = local.taps_specs[count.index].image
+  container_command    = local.taps_specs[count.index].sync_command
+  container_ram_gb     = var.container_ram_gb
+  container_num_cores  = var.container_num_cores
+  use_private_subnet   = var.use_private_subnet
+  use_fargate          = true
+  permitted_s3_buckets = local.needed_s3_buckets
   environment_vars = merge(
     {
       TAP_CONFIG_DIR                                    = "${var.data_lake_metadata_path}/tap-snapshot-${local.unique_hash}",
       TAP_STATE_FILE                                    = "${coalesce(var.data_lake_storage_path, var.data_lake_metadata_path)}/${var.state_file_naming_scheme}",
+      TAP_LOG_DIR                                       = "${var.data_lake_logging_path}/tap-${local.taps_specs[count.index].name}/",
       PIPELINE_VERSION_NUMBER                           = var.pipeline_version_number
       "${local.tap_env_prefix[count.index]}CONFIG_FILE" = "False" # Config will be passed via env vars
       "${local.target_env_prefix}CONFIG_FILE"           = "False" # Config will be passed via env vars

Diff for: catalog/aws/singer-taps/s3-path-parsing.tf

@@ -10,4 +10,9 @@ locals {
     )),
     replace(var.data_file_naming_scheme, "{file}", "")
   ])
+  needed_s3_buckets = distinct(flatten([
+    var.data_lake_storage_path == null ? [] : [split("/", split("//", var.data_lake_storage_path)[1])[0]],
+    var.data_lake_logging_path == null ? [] : [split("/", split("//", var.data_lake_logging_path)[1])[0]],
+    var.data_lake_metadata_path == null ? [] : [split("/", split("//", var.data_lake_metadata_path)[1])[0]],
+  ]))
 }

Diff for: catalog/aws/singer-taps/variables.tf

@@ -101,6 +101,13 @@ variable "data_lake_type" {
   description = "Specify `S3` if loading to an S3 data lake, otherwise leave blank."
   default     = null
 }
+variable "data_lake_logging_path" {
+  description = <<EOF
+The remote folder for storing tap execution logs and log artifacts.
+Currently only S3 paths (s3://...) are supported.
+EOF
+  type        = string
+}
 variable "data_lake_metadata_path" {
   description = <<EOF
 The remote folder for storing tap definitions files.