chore(ci): switch eks CI to FIPS ami, update to 1.31 k8s testing

.github/test-infra/aws/eks/cluster.tf

@@ -0,0 +1,103 @@
+# Copyright 2025 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+
+# Create EKS Cluster
+module "eks" {
+  source  = "terraform-aws-modules/eks/aws"
+  version = "~> 20.0"
+
+  cluster_name                   = var.name
+  cluster_version                = var.kubernetes_version
+  cluster_endpoint_public_access = true
+
+  vpc_id     = data.aws_vpc.vpc.id
+  subnet_ids = local.subnet_ids
+
+  # IAM
+  iam_role_permissions_boundary = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/${var.permissions_boundary_name}"
+
+  # Add CloudWatch logging
+  cluster_enabled_log_types              = ["api", "audit", "authenticator", "controllerManager", "scheduler"]
+  cloudwatch_log_group_retention_in_days = 1
+
+  # Authentication mode
+  authentication_mode = "API_AND_CONFIG_MAP"
+
+  # Enable cluster creator admin permissions
+  enable_cluster_creator_admin_permissions = true
+
+  # Security groups
+  create_cluster_security_group                = true
+  create_node_security_group                   = true
+  node_security_group_enable_recommended_rules = true
+  node_security_group_additional_rules = {
+    clusterapi_ingress = {
+      description                   = "Cluster API Ingress on non-privileged ports"
+      protocol                      = "tcp"
+      from_port                     = 1025
+      to_port                       = 65535
+      type                          = "ingress"
+      source_cluster_security_group = true
+    }
+  }
+
+  # Add tags to all resources
+  tags = local.tags
+
+  # Node groups
+  eks_managed_node_groups = {
+    main = {
+      name           = var.name
+      instance_types = [var.instance_type]
+      ami_type       = "BOTTLEROCKET_x86_64_FIPS"
+
+      min_size     = var.node_group_min_size
+      max_size     = var.node_group_max_size
+      desired_size = var.node_group_desired_size
+
+      disk_size = var.node_disk_size
+
+      # Let the module create the IAM role with permissions boundary
+      create_iam_role               = true
+      iam_role_use_name_prefix      = false
+      iam_role_name                 = "${substr(var.name, 0, 30)}-eks-node-role"
+      iam_role_permissions_boundary = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/${var.permissions_boundary_name}"
+
+      # Add required policies for node functionality
+      iam_role_additional_policies = {
+        AmazonSSMManagedInstanceCore = "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore"
+        AmazonEBSCSIDriverPolicy     = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
+      }
+
+      tags = merge(local.tags, {
+        PermissionsBoundary = var.permissions_boundary_name
+      })
+    }
+  }
+
+  # EKS Addons
+  cluster_addons = {
+    vpc-cni = {
+      most_recent = true
+      configuration_values = jsonencode({
+        enableNetworkPolicy = "true"
+      })
+    }
+    aws-ebs-csi-driver = {
+      most_recent = true
+    }
+    kube-proxy = {
+      most_recent = true
+    }
+    coredns = {
+      most_recent = true
+    }
+  }
+
+  # Explicit dependency on subnet and IAM
+  depends_on = [
+    aws_subnet.cluster_subnet,
+    aws_subnet.cluster_subnet_second,
+  ]
+}

.github/test-infra/aws/eks/data.tf

@@ -0,0 +1,46 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+# Common data sources
+data "aws_caller_identity" "current" {}
+data "aws_partition" "current" {}
+data "aws_region" "current" {}
+
+# Use existing VPC and subnets or create new ones
+data "aws_vpc" "vpc" {
+  filter {
+    name   = "tag:Name"
+    values = [var.vpc_name]
+  }
+}
+
+# Find the public route table in the VPC
+data "aws_route_table" "public" {
+  vpc_id = data.aws_vpc.vpc.id
+  filter {
+    name   = "tag:Name"
+    values = ["*public*"]
+  }
+}
+
+# Random identifiers
+resource "random_id" "default" {
+  byte_length = 2
+}
+
+resource "random_id" "unique_id" {
+  byte_length = 4
+}
+
+# Generate a random subnet CIDR that's valid for the VPC
+resource "random_integer" "subnet_octet_3" {
+  min  = 0
+  max  = 250
+  seed = var.name
+}
+
+resource "random_integer" "subnet_octet_3_second" {
+  min  = 0
+  max  = 250
+  seed = "${var.name}-second"
+}

.github/test-infra/aws/eks/main.tf

@@ -1,25 +1,23 @@
 # Copyright 2024 Defense Unicorns
 # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+locals {
+  # Generate deterministic subnet CIDRs based on cluster name
+  # Use 172.168.x.x/24 to match the VPC CIDR (172.168.0.0/16)
+  generated_subnet_cidr = "172.168.${random_integer.subnet_octet_3.result}.0/24"
+  subnet_cidr           = var.subnet_cidr != "" ? var.subnet_cidr : local.generated_subnet_cidr
+  subnet_cidr_second    = "172.168.${random_integer.subnet_octet_3_second.result}.0/24"
 
-resource "random_id" "default" {
-  byte_length = 2
-}
-
-data "aws_eks_cluster" "existing" {
-  name = var.name
-}
-
-data "aws_caller_identity" "current" {}
-
-data "aws_partition" "current" {}
-
-data "aws_region" "current" {}
+  # Combine subnet IDs for EKS
+  subnet_ids = [aws_subnet.cluster_subnet.id, aws_subnet.cluster_subnet_second.id]
 
-locals {
-  oidc_url_without_protocol     = substr(data.aws_eks_cluster.existing.identity[0].oidc[0].issuer, 8, -1)
-  oidc_arn                      = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_url_without_protocol}"
-  iam_role_permissions_boundary = var.use_permissions_boundary ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/${var.permissions_boundary_name}" : null
+  # Tags for resources
+  tags = {
+    Name                = var.name
+    Environment         = "ci"
+    PermissionsBoundary = var.permissions_boundary_name
+  }
 
+  # Bucket configurations for IRSA
   bucket_configurations = {
     for instance in var.bucket_configurations :
     instance.name => {
@@ -29,18 +27,13 @@ locals {
     }
   }
 
-  kms_key_arns = module.generate_kms
-
+  # IAM policies for IRSA
   iam_policies = {
     "loki"   = resource.aws_iam_policy.loki_policy.arn
     "velero" = resource.aws_iam_policy.velero_policy.arn
   }
 }
 
-resource "random_id" "unique_id" {
-  byte_length = 4
-}
-
 module "generate_kms" {
   for_each                  = local.bucket_configurations
   source                    = "../modules/kms"
@@ -51,6 +44,11 @@ module "generate_kms" {
   tags = {
     Deployment = "UDS Core ${each.value.name}"
   }
+
+  # Explicit dependency on EKS cluster
+  depends_on = [
+    module.eks
+  ]
 }
 
 module "S3" {
@@ -59,24 +57,36 @@ module "S3" {
   bucket_prefix = "${each.value.name}-"
   kms_key_arn   = module.generate_kms[each.key].kms_key_arn
   irsa_role_arn = module.irsa[each.key].role_arn
+
+  # Explicit dependency on KMS
+  depends_on = [
+    module.generate_kms
+  ]
 }
 
 module "irsa" {
   for_each                      = local.bucket_configurations
   source                        = "../modules/irsa"
   name                          = each.value.name
   kubernetes_service_account    = each.value.service_account
-  role_permissions_boundary_arn = local.iam_role_permissions_boundary
+  role_permissions_boundary_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/${var.permissions_boundary_name}"
   account_id                    = data.aws_caller_identity.current.account_id
   current_partition             = data.aws_partition.current.partition
 
   oidc_providers = {
     main = {
-      provider_arn               = local.oidc_arn
+      provider_arn               = module.eks.oidc_provider_arn
       namespace_service_accounts = [format("%s:%s", each.value.namespace, each.value.service_account)]
     }
   }
   role_policy_arns = tomap({
-    "${each.key}" = local.iam_policies[each.key]
+    (each.key) = local.iam_policies[each.key]
   })
+
+  # Explicit dependency on EKS cluster
+  depends_on = [
+    module.eks,
+    aws_iam_policy.loki_policy,
+    aws_iam_policy.velero_policy
+  ]
 }

.github/test-infra/aws/eks/networking.tf

@@ -0,0 +1,38 @@
+# Copyright 2025 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+# Create a new subnet for this cluster
+resource "aws_subnet" "cluster_subnet" {
+  vpc_id                  = data.aws_vpc.vpc.id
+  cidr_block              = local.subnet_cidr
+  availability_zone       = "${var.region}b"
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "${var.name}-subnet-1"
+  }
+}
+
+# Create a second subnet in a different AZ
+resource "aws_subnet" "cluster_subnet_second" {
+  vpc_id                  = data.aws_vpc.vpc.id
+  cidr_block              = local.subnet_cidr_second
+  availability_zone       = "${var.region}c" # Different AZ
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "${var.name}-subnet-2"
+  }
+}
+
+# Associate the public route table with the first subnet
+resource "aws_route_table_association" "rta1" {
+  subnet_id      = aws_subnet.cluster_subnet.id
+  route_table_id = data.aws_route_table.public.id
+}
+
+# Associate the public route table with the second subnet
+resource "aws_route_table_association" "rta2" {
+  subnet_id      = aws_subnet.cluster_subnet_second.id
+  route_table_id = data.aws_route_table.public.id
+}

.github/test-infra/aws/eks/outputs.tf

@@ -57,4 +57,4 @@ output "grafana_pg_password" {
 
 output "grafana_ha" {
   value = true
-}
+}

.github/test-infra/aws/eks/rds.tf

@@ -22,7 +22,7 @@ module "db" {
   source  = "terraform-aws-modules/rds/aws"
   version = "6.11.0"
 
-  identifier                     = "${var.db_name}-db"
+  identifier                     = "${var.name}-db"
   instance_use_identifier_prefix = true
 
   allocated_storage       = var.db_allocated_storage
@@ -34,14 +34,14 @@ module "db" {
   engine               = "postgres"
   engine_version       = var.db_engine_version
   major_engine_version = split(".", var.db_engine_version)[0]
-  family               = "postgres15"
+  family               = "postgres16"
   instance_class       = var.db_instance_class
 
   db_name  = var.db_name
   username = var.username
   port     = var.db_port
 
-  subnet_ids                  = data.aws_subnets.subnets.ids
+  subnet_ids                  = data.aws_subnets.rds_subnets.ids
   create_db_subnet_group      = true
   create_db_parameter_group   = false
   manage_master_user_password = false
@@ -55,7 +55,7 @@ module "db" {
 }
 
 resource "aws_security_group" "rds_sg" {
-  vpc_id = local.vpc_id
+  vpc_id = data.aws_vpc.rds_vpc.id
 
   egress {
     from_port        = 0
@@ -75,20 +75,16 @@ resource "aws_vpc_security_group_ingress_rule" "rds_ingress" {
   to_port     = 5432
 }
 
-data "aws_vpc" "vpc" {
+data "aws_vpc" "rds_vpc" {
   filter {
     name   = "tag:Name"
-    values = ["eksctl-${var.name}-cluster/VPC"]
+    values = [var.vpc_name]
   }
 }
 
-data "aws_subnets" "subnets" {
+data "aws_subnets" "rds_subnets" {
   filter {
     name   = "vpc-id"
-    values = [data.aws_vpc.vpc.id]
+    values = [data.aws_vpc.rds_vpc.id]
   }
 }
-
-locals {
-  vpc_id = data.aws_vpc.vpc.id
-}

.github/test-infra/aws/eks/uds-config.tf

@@ -28,6 +28,10 @@ resource "local_sensitive_file" "uds_config" {
         "grafana_pg_password" : random_password.db_password.result,
         "grafana_pg_user" : var.username
       }
+      "init" : {
+        # Workaround for Bottlerocket EBS issue - https://github.com/bottlerocket-os/bottlerocket/issues/2417
+        "registry_hpa_enable" : false
+      }
     }
   })
 }