Tests for Cluster and FargateProfile (#2)

Partly accomplishes: aws-controllers-k8s/community#858

Description of changes:
Add initial smoke tests for creating and then deleting a `Cluster`
Add smoke test for creating and deleting a `FargateProfile` - using a `Cluster` as the fixture

By submitting this pull request, I confirm that my contribution is made under
the terms of the Apache 2.0 license.

.gitignore

@@ -4,4 +4,5 @@
 .idea
 /docs/site
 bin
-build
+build
+.env

test/e2e/.gitignore

@@ -1,3 +1,4 @@
 __pycache__/
 *.py[cod]
-**/bootstrap.yaml
+**/bootstrap.yaml
+**/bootstrap.pkl

test/e2e/bootstrap_resources.py

@@ -16,19 +16,21 @@
 """
 
 from dataclasses import dataclass
-from acktest.resources import read_bootstrap_config
+from acktest.bootstrapping import Resources
+from acktest.bootstrapping.iam import Role
+from acktest.bootstrapping.vpc import VPC
 from e2e import bootstrap_directory
 
 @dataclass
-class TestBootstrapResources:
-    pass
+class TestBootstrapResources(Resources):
+    ClusterVPC: VPC
+    ClusterRole: Role
+    FargatePodRole: Role
 
 _bootstrap_resources = None
 
-def get_bootstrap_resources(bootstrap_file_name: str = "bootstrap.yaml"):
+def get_bootstrap_resources(bootstrap_file_name: str = "bootstrap.pkl") -> TestBootstrapResources:
     global _bootstrap_resources
     if _bootstrap_resources is None:
-        _bootstrap_resources = TestBootstrapResources(
-            **read_bootstrap_config(bootstrap_directory, bootstrap_file_name=bootstrap_file_name),
-        )
+        _bootstrap_resources = TestBootstrapResources.deseralize(bootstrap_directory, bootstrap_file_name=bootstrap_file_name)
     return _bootstrap_resources

test/e2e/common/types.py

@@ -0,0 +1 @@
+CLUSTER_RESOURCE_PLURAL = 'clusters'

test/e2e/conftest.py

@@ -12,6 +12,7 @@
 # permissions and limitations under the License.
 
 import os
+import boto3
 import pytest
 
 from acktest import k8s
@@ -44,3 +45,7 @@ def pytest_collection_modifyitems(config, items):
 @pytest.fixture(scope='class')
 def k8s_client():
     return k8s._get_k8s_api_client()
+
+@pytest.fixture(scope='module')
+def eks_client():
+    return boto3.client('eks')

test/e2e/replacement_values.py

@@ -14,5 +14,15 @@
 EKS-specific test variables.
 """
 
+from e2e.bootstrap_resources import get_bootstrap_resources
+
+
 REPLACEMENT_VALUES = {
-}
+    "CLUSTER_ROLE": get_bootstrap_resources().ClusterRole.arn,
+    "FARGATE_POD_ROLE": get_bootstrap_resources().FargatePodRole.arn,
+
+    "PUBLIC_SUBNET_1": get_bootstrap_resources().ClusterVPC.public_subnets.subnet_ids[0],
+    "PUBLIC_SUBNET_2": get_bootstrap_resources().ClusterVPC.public_subnets.subnet_ids[1],
+    "PRIVATE_SUBNET_1": get_bootstrap_resources().ClusterVPC.private_subnets.subnet_ids[0],
+    "PRIVATE_SUBNET_2": get_bootstrap_resources().ClusterVPC.private_subnets.subnet_ids[1]
+}

test/e2e/requirements.txt

@@ -1 +1 @@
-acktest @ git+https://github.com/aws-controllers-k8s/test-infra.git@955d7831ee374a212250179e95a5f3b75e555fd9
+acktest @ git+https://github.com/aws-controllers-k8s/test-infra.git@955d7831ee374a212250179e95a5f3b75e555fd9

test/e2e/resources/cluster_simple.yaml

@@ -0,0 +1,13 @@
+apiVersion: eks.services.k8s.aws/v1alpha1
+kind: Cluster
+metadata:
+  name: $CLUSTER_NAME
+spec:
+  name: $CLUSTER_NAME
+  roleARN: $CLUSTER_ROLE
+  resourcesVPCConfig:
+    endpointPrivateAccess: true
+    endpointPublicAccess: true
+    subnetIDs:
+      - "$PUBLIC_SUBNET_1"
+      - "$PUBLIC_SUBNET_2"

test/e2e/resources/fargateprofile_default.yaml

@@ -0,0 +1,13 @@
+apiVersion: eks.services.k8s.aws/v1alpha1
+kind: FargateProfile
+metadata:
+  name: $PROFILE_NAME
+spec:
+  name: $PROFILE_NAME
+  clusterName: $CLUSTER_NAME
+  podExecutionRoleARN: $FARGATE_POD_ROLE
+  subnets:
+    - "$PRIVATE_SUBNET_1"
+    - "$PRIVATE_SUBNET_2"
+  selectors:
+    - namespace: default

test/e2e/service_bootstrap.py

@@ -12,25 +12,33 @@
 # permissions and limitations under the License.
 """Bootstraps the resources required to run the EKS integration tests.
 """
-
-import boto3
 import logging
-import time
 
-from acktest import resources
-from acktest.aws.identity import get_region
+from acktest.bootstrapping import Resources, BootstrapFailureException
+from acktest.bootstrapping.iam import Role
+from acktest.bootstrapping.vpc import VPC
 from e2e import bootstrap_directory
 from e2e.bootstrap_resources import (
     TestBootstrapResources,
 )
 
-def service_bootstrap() -> dict:
+def service_bootstrap() -> Resources:
     logging.getLogger().setLevel(logging.INFO)
+
+    resources = TestBootstrapResources(
+        ClusterRole=Role("cluster-role", "eks.amazonaws.com", ["arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"]),
+        FargatePodRole=Role("fargate-pod-role", "eks-fargate-pods.amazonaws.com", ["arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy"]),
+        ClusterVPC=VPC(name_prefix="cluster-vpc", num_public_subnet=2, num_private_subnet=2)
+    )
+
+    try:
+        resources.bootstrap()
+    except BootstrapFailureException as ex:
+        exit(254)
 
-    return TestBootstrapResources(
-    ).__dict__
+    return resources
 
 if __name__ == "__main__":
     config = service_bootstrap()
     # Write config to current directory by default
-    resources.write_bootstrap_config(config, bootstrap_directory)
+    config.serialize(bootstrap_directory)

test/e2e/service_cleanup.py

@@ -14,22 +14,17 @@
 """Cleans up the resources created by the bootstrapping process.
 """
 
-import boto3
 import logging
 
-from acktest import resources
-from acktest.aws.identity import get_region
+from acktest.bootstrapping import Resources
 
 from e2e import bootstrap_directory
-from e2e.bootstrap_resources import TestBootstrapResources
 
-def service_cleanup(config: dict):
+def service_cleanup():
     logging.getLogger().setLevel(logging.INFO)
 
-    resources = TestBootstrapResources(
-        **config
-    )
+    resources = Resources.deseralize(bootstrap_directory)
+    resources.cleanup()
 
 if __name__ == "__main__":   
-    bootstrap_config = resources.read_bootstrap_config(bootstrap_directory)
-    service_cleanup(bootstrap_config) 
+    service_cleanup() 

test/e2e/tests/test_cluster.py

@@ -0,0 +1,99 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+#	 http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+"""Integration tests for the EKS Cluster resource
+"""
+
+import boto3
+import datetime
+import logging
+import time
+from typing import Dict
+
+import pytest
+
+from acktest.k8s import resource as k8s
+from acktest.resources import random_suffix_name
+from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_eks_resource
+from e2e.common.types import CLUSTER_RESOURCE_PLURAL
+from e2e.replacement_values import REPLACEMENT_VALUES
+
+DELETE_WAIT_AFTER_SECONDS = 30
+
+def wait_for_cluster_active(eks_client, cluster_name):
+    waiter = eks_client.get_waiter('cluster_active')
+    waiter.wait(name=cluster_name)
+
+@pytest.fixture(scope="module")
+def eks_client():
+    return boto3.client('eks')
+
+@pytest.fixture
+def simple_cluster():
+    cluster_name = random_suffix_name("simple-cluster", 32)
+
+    replacements = REPLACEMENT_VALUES.copy()
+    replacements["CLUSTER_NAME"] = cluster_name
+
+    resource_data = load_eks_resource(
+        "cluster_simple",
+        additional_replacements=replacements,
+    )
+    logging.debug(resource_data)
+
+    # Create the k8s resource
+    ref = k8s.CustomResourceReference(
+        CRD_GROUP, CRD_VERSION, CLUSTER_RESOURCE_PLURAL,
+        cluster_name, namespace="default",
+    )
+    k8s.create_custom_resource(ref, resource_data)
+    cr = k8s.wait_resource_consumed_by_controller(ref)
+
+    assert cr is not None
+    assert k8s.get_resource_exists(ref)
+
+    yield (ref, cr)
+
+    # Try to delete, if doesn't already exist
+    try:
+        _, deleted = k8s.delete_custom_resource(ref, 3, 10)
+        assert deleted
+    except:
+        pass
+
+@service_marker
+@pytest.mark.canary
+class TestCluster:
+    def test_create_delete_cluster(self, eks_client, simple_cluster):
+        (ref, cr) = simple_cluster
+
+        cluster_name = cr["spec"]["name"]
+
+        try:
+            aws_res = eks_client.describe_cluster(name=cluster_name)
+            assert aws_res is not None
+        except eks_client.exceptions.ResourceNotFoundException:
+            pytest.fail(f"Could not find cluster '{cluster_name}' in EKS")
+
+        # Delete the k8s resource on teardown of the module
+        k8s.delete_custom_resource(ref)
+
+        time.sleep(DELETE_WAIT_AFTER_SECONDS)
+
+        # Cluster should no longer appear in EKS
+        try:
+            aws_res = eks_client.describe_cluster(name=cluster_name)
+            assert aws_res is not None
+            pytest.fail(f"Cluster '{cluster_name}' was not deleted from EKS")
+        except eks_client.exceptions.ResourceNotFoundException:
+            pass

test/e2e/tests/test_fargate_profile.py

@@ -0,0 +1,99 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+#	 http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+"""Integration tests for the EKS FargateProfile resource
+"""
+
+from dataclasses import replace
+import boto3
+import datetime
+import logging
+import time
+from typing import Dict
+
+import pytest
+
+from acktest.k8s import resource as k8s
+from acktest.resources import random_suffix_name
+from e2e import CRD_VERSION, service_marker, CRD_GROUP, load_eks_resource
+from e2e.common.types import CLUSTER_RESOURCE_PLURAL
+from e2e.replacement_values import REPLACEMENT_VALUES
+from e2e.bootstrap_resources import get_bootstrap_resources
+
+from .test_cluster import simple_cluster, wait_for_cluster_active
+
+RESOURCE_PLURAL = 'fargateprofiles'
+
+DELETE_WAIT_AFTER_SECONDS = 10
+
+def wait_for_profile_active(eks_client, cluster_name, profile_name):
+    waiter = eks_client.get_waiter('fargate_profile_active')
+    waiter.wait(clusterName=cluster_name, fargateProfileName=profile_name)
+
+def wait_for_profile_deleted(eks_client, cluster_name, profile_name):
+    waiter = eks_client.get_waiter('fargate_profile_deleted')
+    waiter.wait(clusterName=cluster_name, fargateProfileName=profile_name)
+
+@pytest.fixture(scope="module")
+def eks_client():
+    return boto3.client('eks')
+
+@service_marker
+@pytest.mark.canary
+class TestFargateProfile:
+    def test_create_delete_fargate_profile(self, simple_cluster, eks_client):
+        (ref, cr) = simple_cluster
+        cluster_name = cr["spec"]["name"]
+
+        wait_for_cluster_active(eks_client, cluster_name)
+
+        profile_name = random_suffix_name("profile", 32)
+
+        replacements = REPLACEMENT_VALUES.copy()
+        replacements["CLUSTER_NAME"] = cluster_name
+        replacements["PROFILE_NAME"] = profile_name
+
+        resource_data = load_eks_resource(
+            "fargateprofile_default",
+            additional_replacements=replacements,
+        )
+        logging.debug(resource_data)
+
+        # Create the k8s resource
+        ref = k8s.CustomResourceReference(
+            CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL,
+            profile_name, namespace="default",
+        )
+        k8s.create_custom_resource(ref, resource_data)
+        cr = k8s.wait_resource_consumed_by_controller(ref)
+
+        assert cr is not None
+        assert k8s.get_resource_exists(ref)
+
+        try:
+            aws_res = eks_client.describe_fargate_profile(
+                clusterName=cluster_name,
+                fargateProfileName=profile_name
+            )
+            assert aws_res is not None
+
+            assert aws_res["fargateProfile"]["selectors"][0]["namespace"] == "default"
+        except eks_client.exceptions.ResourceNotFoundException:
+            pytest.fail(f"Could not find fargate profile '{profile_name}' in EKS")
+
+        wait_for_profile_active(eks_client, cluster_name, profile_name)
+
+        _, deleted = k8s.delete_custom_resource(ref, 3, 10)
+        assert deleted
+
+        wait_for_profile_deleted(eks_client, cluster_name, profile_name)