diff --git a/cmd/bastion/aws/create.go b/cmd/bastion/aws/create.go index f5abf0965f39..8f3deb23fab1 100644 --- a/cmd/bastion/aws/create.go +++ b/cmd/bastion/aws/create.go @@ -13,6 +13,7 @@ import ( awsutil "github.com/openshift/hypershift/cmd/infra/aws/util" "github.com/openshift/hypershift/cmd/log" "github.com/openshift/hypershift/cmd/util" + supportawsutil "github.com/openshift/hypershift/support/awsutil" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" @@ -157,18 +158,18 @@ func (o *CreateBastionOpts) Run(ctx context.Context, logger logr.Logger) (string }) // Ensure security group exists - sgID, err := ensureBastionSecurityGroup(ctx, logger, ec2Client, infraID) + sgID, err := ensureBastionSecurityGroup(ctx, logger, ec2Client, infraID, o.Name) if err != nil { return "", "", fmt.Errorf("failed to ensure security group for bastion: %w", err) } // Ensure keypair exists - if err := ensureBastionKeyPair(ctx, logger, ec2Client, infraID, sshPublicKey); err != nil { + if err := ensureBastionKeyPair(ctx, logger, ec2Client, infraID, o.Name, sshPublicKey); err != nil { return "", "", fmt.Errorf("failed to ensure bastion keypair: %w", err) } // Create ec2 instance - instanceID, err := runEC2BastionInstance(ctx, logger, ec2Client, sgID, infraID) + instanceID, err := runEC2BastionInstance(ctx, logger, ec2Client, sgID, infraID, o.Name) if err != nil { return "", "", fmt.Errorf("failed to run bastion machine instance: %w", err) } @@ -185,7 +186,7 @@ func (o *CreateBastionOpts) Run(ctx context.Context, logger logr.Logger) (string return instanceID, publicIP, nil } -func ensureBastionSecurityGroup(ctx context.Context, logger logr.Logger, ec2Client *ec2.Client, infraID string) (string, error) { +func ensureBastionSecurityGroup(ctx context.Context, logger logr.Logger, ec2Client *ec2.Client, infraID, clusterName string) (string, error) { // find VPC vpcID, err := existingVPC(ctx, ec2Client, infraID) if err != nil { @@ -218,6 +219,14 @@ func ensureBastionSecurityGroup(ctx context.Context, logger logr.Logger, ec2Clie Key: aws.String("Name"), Value: aws.String(name), }, + { + Key: aws.String(supportawsutil.HypershiftInfraIDTagKey), + Value: aws.String(infraID), + }, + { + Key: aws.String(supportawsutil.HypershiftClusterNameTagKey), + Value: aws.String(clusterName), + }, }, }, }, @@ -340,7 +349,7 @@ func existingVPC(ctx context.Context, ec2Client *ec2.Client, infraID string) (st return vpcID, nil } -func ensureBastionKeyPair(ctx context.Context, logger logr.Logger, ec2Client *ec2.Client, infraID string, publicKey []byte) error { +func ensureBastionKeyPair(ctx context.Context, logger logr.Logger, ec2Client *ec2.Client, infraID, clusterName string, publicKey []byte) error { keyPairID, err := existingKeyPair(ctx, ec2Client, infraID) if err != nil { return fmt.Errorf("failed to check for existing keypair: %w", err) @@ -366,6 +375,14 @@ func ensureBastionKeyPair(ctx context.Context, logger logr.Logger, ec2Client *ec Key: aws.String("Name"), Value: aws.String(keyPairName(infraID)), }, + { + Key: aws.String(supportawsutil.HypershiftInfraIDTagKey), + Value: aws.String(infraID), + }, + { + Key: aws.String(supportawsutil.HypershiftClusterNameTagKey), + Value: aws.String(clusterName), + }, }, }, }, @@ -437,7 +454,7 @@ func getLatestAmazonLinux2AMI(ctx context.Context, ec2Client *ec2.Client) (strin return aws.ToString(latestAMI.ImageId), nil } -func runEC2BastionInstance(ctx context.Context, logger logr.Logger, ec2Client *ec2.Client, sgID, infraID string) (string, error) { +func runEC2BastionInstance(ctx context.Context, logger logr.Logger, ec2Client *ec2.Client, sgID, infraID, clusterName string) (string, error) { // find existing instance instanceID, err := existingInstance(ctx, ec2Client, infraID) if err != nil { @@ -493,6 +510,14 @@ func runEC2BastionInstance(ctx context.Context, logger logr.Logger, ec2Client *e Key: aws.String("Name"), Value: aws.String(instanceName(infraID)), }, + { + Key: aws.String(supportawsutil.HypershiftInfraIDTagKey), + Value: aws.String(infraID), + }, + { + Key: aws.String(supportawsutil.HypershiftClusterNameTagKey), + Value: aws.String(clusterName), + }, }, }, }, diff --git a/cmd/cluster/aws/create.go b/cmd/cluster/aws/create.go index 2eb4910995e8..19946c86829a 100644 --- a/cmd/cluster/aws/create.go +++ b/cmd/cluster/aws/create.go @@ -14,6 +14,7 @@ import ( awsinfra "github.com/openshift/hypershift/cmd/infra/aws" awsutil "github.com/openshift/hypershift/cmd/infra/aws/util" "github.com/openshift/hypershift/cmd/util" + supportawsutil "github.com/openshift/hypershift/support/awsutil" configv1 "github.com/openshift/api/config/v1" @@ -555,7 +556,7 @@ func CreateInfraOptions(awsOpts *ValidatedCreateOptions, opts *core.CreateOption BaseDomain: opts.BaseDomain, BaseDomainPrefix: opts.BaseDomainPrefix, RedactBaseDomain: opts.RedactBaseDomain, - AdditionalTags: awsOpts.AdditionalTags, + AdditionalTags: append(awsOpts.AdditionalTags, supportawsutil.HypershiftSourceTagKey+"=cli"), Zones: awsOpts.Zones, EnableProxy: awsOpts.EnableProxy, EnableSecureProxy: awsOpts.EnableSecureProxy, @@ -574,7 +575,7 @@ func CreateIAMOptions(awsOpts *ValidatedCreateOptions, infra *awsinfra.CreateInf AWSCredentialsOpts: awsOpts.Credentials, InfraID: infra.InfraID, IssuerURL: awsOpts.IssuerURL, - AdditionalTags: awsOpts.AdditionalTags, + AdditionalTags: append(awsOpts.AdditionalTags, supportawsutil.HypershiftSourceTagKey+"=cli", supportawsutil.HypershiftClusterNameTagKey+"="+infra.Name), PrivateZoneID: infra.PrivateZoneID, PublicZoneID: infra.PublicZoneID, LocalZoneID: infra.LocalZoneID, diff --git a/cmd/fix/dr_oidc_iam.go b/cmd/fix/dr_oidc_iam.go index 381a3b8905e6..6fd9c603d8c4 100644 --- a/cmd/fix/dr_oidc_iam.go +++ b/cmd/fix/dr_oidc_iam.go @@ -7,6 +7,7 @@ import ( stderrors "errors" "fmt" "net/http" + "net/url" "os" "strings" "time" @@ -14,11 +15,13 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" awsutil "github.com/openshift/hypershift/cmd/infra/aws/util" "github.com/openshift/hypershift/hypershift-operator/controllers/manifests" + supportawsutil "github.com/openshift/hypershift/support/awsutil" "github.com/openshift/hypershift/support/infraid" "github.com/openshift/hypershift/support/oidc" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" + iamtypes "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" @@ -756,6 +759,19 @@ func (o *DrOidcIamOptions) ensureOIDCBucket(ctx context.Context, s3Client *s3.Cl return fmt.Errorf("failed to create S3 bucket: %w", err) } + _, err = s3Client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{ + Bucket: aws.String(o.OIDCStorageProviderS3Bucket), + Tagging: &s3types.Tagging{ + TagSet: []s3types.Tag{ + {Key: aws.String(supportawsutil.HypershiftInfraIDTagKey), Value: aws.String(o.InfraID)}, + {Key: aws.String(supportawsutil.HypershiftClusterNameTagKey), Value: aws.String(o.HostedClusterName)}, + }, + }, + }) + if err != nil { + return fmt.Errorf("failed to tag S3 bucket: %w", err) + } + return configureBucketPublicAccess(ctx, s3Client, o.OIDCStorageProviderS3Bucket) } @@ -798,6 +814,7 @@ func (o *DrOidcIamOptions) generateAndUploadOIDCDocuments(ctx context.Context, k Key: aws.String(o.InfraID + path), Body: bodyReader, ContentType: aws.String("application/json"), + Tagging: aws.String(supportawsutil.HypershiftInfraIDTagKey + "=" + url.QueryEscape(o.InfraID) + "&" + supportawsutil.HypershiftClusterNameTagKey + "=" + url.QueryEscape(o.HostedClusterName)), }) if err != nil { return fmt.Errorf("failed to upload OIDC document %s: %w", path, err) @@ -890,6 +907,10 @@ func (o *DrOidcIamOptions) createOIDCProvider(ctx context.Context, iamClient *ia ThumbprintList: []string{ thumbprint, }, + Tags: []iamtypes.Tag{ + {Key: aws.String(supportawsutil.HypershiftInfraIDTagKey), Value: aws.String(o.InfraID)}, + {Key: aws.String(supportawsutil.HypershiftClusterNameTagKey), Value: aws.String(o.HostedClusterName)}, + }, Url: aws.String(o.Issuer), } diff --git a/cmd/infra/aws/create.go b/cmd/infra/aws/create.go index 1c2e35eaca73..88f6a863e70d 100644 --- a/cmd/infra/aws/create.go +++ b/cmd/infra/aws/create.go @@ -14,6 +14,7 @@ import ( "github.com/openshift/hypershift/cmd/log" "github.com/openshift/hypershift/cmd/util" "github.com/openshift/hypershift/support/awsapi" + supportawsutil "github.com/openshift/hypershift/support/awsutil" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" @@ -138,6 +139,7 @@ func NewCreateCommand() *cobra.Command { if err = opts.Validate(); err != nil { return err } + opts.AdditionalTags = append(opts.AdditionalTags, supportawsutil.HypershiftSourceTagKey+"=cli") if err := opts.Run(cmd.Context(), l); err != nil { l.Error(err, "Failed to create infrastructure") return err @@ -675,6 +677,14 @@ func (o *CreateInfraOptions) shareSubnets(ctx context.Context, l logr.Logger, vp Key: aws.String(clusterTag(o.InfraID)), Value: aws.String(clusterTagValue), }, + { + Key: aws.String(supportawsutil.HypershiftInfraIDTagKey), + Value: aws.String(o.InfraID), + }, + { + Key: aws.String(supportawsutil.HypershiftClusterNameTagKey), + Value: aws.String(o.Name), + }, }, }); err != nil { return err diff --git a/cmd/infra/aws/create_cli_role.go b/cmd/infra/aws/create_cli_role.go index 22a7d73b0072..2b4146a6a051 100644 --- a/cmd/infra/aws/create_cli_role.go +++ b/cmd/infra/aws/create_cli_role.go @@ -131,6 +131,7 @@ const ( "route53:CreateHostedZone", "route53:ListHostedZones", "route53:ChangeResourceRecordSets", + "route53:ChangeTagsForResource", "route53:ListResourceRecordSets", "route53:DeleteHostedZone", "route53:AssociateVPCWithHostedZone", diff --git a/cmd/infra/aws/create_iam.go b/cmd/infra/aws/create_iam.go index 87f5c88a5d6d..5e124ebd1ff7 100644 --- a/cmd/infra/aws/create_iam.go +++ b/cmd/infra/aws/create_iam.go @@ -11,6 +11,7 @@ import ( awsutil "github.com/openshift/hypershift/cmd/infra/aws/util" "github.com/openshift/hypershift/cmd/log" "github.com/openshift/hypershift/cmd/util" + supportawsutil "github.com/openshift/hypershift/support/awsutil" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" @@ -115,6 +116,7 @@ func NewCreateIAMCommand() *cobra.Command { logger.Error(err, "failed to create client") return err } + opts.AdditionalTags = append(opts.AdditionalTags, supportawsutil.HypershiftSourceTagKey+"=cli") if err := opts.Run(cmd.Context(), client, logger); err != nil { logger.Error(err, "Failed to create infrastructure") return err @@ -168,6 +170,12 @@ func (o *CreateIAMOptions) CreateIAM(ctx context.Context, client crclient.Client if err = o.ParseAdditionalTags(); err != nil { return nil, err } + if len(o.InfraID) > 0 { + o.additionalIAMTags = append(o.additionalIAMTags, iamtypes.Tag{ + Key: aws.String(supportawsutil.HypershiftInfraIDTagKey), + Value: aws.String(o.InfraID), + }) + } if o.OIDCStorageProviderS3BucketName == "" || o.OIDCStorageProviderS3Region == "" { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: "kube-public", Name: "oidc-storage-provider-s3-config"}, diff --git a/cmd/infra/aws/delegatingclientgenerator/main.go b/cmd/infra/aws/delegatingclientgenerator/main.go index d69caa8d64f7..8f9399ff27b4 100644 --- a/cmd/infra/aws/delegatingclientgenerator/main.go +++ b/cmd/infra/aws/delegatingclientgenerator/main.go @@ -396,6 +396,7 @@ var extendedAPIs = map[string][]string{ }, "route53": { "AssociateVPCWithHostedZone", + "ChangeTagsForResource", "CreateHostedZone", "CreateVPCAssociationAuthorization", "DeleteHostedZone", diff --git a/cmd/infra/aws/ec2.go b/cmd/infra/aws/ec2.go index 1c78d68421e7..cfcd47b1583e 100644 --- a/cmd/infra/aws/ec2.go +++ b/cmd/infra/aws/ec2.go @@ -9,6 +9,7 @@ import ( "github.com/openshift/hypershift/cmd/util" "github.com/openshift/hypershift/support/awsapi" + supportawsutil "github.com/openshift/hypershift/support/awsutil" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" @@ -372,7 +373,8 @@ func (o *CreateInfraOptions) CreateNATGateway(ctx context.Context, l logr.Logger } eipResult, err := client.AllocateAddress(ctx, &ec2.AllocateAddressInput{ - Domain: ec2types.DomainTypeVpc, + Domain: ec2types.DomainTypeVpc, + TagSpecifications: o.ec2TagSpecifications("elastic-ip", fmt.Sprintf("%s-eip-%s", o.InfraID, availabilityZone)), }) if err != nil { return "", fmt.Errorf("cannot allocate EIP for NAT gateway: %w", err) @@ -380,26 +382,6 @@ func (o *CreateInfraOptions) CreateNATGateway(ctx context.Context, l logr.Logger allocationID := aws.ToString(eipResult.AllocationId) l.Info("Created elastic IP for NAT gateway", "id", allocationID) - // NOTE: there's a potential to leak EIP addresses if the following tag operation fails, since we have no way of - // recognizing the EIP as belonging to the cluster - isRetriable := func(err error) bool { - var apiErr smithy.APIError - if errors.As(err, &apiErr) { - return strings.EqualFold(apiErr.ErrorCode(), invalidElasticIPNotFound) - } - return false - } - err = retry.OnError(retryBackoff, isRetriable, func() error { - _, err = client.CreateTags(ctx, &ec2.CreateTagsInput{ - Resources: []string{allocationID}, - Tags: append(ec2Tags(o.InfraID, fmt.Sprintf("%s-eip-%s", o.InfraID, availabilityZone)), o.additionalEC2Tags...), - }) - return err - }) - if err != nil { - return "", fmt.Errorf("cannot tag NAT gateway EIP: %w", err) - } - isNATGatewayRetriable := func(err error) bool { var apiErr smithy.APIError if errors.As(err, &apiErr) { @@ -641,7 +623,7 @@ func (o *CreateInfraOptions) ec2TagSpecifications(resourceType, name string) []e return []ec2types.TagSpecification{ { ResourceType: ec2types.ResourceType(resourceType), - Tags: append(ec2Tags(o.InfraID, name), o.additionalEC2Tags...), + Tags: append(ec2Tags(o.InfraID, o.Name, name), o.additionalEC2Tags...), }, } } @@ -680,13 +662,25 @@ func clusterTag(infraID string) string { return fmt.Sprintf("kubernetes.io/cluster/%s", infraID) } -func ec2Tags(infraID, name string) []ec2types.Tag { +func ec2Tags(infraID, clusterName, name string) []ec2types.Tag { tags := []ec2types.Tag{ { Key: aws.String(clusterTag(infraID)), Value: aws.String(clusterTagValue), }, } + if len(infraID) > 0 { + tags = append(tags, ec2types.Tag{ + Key: aws.String(supportawsutil.HypershiftInfraIDTagKey), + Value: aws.String(infraID), + }) + } + if len(clusterName) > 0 { + tags = append(tags, ec2types.Tag{ + Key: aws.String(supportawsutil.HypershiftClusterNameTagKey), + Value: aws.String(clusterName), + }) + } if len(name) > 0 { tags = append(tags, ec2types.Tag{ Key: aws.String("Name"), @@ -694,5 +688,4 @@ func ec2Tags(infraID, name string) []ec2types.Tag { }) } return tags - } diff --git a/cmd/infra/aws/route53.go b/cmd/infra/aws/route53.go index 048055579ca5..aa14944f5104 100644 --- a/cmd/infra/aws/route53.go +++ b/cmd/infra/aws/route53.go @@ -8,6 +8,7 @@ import ( "time" "github.com/openshift/hypershift/support/awsapi" + supportawsutil "github.com/openshift/hypershift/support/awsutil" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/route53" @@ -114,6 +115,22 @@ func (o *CreateInfraOptions) CreatePrivateZone(ctx context.Context, logger logr. id = cleanZoneID(aws.ToString(res.HostedZone.Id)) logger.Info("Created private zone", "name", name, "id", id) + r53Tags := []route53types.Tag{ + {Key: aws.String(clusterTag(o.InfraID)), Value: aws.String(clusterTagValue)}, + {Key: aws.String(supportawsutil.HypershiftInfraIDTagKey), Value: aws.String(o.InfraID)}, + {Key: aws.String(supportawsutil.HypershiftClusterNameTagKey), Value: aws.String(o.Name)}, + } + for _, t := range o.additionalEC2Tags { + r53Tags = append(r53Tags, route53types.Tag{Key: t.Key, Value: t.Value}) + } + if _, err := client.ChangeTagsForResource(ctx, &route53.ChangeTagsForResourceInput{ + ResourceId: aws.String(id), + ResourceType: route53types.TagResourceTypeHostedzone, + AddTags: r53Tags, + }); err != nil { + return "", fmt.Errorf("failed to tag hosted zone: %w", err) + } + err = setSOAMinimum(ctx, client, id, name) if err != nil { return "", err diff --git a/cmd/infra/aws/route53_test.go b/cmd/infra/aws/route53_test.go index 01c1913a7261..99e0d8e1e1b5 100644 --- a/cmd/infra/aws/route53_test.go +++ b/cmd/infra/aws/route53_test.go @@ -208,6 +208,8 @@ func TestCreatePrivateZone(t *testing.T) { Name: aws.String(testZoneName + "."), }, }, nil) + m.EXPECT().ChangeTagsForResource(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&route53.ChangeTagsForResourceOutput{}, nil) // setSOAMinimum: findRecord + update m.EXPECT().ListResourceRecordSets(gomock.Any(), gomock.Any(), gomock.Any()). Return(soaRecordFor(testZoneName), nil) @@ -233,6 +235,8 @@ func TestCreatePrivateZone(t *testing.T) { Name: aws.String(testZoneName + "."), }, }, nil) + m.EXPECT().ChangeTagsForResource(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&route53.ChangeTagsForResourceOutput{}, nil) m.EXPECT().ListResourceRecordSets(gomock.Any(), gomock.Any(), gomock.Any()). Return(soaRecordFor(testZoneName), nil) m.EXPECT().ChangeResourceRecordSets(gomock.Any(), gomock.Any(), gomock.Any()). diff --git a/support/awsapi/route53.go b/support/awsapi/route53.go index 9451e416501d..ca525c05ef26 100644 --- a/support/awsapi/route53.go +++ b/support/awsapi/route53.go @@ -20,6 +20,7 @@ import ( type ROUTE53API interface { AssociateVPCWithHostedZone(ctx context.Context, input *route53.AssociateVPCWithHostedZoneInput, optFns ...func(*route53.Options)) (*route53.AssociateVPCWithHostedZoneOutput, error) ChangeResourceRecordSets(ctx context.Context, input *route53.ChangeResourceRecordSetsInput, optFns ...func(*route53.Options)) (*route53.ChangeResourceRecordSetsOutput, error) + ChangeTagsForResource(ctx context.Context, input *route53.ChangeTagsForResourceInput, optFns ...func(*route53.Options)) (*route53.ChangeTagsForResourceOutput, error) CreateHostedZone(ctx context.Context, input *route53.CreateHostedZoneInput, optFns ...func(*route53.Options)) (*route53.CreateHostedZoneOutput, error) CreateVPCAssociationAuthorization(ctx context.Context, input *route53.CreateVPCAssociationAuthorizationInput, optFns ...func(*route53.Options)) (*route53.CreateVPCAssociationAuthorizationOutput, error) DeleteHostedZone(ctx context.Context, input *route53.DeleteHostedZoneInput, optFns ...func(*route53.Options)) (*route53.DeleteHostedZoneOutput, error) diff --git a/support/awsutil/tags.go b/support/awsutil/tags.go new file mode 100644 index 000000000000..546788a01915 --- /dev/null +++ b/support/awsutil/tags.go @@ -0,0 +1,8 @@ +package awsutil + +const ( + HypershiftClusterNameTagKey = "hypershift.openshift.io/cluster-name" + HypershiftInfraIDTagKey = "hypershift.openshift.io/infra-id" + HypershiftSourceTagKey = "hypershift.openshift.io/source" + HypershiftProwJobIDTagKey = "hypershift.openshift.io/prow-job-id" +) diff --git a/test/e2e/karpenter_test.go b/test/e2e/karpenter_test.go index 8906e7750466..e61617d6cded 100644 --- a/test/e2e/karpenter_test.go +++ b/test/e2e/karpenter_test.go @@ -772,6 +772,9 @@ func testCapacityReservation(ctx context.Context, mgtClient, guestClient crclien "t3.xlarge", targetAZ, 1, + hc.Spec.InfraID, + hc.Name, + e2eutil.E2ETagsFromEnvironment(), ) g.Expect(err).NotTo(HaveOccurred(), "failed to create capacity reservation") t.Logf("Created capacity reservation %s in %s", crID, targetAZ) @@ -955,7 +958,7 @@ func testArbitrarySubnet(ctx context.Context, mgtClient, guestClient crclient.Cl t.Logf("Selected AZ %s for test subnet (supported by endpoint service, not in VPC)", az) // Create a small test subnet in the VPC. - subnetID, cleanupSubnet := e2eutil.CreateTestSubnet(ctx, t, ec2client, vpcID, az, hc.Spec.InfraID) + subnetID, cleanupSubnet := e2eutil.CreateTestSubnet(ctx, t, ec2client, vpcID, az, hc.Spec.InfraID, hc.Name, e2eutil.E2ETagsFromEnvironment()) t.Logf("Created test subnet %s in AZ %s", subnetID, az) // Create an OpenshiftEC2NodeClass that selects the subnet by ID. diff --git a/test/e2e/nodepool_spot_termination_handler_test.go b/test/e2e/nodepool_spot_termination_handler_test.go index 2ade4368de71..e75fe3f8bfae 100644 --- a/test/e2e/nodepool_spot_termination_handler_test.go +++ b/test/e2e/nodepool_spot_termination_handler_test.go @@ -15,6 +15,7 @@ import ( "github.com/google/uuid" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/hypershift-operator/controllers/manifests" + supportawsutil "github.com/openshift/hypershift/support/awsutil" "github.com/openshift/hypershift/support/podspec" e2eutil "github.com/openshift/hypershift/test/e2e/util" appsv1 "k8s.io/api/apps/v1" @@ -146,8 +147,12 @@ func (s *SpotTerminationHandlerTest) Run(t *testing.T, nodePool hyperv1.NodePool sqsClient := e2eutil.GetSQSClient(s.ctx, s.clusterOpts.AWSPlatform.Credentials.AWSCredentialsFile, s.clusterOpts.AWSPlatform.Region) sqsQueueName := s.hostedCluster.Name + "-nth-queue" t.Logf("Creating SQS queue %s", sqsQueueName) + sqsTags := e2eutil.E2ETagsFromEnvironment() + sqsTags[supportawsutil.HypershiftInfraIDTagKey] = s.hostedCluster.Spec.InfraID + sqsTags[supportawsutil.HypershiftClusterNameTagKey] = s.hostedCluster.Name createQueueResult, err := sqsClient.CreateQueue(s.ctx, &sqs.CreateQueueInput{ QueueName: aws.String(sqsQueueName), + Tags: sqsTags, }) if err != nil { t.Fatalf("failed to create SQS queue %s: %v", sqsQueueName, err) diff --git a/test/e2e/util/aws.go b/test/e2e/util/aws.go index fef417b80161..2d24d2d73393 100644 --- a/test/e2e/util/aws.go +++ b/test/e2e/util/aws.go @@ -6,12 +6,14 @@ import ( "errors" "fmt" "net" + "os" "strings" "testing" "time" awsutil "github.com/openshift/hypershift/cmd/infra/aws/util" "github.com/openshift/hypershift/support/awsapi" + supportawsutil "github.com/openshift/hypershift/support/awsutil" "github.com/openshift/hypershift/support/oidc" "github.com/openshift/hypershift/support/util" @@ -31,6 +33,19 @@ import ( "github.com/go-logr/logr" ) +// E2ETagsFromEnvironment returns a tag map with e2e provenance tags. +// It reads PROW_JOB_ID from the environment once; callers pass the result +// to resource-creation helpers so they don't read os.Getenv themselves. +func E2ETagsFromEnvironment() map[string]string { + tags := map[string]string{ + supportawsutil.HypershiftSourceTagKey: "e2e", + } + if prowJobID := os.Getenv("PROW_JOB_ID"); prowJobID != "" { + tags[supportawsutil.HypershiftProwJobIDTagKey] = prowJobID + } + return tags +} + func GetKMSKeyArn(ctx context.Context, awsCreds, awsRegion, alias string) (*string, error) { if alias == "" { return awsv2.String(""), nil @@ -168,7 +183,7 @@ func DestroyOIDCProvider(ctx context.Context, log logr.Logger, iamClient awsapi. // associates it with an existing private route table (one with a NAT gateway route), // and returns the subnet ID plus a cleanup function that disassociates and deletes it. // The subnet CIDR is chosen dynamically to avoid overlapping with any existing subnets. -func CreateTestSubnet(ctx context.Context, t *testing.T, client *ec2v2.Client, vpcID, az, infraID string) (string, func()) { +func CreateTestSubnet(ctx context.Context, t *testing.T, client *ec2v2.Client, vpcID, az, infraID, clusterName string, additionalTags map[string]string) (string, func()) { t.Helper() // Fetch all existing subnets in the VPC to find a non-overlapping CIDR. @@ -200,6 +215,15 @@ func CreateTestSubnet(ctx context.Context, t *testing.T, client *ec2v2.Client, v } subnetName := fmt.Sprintf("%s-karpenter-test-subnet", infraID) + subnetTags := []ec2types.Tag{ + {Key: awsv2.String("Name"), Value: awsv2.String(subnetName)}, + {Key: awsv2.String(fmt.Sprintf("kubernetes.io/cluster/%s", infraID)), Value: awsv2.String("owned")}, + {Key: awsv2.String(supportawsutil.HypershiftInfraIDTagKey), Value: awsv2.String(infraID)}, + {Key: awsv2.String(supportawsutil.HypershiftClusterNameTagKey), Value: awsv2.String(clusterName)}, + } + for k, v := range additionalTags { + subnetTags = append(subnetTags, ec2types.Tag{Key: awsv2.String(k), Value: awsv2.String(v)}) + } createOut, err := client.CreateSubnet(ctx, &ec2v2.CreateSubnetInput{ VpcId: awsv2.String(vpcID), CidrBlock: awsv2.String(candidateCIDR), @@ -207,10 +231,7 @@ func CreateTestSubnet(ctx context.Context, t *testing.T, client *ec2v2.Client, v TagSpecifications: []ec2types.TagSpecification{ { ResourceType: ec2types.ResourceTypeSubnet, - Tags: []ec2types.Tag{ - {Key: awsv2.String("Name"), Value: awsv2.String(subnetName)}, - {Key: awsv2.String(fmt.Sprintf("kubernetes.io/cluster/%s", infraID)), Value: awsv2.String("owned")}, - }, + Tags: subnetTags, }, }, }) @@ -348,13 +369,20 @@ func CleanupOIDCBucketObjects(ctx context.Context, log logr.Logger, s3Client aws // CreateCapacityReservation creates an EC2 capacity reservation and returns its ID and a cleanup function // that cancels the reservation. The caller is responsible for calling the cleanup function. -func CreateCapacityReservation(ctx context.Context, awsCreds, awsRegion, instanceType, availabilityZone string, instanceCount int32) (string, func() error, error) { +func CreateCapacityReservation(ctx context.Context, awsCreds, awsRegion, instanceType, availabilityZone string, instanceCount int32, infraID, clusterName string, additionalTags map[string]string) (string, func() error, error) { awsSession := awsutil.NewSession(ctx, "e2e-capacity-reservation", awsCreds, "", "", awsRegion) awsConfig := awsutil.NewConfig() ec2Client := ec2v2.NewFromConfig(*awsSession, func(o *ec2v2.Options) { o.Retryer = awsConfig() }) + crTags := []ec2types.Tag{ + {Key: awsv2.String(supportawsutil.HypershiftInfraIDTagKey), Value: awsv2.String(infraID)}, + {Key: awsv2.String(supportawsutil.HypershiftClusterNameTagKey), Value: awsv2.String(clusterName)}, + } + for k, v := range additionalTags { + crTags = append(crTags, ec2types.Tag{Key: awsv2.String(k), Value: awsv2.String(v)}) + } result, err := ec2Client.CreateCapacityReservation(ctx, &ec2v2.CreateCapacityReservationInput{ InstanceType: awsv2.String(instanceType), InstancePlatform: ec2types.CapacityReservationInstancePlatformLinuxUnix, @@ -363,6 +391,12 @@ func CreateCapacityReservation(ctx context.Context, awsCreds, awsRegion, instanc InstanceMatchCriteria: ec2types.InstanceMatchCriteriaTargeted, EndDateType: ec2types.EndDateTypeLimited, EndDate: awsv2.Time(time.Now().Add(2 * time.Hour)), + TagSpecifications: []ec2types.TagSpecification{ + { + ResourceType: ec2types.ResourceTypeCapacityReservation, + Tags: crTags, + }, + }, }) if err != nil { return "", nil, fmt.Errorf("failed to create capacity reservation: %w", err) diff --git a/test/e2e/util/fixture.go b/test/e2e/util/fixture.go index e12aae01d1fb..dba166453493 100644 --- a/test/e2e/util/fixture.go +++ b/test/e2e/util/fixture.go @@ -108,7 +108,14 @@ func createCluster(ctx context.Context, hc *hyperv1.HostedCluster, opts *Platfor } validOpts := completer.(*aws.ValidatedCreateOptions) + e2eTags := E2ETagsFromEnvironment() + var e2eTagsList []string + for k, v := range e2eTags { + e2eTagsList = append(e2eTagsList, k+"="+v) + } + infraOpts := aws.CreateInfraOptions(validOpts, coreOpts) + infraOpts.AdditionalTags = append(infraOpts.AdditionalTags, e2eTagsList...) infraOpts.OutputFile = infraFile infra, err := infraOpts.CreateInfra(ctx, zapr.NewLogger(infraLogger)) if err != nil { @@ -123,6 +130,7 @@ func createCluster(ctx context.Context, hc *hyperv1.HostedCluster, opts *Platfor return err } iamOpts := aws.CreateIAMOptions(validOpts, infra) + iamOpts.AdditionalTags = append(iamOpts.AdditionalTags, e2eTagsList...) iamOpts.OutputFile = iamFile iam, err := iamOpts.CreateIAM(ctx, client, zapr.NewLogger(iamLogger)) if err != nil {