diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index f9ba346915..3fea1e189d 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -604,6 +604,53 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat log.Debugf("Found ENI: %s, MAC %s, device %d", eniID, eniMAC, deviceNum) + // Get IMDS fields for the interface + macImdsFields, err := cache.imds.GetMACImdsFields(ctx, eniMAC) + if err != nil { + awsAPIErrInc("GetMACImdsFields", err) + return ENIMetadata{}, err + } + ipInfoAvailable := false + // Efa-only interfaces do not have any ipv4s or ipv6s associated with it. If we don't find any local-ipv4 or ipv6 info in imds we assume it to be efa-only interface and validate this later via ec2 call + for _, field := range macImdsFields { + if field == "local-ipv4s" { + imdsIPv4s, err := cache.imds.GetLocalIPv4s(ctx, eniMAC) + if err != nil { + awsAPIErrInc("GetLocalIPv4s", err) + return ENIMetadata{}, err + } + if len(imdsIPv4s) > 0 { + ipInfoAvailable = true + log.Debugf("Found IPv4 addresses associated with interface. This is not efa-only interface") + break + } + } + if field == "ipv6s" { + imdsIPv6s, err := cache.imds.GetIPv6s(ctx, eniMAC) + if err != nil { + awsAPIErrInc("GetIPv6s", err) + } else if len(imdsIPv6s) > 0 { + ipInfoAvailable = true + log.Debugf("Found IPv6 addresses associated with interface. This is not efa-only interface") + break + } + } + } + + if !ipInfoAvailable { + return ENIMetadata{ + ENIID: eniID, + MAC: eniMAC, + DeviceNumber: deviceNum, + SubnetIPv4CIDR: "", + IPv4Addresses: make([]*ec2.NetworkInterfacePrivateIpAddress, 0), + IPv4Prefixes: make([]*ec2.Ipv4PrefixSpecification, 0), + SubnetIPv6CIDR: "", + IPv6Addresses: make([]*ec2.NetworkInterfaceIpv6Address, 0), + IPv6Prefixes: make([]*ec2.Ipv6PrefixSpecification, 0), + }, nil + } + // Get IPv4 and IPv6 addresses assigned to interface cidr, err := cache.imds.GetSubnetIPv4CIDRBlock(ctx, eniMAC) if err != nil { @@ -1356,9 +1403,16 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult, if interfaceType == "trunk" { trunkENI = eniID } - if interfaceType == "efa" { + if interfaceType == "efa" || interfaceType == "efa-only" { efaENIs[eniID] = true } + if interfaceType != "efa-only" { + if len(eniMetadata.IPv4Addresses) == 0 { + log.Errorf("Missing IP addresses from IMDS. Non efa-only interface should have IP address associated with it %s", eniID) + outOfSyncErr := errors.New("DescribeAllENIs: No IPv4 address found") + return DescribeAllENIsResult{}, outOfSyncErr + } + } // Check IPv4 addresses logOutOfSyncState(eniID, eniMetadata.IPv4Addresses, ec2res.PrivateIpAddresses) tagMap[eniMetadata.ENIID] = convertSDKTagsToTags(ec2res.TagSet) diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index cf93040526..e923a82bcc 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -80,16 +80,19 @@ const ( eni2ID = "eni-12341234" metadataVPCIPv4CIDRs = "192.168.0.0/16 100.66.0.0/1" myNodeName = "testNodeName" + imdsMACFields = "security-group-ids subnet-id vpc-id vpc-ipv4-cidr-blocks device-number interface-id subnet-ipv4-cidr-block local-ipv4s ipv4-prefix ipv6-prefix" + imdsMACFieldsEfaOnly = "security-group-ids subnet-id vpc-id vpc-ipv4-cidr-blocks device-number interface-id subnet-ipv4-cidr-block ipv4-prefix ipv6-prefix" ) func testMetadata(overrides map[string]interface{}) FakeIMDS { data := map[string]interface{}{ - metadataAZ: az, - metadataLocalIP: localIP, - metadataInstanceID: instanceID, - metadataInstanceType: instanceType, - metadataMAC: primaryMAC, - metadataMACPath: primaryMAC, + metadataAZ: az, + metadataLocalIP: localIP, + metadataInstanceID: instanceID, + metadataInstanceType: instanceType, + metadataMAC: primaryMAC, + metadataMACPath: primaryMAC, + metadataMACPath + primaryMAC: imdsMACFields, metadataMACPath + primaryMAC + metadataDeviceNum: eni1Device, metadataMACPath + primaryMAC + metadataInterface: primaryeniID, metadataMACPath + primaryMAC + metadataSGs: sgs, @@ -109,12 +112,13 @@ func testMetadata(overrides map[string]interface{}) FakeIMDS { func testMetadataWithPrefixes(overrides map[string]interface{}) FakeIMDS { data := map[string]interface{}{ - metadataAZ: az, - metadataLocalIP: localIP, - metadataInstanceID: instanceID, - metadataInstanceType: instanceType, - metadataMAC: primaryMAC, - metadataMACPath: primaryMAC, + metadataAZ: az, + metadataLocalIP: localIP, + metadataInstanceID: instanceID, + metadataInstanceType: instanceType, + metadataMAC: primaryMAC, + metadataMACPath: primaryMAC, + metadataMACPath + primaryMAC: imdsMACFields, metadataMACPath + primaryMAC + metadataDeviceNum: eni1Device, metadataMACPath + primaryMAC + metadataInterface: primaryeniID, metadataMACPath + primaryMAC + metadataSGs: sgs, @@ -203,7 +207,8 @@ func TestInitWithEC2metadataErr(t *testing.T) { func TestGetAttachedENIs(t *testing.T) { mockMetadata := testMetadata(map[string]interface{}{ - metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath + eni2MAC: imdsMACFields, metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, metadataMACPath + eni2MAC + metadataInterface: eni2ID, metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, @@ -217,9 +222,26 @@ func TestGetAttachedENIs(t *testing.T) { } } +func TestGetAttachedENIsWithEfaOnly(t *testing.T) { + mockMetadata := testMetadata(map[string]interface{}{ + metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath + eni2MAC: imdsMACFieldsEfaOnly, + metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, + metadataMACPath + eni2MAC + metadataInterface: eni2ID, + metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, + }) + + cache := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}} + ens, err := cache.GetAttachedENIs() + if assert.NoError(t, err) { + assert.Equal(t, len(ens), 2) + } +} + func TestGetAttachedENIsWithPrefixes(t *testing.T) { mockMetadata := testMetadata(map[string]interface{}{ - metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath + eni2MAC: imdsMACFields, metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, metadataMACPath + eni2MAC + metadataInterface: eni2ID, metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, @@ -343,6 +365,7 @@ func TestDescribeAllENIs(t *testing.T) { Attachment: &ec2.NetworkInterfaceAttachment{ NetworkCardIndex: aws.Int64(0), }, + NetworkInterfaceId: aws.String(primaryeniID), }}, } @@ -357,7 +380,7 @@ func TestDescribeAllENIs(t *testing.T) { awsErr error expErr error }{ - {"Success DescribeENI", map[string]TagMap{"": {"foo": "foo-value"}}, 1, nil, nil}, + {"Success DescribeENI", map[string]TagMap{"eni-00000000": {"foo": "foo-value"}}, 1, nil, nil}, {"Not found error", nil, maxENIEC2APIRetries, awserr.New("InvalidNetworkInterfaceID.NotFound", "no 'eni-xxx'", nil), expectedError}, {"Not found, no message", nil, maxENIEC2APIRetries, awserr.New("InvalidNetworkInterfaceID.NotFound", "no message", nil), noMessageError}, {"Other error", nil, maxENIEC2APIRetries, err, err}, @@ -1006,7 +1029,8 @@ func TestEC2InstanceMetadataCache_waitForENIAndIPsAttached(t *testing.T) { } fmt.Println("eniips", eniIPs) mockMetadata := testMetadata(map[string]interface{}{ - metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath + eni2MAC: imdsMACFields, metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, metadataMACPath + eni2MAC + metadataInterface: eni2ID, metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, @@ -1101,7 +1125,8 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) { eniPrefixes = "" } mockMetadata := testMetadata(map[string]interface{}{ - metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath + eni2MAC: imdsMACFields, metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, metadataMACPath + eni2MAC + metadataInterface: eni2ID, metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, diff --git a/pkg/awsutils/imds.go b/pkg/awsutils/imds.go index 69c9343501..ab845eeb45 100644 --- a/pkg/awsutils/imds.go +++ b/pkg/awsutils/imds.go @@ -136,6 +136,24 @@ func (imds TypedIMDS) GetMACs(ctx context.Context) ([]string, error) { return list, err } +// GetMACImdsFields returns the imds fields present for a MAC +func (imds TypedIMDS) GetMACImdsFields(ctx context.Context, mac string) ([]string, error) { + key := fmt.Sprintf("network/interfaces/macs/%s", mac) + list, err := imds.getList(ctx, key) + if err != nil { + if imdsErr, ok := err.(*imdsRequestError); ok { + log.Warnf("%v", err) + return nil, imdsErr.err + } + return nil, err + } + // Remove trailing / + for i, item := range list { + list[i] = strings.TrimSuffix(item, "/") + } + return list, err +} + // GetInterfaceID returns the ID of the network interface. func (imds TypedIMDS) GetInterfaceID(ctx context.Context, mac string) (string, error) { key := fmt.Sprintf("network/interfaces/macs/%s/interface-id", mac)