identity: Add Entity-CIDR Selectability

- identity: introduce concept of identity scopes

  We already use the high-8 bits of the numeric identity space to indicate
  what "kind" of identity is represented. This PR makes this more
  explicit, and defines it as an identity "scope".

  We will be adding another identity scope: remote-node, in future
  commits.

  No behavior changed, only identifiers.

- identity: prepare for scoped allocators

  Since we would like to support multiple local identity scopes, it means
  that the local identity allocator needs to be scope-aware.

  This change restructures the code a bit to make it clear that local
  allocators have a specific scope.

  This does not change any behavior.

- identity / bpf: reserve 0x02 for the remote-node scope

  This sets aside the identity scope of 0x02 for remote nodes. This scope
  is currently unused, but will be in future commits.

  It also updates the datapath to treat this as a remote node.

Signed-off-by: Casey Callendrello <[email protected]>
Signed-off-by: Nathan Sweet <[email protected]>

bpf/lib/identity.h

@@ -11,6 +11,9 @@ static __always_inline bool identity_in_range(__u32 identity, __u32 range_start,
 	return range_start <= identity && identity <= range_end;
 }
 
+#define IDENTITY_SCOPE_MASK 0xFF000000
+#define IDENTITY_SCOPE_REMOTE_NODE 0x02000000
+
 static __always_inline bool identity_is_remote_node(__u32 identity)
 {
 	/* KUBE_APISERVER_NODE_ID is the reserved identity that corresponds to
@@ -23,12 +26,18 @@ static __always_inline bool identity_is_remote_node(__u32 identity)
 	 * identities. But for now, this is good enough to capture the notion
 	 * of 'remote nodes in the cluster' for routing decisions.
 	 *
+	 * Remote nodes may also have, instead, an identity allocated from the
+	 * remote node identity scope, which is identified by the top 8 bits
+	 * being 0x02.
+	 *
 	 * Note that kube-apiserver policy is handled entirely separately by
 	 * the standard policymap enforcement logic and has no relationship to
 	 * the identity as used here. If the apiserver is outside the cluster,
 	 * then the KUBE_APISERVER_NODE_ID case should not ever be hit.
 	 */
-	return identity == REMOTE_NODE_ID || identity == KUBE_APISERVER_NODE_ID;
+	return identity == REMOTE_NODE_ID ||
+		identity == KUBE_APISERVER_NODE_ID ||
+		(identity & IDENTITY_SCOPE_MASK) == IDENTITY_SCOPE_REMOTE_NODE;
 }
 
 static __always_inline bool identity_is_node(__u32 identity)

pkg/endpoint/endpoint_status_test.go

@@ -376,7 +376,7 @@ func (s *EndpointSuite) TestgetEndpointPolicyMapState(c *check.C) {
 			name: "World shadows CIDR ingress",
 			args: []args{
 				{uint32(identity.ReservedIdentityWorld), 0, 0, trafficdirection.Ingress},
-				{uint32(identity.LocalIdentityFlag), 0, 0, trafficdirection.Ingress},
+				{uint32(identity.IdentityScopeLocal), 0, 0, trafficdirection.Ingress},
 			},
 			ingressResult: []apiResult{
 				{"reserved:world", uint64(identity.ReservedIdentityWorld), 0, 0},
@@ -387,7 +387,7 @@ func (s *EndpointSuite) TestgetEndpointPolicyMapState(c *check.C) {
 			name: "World shadows CIDR egress",
 			args: []args{
 				{uint32(identity.ReservedIdentityWorld), 0, 0, trafficdirection.Egress},
-				{uint32(identity.LocalIdentityFlag), 0, 0, trafficdirection.Egress},
+				{uint32(identity.IdentityScopeLocal), 0, 0, trafficdirection.Egress},
 			},
 			ingressResult: nil,
 			egressResult: []apiResult{

pkg/hubble/parser/threefour/parser_test.go

@@ -515,8 +515,8 @@ func TestDecodeDropReason(t *testing.T) {
 func TestDecodeLocalIdentity(t *testing.T) {
 	tn := monitor.TraceNotifyV0{
 		Type:     byte(monitorAPI.MessageTypeTrace),
-		SrcLabel: 123 | identity.LocalIdentityFlag,
-		DstLabel: 456 | identity.LocalIdentityFlag,
+		SrcLabel: 123 | identity.IdentityScopeLocal,
+		DstLabel: 456 | identity.IdentityScopeLocal,
 	}
 	data, err := testutils.CreateL3L4Payload(tn)
 	require.NoError(t, err)

pkg/identity/cache/allocator.go

@@ -253,7 +253,7 @@ func NewCachingIdentityAllocator(owner IdentityAllocatorOwner) *CachingIdentityA
 
 	// Local identity cache can be created synchronously since it doesn't
 	// rely upon any external resources (e.g., external kvstore).
-	m.localIdentities = newLocalIdentityCache(identity.MinAllocatorLocalIdentity, identity.MaxAllocatorLocalIdentity, m.events)
+	m.localIdentities = newLocalIdentityCache(identity.IdentityScopeLocal, identity.MinAllocatorLocalIdentity, identity.MaxAllocatorLocalIdentity, m.events)
 
 	return m
 }
@@ -349,7 +349,10 @@ func (m *CachingIdentityAllocator) AllocateIdentity(ctx context.Context, lbls la
 		return reservedIdentity, false, nil
 	}
 
-	if !identity.RequiresGlobalIdentity(lbls) {
+	// If the set of labels uses non-global scope,
+	// then allocate with the appropriate local allocator and return.
+	switch identity.ScopeForLabels(lbls) {
+	case identity.IdentityScopeLocal:
 		return m.localIdentities.lookupOrCreate(lbls, oldNID)
 	}
 
@@ -411,7 +414,8 @@ func (m *CachingIdentityAllocator) Release(ctx context.Context, id *identity.Ide
 		return false, nil
 	}
 
-	if !identity.RequiresGlobalIdentity(id.Labels) {
+	switch identity.ScopeForLabels(id.Labels) {
+	case identity.IdentityScopeLocal:
 		return m.localIdentities.release(id), nil
 	}
 

pkg/identity/cache/cache.go

@@ -208,7 +208,8 @@ func (m *CachingIdentityAllocator) LookupIdentity(ctx context.Context, lbls labe
 		return reservedIdentity
 	}
 
-	if !identity.RequiresGlobalIdentity(lbls) {
+	switch identity.ScopeForLabels(lbls) {
+	case identity.IdentityScopeLocal:
 		return m.localIdentities.lookup(lbls)
 	}
 
@@ -247,7 +248,8 @@ func (m *CachingIdentityAllocator) LookupIdentityByID(ctx context.Context, id id
 		return identity
 	}
 
-	if id.HasLocalScope() {
+	switch id.Scope() {
+	case identity.IdentityScopeLocal:
 		return m.localIdentities.lookupByID(id)
 	}
 

pkg/identity/cache/local.go

@@ -20,16 +20,18 @@ type localIdentityCache struct {
 	identitiesByID      map[identity.NumericIdentity]*identity.Identity
 	identitiesByLabels  map[string]*identity.Identity
 	nextNumericIdentity identity.NumericIdentity
+	scope               identity.NumericIdentity
 	minID               identity.NumericIdentity
 	maxID               identity.NumericIdentity
 	events              allocator.AllocatorEventSendChan
 }
 
-func newLocalIdentityCache(minID, maxID identity.NumericIdentity, events allocator.AllocatorEventSendChan) *localIdentityCache {
+func newLocalIdentityCache(scope, minID, maxID identity.NumericIdentity, events allocator.AllocatorEventSendChan) *localIdentityCache {
 	return &localIdentityCache{
 		identitiesByID:      map[identity.NumericIdentity]*identity.Identity{},
 		identitiesByLabels:  map[string]*identity.Identity{},
 		nextNumericIdentity: minID,
+		scope:               scope,
 		minID:               minID,
 		maxID:               maxID,
 		events:              events,
@@ -50,16 +52,16 @@ func (l *localIdentityCache) bumpNextNumericIdentity() {
 // The l.mutex must be held
 func (l *localIdentityCache) getNextFreeNumericIdentity(idCandidate identity.NumericIdentity) (identity.NumericIdentity, error) {
 	// Try first with the given candidate
-	if idCandidate.HasLocalScope() {
+	if idCandidate.Scope() == l.scope {
 		if _, taken := l.identitiesByID[idCandidate]; !taken {
 			// let nextNumericIdentity be, allocated identities will be skipped anyway
-			log.Debugf("Reallocated restored CIDR identity: %d", idCandidate)
+			log.Debugf("Reallocated restored local identity: %d", idCandidate)
 			return idCandidate, nil
 		}
 	}
 	firstID := l.nextNumericIdentity
 	for {
-		idCandidate = l.nextNumericIdentity | identity.LocalIdentityFlag
+		idCandidate = l.nextNumericIdentity | l.scope
 		if _, taken := l.identitiesByID[idCandidate]; !taken {
 			l.bumpNextNumericIdentity()
 			return idCandidate, nil

pkg/identity/cache/local_test.go

@@ -15,7 +15,8 @@ import (
 
 func (s *IdentityCacheTestSuite) TestBumpNextNumericIdentity(c *C) {
 	minID, maxID := identity.NumericIdentity(1), identity.NumericIdentity(5)
-	cache := newLocalIdentityCache(minID, maxID, nil)
+	scope := identity.NumericIdentity(0x42_00_00_00)
+	cache := newLocalIdentityCache(scope, minID, maxID, nil)
 
 	for i := minID; i <= maxID; i++ {
 		c.Assert(cache.nextNumericIdentity, Equals, i)
@@ -28,7 +29,8 @@ func (s *IdentityCacheTestSuite) TestBumpNextNumericIdentity(c *C) {
 
 func (s *IdentityCacheTestSuite) TestLocalIdentityCache(c *C) {
 	minID, maxID := identity.NumericIdentity(1), identity.NumericIdentity(5)
-	cache := newLocalIdentityCache(minID, maxID, nil)
+	scope := identity.NumericIdentity(0x42_00_00_00)
+	cache := newLocalIdentityCache(scope, minID, maxID, nil)
 
 	identities := map[identity.NumericIdentity]*identity.Identity{}
 
@@ -38,6 +40,7 @@ func (s *IdentityCacheTestSuite) TestLocalIdentityCache(c *C) {
 		id, isNew, err := cache.lookupOrCreate(labels.NewLabelsFromModel([]string{fmt.Sprintf("%d", i)}), identity.InvalidIdentity)
 		c.Assert(err, IsNil)
 		c.Assert(isNew, Equals, true)
+		c.Assert(id.ID, Equals, scope+i)
 		identities[id.ID] = id
 	}
 
@@ -64,7 +67,7 @@ func (s *IdentityCacheTestSuite) TestLocalIdentityCache(c *C) {
 	// lookup must still be successful
 	for i := minID; i <= maxID; i++ {
 		c.Assert(cache.lookup(labels.NewLabelsFromModel([]string{fmt.Sprintf("%d", i)})), Not(IsNil))
-		c.Assert(cache.lookupByID(i|identity.LocalIdentityFlag), Not(IsNil))
+		c.Assert(cache.lookupByID(i|scope), Not(IsNil))
 	}
 
 	// release the identities a second time, this must cause the identity
@@ -82,7 +85,7 @@ func (s *IdentityCacheTestSuite) TestLocalIdentityCache(c *C) {
 	}
 
 	// release a random identity in the middle
-	randomID := identity.NumericIdentity(3) | identity.LocalIdentityFlag
+	randomID := identity.NumericIdentity(3) | scope
 	c.Assert(cache.release(identities[randomID]), Equals, true)
 
 	id, isNew, err := cache.lookupOrCreate(labels.NewLabelsFromModel([]string{"foo"}), identity.InvalidIdentity)

pkg/identity/identity.go

@@ -180,18 +180,26 @@ func (pair *IPIdentityPair) PrefixString() string {
 // RequiresGlobalIdentity returns true if the label combination requires a
 // global identity
 func RequiresGlobalIdentity(lbls labels.Labels) bool {
-	needsGlobal := true
+	return ScopeForLabels(lbls) == IdentityScopeGlobal
+}
+
+// ScopeForLabels returns the identity scope to be used for the label set.
+// If all labels are either CIDR or reserved, then returns the CIDR scope.
+// Note: This assumes the caller has already called LookupReservedIdentityByLabels;
+// it does not handle that case.
+func ScopeForLabels(lbls labels.Labels) NumericIdentity {
+	scope := IdentityScopeGlobal
 
 	for _, label := range lbls {
 		switch label.Source {
 		case labels.LabelSourceCIDR, labels.LabelSourceReserved:
-			needsGlobal = false
+			scope = IdentityScopeLocal
 		default:
-			return true
+			return IdentityScopeGlobal
 		}
 	}
 
-	return needsGlobal
+	return scope
 }
 
 // AddUserDefinedNumericIdentitySet adds all key-value pairs from the given map

pkg/identity/identity_test.go

@@ -81,6 +81,64 @@ func (s *IdentityTestSuite) TestRequiresGlobalIdentity(c *C) {
 	c.Assert(RequiresGlobalIdentity(labels.NewLabelsFromModel([]string{"k8s:foo=bar"})), Equals, true)
 }
 
+func (s *IdentityTestSuite) TestScopeForLabels(c *C) {
+	tests := []struct {
+		lbls  labels.Labels
+		scope NumericIdentity
+	}{
+		{
+			lbls:  cidr.GetCIDRLabels(netip.MustParsePrefix("0.0.0.0/0")),
+			scope: IdentityScopeLocal,
+		},
+		{
+			lbls:  cidr.GetCIDRLabels(netip.MustParsePrefix("192.168.23.0/24")),
+			scope: IdentityScopeLocal,
+		},
+		{
+			lbls:  labels.NewLabelsFromModel([]string{"k8s:foo=bar"}),
+			scope: IdentityScopeGlobal,
+		},
+		{
+			lbls:  labels.NewLabelsFromModel([]string{"reserved:world"}),
+			scope: IdentityScopeGlobal,
+		},
+		{
+			lbls:  labels.NewLabelsFromModel([]string{"reserved:unmanaged"}),
+			scope: IdentityScopeGlobal,
+		},
+		{
+			lbls:  labels.NewLabelsFromModel([]string{"reserved:health"}),
+			scope: IdentityScopeGlobal,
+		},
+		{
+			lbls:  labels.NewLabelsFromModel([]string{"reserved:init"}),
+			scope: IdentityScopeGlobal,
+		},
+		{
+			lbls:  labels.NewLabelsFromModel([]string{"reserved:ingress"}),
+			scope: IdentityScopeGlobal,
+		},
+		{
+			lbls:  labels.NewLabelsFromModel([]string{"reserved:remote-node"}),
+			scope: IdentityScopeGlobal,
+		},
+		{
+			lbls:  labels.NewLabelsFromModel([]string{"reserved:remote-node", "reserved:kube-apiserver"}),
+			scope: IdentityScopeGlobal,
+		},
+	}
+
+	for i, test := range tests {
+		// ScopeForLabels requires this to return nil
+		id := LookupReservedIdentityByLabels(test.lbls)
+		if id != nil {
+			continue
+		}
+		scope := ScopeForLabels(test.lbls)
+		c.Assert(scope, Equals, test.scope, Commentf("%d / labels %s", i, test.lbls.String()))
+	}
+}
+
 func (s *IdentityTestSuite) TestNewIdentityFromLabelArray(c *C) {
 	id := NewIdentityFromLabelArray(NumericIdentity(1001),
 		labels.NewLabelArrayFromSortedList("unspec:a=;unspec:b;unspec:c=d"))

pkg/identity/numericidentity.go

@@ -23,9 +23,28 @@ const (
 	// shifted
 	ClusterIDShift = 16
 
-	// LocalIdentityFlag is the bit in the numeric identity that identifies
-	// a numeric identity to have local scope
-	LocalIdentityFlag = NumericIdentity(1 << 24)
+	// Identities also have scopes, which is defined by the high 8 bits.
+	// 0x00 -- Global and reserved identities. Reserved identities are
+	//         not allocated like global identities, but are known
+	//         because they are hardcoded in Cilium. Older versions of
+	//         Cilium will not be aware of any "new" reserved identities
+	//         that are added.
+	// 0x01 -- local (CIDR) identities
+	// 0x02 -- remote nodes
+
+	// IdentityScopeMask is the top 8 bits of the 32 bit identity
+	IdentityScopeMask = NumericIdentity(0xFF_00_00_00)
+
+	// IdentityScopeGlobal is the identity scope used by global and reserved identities.
+	IdentityScopeGlobal = NumericIdentity(0)
+
+	// IdentityScopeLocal is the tag in the numeric identity that identifies
+	// a numeric identity to have local (CIDR) scope.
+	IdentityScopeLocal = NumericIdentity(1 << 24)
+
+	// IdentityScopeRemoteNode is the tag in the numeric identity that identifies
+	// an identity to be a remote in-cluster node.
+	IdentityScopeRemoteNode = NumericIdentity(2 << 24)
 
 	// MinAllocatorLocalIdentity represents the minimal numeric identity
 	// that the localIdentityCache allocator can allocate for a local (CIDR)
@@ -38,7 +57,7 @@ const (
 
 	// MinLocalIdentity represents the actual minimal numeric identity value
 	// for a local (CIDR) identity.
-	MinLocalIdentity = MinAllocatorLocalIdentity | LocalIdentityFlag
+	MinLocalIdentity = MinAllocatorLocalIdentity | IdentityScopeLocal
 
 	// MaxAllocatorLocalIdentity represents the maximal numeric identity
 	// that the localIdentityCache allocator can allocate for a local (CIDR)
@@ -51,7 +70,7 @@ const (
 
 	// MaxLocalIdentity represents the actual maximal numeric identity value
 	// for a local (CIDR) identity.
-	MaxLocalIdentity = MaxAllocatorLocalIdentity | LocalIdentityFlag
+	MaxLocalIdentity = MaxAllocatorLocalIdentity | IdentityScopeLocal
 
 	// MinimalNumericIdentity represents the minimal numeric identity not
 	// used for reserved purposes.
@@ -583,9 +602,18 @@ func iterateReservedIdentityLabels(f func(_ NumericIdentity, _ labels.Labels)) {
 	}
 }
 
-// HasLocalScope returns true if the identity has a local scope
+// HasLocalScope returns true if the identity is in the Local (CIDR) scope
 func (id NumericIdentity) HasLocalScope() bool {
-	return (id & LocalIdentityFlag) != 0
+	return id.Scope() == IdentityScopeLocal
+}
+
+func (id NumericIdentity) HasRemoteNodeScope() bool {
+	return id.Scope() == IdentityScopeRemoteNode
+}
+
+// Scope returns the identity scope of this given numeric ID.
+func (id NumericIdentity) Scope() NumericIdentity {
+	return id & IdentityScopeMask
 }
 
 // IsWorld returns true if the identity is one of the world identities

pkg/identity/numericidentity_test.go

@@ -14,7 +14,7 @@ import (
 )
 
 func (s *IdentityTestSuite) TestLocalIdentity(c *C) {
-	localID := NumericIdentity(LocalIdentityFlag | 1)
+	localID := NumericIdentity(IdentityScopeLocal | 1)
 	c.Assert(localID.HasLocalScope(), Equals, true)
 
 	maxClusterID := NumericIdentity(types.ClusterIDMax | 1)

pkg/ipcache/metadata_test.go

@@ -346,7 +346,7 @@ func TestRemoveLabelsFromIPs(t *testing.T) {
 	assert.Len(t, remaining, 0)
 	id := IPIdentityCache.IdentityAllocator.LookupIdentityByID(
 		context.TODO(),
-		identity.LocalIdentityFlag, // we assume first local ID
+		identity.IdentityScopeLocal, // we assume first local ID
 	)
 	assert.NotNil(t, id)
 	assert.Equal(t, 1, id.ReferenceCount)

pkg/policy/distillery_test.go

@@ -42,7 +42,7 @@ var (
 )
 
 func localIdentity(n uint32) identity.NumericIdentity {
-	return identity.NumericIdentity(n) | identity.LocalIdentityFlag
+	return identity.NumericIdentity(n) | identity.IdentityScopeLocal
 
 }
 func (s *DistilleryTestSuite) TestCacheManagement(c *C) {

pkg/testutils/identity/allocator.go

@@ -43,7 +43,7 @@ func NewMockIdentityAllocator(c cache.IdentityCache) *MockIdentityAllocator {
 		IdentityCache: c,
 
 		currentID:        1000,
-		localID:          int(identity.LocalIdentityFlag),
+		localID:          int(identity.IdentityScopeLocal),
 		ipToIdentity:     make(map[string]int),
 		idToIdentity:     make(map[int]*identity.Identity),
 		labelsToIdentity: make(map[string]int),