Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Encryption status output enriched with IPsec details #2454

Merged
merged 1 commit into from
Apr 10, 2024
Merged

Encryption status output enriched with IPsec details #2454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cli/encrypt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ func newCmdEncryptStatus() *cobra.Command {
RunE: func(_ *cobra.Command, _ []string) error {
params.CiliumNamespace = namespace
s := encrypt.NewEncrypt(k8sClient, params)
if err := s.GetEncryptStatus(context.Background()); err != nil {
fatalf("Unable to get encrypt status: %s", err)
if err := s.PrintEncryptStatus(context.Background()); err != nil {
fatalf("Unable to print encryption status: %s", err)
}
return nil
},
Expand Down
14 changes: 11 additions & 3 deletions encrypt/ipsec_key_status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,23 @@ func (s *Encrypt) IPsecKeyStatus(ctx context.Context) error {
ctx, cancelFn := context.WithTimeout(ctx, s.params.WaitDuration)
defer cancelFn()

key, err := s.readIPsecKey(ctx)
if err != nil {
return err
}
return printIPsecKey(key, s.params.Output)
}

func (s *Encrypt) readIPsecKey(ctx context.Context) (string, error) {
secret, err := s.client.GetSecret(ctx, s.params.CiliumNamespace, defaults.EncryptionSecretName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to fetch IPsec secret: %s", err)
return "", fmt.Errorf("failed to fetch IPsec secret: %s", err)
}

if key, ok := secret.Data["keys"]; ok {
return printIPsecKey(string(key), s.params.Output)
return string(key), nil
}
return fmt.Errorf("IPsec keys not found in the secret: %s", defaults.EncryptionSecretName)
return "", fmt.Errorf("IPsec keys not found in the secret: %s", defaults.EncryptionSecretName)
}

type ipsecKeyStatus struct {
Expand Down
30 changes: 21 additions & 9 deletions encrypt/model.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,26 @@

package encrypt

import "github.com/cilium/cilium/api/v1/models"

type clusterStatus struct {
TotalNodeCount int `json:"total-node-count,omitempty"`
EncDisabledNodeCount int `json:"enc-disabled-node-count,omitempty"`
EncIPsecNodeCount int `json:"enc-ipsec-node-count,omitempty"`
EncWireguardNodeCount int `json:"enc-wireguard-node-count,omitempty"`
IPsecKeysInUseNodeCount map[int64]int64 `json:"ipsec-keys-in-use-node-count,omitempty"`
IPsecMaxSeqNum string `json:"ipsec-max-seq-num,omitempty"`
IPsecErrCount int64 `json:"ipsec-err-count,omitempty"`
XfrmErrors map[string]int64 `json:"xfrm-errors,omitempty"`
XfrmErrorNodeCount map[string]int64 `json:"xfrm-error-node-count,omitempty"`
TotalNodeCount int `json:"total-node-count,omitempty"`
EncDisabledNodeCount int `json:"enc-disabled-node-count,omitempty"`
EncIPsecNodeCount int `json:"enc-ipsec-node-count,omitempty"`
EncWireguardNodeCount int `json:"enc-wireguard-node-count,omitempty"`
IPsecKeysInUseNodeCount map[int64]int64 `json:"ipsec-keys-in-use-node-count,omitempty"`
IPsecMaxSeqNum string `json:"ipsec-max-seq-num,omitempty"`
IPsecErrCount int64 `json:"ipsec-err-count,omitempty"`
IPsecPerNodeKey bool `json:"ipsec-per-node-key,omitempty"`
IPsecKeyRotationInProgress bool `json:"ipsec-key-rotation-in-progress,omitempty"`
IPsecExpectedKeyCount int `json:"ipsec-expected-key-count,omitempty"`
XfrmErrors map[string]int64 `json:"xfrm-errors,omitempty"`
XfrmErrorNodeCount map[string]int64 `json:"xfrm-error-node-count,omitempty"`
}

type nodeStatus struct {
models.EncryptionStatus
IPsecPerNodeKey bool `json:"ipsec-per-node-key,omitempty"`
IPsecKeyRotationInProgress bool `json:"ipsec-key-rotation-in-progress,omitempty"`
IPsecExpectedKeyCount int `json:"ipsec-expected-key-count,omitempty"`
}
184 changes: 127 additions & 57 deletions encrypt/status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,15 @@ import (

"github.com/cilium/cilium/api/v1/models"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/cilium/cilium-cli/defaults"
"github.com/cilium/cilium-cli/status"
"github.com/cilium/cilium-cli/utils/features"
)

// GetEncryptStatus gets encryption status from all/specific cilium agent pods.
func (s *Encrypt) GetEncryptStatus(ctx context.Context) error {
// PrintEncryptStatus prints encryption status from all/specific cilium agent pods.
func (s *Encrypt) PrintEncryptStatus(ctx context.Context) error {
ctx, cancelFn := context.WithTimeout(ctx, s.params.WaitDuration)
defer cancelFn()

Expand All @@ -29,12 +31,25 @@ func (s *Encrypt) GetEncryptStatus(ctx context.Context) error {
return err
}

res, err := s.fetchEncryptStatusConcurrently(ctx, pods)
nodeMap, err := s.fetchEncryptStatusConcurrently(ctx, pods)
if err != nil {
return err
}

return s.writeStatus(res)
ikProps, err := s.getIPsecKeyProps(ctx, len(pods))
if err != nil {
return err
}

if s.params.PerNodeDetails {
return printPerNodeStatus(nodeMap, ikProps, s.params.Output)
}

cs, err := getClusterStatus(nodeMap, ikProps)
if err != nil {
return err
}
return printClusterStatus(cs, s.params.Output)
}

func (s *Encrypt) fetchEncryptStatusConcurrently(ctx context.Context, pods []corev1.Pod) (map[string]models.EncryptionStatus, error) {
Expand Down Expand Up @@ -150,30 +165,100 @@ func nodeStatusFromText(str string) (models.EncryptionStatus, error) {
return res, nil
}

func (s *Encrypt) writeStatus(res map[string]models.EncryptionStatus) error {
if s.params.PerNodeDetails {
for nodeName, n := range res {
if err := printStatus(nodeName, n, s.params.Output); err != nil {
return err
}
}
return nil
type ipsecKeyProps struct {
perNode bool
expectedCount int
}

func (s *Encrypt) getIPsecKeyProps(ctx context.Context, nodeCount int) (ipsecKeyProps, error) {
cm, err := s.client.GetConfigMap(ctx, s.params.CiliumNamespace, defaults.ConfigMapName, metav1.GetOptions{})
if err != nil {
return ipsecKeyProps{}, fmt.Errorf("unable get ConfigMap %q: %w", defaults.ConfigMapName, err)
}
cs, err := clusterNodeStatus(res)

fs := features.Set{}
fs.ExtractFromConfigMap(cm)
if !fs[features.IPsecEnabled].Enabled {
return ipsecKeyProps{}, nil
}

key, err := s.readIPsecKey(ctx)
if err != nil {
return ipsecKeyProps{}, err
}

perNode := strings.Contains(key, "+")
return ipsecKeyProps{
perNode: perNode,
expectedCount: expectedIPsecKeyCount(nodeCount, fs, perNode),
}, nil
}

func expectedIPsecKeyCount(ciliumPods int, fs features.Set, perNodeKey bool) int {
if !perNodeKey {
return 1
}
// IPsec key states for `local_cilium_internal_ip` and `remote_node_ip`
xfrmStates := 2
if fs[features.CiliumIPAMMode].Mode == "eni" || fs[features.CiliumIPAMMode].Mode == "azure" {
xfrmStates++
}
if fs[features.IPv6].Enabled {
// multiply by 2 because of dual state: IPv4 & IPv6
xfrmStates *= 2
}
// subtract 1 to count remote nodes only
return (ciliumPods - 1) * xfrmStates
}

func printPerNodeStatus(nodeMap map[string]models.EncryptionStatus, ikProps ipsecKeyProps, format string) error {
for node, st := range nodeMap {
if format == status.OutputJSON {
var ns any = st
if st.Mode == "IPsec" {
ns = nodeStatus{
EncryptionStatus: st,
IPsecPerNodeKey: ikProps.perNode,
IPsecExpectedKeyCount: ikProps.expectedCount,
IPsecKeyRotationInProgress: int64(ikProps.expectedCount) != st.Ipsec.KeysInUse,
}
}
return printJSONStatus(ns)
}

builder := strings.Builder{}
builder.WriteString(fmt.Sprintf("Node: %s\n", node))
builder.WriteString(fmt.Sprintf("Encryption: %s\n", st.Mode))
if st.Mode == "IPsec" {
if st.Ipsec.KeysInUse > 0 {
builder.WriteString(fmt.Sprintf("IPsec keys in use: %d\n", st.Ipsec.KeysInUse))
}
if st.Ipsec.MaxSeqNumber != "" {
builder.WriteString(fmt.Sprintf("IPsec highest Seq. Number: %s\n", st.Ipsec.MaxSeqNumber))
}
builder.WriteString(fmt.Sprintf("IPsec expected key count: %d\n", ikProps.expectedCount))
builder.WriteString(fmt.Sprintf("IPsec key rotation in progress: %t\n", int64(ikProps.expectedCount) != st.Ipsec.KeysInUse))
builder.WriteString(fmt.Sprintf("IPsec per-node key: %t\n", ikProps.perNode))
builder.WriteString(fmt.Sprintf("IPsec errors: %d\n", st.Ipsec.ErrorCount))
for k, v := range st.Ipsec.XfrmErrors {
builder.WriteString(fmt.Sprintf("\t%s: %d\n", k, v))
}
}
_, err := fmt.Println(builder.String())
return err
}
return cs.printStatus(s.params.Output)
return nil
}

func clusterNodeStatus(res map[string]models.EncryptionStatus) (clusterStatus, error) {
func getClusterStatus(nodeMap map[string]models.EncryptionStatus, ikProps ipsecKeyProps) (clusterStatus, error) {
cs := clusterStatus{
TotalNodeCount: len(res),
TotalNodeCount: len(nodeMap),
IPsecKeysInUseNodeCount: make(map[int64]int64),
XfrmErrors: make(map[string]int64),
XfrmErrorNodeCount: make(map[string]int64),
}
for _, v := range res {
keyRotationInProgress := false
for _, v := range nodeMap {
if v.Mode == "Disabled" {
cs.EncDisabledNodeCount++
continue
Expand All @@ -184,6 +269,8 @@ func clusterNodeStatus(res map[string]models.EncryptionStatus) (clusterStatus, e
}
if v.Mode == "IPsec" {
cs.EncIPsecNodeCount++
cs.IPsecExpectedKeyCount = ikProps.expectedCount
cs.IPsecPerNodeKey = ikProps.perNode
}
cs.IPsecKeysInUseNodeCount[v.Ipsec.KeysInUse]++
maxSeqNum, err := maxSequenceNumber(v.Ipsec.MaxSeqNumber, cs.IPsecMaxSeqNum)
Expand All @@ -196,67 +283,50 @@ func clusterNodeStatus(res map[string]models.EncryptionStatus) (clusterStatus, e
cs.XfrmErrors[k] += e
cs.XfrmErrorNodeCount[k]++
}
if int64(ikProps.expectedCount) != v.Ipsec.KeysInUse {
keyRotationInProgress = true
}
}
cs.IPsecKeyRotationInProgress = keyRotationInProgress
return cs, nil
}

func (c clusterStatus) printStatus(format string) error {
func printClusterStatus(cs clusterStatus, format string) error {
if format == status.OutputJSON {
return printJSONStatus(c)
return printJSONStatus(cs)
}

builder := strings.Builder{}
if c.EncDisabledNodeCount > 0 {
builder.WriteString(fmt.Sprintf("Encryption: Disabled (%d/%d nodes)\n", c.EncDisabledNodeCount, c.TotalNodeCount))
if cs.EncDisabledNodeCount > 0 {
builder.WriteString(fmt.Sprintf("Encryption: Disabled (%d/%d nodes)\n", cs.EncDisabledNodeCount, cs.TotalNodeCount))
}
if c.EncIPsecNodeCount > 0 {
builder.WriteString(fmt.Sprintf("Encryption: IPsec (%d/%d nodes)\n", c.EncIPsecNodeCount, c.TotalNodeCount))
if len(c.IPsecKeysInUseNodeCount) > 0 {
keys := make([]int64, 0, len(c.IPsecKeysInUseNodeCount))
for k := range c.IPsecKeysInUseNodeCount {
if cs.EncIPsecNodeCount > 0 {
builder.WriteString(fmt.Sprintf("Encryption: IPsec (%d/%d nodes)\n", cs.EncIPsecNodeCount, cs.TotalNodeCount))
if len(cs.IPsecKeysInUseNodeCount) > 0 {
keys := make([]int64, 0, len(cs.IPsecKeysInUseNodeCount))
for k := range cs.IPsecKeysInUseNodeCount {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
return keys[i] < keys[j]
})
keyStrs := make([]string, 0, len(keys))
for _, k := range keys {
keyStrs = append(keyStrs, fmt.Sprintf("%d on %d/%d", k, c.IPsecKeysInUseNodeCount[k], c.TotalNodeCount))
keyStrs = append(keyStrs, fmt.Sprintf("%d on %d/%d", k, cs.IPsecKeysInUseNodeCount[k], cs.TotalNodeCount))
}
builder.WriteString(fmt.Sprintf("IPsec keys in use: %s\n", strings.Join(keyStrs, ", ")))
}
builder.WriteString(fmt.Sprintf("IPsec highest Seq. Number: %s across all nodes\n", c.IPsecMaxSeqNum))
builder.WriteString(fmt.Sprintf("IPsec errors: %d across all nodes\n", c.IPsecErrCount))
for k, v := range c.XfrmErrors {
builder.WriteString(fmt.Sprintf("\t%s: %d on %d/%d nodes\n", k, v, c.XfrmErrorNodeCount[k], c.TotalNodeCount))
builder.WriteString(fmt.Sprintf("IPsec highest Seq. Number: %s across all nodes\n", cs.IPsecMaxSeqNum))
builder.WriteString(fmt.Sprintf("IPsec expected key count: %d\n", cs.IPsecExpectedKeyCount))
builder.WriteString(fmt.Sprintf("IPsec key rotation in progress: %t\n", cs.IPsecKeyRotationInProgress))
builder.WriteString(fmt.Sprintf("IPsec per-node key: %t\n", cs.IPsecPerNodeKey))
builder.WriteString(fmt.Sprintf("IPsec errors: %d across all nodes\n", cs.IPsecErrCount))
for k, v := range cs.XfrmErrors {
builder.WriteString(fmt.Sprintf("\t%s: %d on %d/%d nodes\n", k, v, cs.XfrmErrorNodeCount[k], cs.TotalNodeCount))
}
}
if c.EncWireguardNodeCount > 0 {
builder.WriteString(fmt.Sprintf("Encryption: Wireguard (%d/%d nodes)\n", c.EncWireguardNodeCount, c.TotalNodeCount))
}
_, err := fmt.Println(builder.String())
return err
}

func printStatus(nodeName string, n models.EncryptionStatus, format string) error {
if format == status.OutputJSON {
return printJSONStatus(n)
}

builder := strings.Builder{}
builder.WriteString(fmt.Sprintf("Node: %s\n", nodeName))
builder.WriteString(fmt.Sprintf("Encryption: %s\n", n.Mode))
if n.Mode == "IPsec" {
if n.Ipsec.KeysInUse > 0 {
builder.WriteString(fmt.Sprintf("IPsec keys in use: %d\n", n.Ipsec.KeysInUse))
}
if n.Ipsec.MaxSeqNumber != "" {
builder.WriteString(fmt.Sprintf("IPsec highest Seq. Number: %s\n", n.Ipsec.MaxSeqNumber))
}
builder.WriteString(fmt.Sprintf("IPsec errors: %d\n", n.Ipsec.ErrorCount))
for k, v := range n.Ipsec.XfrmErrors {
builder.WriteString(fmt.Sprintf("\t%s: %d\n", k, v))
}
if cs.EncWireguardNodeCount > 0 {
builder.WriteString(fmt.Sprintf("Encryption: Wireguard (%d/%d nodes)\n", cs.EncWireguardNodeCount, cs.TotalNodeCount))
}
_, err := fmt.Println(builder.String())
return err
Expand Down
Loading
Loading