core.go

// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package status

import (
	"fmt"
	"math"
	"strings"
	"time"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)

// GetConditionsFn defines the signature for functions to compute the
// status of a built-in resource.
type GetConditionsFn func(*unstructured.Unstructured) (*Result, error)

// legacyTypes defines the mapping from GroupKind to a function that can
// compute the status for the given resource.
var legacyTypes = map[string]GetConditionsFn{
	"Service":                    serviceConditions,
	"Pod":                        podConditions,
	"Secret":                     alwaysReady,
	"PersistentVolumeClaim":      pvcConditions,
	"apps/StatefulSet":           stsConditions,
	"apps/DaemonSet":             daemonsetConditions,
	"extensions/DaemonSet":       daemonsetConditions,
	"apps/Deployment":            deploymentConditions,
	"extensions/Deployment":      deploymentConditions,
	"apps/ReplicaSet":            replicasetConditions,
	"extensions/ReplicaSet":      replicasetConditions,
	"policy/PodDisruptionBudget": pdbConditions,
	"batch/CronJob":              alwaysReady,
	"ConfigMap":                  alwaysReady,
	"batch/Job":                  jobConditions,
	"apiextensions.k8s.io/CustomResourceDefinition": crdConditions,
}

const (
	tooFewReady     = "LessReady"
	tooFewAvailable = "LessAvailable"
	tooFewUpdated   = "LessUpdated"
	tooFewReplicas  = "LessReplicas"
	extraPods       = "ExtraPods"

	onDeleteUpdateStrategy = "OnDelete"

	// How long a pod can be unscheduled before it is reported as
	// unschedulable.
	ScheduleWindow = 15 * time.Second
)

// GetLegacyConditionsFn returns a function that can compute the status for the
// given resource, or nil if the resource type is not known.
func GetLegacyConditionsFn(u *unstructured.Unstructured) GetConditionsFn {
	gvk := u.GroupVersionKind()
	g := gvk.Group
	k := gvk.Kind
	key := g + "/" + k
	if g == "" {
		key = k
	}
	return legacyTypes[key]
}

// alwaysReady Used for resources that are always ready
func alwaysReady(u *unstructured.Unstructured) (*Result, error) {
	return &Result{
		Status:     CurrentStatus,
		Message:    "Resource is always ready",
		Conditions: []Condition{},
	}, nil
}

// stsConditions return standardized Conditions for Statefulset
//
// StatefulSet does define the .status.conditions property, but the controller never
// actually sets any Conditions. Thus, status must be computed only based on the other
// properties under .status. We don't have any way to find out if a reconcile for a
// StatefulSet has failed.
func stsConditions(u *unstructured.Unstructured) (*Result, error) {
	obj := u.UnstructuredContent()

	// updateStrategy==ondelete is a user managed statefulset.
	updateStrategy := GetStringField(obj, ".spec.updateStrategy.type", "")
	if updateStrategy == onDeleteUpdateStrategy {
		return &Result{
			Status:     CurrentStatus,
			Message:    "StatefulSet is using the ondelete update strategy",
			Conditions: []Condition{},
		}, nil
	}

	// Replicas
	specReplicas := GetIntField(obj, ".spec.replicas", 1)
	readyReplicas := GetIntField(obj, ".status.readyReplicas", 0)
	currentReplicas := GetIntField(obj, ".status.currentReplicas", 0)
	updatedReplicas := GetIntField(obj, ".status.updatedReplicas", 0)
	statusReplicas := GetIntField(obj, ".status.replicas", 0)
	partition := GetIntField(obj, ".spec.updateStrategy.rollingUpdate.partition", -1)

	if specReplicas > statusReplicas {
		message := fmt.Sprintf("Replicas: %d/%d", statusReplicas, specReplicas)
		return newInProgressStatus(tooFewReplicas, message), nil
	}

	if specReplicas > readyReplicas {
		message := fmt.Sprintf("Ready: %d/%d", readyReplicas, specReplicas)
		return newInProgressStatus(tooFewReady, message), nil
	}

	if statusReplicas > specReplicas {
		message := fmt.Sprintf("Pending termination: %d", statusReplicas-specReplicas)
		return newInProgressStatus(extraPods, message), nil
	}

	// https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions
	if partition != -1 {
		if updatedReplicas < (specReplicas - partition) {
			message := fmt.Sprintf("updated: %d/%d", updatedReplicas, specReplicas-partition)
			return newInProgressStatus("PartitionRollout", message), nil
		}
		// Partition case All ok
		return &Result{
			Status:     CurrentStatus,
			Message:    fmt.Sprintf("Partition rollout complete. updated: %d", updatedReplicas),
			Conditions: []Condition{},
		}, nil
	}

	if specReplicas > currentReplicas {
		message := fmt.Sprintf("current: %d/%d", currentReplicas, specReplicas)
		return newInProgressStatus("LessCurrent", message), nil
	}

	// Revision
	currentRevision := GetStringField(obj, ".status.currentRevision", "")
	updatedRevision := GetStringField(obj, ".status.updateRevision", "")
	if currentRevision != updatedRevision {
		message := "Waiting for updated revision to match current"
		return newInProgressStatus("RevisionMismatch", message), nil
	}

	// All ok
	return &Result{
		Status:     CurrentStatus,
		Message:    fmt.Sprintf("All replicas scheduled as expected. Replicas: %d", statusReplicas),
		Conditions: []Condition{},
	}, nil
}

// deploymentConditions return standardized Conditions for Deployment.
//
// For Deployments, we look at .status.conditions as well as the other properties
// under .status. Status will be Failed if the progress deadline has been exceeded.
func deploymentConditions(u *unstructured.Unstructured) (*Result, error) {
	obj := u.UnstructuredContent()

	progressing := false

	// Check if progressDeadlineSeconds is set. If not, the controller will not set
	// the `Progressing` condition, so it will always consider a deployment to be
	// progressing. The use of math.MaxInt32 is due to special handling in the
	// controller:
	// https://github.com/kubernetes/kubernetes/blob/a3ccea9d8743f2ff82e41b6c2af6dc2c41dc7b10/pkg/controller/deployment/util/deployment_util.go#L886
	progressDeadline := GetIntField(obj, ".spec.progressDeadlineSeconds", math.MaxInt32)
	if progressDeadline == math.MaxInt32 {
		progressing = true
	}

	available := false

	objc, err := GetObjectWithConditions(obj)
	if err != nil {
		return nil, err
	}

	for _, c := range objc.Status.Conditions {
		switch c.Type {
		case "Progressing": // appsv1.DeploymentProgressing:
			// https://github.com/kubernetes/kubernetes/blob/a3ccea9d8743f2ff82e41b6c2af6dc2c41dc7b10/pkg/controller/deployment/progress.go#L52
			if c.Reason == "ProgressDeadlineExceeded" {
				return &Result{
					Status:     FailedStatus,
					Message:    "Progress deadline exceeded",
					Conditions: []Condition{{ConditionStalled, corev1.ConditionTrue, c.Reason, c.Message}},
				}, nil
			}
			if c.Status == corev1.ConditionTrue && c.Reason == "NewReplicaSetAvailable" {
				progressing = true
			}
		case "Available": // appsv1.DeploymentAvailable:
			if c.Status == corev1.ConditionTrue {
				available = true
			}
		}
	}

	// replicas
	specReplicas := GetIntField(obj, ".spec.replicas", 1) // Controller uses 1 as default if not specified.
	statusReplicas := GetIntField(obj, ".status.replicas", 0)
	updatedReplicas := GetIntField(obj, ".status.updatedReplicas", 0)
	readyReplicas := GetIntField(obj, ".status.readyReplicas", 0)
	availableReplicas := GetIntField(obj, ".status.availableReplicas", 0)

	// TODO spec.replicas zero case ??

	if specReplicas > statusReplicas {
		message := fmt.Sprintf("Replicas: %d/%d", statusReplicas, specReplicas)
		return newInProgressStatus(tooFewReplicas, message), nil
	}

	if specReplicas > updatedReplicas {
		message := fmt.Sprintf("Updated: %d/%d", updatedReplicas, specReplicas)
		return newInProgressStatus(tooFewUpdated, message), nil
	}

	if statusReplicas > specReplicas {
		message := fmt.Sprintf("Pending termination: %d", statusReplicas-specReplicas)
		return newInProgressStatus(extraPods, message), nil
	}

	if updatedReplicas > availableReplicas {
		message := fmt.Sprintf("Available: %d/%d", availableReplicas, updatedReplicas)
		return newInProgressStatus(tooFewAvailable, message), nil
	}

	if specReplicas > readyReplicas {
		message := fmt.Sprintf("Ready: %d/%d", readyReplicas, specReplicas)
		return newInProgressStatus(tooFewReady, message), nil
	}

	// check conditions
	if !progressing {
		message := "ReplicaSet not Available"
		return newInProgressStatus("ReplicaSetNotAvailable", message), nil
	}
	if !available {
		message := "Deployment not Available"
		return newInProgressStatus("DeploymentNotAvailable", message), nil
	}
	// All ok
	return &Result{
		Status:     CurrentStatus,
		Message:    fmt.Sprintf("Deployment is available. Replicas: %d", statusReplicas),
		Conditions: []Condition{},
	}, nil
}

// replicasetConditions return standardized Conditions for Replicaset
func replicasetConditions(u *unstructured.Unstructured) (*Result, error) {
	obj := u.UnstructuredContent()

	// Conditions
	objc, err := GetObjectWithConditions(obj)
	if err != nil {
		return nil, err
	}

	for _, c := range objc.Status.Conditions {
		// https://github.com/kubernetes/kubernetes/blob/a3ccea9d8743f2ff82e41b6c2af6dc2c41dc7b10/pkg/controller/replicaset/replica_set_utils.go
		if c.Type == "ReplicaFailure" && c.Status == corev1.ConditionTrue {
			message := "Replica Failure condition. Check Pods"
			return newInProgressStatus("ReplicaFailure", message), nil
		}
	}

	// Replicas
	specReplicas := GetIntField(obj, ".spec.replicas", 1) // Controller uses 1 as default if not specified.
	statusReplicas := GetIntField(obj, ".status.replicas", 0)
	readyReplicas := GetIntField(obj, ".status.readyReplicas", 0)
	availableReplicas := GetIntField(obj, ".status.availableReplicas", 0)
	fullyLabelledReplicas := GetIntField(obj, ".status.fullyLabeledReplicas", 0)

	if specReplicas > fullyLabelledReplicas {
		message := fmt.Sprintf("Labelled: %d/%d", fullyLabelledReplicas, specReplicas)
		return newInProgressStatus("LessLabelled", message), nil
	}

	if specReplicas > availableReplicas {
		message := fmt.Sprintf("Available: %d/%d", availableReplicas, specReplicas)
		return newInProgressStatus(tooFewAvailable, message), nil
	}

	if specReplicas > readyReplicas {
		message := fmt.Sprintf("Ready: %d/%d", readyReplicas, specReplicas)
		return newInProgressStatus(tooFewReady, message), nil
	}

	if statusReplicas > specReplicas {
		message := fmt.Sprintf("Pending termination: %d", statusReplicas-specReplicas)
		return newInProgressStatus(extraPods, message), nil
	}
	// All ok
	return &Result{
		Status:     CurrentStatus,
		Message:    fmt.Sprintf("ReplicaSet is available. Replicas: %d", statusReplicas),
		Conditions: []Condition{},
	}, nil
}

// daemonsetConditions return standardized Conditions for DaemonSet
func daemonsetConditions(u *unstructured.Unstructured) (*Result, error) {
	// We check that the latest generation is equal to observed generation as
	// part of checking generic properties but in that case, we are lenient and
	// skip the check if those fields are unset. For daemonset, we know that if
	// the daemonset controller has acted on a resource, these fields would not
	// be unset. So, we ensure that here.
	res, err := checkGenerationSet(u)
	if err != nil || res != nil {
		return res, err
	}

	obj := u.UnstructuredContent()

	// replicas
	desiredNumberScheduled := GetIntField(obj, ".status.desiredNumberScheduled", -1)
	currentNumberScheduled := GetIntField(obj, ".status.currentNumberScheduled", 0)
	updatedNumberScheduled := GetIntField(obj, ".status.updatedNumberScheduled", 0)
	numberAvailable := GetIntField(obj, ".status.numberAvailable", 0)
	numberReady := GetIntField(obj, ".status.numberReady", 0)

	if desiredNumberScheduled == -1 {
		message := "Missing .status.desiredNumberScheduled"
		return newInProgressStatus("NoDesiredNumber", message), nil
	}

	if desiredNumberScheduled > currentNumberScheduled {
		message := fmt.Sprintf("Current: %d/%d", currentNumberScheduled, desiredNumberScheduled)
		return newInProgressStatus("LessCurrent", message), nil
	}

	if desiredNumberScheduled > updatedNumberScheduled {
		message := fmt.Sprintf("Updated: %d/%d", updatedNumberScheduled, desiredNumberScheduled)
		return newInProgressStatus(tooFewUpdated, message), nil
	}

	if desiredNumberScheduled > numberAvailable {
		message := fmt.Sprintf("Available: %d/%d", numberAvailable, desiredNumberScheduled)
		return newInProgressStatus(tooFewAvailable, message), nil
	}

	if desiredNumberScheduled > numberReady {
		message := fmt.Sprintf("Ready: %d/%d", numberReady, desiredNumberScheduled)
		return newInProgressStatus(tooFewReady, message), nil
	}

	// All ok
	return &Result{
		Status:     CurrentStatus,
		Message:    fmt.Sprintf("All replicas scheduled as expected. Replicas: %d", desiredNumberScheduled),
		Conditions: []Condition{},
	}, nil
}

// checkGenerationSet checks that the metadata.generation and
// status.observedGeneration fields are set.
func checkGenerationSet(u *unstructured.Unstructured) (*Result, error) {
	_, found, err := unstructured.NestedInt64(u.Object, "metadata", "generation")
	if err != nil {
		return nil, fmt.Errorf("looking up metadata.generation from resource: %w", err)
	}
	if !found {
		message := fmt.Sprintf("%s metadata.generation not found", u.GetKind())
		return &Result{
			Status:     InProgressStatus,
			Message:    message,
			Conditions: []Condition{newReconcilingCondition("NoGeneration", message)},
		}, nil
	}

	_, found, err = unstructured.NestedInt64(u.Object, "status", "observedGeneration")
	if err != nil {
		return nil, fmt.Errorf("looking up status.observedGeneration from resource: %w", err)
	}
	if !found {
		message := fmt.Sprintf("%s status.observedGeneration not found", u.GetKind())
		return &Result{
			Status:     InProgressStatus,
			Message:    message,
			Conditions: []Condition{newReconcilingCondition("NoObservedGeneration", message)},
		}, nil
	}

	return nil, nil
}

// pvcConditions return standardized Conditions for PVC
func pvcConditions(u *unstructured.Unstructured) (*Result, error) {
	obj := u.UnstructuredContent()

	phase := GetStringField(obj, ".status.phase", "unknown")
	if phase != "Bound" { // corev1.ClaimBound
		message := fmt.Sprintf("PVC is not Bound. phase: %s", phase)
		return newInProgressStatus("NotBound", message), nil
	}
	// All ok
	return &Result{
		Status:     CurrentStatus,
		Message:    "PVC is Bound",
		Conditions: []Condition{},
	}, nil
}

// podConditions return standardized Conditions for Pod
func podConditions(u *unstructured.Unstructured) (*Result, error) {
	obj := u.UnstructuredContent()
	objc, err := GetObjectWithConditions(obj)
	if err != nil {
		return nil, err
	}
	phase := GetStringField(obj, ".status.phase", "")

	switch phase {
	case "Succeeded":
		return &Result{
			Status:     CurrentStatus,
			Message:    "Pod has completed successfully",
			Conditions: []Condition{},
		}, nil
	case "Failed":
		return &Result{
			Status:     CurrentStatus,
			Message:    "Pod has completed, but not successfully",
			Conditions: []Condition{},
		}, nil
	case "Running":
		if hasConditionWithStatus(objc.Status.Conditions, "Ready", corev1.ConditionTrue) {
			return &Result{
				Status:     CurrentStatus,
				Message:    "Pod is Ready",
				Conditions: []Condition{},
			}, nil
		}

		containerNames, isCrashLooping, err := getCrashLoopingContainers(obj)
		if err != nil {
			return nil, err
		}
		if isCrashLooping {
			return newFailedStatus("ContainerCrashLooping",
				fmt.Sprintf("Containers in CrashLoop state: %s", strings.Join(containerNames, ","))), nil
		}

		return newInProgressStatus("PodRunningNotReady", "Pod is running but is not Ready"), nil
	case "Pending":
		c, found := getConditionWithStatus(objc.Status.Conditions, "PodScheduled", corev1.ConditionFalse)
		if found && c.Reason == "Unschedulable" {
			if time.Now().Add(-ScheduleWindow).Before(u.GetCreationTimestamp().Time) {
				// We give the pod 15 seconds to be scheduled before we report it
				// as unschedulable.
				return newInProgressStatus("PodNotScheduled", "Pod has not been scheduled"), nil
			}
			return newFailedStatus("PodUnschedulable", "Pod could not be scheduled"), nil
		}
		return newInProgressStatus("PodPending", "Pod is in the Pending phase"), nil
	default:
		// If the controller hasn't observed the pod yet, there is no phase. We consider this as it
		// still being in progress.
		if phase == "" {
			return newInProgressStatus("PodNotObserved", "Pod phase not available"), nil
		}
		return nil, fmt.Errorf("unknown phase %s", phase)
	}
}

func getCrashLoopingContainers(obj map[string]interface{}) ([]string, bool, error) {
	var containerNames []string
	css, found, err := unstructured.NestedSlice(obj, "status", "containerStatuses")
	if !found || err != nil {
		return containerNames, found, err
	}
	for _, item := range css {
		cs := item.(map[string]interface{})
		n, found := cs["name"]
		if !found {
			continue
		}
		name := n.(string)
		s, found := cs["state"]
		if !found {
			continue
		}
		state := s.(map[string]interface{})

		ws, found := state["waiting"]
		if !found {
			continue
		}
		waitingState := ws.(map[string]interface{})

		r, found := waitingState["reason"]
		if !found {
			continue
		}
		reason := r.(string)
		if reason == "CrashLoopBackOff" {
			containerNames = append(containerNames, name)
		}
	}
	if len(containerNames) > 0 {
		return containerNames, true, nil
	}
	return containerNames, false, nil
}

// pdbConditions computes the status for PodDisruptionBudgets. A PDB
// is currently considered Current if the disruption controller has
// observed the latest version of the PDB resource and has computed
// the AllowedDisruptions. PDBs do have ObservedGeneration in the
// Status object, so if this function gets called we know that
// the controller has observed the latest changes.
// The disruption controller does not set any conditions if
// computing the AllowedDisruptions fails (and there are many ways
// it can fail), but there is PR against OSS Kubernetes to address
// this: https://github.com/kubernetes/kubernetes/pull/86929
func pdbConditions(_ *unstructured.Unstructured) (*Result, error) {
	// All ok
	return &Result{
		Status:     CurrentStatus,
		Message:    "AllowedDisruptions has been computed.",
		Conditions: []Condition{},
	}, nil
}

// jobConditions return standardized Conditions for Job
//
// A job will have the InProgress status until it starts running. Then it will have the Current
// status while the job is running and after it has been completed successfully. It
// will have the Failed status if it the job has failed.
func jobConditions(u *unstructured.Unstructured) (*Result, error) {
	obj := u.UnstructuredContent()

	parallelism := GetIntField(obj, ".spec.parallelism", 1)
	completions := GetIntField(obj, ".spec.completions", parallelism)
	succeeded := GetIntField(obj, ".status.succeeded", 0)
	active := GetIntField(obj, ".status.active", 0)
	failed := GetIntField(obj, ".status.failed", 0)
	starttime := GetStringField(obj, ".status.startTime", "")

	// Conditions
	// https://github.com/kubernetes/kubernetes/blob/master/pkg/controller/job/utils.go#L24
	objc, err := GetObjectWithConditions(obj)
	if err != nil {
		return nil, err
	}
	for _, c := range objc.Status.Conditions {
		switch c.Type {
		case "Complete":
			if c.Status == corev1.ConditionTrue {
				message := fmt.Sprintf("Job Completed. succeeded: %d/%d", succeeded, completions)
				return &Result{
					Status:     CurrentStatus,
					Message:    message,
					Conditions: []Condition{},
				}, nil
			}
		case "Failed":
			if c.Status == corev1.ConditionTrue {
				return newFailedStatus("JobFailed",
					fmt.Sprintf("Job Failed. failed: %d/%d", failed, completions)), nil
			}
		}
	}

	// replicas
	if starttime == "" {
		message := "Job not started"
		return newInProgressStatus("JobNotStarted", message), nil
	}
	return &Result{
		Status:     CurrentStatus,
		Message:    fmt.Sprintf("Job in progress. success:%d, active: %d, failed: %d", succeeded, active, failed),
		Conditions: []Condition{},
	}, nil
}

// serviceConditions return standardized Conditions for Service
func serviceConditions(u *unstructured.Unstructured) (*Result, error) {
	obj := u.UnstructuredContent()

	specType := GetStringField(obj, ".spec.type", "ClusterIP")
	specClusterIP := GetStringField(obj, ".spec.clusterIP", "")

	if specType == "LoadBalancer" {
		if specClusterIP == "" {
			message := "ClusterIP not set. Service type: LoadBalancer"
			return newInProgressStatus("NoIPAssigned", message), nil
		}
	}

	return &Result{
		Status:     CurrentStatus,
		Message:    "Service is ready",
		Conditions: []Condition{},
	}, nil
}

func crdConditions(u *unstructured.Unstructured) (*Result, error) {
	obj := u.UnstructuredContent()

	objc, err := GetObjectWithConditions(obj)
	if err != nil {
		return nil, err
	}

	for _, c := range objc.Status.Conditions {
		if c.Type == "NamesAccepted" && c.Status == corev1.ConditionFalse {
			return newFailedStatus(c.Reason, c.Message), nil
		}
		if c.Type == "Established" {
			if c.Status == corev1.ConditionFalse && c.Reason != "Installing" {
				return newFailedStatus(c.Reason, c.Message), nil
			}
			if c.Status == corev1.ConditionTrue {
				return &Result{
					Status:     CurrentStatus,
					Message:    "CRD is established",
					Conditions: []Condition{},
				}, nil
			}
		}
	}
	return newInProgressStatus("Installing", "Install in progress"), nil
}