Add instrumentation for contextual logging to include service provider keys

hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go

@@ -42,7 +42,7 @@ import (
 	"github.com/openshift/hypershift/hypershift-operator/controllers/hostedcluster/internal/platform"
 	platformaws "github.com/openshift/hypershift/hypershift-operator/controllers/hostedcluster/internal/platform/aws"
 	hcmetrics "github.com/openshift/hypershift/hypershift-operator/controllers/hostedcluster/metrics"
-	validations "github.com/openshift/hypershift/hypershift-operator/controllers/hostedcluster/validations"
+	"github.com/openshift/hypershift/hypershift-operator/controllers/hostedcluster/validations"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests/clusterapi"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator"
@@ -58,6 +58,7 @@ import (
 	controlplanecomponent "github.com/openshift/hypershift/support/controlplane-component"
 	"github.com/openshift/hypershift/support/globalconfig"
 	"github.com/openshift/hypershift/support/infraid"
+	"github.com/openshift/hypershift/support/logcontext"
 	"github.com/openshift/hypershift/support/metrics"
 	"github.com/openshift/hypershift/support/oidc"
 	"github.com/openshift/hypershift/support/releaseinfo"
@@ -344,6 +345,8 @@ func (r *HostedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		}
 		return ctrl.Result{}, fmt.Errorf("failed to get cluster %q: %w", req.NamespacedName, err)
 	}
+	log = logcontext.AddAnnotationContext(log, hcluster.Annotations)
+	ctx = ctrl.LoggerInto(ctx, log)
 
 	var res reconcile.Result
 	if r.overwriteReconcile != nil {
@@ -1304,6 +1307,11 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques
 		}
 		controlPlaneNamespace.Labels["security.openshift.io/scc.podSecurityLabelSync"] = "false"
 
+		if controlPlaneNamespace.Annotations == nil {
+			controlPlaneNamespace.Annotations = make(map[string]string)
+		}
+		logcontext.AddServiceProviderAnnotations(controlPlaneNamespace.Annotations, hcluster.Annotations)
+
 		// Enable monitoring for hosted control plane namespaces
 		if r.EnableOCPClusterMonitoring {
 			controlPlaneNamespace.Labels["openshift.io/cluster-monitoring"] = "true"

hypershift-operator/controllers/hostedcluster/hostedcluster_webhook.go

@@ -6,6 +6,7 @@ import (
 
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	"github.com/openshift/hypershift/cmd/cluster/core"
+	"github.com/openshift/hypershift/support/logcontext"
 	"github.com/openshift/hypershift/support/supportedversion"
 	hyperutil "github.com/openshift/hypershift/support/util"
 
@@ -34,6 +35,10 @@ func (defaulter *hostedClusterDefaulter) Default(ctx context.Context, obj runtim
 	if !ok {
 		return apierrors.NewBadRequest(fmt.Sprintf("expected a HostedCluster but got a %T", obj))
 	}
+	// we do this so that others pulling the logger from the context will get the full context.
+	log := ctrl.LoggerFrom(ctx)
+	log = logcontext.AddAnnotationContext(log, hcluster.Annotations)
+	ctx = ctrl.LoggerInto(ctx, log)
 
 	if hcluster.Spec.Release.Image == "" {
 		pullSpec, err := supportedversion.LookupLatestSupportedRelease(ctx, hcluster)

hypershift-operator/controllers/hostedclustersizing/hostedclustersizing_controller.go

@@ -11,6 +11,7 @@ import (
 	hypershiftv1beta1applyconfigurations "github.com/openshift/hypershift/client/applyconfiguration/hypershift/v1beta1"
 	hypershiftclient "github.com/openshift/hypershift/client/clientset/clientset"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests"
+	"github.com/openshift/hypershift/support/logcontext"
 	"github.com/openshift/hypershift/support/releaseinfo"
 	hyperutil "github.com/openshift/hypershift/support/util"
 
@@ -134,6 +135,8 @@ func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) (
 		}
 		return reconcile.Result{}, err
 	}
+	logger = logcontext.AddAnnotationContext(logger, hostedCluster.Annotations)
+	ctx = ctrl.LoggerInto(ctx, logger)
 
 	action, err := r.reconcile(ctx, request, config, hostedCluster)
 	if err != nil {

hypershift-operator/controllers/nodepool/nodepool_controller.go

@@ -17,6 +17,7 @@ import (
 	kvinfra "github.com/openshift/hypershift/kubevirtexternalinfra"
 	"github.com/openshift/hypershift/support/capabilities"
 	"github.com/openshift/hypershift/support/images"
+	"github.com/openshift/hypershift/support/logcontext"
 	"github.com/openshift/hypershift/support/releaseinfo"
 	"github.com/openshift/hypershift/support/supportedversion"
 	"github.com/openshift/hypershift/support/upsert"
@@ -218,6 +219,8 @@ func (r *NodePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	if err != nil {
 		return ctrl.Result{}, err
 	}
+	log = logcontext.AddAnnotationContext(log, hcluster.Annotations)
+	ctx = ctrl.LoggerInto(ctx, log)
 
 	// Ensure the nodePool has a finalizer for cleanup
 	if !controllerutil.ContainsFinalizer(nodePool, finalizer) {

hypershift-operator/controllers/platform/aws/controller.go

@@ -11,6 +11,7 @@ import (
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	awsutil "github.com/openshift/hypershift/cmd/infra/aws/util"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
+	"github.com/openshift/hypershift/support/logcontext"
 	"github.com/openshift/hypershift/support/upsert"
 	supportutil "github.com/openshift/hypershift/support/util"
 
@@ -256,6 +257,8 @@ func (r *AWSEndpointServiceReconciler) Reconcile(ctx context.Context, req ctrl.R
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to get hosted cluster: %w", err)
 	}
+	log = logcontext.AddAnnotationContext(log, hc.Annotations)
+	ctx = ctrl.LoggerInto(ctx, log)
 
 	if isPaused, duration := supportutil.IsReconciliationPaused(log, hc.Spec.PausedUntil); isPaused {
 		log.Info("Reconciliation paused", "pausedUntil", *hc.Spec.PausedUntil)

hypershift-operator/controllers/resourcebasedcpautoscaler/controller.go

@@ -8,6 +8,7 @@ import (
 	schedulingv1alpha1 "github.com/openshift/hypershift/api/scheduling/v1alpha1"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests"
 	controlplaneautoscalermanifests "github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneautoscaler"
+	"github.com/openshift/hypershift/support/logcontext"
 	"github.com/openshift/hypershift/support/util"
 
 	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
@@ -119,6 +120,9 @@ func (r *ControlPlaneAutoscalerController) Reconcile(ctx context.Context, reques
 		}
 		return ctrl.Result{}, err
 	}
+	log := ctrl.LoggerFrom(ctx)
+	log = logcontext.AddAnnotationContext(log, hc.Annotations)
+	ctx = ctrl.LoggerInto(ctx, log)
 
 	if !hc.DeletionTimestamp.IsZero() {
 		// No need to process deleting HostedClusters

hypershift-operator/controllers/scheduler/aws/autoscaler.go

@@ -12,6 +12,7 @@ import (
 
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	schedulingv1alpha1 "github.com/openshift/hypershift/api/scheduling/v1alpha1"
+	"github.com/openshift/hypershift/support/logcontext"
 	"github.com/openshift/hypershift/support/util"
 
 	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
@@ -168,6 +169,8 @@ func (r *MachineSetDescaler) Reconcile(ctx context.Context, req ctrl.Request) (c
 			return ctrl.Result{}, fmt.Errorf("failed to get hosted cluster: %w", err)
 		}
 	}
+	log = logcontext.AddAnnotationContext(log, hostedCluster.Annotations)
+	ctx = ctrl.LoggerInto(ctx, log)
 	machineSetList := &machinev1beta1.MachineSetList{}
 	if err := r.List(ctx, machineSetList, client.InNamespace(machineSetNamespace)); err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to list machinesets: %w", err)

hypershift-operator/controllers/scheduler/aws/dedicated_request_serving_nodes.go

@@ -10,6 +10,7 @@ import (
 	schedulingv1alpha1 "github.com/openshift/hypershift/api/scheduling/v1alpha1"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/hostedcluster"
 	schedulerutil "github.com/openshift/hypershift/hypershift-operator/controllers/scheduler/util"
+	"github.com/openshift/hypershift/support/logcontext"
 	"github.com/openshift/hypershift/support/upsert"
 	"github.com/openshift/hypershift/support/util"
 
@@ -151,6 +152,8 @@ func (r *DedicatedServingComponentScheduler) Reconcile(ctx context.Context, req
 		}
 		return ctrl.Result{}, fmt.Errorf("failed to get cluster %q: %w", req.NamespacedName, err)
 	}
+	log = logcontext.AddAnnotationContext(log, hcluster.Annotations)
+	ctx = ctrl.LoggerInto(ctx, log)
 	if !hcluster.DeletionTimestamp.IsZero() {
 		log.Info("hostedcluster is deleted, nothing to do")
 		return ctrl.Result{}, nil
@@ -428,6 +431,8 @@ func (r *DedicatedServingComponentSchedulerAndSizer) Reconcile(ctx context.Conte
 		}
 		return ctrl.Result{}, fmt.Errorf("failed to get cluster %q: %w", req.NamespacedName, err)
 	}
+	log = logcontext.AddAnnotationContext(log, hc.Annotations)
+	ctx = ctrl.LoggerInto(ctx, log)
 	if !hc.DeletionTimestamp.IsZero() {
 		log.Info("hostedcluster is deleted, cleaning up")
 		if controllerutil.ContainsFinalizer(hc, schedulerFinalizer) {

hypershift-operator/controllers/scheduler/azure/controller.go

@@ -8,6 +8,7 @@ import (
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	schedulingv1alpha1 "github.com/openshift/hypershift/api/scheduling/v1alpha1"
 	schedulerutil "github.com/openshift/hypershift/hypershift-operator/controllers/scheduler/util"
+	"github.com/openshift/hypershift/support/logcontext"
 	"github.com/openshift/hypershift/support/util"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -65,6 +66,8 @@ func (r *Scheduler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul
 		}
 		return ctrl.Result{}, fmt.Errorf("failed to get cluster %q: %w", req.NamespacedName, err)
 	}
+	log = logcontext.AddAnnotationContext(log, hc.Annotations)
+	ctx = ctrl.LoggerInto(ctx, log)
 
 	if !hc.DeletionTimestamp.IsZero() {
 		log.Info("hostedcluster is being deleted, aborting reconcile")

support/logcontext/service_provider_logging.go

@@ -0,0 +1,38 @@
+package logcontext
+
+import (
+	"strings"
+
+	"github.com/go-logr/logr"
+)
+
+// AddAnnotationContext takes annotations, extracts context desired by the service provider, and adds that context to the logger.
+// This is useful when the service provider has keys like resourceGroupName, resourceName, hcpClusterName, clusterServiceID
+// and wants to be able to select all log lines that contain those keys.
+// We use annotations because they can hold more values and are applicable to all resource types.
+func AddAnnotationContext(log logr.Logger, annotations map[string]string) logr.Logger {
+	for k, v := range annotations {
+		if !strings.HasPrefix(k, "context.serviceprovider.hypershift.openshift.io/") {
+			continue
+		}
+		logKey, _ := strings.CutPrefix(k, "context.serviceprovider.hypershift.openshift.io/")
+
+		log = log.WithValues(logKey, v)
+	}
+	return log
+}
+
+// AddServiceProviderAnnotations adds service provider annotations from the hosted cluster (or other authoritative annotations)
+// and sets them on the target if they are not already set.  This is an easy way to take serviceprovider annotations used for log
+// and metric labeling during aggregated ingestion and placing them on namespaces where they can be accessed.
+func AddServiceProviderAnnotations(targetAnnotations map[string]string, hostedClusterAnnotations map[string]string) {
+	for k, v := range hostedClusterAnnotations {
+		if !strings.HasPrefix(k, "context.serviceprovider.hypershift.openshift.io/") {
+			continue
+		}
+		if _, exists := targetAnnotations[k]; exists {
+			continue
+		}
+		targetAnnotations[k] = v
+	}
+}