odiglet: Initialize clientset at startup and do OpenShift selinux cha…

…nges last (#2405)

This does 2 things in the Odiglet:

* **Make SELinux calls for openshift last in the init phase.** These
functions `chroot` to the host directory to run selinux commands that
update the agent permissions, so they are readable by pods. This
`chroot` was preventing the k8s client from initializing with `open
/var/run/secrets/kubernetes.io/serviceaccount/token: no such file or
directory`. Moved last so the `chroot` doesn't mess up future changes.
* **Initializes the k8s clientset in the main odiglet function.** This
allows the same clientset to be passed through to the init phase (which
uses it to apply labels to nodes) and the odiglet controller. Doing so
makes `clientset` an argument to functions like `odiglet.New` and
`k8snode.AddLabelToNode`, the latter of which is just a helper which
should not be initializing its own clientset every call anyway.

Technically, either one of these changes would have fixed the issue. But
doing both is even better to help prevent future issues like this.

k8sutils/pkg/node/node.go

@@ -6,27 +6,16 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 
 	"github.com/odigos-io/odigos/api/k8sconsts"
 	"github.com/odigos-io/odigos/common"
 	"github.com/odigos-io/odigos/k8sutils/pkg/env"
 )
 
-func AddLabelToNode(nodeName string, labelKey string, labelValue string) error {
+func AddLabelToNode(clientset *kubernetes.Clientset, nodeName string, labelKey string, labelValue string) error {
 	// Add odiglet installed label to node
-	cfg, err := rest.InClusterConfig()
-	if err != nil {
-		return err
-	}
-
-	clientset, err := kubernetes.NewForConfig(cfg)
-	if err != nil {
-		return err
-	}
-
 	patch := []byte(`{"metadata": {"labels": {"` + labelKey + `": "` + labelValue + `"}}}`)
-	_, err = clientset.CoreV1().Nodes().Patch(context.Background(), nodeName, types.StrategicMergePatchType, patch, metav1.PatchOptions{})
+	_, err := clientset.CoreV1().Nodes().Patch(context.Background(), nodeName, types.StrategicMergePatchType, patch, metav1.PatchOptions{})
 	if err != nil {
 		return err
 	}

odiglet/cmd/main.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/odigos-io/odigos/odiglet"
 	"github.com/odigos-io/odigos/odiglet/pkg/ebpf/sdks"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 
 	"github.com/odigos-io/odigos/common"
 	commonInstrumentation "github.com/odigos-io/odigos/instrumentation"
@@ -18,9 +20,19 @@ import (
 )
 
 func main() {
+	// Init Kubernetes clientset
+	cfg, err := rest.InClusterConfig()
+	if err != nil {
+		panic(err)
+	}
+	clientset, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		panic(err)
+	}
+
 	// If started in init mode
 	if len(os.Args) == 2 && os.Args[1] == "init" {
-		odiglet.OdigletInitPhase()
+		odiglet.OdigletInitPhase(clientset)
 	}
 
 	if err := log.Init(); err != nil {
@@ -35,7 +47,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	o, err := odiglet.New(deviceInjectionCallbacks(), ebpfInstrumentationFactories())
+	o, err := odiglet.New(clientset, deviceInjectionCallbacks(), ebpfInstrumentationFactories())
 	if err != nil {
 		log.Logger.Error(err, "Failed to initialize odiglet")
 		os.Exit(1)

odiglet/odiglet.go

@@ -21,7 +21,6 @@ import (
 	"github.com/odigos-io/odigos/opampserver/pkg/server"
 	"golang.org/x/sync/errgroup"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 )
 
@@ -39,18 +38,7 @@ const (
 )
 
 // New creates a new Odiglet instance.
-func New(deviceInjectionCallbacks instrumentation.OtelSdksLsf, factories map[commonInstrumentation.OtelDistribution]commonInstrumentation.Factory) (*Odiglet, error) {
-	// Init Kubernetes API client
-	cfg, err := rest.InClusterConfig()
-	if err != nil {
-		return nil, fmt.Errorf("failed to create in-cluster config for Kubernetes client %w", err)
-	}
-
-	clientset, err := kubernetes.NewForConfig(cfg)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create Kubernetes client %w", err)
-	}
-
+func New(clientset *kubernetes.Clientset, deviceInjectionCallbacks instrumentation.OtelSdksLsf, factories map[commonInstrumentation.OtelDistribution]commonInstrumentation.Factory) (*Odiglet, error) {
 	mgr, err := kube.CreateManager()
 	if err != nil {
 		return nil, fmt.Errorf("failed to create controller-runtime manager %w", err)
@@ -180,7 +168,7 @@ func runDeviceManager(clientset *kubernetes.Clientset, otelSdkLsf instrumentatio
 	return nil
 }
 
-func OdigletInitPhase() {
+func OdigletInitPhase(clientset *kubernetes.Clientset) {
 	if err := log.Init(); err != nil {
 		panic(err)
 	}
@@ -200,10 +188,17 @@ func OdigletInitPhase() {
 
 	log.Logger.V(0).Info("Adding Label to Node", "odigletLabel", odigletInstalledLabel)
 
-	if err := k8snode.AddLabelToNode(nn, odigletInstalledLabel, k8sconsts.OdigletInstalledLabelValue); err != nil {
+	if err := k8snode.AddLabelToNode(clientset, nn, odigletInstalledLabel, k8sconsts.OdigletInstalledLabelValue); err != nil {
 		log.Logger.Error(err, "Failed to add Odiglet installed label to the node")
 		os.Exit(-1)
 	}
 
+	// SELinux settings should be applied last. This function chroot's to use the host's PATH for
+	// executing selinux commands to make agents readable by pods.
+	if err := fs.ApplyOpenShiftSELinuxSettings(); err != nil {
+		log.Logger.Error(err, "Failed to apply SELinux settings on RHEL host")
+		os.Exit(-1)
+	}
+
 	os.Exit(0)
 }

odiglet/pkg/instrumentation/fs/agents.go

@@ -55,37 +55,6 @@ func CopyAgentsDirectoryToHost() error {
 		return err
 	}
 
-	// Check if the semanage command exists when running on RHEL/CoreOS
-	_, err = exec.LookPath(filepath.Join(chrootDir, semanagePath))
-	if err == nil {
-		// Run the semanage command to add the new directory to the container_ro_file_t context
-		// semanage writes SELinux config to host
-		syscall.Chroot(chrootDir)
-		cmd := exec.Command(semanagePath, "fcontext", "-a", "-t", "container_ro_file_t", "/var/odigos(/.*)?")
-		err = cmd.Run()
-		if err != nil {
-			log.Logger.Error(err, "Error running semanage command")
-		}
-
-		// Check if the restorecon command exists when running on RHEL/CoreOS
-		// restorecon applies the SELinux settings we just created to the host
-		// And we are already chrooted to the host path, so we can just look for restoreconPath now
-		_, err = exec.LookPath(restoreconPath)
-		if err == nil {
-			// Run the restorecon command to apply the new context
-			cmd := exec.Command(restoreconPath, "-r", k8sconsts.OdigosAgentsDirectory)
-			err = cmd.Run()
-			if err != nil {
-				log.Logger.Error(err, "Error running restorecon command")
-			}
-		} else {
-			log.Logger.Error(err, "Unable to find restorecon path")
-		}
-
-	} else {
-		log.Logger.Error(err, "Unable to find semanage path")
-	}
-
 	return nil
 }
 
@@ -173,3 +142,42 @@ func calculateFileHash(filePath string) (string, error) {
 
 	return fmt.Sprintf("%x", hasher.Sum(nil)), nil
 }
+
+// ApplyOpenShiftSELinuxSettings makes auto-instrumentation agents readable by containers on RHEL hosts.
+// Note: This function calls chroot to use the host's PATH to execute selinux commands. Calling it will
+// affect the odiglet running process's apparent filesystem.
+func ApplyOpenShiftSELinuxSettings() error {
+	// Check if the semanage command exists when running on RHEL/CoreOS
+	_, err := exec.LookPath(filepath.Join(chrootDir, semanagePath))
+	if err == nil {
+		// Run the semanage command to add the new directory to the container_ro_file_t context
+		// semanage writes SELinux config to host
+		syscall.Chroot(chrootDir)
+		cmd := exec.Command(semanagePath, "fcontext", "-a", "-t", "container_ro_file_t", "/var/odigos(/.*)?")
+		err = cmd.Run()
+		if err != nil {
+			log.Logger.Error(err, "Error running semanage command")
+			return err
+		}
+
+		// Check if the restorecon command exists when running on RHEL/CoreOS
+		// restorecon applies the SELinux settings we just created to the host
+		// And we are already chrooted to the host path, so we can just look for restoreconPath now
+		_, err = exec.LookPath(restoreconPath)
+		if err == nil {
+			// Run the restorecon command to apply the new context
+			cmd := exec.Command(restoreconPath, "-r", k8sconsts.OdigosAgentsDirectory)
+			err = cmd.Run()
+			if err != nil {
+				log.Logger.Error(err, "Error running restorecon command")
+				return err
+			}
+		} else {
+			log.Logger.Error(err, "Unable to find restorecon path")
+			return err
+		}
+	} else {
+		log.Logger.Info("Unable to find semanage path, possibly not on RHEL host")
+	}
+	return nil
+}