From bb8f9f7fffdba226506c282d38eeeff2c877daa4 Mon Sep 17 00:00:00 2001
From: "Jonathan A. Sternberg" <jonathan.sternberg@docker.com>
Date: Mon, 23 Oct 2023 13:57:35 -0500
Subject: [PATCH] buildkitd: use default config file location when run as root
 in rootless

When running buildkitd in a rootless environment but as root, buildkitd
misinterprets how it is being run and loads the config file from the
home directory instead of the global location in `/etc/buildkitd`.

This happens when buildkit is being run in docker and docker itself is
being run as rootless. The buildkit daemon sees the user remapping and
assumes that the remapping belongs to itself rather than to docker.

This causes it to load the wrong configuration file as buildkitd is
still being run as "root" but a remapped root that docker created.

Signed-off-by: Jonathan A. Sternberg <jonathan.sternberg@docker.com>
---
 cmd/buildkitd/main.go | 44 +++++++++++++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 14 deletions(-)

diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go
index 3375c438a18c..f80c62d9a399 100644
--- a/cmd/buildkitd/main.go
+++ b/cmd/buildkitd/main.go
@@ -12,6 +12,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/containerd/containerd/pkg/seed" //nolint:staticcheck // SA1019 deprecated
 	"github.com/containerd/containerd/pkg/userns"
@@ -412,7 +413,7 @@ func serveGRPC(cfg config.GRPCConfig, server *grpc.Server, errCh chan error) err
 }
 
 func defaultConfigPath() string {
-	if userns.RunningInUserNS() {
+	if isRootlessConfig() {
 		return filepath.Join(appdefaults.UserConfigDir(), "buildkitd.toml")
 	}
 	return filepath.Join(appdefaults.ConfigDir, "buildkitd.toml")
@@ -466,26 +467,41 @@ func setDefaultConfig(cfg *config.Config) {
 	cfg.Workers.OCI.NetworkConfig = setDefaultNetworkConfig(cfg.Workers.OCI.NetworkConfig)
 	cfg.Workers.Containerd.NetworkConfig = setDefaultNetworkConfig(cfg.Workers.Containerd.NetworkConfig)
 
-	if userns.RunningInUserNS() {
-		// if buildkitd is being executed as the mapped-root (not only EUID==0 but also $USER==root)
-		// in a user namespace, we need to enable the rootless mode but
-		// we don't want to honor $HOME for setting up default paths.
-		if u := os.Getenv("USER"); u != "" && u != "root" {
-			if orig.Root == "" {
-				cfg.Root = appdefaults.UserRoot()
-			}
-			if len(orig.GRPC.Address) == 0 {
-				cfg.GRPC.Address = []string{appdefaults.UserAddress()}
-			}
-			appdefaults.EnsureUserAddressDir()
+	if isRootlessConfig() {
+		if orig.Root == "" {
+			cfg.Root = appdefaults.UserRoot()
+		}
+		if len(orig.GRPC.Address) == 0 {
+			cfg.GRPC.Address = []string{appdefaults.UserAddress()}
 		}
+		appdefaults.EnsureUserAddressDir()
 	}
 
 	if cfg.OTEL.SocketPath == "" {
-		cfg.OTEL.SocketPath = appdefaults.TraceSocketPath(userns.RunningInUserNS())
+		cfg.OTEL.SocketPath = appdefaults.TraceSocketPath(isRootlessConfig())
 	}
 }
 
+var isRootlessConfigOnce sync.Once
+var isRootlessConfigValue bool
+
+// isRootlessConfig is true if we should be using the rootless config
+// defaults instead of the normal defaults.
+func isRootlessConfig() bool {
+	isRootlessConfigOnce.Do(func() {
+		if !userns.RunningInUserNS() {
+			// Default value is false so keep it that way.
+			return
+		}
+		// if buildkitd is being executed as the mapped-root (not only EUID==0 but also $USER==root)
+		// in a user namespace, we don't want to load the rootless changes in the
+		// configuration.
+		u := os.Getenv("USER")
+		isRootlessConfigValue = u != "" && u != "root"
+	})
+	return isRootlessConfigValue
+}
+
 func applyMainFlags(c *cli.Context, cfg *config.Config) error {
 	if c.IsSet("debug") {
 		cfg.Debug = c.Bool("debug")