feat: support auth from docker config

.circleci/config.yml

@@ -765,9 +765,9 @@ jobs:
               false
             fi
       - run:
-          name: "Verify Kurtosis cleaned up all its volumes (except for the log storage and github auth storage volumes)"
+          name: "Verify Kurtosis cleaned up all its volumes (except for the log storage, github auth storage, docker auth config volumes)"
           command: |
-            if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l ) -eq 2 ]; then
+            if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l ) -eq 3 ]; then
               docker volume ls
               false
             fi
@@ -1004,9 +1004,9 @@ jobs:
                     false
                   fi
             - run:
-                name: "Verify Kurtosis cleaned up all its volumes (except for the log storage and github auth storage volumes)"
+                name: "Verify Kurtosis cleaned up all its volumes (except for the log storage, github auth storage, docker auth config volumes)"
                 command: |
-                  if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l) -eq 2 ]; then
+                  if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l) -eq 3 ]; then
                     docker volume ls
                     false
                   fi

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go

@@ -3,11 +3,12 @@ package backend_creator
 import (
 	"context"
 	"fmt"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db"
 	"net"
 	"os"
 	"path"
 
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db"
+
 	"github.com/docker/docker/client"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions"

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go

@@ -32,7 +32,8 @@ const (
 	NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge"
 	HttpApplicationProtocol                             = "http"
 
-	GitHubAuthStorageDirPath = "/kurtosis-data/github-auth/"
+	GitHubAuthStorageDirPath   = "/kurtosis-data/github-auth/"
+	DockerConfigStorageDirPath = "/root/.docker/"
 
 	EmptyApplicationURL = ""
 )

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go

@@ -618,3 +618,22 @@ func (backend *DockerKurtosisBackend) getGitHubAuthStorageVolume(ctx context.Con
 	volume := foundVolumes[0]
 	return volume.Name, nil
 }
+
+// Guaranteed to either return a Docker config storage volume name or throw an error
+func (backend *DockerKurtosisBackend) getDockerConfigStorageVolume(ctx context.Context) (string, error) {
+	volumeSearchLabels := map[string]string{
+		docker_label_key.VolumeTypeDockerLabelKey.GetString(): label_value_consts.DockerConfigStorageVolumeTypeDockerLabelValue.GetString(),
+	}
+	foundVolumes, err := backend.dockerManager.GetVolumesByLabels(ctx, volumeSearchLabels)
+	if err != nil {
+		return "", stacktrace.Propagate(err, "An error occurred getting Docker config storage volumes matching labels '%+v'", volumeSearchLabels)
+	}
+	if len(foundVolumes) > 1 {
+		return "", stacktrace.NewError("Found multiple Docker config storage volumes. This should never happen")
+	}
+	if len(foundVolumes) == 0 {
+		return "", stacktrace.NewError("No Docker config storage volume found.")
+	}
+	volume := foundVolumes[0]
+	return volume.Name, nil
+}

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go

@@ -3,10 +3,11 @@ package docker_kurtosis_backend
 import (
 	"context"
 	"encoding/json"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec"
 	"net"
 	"time"
 
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec"
+
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key"
 
 	"github.com/docker/go-connections/nat"
@@ -81,6 +82,11 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer(
 		return nil, stacktrace.Propagate(err, "An error occurred getting the GitHub auth storage volume name.")
 	}
 
+	dockerConfigStorageVolumeName, err := backend.getDockerConfigStorageVolume(ctx)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred getting the Docker config storage volume name.")
+	}
+
 	// Get the Docker network ID where we'll start the new API container
 	enclaveNetwork, err := backend.getEnclaveNetworkByEnclaveUuid(ctx, enclaveUuid)
 	if err != nil {
@@ -191,8 +197,9 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer(
 	}
 
 	volumeMounts := map[string]string{
-		enclaveDataVolumeName:       enclaveDataVolumeDirpath,
-		githubAuthStorageVolumeName: consts.GitHubAuthStorageDirPath,
+		enclaveDataVolumeName:         enclaveDataVolumeDirpath,
+		githubAuthStorageVolumeName:   consts.GitHubAuthStorageDirPath,
+		dockerConfigStorageVolumeName: consts.DockerConfigStorageDirPath,
 	}
 
 	labelStrs := map[string]string{}

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go

@@ -3,9 +3,11 @@ package engine_functions
 import (
 	"context"
 	"fmt"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/github_auth_storage_creator"
 	"time"
 
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/docker_config_storage_creator"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/github_auth_storage_creator"
+
 	"github.com/docker/go-connections/nat"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions"
@@ -249,14 +251,34 @@ func CreateEngine(
 		return nil, stacktrace.Propagate(err, "An error occurred creating GitHub auth storage.")
 	}
 
+	// Configure Docker Config by writing the provided config files to a volume that's accessible by the engine
+	dockerConfigStorageVolObjAttrs, err := objAttrsProvider.ForDockerConfigStorageVolume()
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred retrieving object attributes for GitHub auth storage.")
+	}
+	dockerConfigStorageVolNameStr := dockerConfigStorageVolObjAttrs.GetName().GetString()
+	dockerConfigStorageVolLabelStrs := map[string]string{}
+	for labelKey, labelValue := range dockerConfigStorageVolObjAttrs.GetLabels() {
+		dockerConfigStorageVolLabelStrs[labelKey.GetString()] = labelValue.GetString()
+	}
+	// This volume is created idempotently (like logs storage volume) and just write the token to the file everytime the engine starts
+	if err = dockerManager.CreateVolume(ctx, dockerConfigStorageVolNameStr, dockerConfigStorageVolLabelStrs); err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred creating Docker config storage volume.")
+	}
+	err = docker_config_storage_creator.CreateDockerConfigStorage(ctx, targetNetworkId, dockerConfigStorageVolNameStr, consts.DockerConfigStorageDirPath, dockerManager)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred creating Docker config storage.")
+	}
+
 	bindMounts := map[string]string{
 		// Necessary so that the engine server can interact with the Docker engine
 		consts.DockerSocketFilepath: consts.DockerSocketFilepath,
 	}
 
 	volumeMounts := map[string]string{
-		logsStorageVolNameStr:       logsStorageDirPath,
-		githubAuthStorageVolNameStr: consts.GitHubAuthStorageDirPath,
+		logsStorageVolNameStr:         logsStorageDirPath,
+		githubAuthStorageVolNameStr:   consts.GitHubAuthStorageDirPath,
+		dockerConfigStorageVolNameStr: consts.DockerConfigStorageDirPath,
 	}
 
 	if serverArgs.OnBastionHost {

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/docker_config_storage_creator/docker_config_storage_creator.go

@@ -0,0 +1,175 @@
+package docker_config_storage_creator
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/docker/docker/api/types/registry"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager"
+	"github.com/kurtosis-tech/stacktrace"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// We use this image and version because we already are using this in other projects so there is a high probability
+	// that the image is in the local machine's cache
+	creatorContainerImage = "alpine:3.17"
+	creatorContainerName  = "kurtosis-docker-config-storage-creator"
+
+	shBinaryFilepath = "/bin/sh"
+	shCmdFlag        = "-c"
+	printfCmdName    = "printf"
+
+	creationSuccessExitCode = 0
+
+	creationCmdMaxRetries     = 2
+	creationCmdDelayInRetries = 200 * time.Millisecond
+
+	configFilePath = "config.json"
+
+	sleepSeconds = 1800
+)
+
+func CreateDockerConfigStorage(
+	ctx context.Context,
+	targetNetworkId string,
+	volumeName string,
+	storageDirPath string,
+	dockerManager *docker_manager.DockerManager,
+) error {
+	entrypointArgs := []string{
+		shBinaryFilepath,
+		shCmdFlag,
+		fmt.Sprintf("sleep %v", sleepSeconds),
+	}
+
+	volumeMounts := map[string]string{
+		volumeName: storageDirPath,
+	}
+
+	createAndStartArgs := docker_manager.NewCreateAndStartContainerArgsBuilder(
+		creatorContainerImage,
+		creatorContainerName,
+		targetNetworkId,
+	).WithEntrypointArgs(
+		entrypointArgs,
+	).WithVolumeMounts(
+		volumeMounts,
+	).Build()
+
+	containerId, _, err := dockerManager.CreateAndStartContainer(ctx, createAndStartArgs)
+	if err != nil {
+		return stacktrace.Propagate(err, "An error occurred starting the Docker Config Storage Creator container with these args '%+v'", createAndStartArgs)
+	}
+	//The killing step has to be executed always in the success and also in the failed case
+	defer func() {
+		if err = dockerManager.RemoveContainer(context.Background(), containerId); err != nil {
+			logrus.Errorf(
+				"Launching the Docker Config Creator container with container ID '%v' didn't complete successfully so we "+
+					"tried to remove the container we started, but doing so exited with an error:\n%v",
+				containerId,
+				err)
+			logrus.Errorf("ACTION REQUIRED: You'll need to manually remove the container with ID '%v'!!!!!!", containerId)
+		}
+	}()
+
+	if err := storeConfigInVolume(
+		ctx,
+		dockerManager,
+		containerId,
+		creationCmdMaxRetries,
+		creationCmdDelayInRetries,
+		storageDirPath,
+	); err != nil {
+		return stacktrace.Propagate(err, "An error occurred creating  Docker config storage in volume.")
+	}
+
+	return nil
+}
+
+func storeConfigInVolume(
+	ctx context.Context,
+	dockerManager *docker_manager.DockerManager,
+	containerId string,
+	maxRetries uint,
+	timeBetweenRetries time.Duration,
+	storageDirPath string,
+) error {
+	// Get all the registries from the Docker config
+	registries, err := docker_manager.GetAllRegistriesFromDockerConfig()
+	if err != nil {
+		return stacktrace.NewError("An error occurred getting all registries from Docker config: %v", err)
+	}
+
+	cfg := struct {
+		Auths map[string]registry.AuthConfig `json:"auths"`
+	}{
+		Auths: make(map[string]registry.AuthConfig),
+	}
+
+	// Add the auths for each registry
+	for _, registry := range registries {
+		creds, err := docker_manager.GetAuthFromDockerConfig(registry)
+		if err != nil {
+			return stacktrace.NewError("An error occurred getting auth for registry '%v' from Docker config: %v", registry, err)
+		}
+		cfg.Auths[registry] = *creds
+	}
+
+	cfgJsonStr, err := json.Marshal(cfg)
+	if err != nil {
+		return stacktrace.NewError("An error occurred marshalling the Docker config into JSON: %v", err)
+	}
+
+	// Write the config.json to the volume
+	commandStr := fmt.Sprintf(
+		"%v '%v' > %v",
+		printfCmdName,
+		string(cfgJsonStr),
+		fmt.Sprintf("%s/%s", storageDirPath, configFilePath),
+	)
+
+	execCmd := []string{
+		shBinaryFilepath,
+		shCmdFlag,
+		commandStr,
+	}
+	for i := uint(0); i < maxRetries; i++ {
+		outputBuffer := &bytes.Buffer{}
+		exitCode, err := dockerManager.RunExecCommand(ctx, containerId, execCmd, outputBuffer)
+		if err == nil {
+			if exitCode == creationSuccessExitCode {
+				logrus.Debugf("The Docker config file was successfully added into the volume.")
+				return nil
+			}
+			logrus.Debugf(
+				"Docker config storage creation command '%v' returned without a Docker error, but exited with non-%v exit code '%v' and logs:\n%v",
+				commandStr,
+				creationSuccessExitCode,
+				exitCode,
+				outputBuffer.String(),
+			)
+		} else {
+			logrus.Debugf(
+				"Docker config storage creation command '%v' experienced a Docker error:\n%v",
+				commandStr,
+				err,
+			)
+		}
+
+		// Tiny optimization to not sleep if we're not going to run the loop again
+		if i < maxRetries {
+			time.Sleep(timeBetweenRetries)
+		}
+	}
+
+	return stacktrace.NewError(
+		"The Docker config storage creation didn't return success (as measured by the command '%v') even after retrying %v times with %v between retries",
+		commandStr,
+		maxRetries,
+		timeBetweenRetries,
+	)
+}