fix: enable GCP deployments (#1474)

Author: julienmancuso

deploy/cloud/operator/internal/consts/consts.go

@@ -69,4 +69,7 @@ const (
 	KubeAnnotationDynamoComponentStorageNS       = "nvidia.com/dynamo-storage-namespace"
 
 	DynamoDeploymentConfigEnvVar = "DYN_DEPLOYMENT_CONFIG"
+
+	DockerConfigVolumeName      = "docker-config"
+	DockerConfigVolumeMountPath = "/docker-config/.docker"
 )

deploy/cloud/operator/internal/controller/common.go

@@ -67,3 +67,10 @@ func getIngressHost(ingressSpec v1alpha1.IngressSpec) string {
 func getK8sName(value string) string {
 	return strings.ReplaceAll(value, ":", "--")
 }
+
+func isGoogleRegistry(host string) bool {
+	return host == "gcr.io" ||
+		strings.HasSuffix(host, ".gcr.io") ||
+		strings.HasSuffix(host, ".pkg.dev") ||
+		strings.HasSuffix(host, ".google.com")
+}

deploy/cloud/operator/internal/controller/dynamocomponent_controller.go

@@ -852,11 +852,15 @@ func (r *DynamoComponentReconciler) generateImageBuilderPodTemplateSpec(ctx cont
 			Name:      "workspace",
 			MountPath: "/workspace",
 		},
+		{
+			Name:      consts.DockerConfigVolumeName,
+			MountPath: consts.DockerConfigVolumeMountPath,
+		},
 	}
 
 	if dockerConfigJSONSecretName != "" {
 		volumes = append(volumes, corev1.Volume{
-			Name: dockerConfigJSONSecretName,
+			Name: consts.DockerConfigVolumeName,
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
 					SecretName: dockerConfigJSONSecretName,
@@ -869,9 +873,12 @@ func (r *DynamoComponentReconciler) generateImageBuilderPodTemplateSpec(ctx cont
 				},
 			},
 		})
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      dockerConfigJSONSecretName,
-			MountPath: "/kaniko/.docker/",
+	} else {
+		volumes = append(volumes, corev1.Volume{
+			Name: consts.DockerConfigVolumeName,
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
 		})
 	}
 
@@ -921,8 +928,6 @@ func (r *DynamoComponentReconciler) generateImageBuilderPodTemplateSpec(ctx cont
 
 	buildEngine := getDynamoComponentImageBuildEngine()
 
-	privileged := buildEngine != DynamoComponentImageBuildEngineBuildkitRootless
-
 	dynamoComponentDownloadCommandTemplate, err := template.New("downloadCommand").Parse(`
 set -e
 
@@ -943,10 +948,6 @@ echo "Extracting dynamoComponent tar file..."
 tar -xvf /tmp/downloaded.tar
 echo "Removing dynamoComponent tar file..."
 rm /tmp/downloaded.tar
-{{if not .Privileged}}
-echo "Changing directory permission..."
-chown -R 1000:1000 /workspace
-{{end}}
 echo "Done"
 	`)
 
@@ -961,7 +962,6 @@ echo "Done"
 		"DynamoComponentDownloadURL":    dynamoComponentDownloadURL,
 		"DynamoComponentRepositoryName": dynamoComponentRepositoryName,
 		"DynamoComponentVersion":        dynamoComponentVersion,
-		"Privileged":                    privileged,
 	})
 	if err != nil {
 		err = errors.Wrap(err, "failed to execute download command template")
@@ -1004,6 +1004,44 @@ echo "Done"
 		},
 	}
 
+	if dockerConfigJSONSecretName == "" {
+		// if no explicit docker config is provided, we need to provide the docker config to the image builder
+		var ref name.Reference
+		ref, err = name.ParseReference(imageName)
+		if err != nil {
+			err = errors.Wrap(err, "failed to parse reference")
+			return
+		}
+		dockerRegistry := ref.Context().RegistryStr()
+		if isGoogleRegistry(dockerRegistry) {
+			// for GCP, we use the google cloud sdk to get the docker config.
+			initContainers = append(initContainers, corev1.Container{
+				Name:  "gcp-init-docker-config",
+				Image: "google/cloud-sdk:slim",
+				Command: []string{
+					"/bin/bash",
+					"-c",
+					fmt.Sprintf(`set -e
+gcloud --quiet config get-value account
+TOKEN=$(gcloud --quiet auth print-access-token)
+cat > %s/config.json <<EOL
+{
+	"auths": {
+		"%s": {
+			"auth": "$(echo -n "oauth2accesstoken:${TOKEN}" | base64 -w 0)"
+		}
+	}
+}
+EOL
+echo 'Docker config.json created successfully'`, consts.DockerConfigVolumeMountPath, dockerRegistry),
+				},
+				Resources:    downloaderContainerResources,
+				EnvFrom:      downloaderContainerEnvFrom,
+				VolumeMounts: volumeMounts,
+			})
+		}
+	}
+
 	containers := make([]corev1.Container, 0)
 
 	var globalExtraPodMetadata *dynamoCommon.ExtraPodMetadata
@@ -1111,13 +1149,10 @@ echo "Done"
 			Name:  "IFS",
 			Value: "''",
 		},
-	}
-
-	if dockerConfigJSONSecretName != "" {
-		builderContainerEnvs = append(builderContainerEnvs, corev1.EnvVar{
+		{
 			Name:  "DOCKER_CONFIG",
-			Value: "/kaniko/.docker/",
-		})
+			Value: consts.DockerConfigVolumeMountPath,
+		},
 	}
 
 	kanikoCacheRepo := os.Getenv("KANIKO_CACHE_REPO")
@@ -1174,9 +1209,6 @@ echo "Done"
 	if isBuildkit {
 		output := fmt.Sprintf("type=image,name=%s,push=true,registry.insecure=%v", imageName, dockerRegistryInsecure)
 		buildkitdFlags := []string{}
-		if !privileged {
-			buildkitdFlags = append(buildkitdFlags, "--oci-worker-no-process-sandbox")
-		}
 		if isEstargzEnabled() {
 			buildkitdFlags = append(buildkitdFlags, "--oci-worker-snapshotter=stargz")
 			output += ",oci-mediatypes=true,compression=estargz,force-compression=true"
@@ -1215,23 +1247,6 @@ echo "Done"
 		}
 	}
 
-	var builderContainerSecurityContext *corev1.SecurityContext
-
-	if buildEngine == DynamoComponentImageBuildEngineBuildkit {
-		builderContainerSecurityContext = &corev1.SecurityContext{
-			Privileged: ptr.To(true),
-		}
-	} else if buildEngine == DynamoComponentImageBuildEngineBuildkitRootless {
-		kubeAnnotations["container.apparmor.security.beta.kubernetes.io/builder"] = "unconfined"
-		builderContainerSecurityContext = &corev1.SecurityContext{
-			SeccompProfile: &corev1.SeccompProfile{
-				Type: corev1.SeccompProfileTypeUnconfined,
-			},
-			RunAsUser:  ptr.To(int64(1000)),
-			RunAsGroup: ptr.To(int64(1000)),
-		}
-	}
-
 	// add build args to pass via --build-arg
 	for _, buildArg := range buildArgs {
 		quotedBuildArg := unix.SingleQuote.Quote(buildArg)
@@ -1261,7 +1276,13 @@ echo "Done"
 		EnvFrom:         builderContainerEnvFrom,
 		TTY:             true,
 		Stdin:           true,
-		SecurityContext: builderContainerSecurityContext,
+	}
+
+	if buildEngine == DynamoComponentImageBuildEngineKaniko {
+		// we need to run as root when using kaniko
+		container.SecurityContext = &corev1.SecurityContext{
+			RunAsUser: ptr.To(int64(0)),
+		}
 	}
 
 	if globalDefaultImageBuilderContainerResources != nil {
@@ -1284,6 +1305,11 @@ echo "Done"
 			Volumes:        volumes,
 			InitContainers: initContainers,
 			Containers:     containers,
+			SecurityContext: &corev1.PodSecurityContext{
+				RunAsUser:  ptr.To(int64(1000)),
+				RunAsGroup: ptr.To(int64(1000)),
+				FSGroup:    ptr.To(int64(1000)),
+			},
 		},
 	}
 

deploy/cloud/operator/internal/controller/dynamocomponentdeployment_controller.go

@@ -1000,21 +1000,23 @@ func (r *DynamoComponentDeploymentReconciler) createOrUpdateOrDeleteServices(ctx
 	return
 }
 
-func (r *DynamoComponentDeploymentReconciler) createOrUpdateOrDeleteIngress(ctx context.Context, opt generateResourceOption) (modified bool, err error) {
-	modified, _, err = commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*networkingv1.Ingress, bool, error) {
+func (r *DynamoComponentDeploymentReconciler) createOrUpdateOrDeleteIngress(ctx context.Context, opt generateResourceOption) (bool, error) {
+	modified, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*networkingv1.Ingress, bool, error) {
 		return r.generateIngress(ctx, opt)
 	})
 	if err != nil {
-		return
+		return false, err
 	}
-	modified_, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*networkingv1beta1.VirtualService, bool, error) {
-		return r.generateVirtualService(ctx, opt)
-	})
-	if err != nil {
-		return
+	if r.UseVirtualService {
+		modified_, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*networkingv1beta1.VirtualService, bool, error) {
+			return r.generateVirtualService(ctx, opt)
+		})
+		if err != nil {
+			return false, err
+		}
+		return modified || modified_, nil
 	}
-	modified = modified || modified_
-	return
+	return modified, nil
 }
 
 func (r *DynamoComponentDeploymentReconciler) generateIngress(ctx context.Context, opt generateResourceOption) (*networkingv1.Ingress, bool, error) {

docs/guides/dynamo_deploy/dynamo_cloud.md

@@ -100,6 +100,12 @@ kubectl get storageclass
 # standard (default)   kubernetes.io/gce-pd    Delete          Immediate              true                   1d
 ```
 
+### Cloud Provider-Specific deployment
+
+#### Google Kubernetes Engine (GKE) deployment
+
+You can find detailed instructions for deployment in GKE [here](../dynamo_deploy/gke_setup.md)
+
 ### Installation
 
 1. Set the required environment variables: