Skip to content

Commit 3184488

Browse files
julienmancusojulienmancuso
committed
fix: enable GCP deployments
1 parent e8e728b commit 3184488

File tree

4 files changed

+73
-48
lines changed

4 files changed

+73
-48
lines changed

‎deploy/cloud/operator/internal/consts/consts.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,7 @@ const (
6969
KubeAnnotationDynamoComponentStorageNS = "nvidia.com/dynamo-storage-namespace"
7070

7171
DynamoDeploymentConfigEnvVar = "DYN_DEPLOYMENT_CONFIG"
72+
73+
DockerConfigVolumeName = "docker-config"
74+
DockerConfigVolumeMountPath = "/docker-config/.docker"
7275
)

‎deploy/cloud/operator/internal/controller/common.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,3 +67,10 @@ func getIngressHost(ingressSpec v1alpha1.IngressSpec) string {
6767
func getK8sName(value string) string {
6868
return strings.ReplaceAll(value, ":", "--")
6969
}
70+
71+
func isGoogleRegistry(host string) bool {
72+
return host == "gcr.io" ||
73+
strings.HasSuffix(host, ".gcr.io") ||
74+
strings.HasSuffix(host, ".pkg.dev") ||
75+
strings.HasSuffix(host, ".google.com")
76+
}

‎deploy/cloud/operator/internal/controller/dynamocomponent_controller.go‎

Lines changed: 51 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -852,11 +852,15 @@ func (r *DynamoComponentReconciler) generateImageBuilderPodTemplateSpec(ctx cont
852852
Name: "workspace",
853853
MountPath: "/workspace",
854854
},
855+
{
856+
Name: consts.DockerConfigVolumeName,
857+
MountPath: consts.DockerConfigVolumeMountPath,
858+
},
855859
}
856860

857861
if dockerConfigJSONSecretName != "" {
858862
volumes = append(volumes, corev1.Volume{
859-
Name: dockerConfigJSONSecretName,
863+
Name: consts.DockerConfigVolumeName,
860864
VolumeSource: corev1.VolumeSource{
861865
Secret: &corev1.SecretVolumeSource{
862866
SecretName: dockerConfigJSONSecretName,
@@ -869,9 +873,12 @@ func (r *DynamoComponentReconciler) generateImageBuilderPodTemplateSpec(ctx cont
869873
},
870874
},
871875
})
872-
volumeMounts = append(volumeMounts, corev1.VolumeMount{
873-
Name: dockerConfigJSONSecretName,
874-
MountPath: "/kaniko/.docker/",
876+
} else {
877+
volumes = append(volumes, corev1.Volume{
878+
Name: consts.DockerConfigVolumeName,
879+
VolumeSource: corev1.VolumeSource{
880+
EmptyDir: &corev1.EmptyDirVolumeSource{},
881+
},
875882
})
876883
}
877884

@@ -921,8 +928,6 @@ func (r *DynamoComponentReconciler) generateImageBuilderPodTemplateSpec(ctx cont
921928

922929
buildEngine := getDynamoComponentImageBuildEngine()
923930

924-
privileged := buildEngine != DynamoComponentImageBuildEngineBuildkitRootless
925-
926931
dynamoComponentDownloadCommandTemplate, err := template.New("downloadCommand").Parse(`
927932
set -e
928933
@@ -943,10 +948,6 @@ echo "Extracting dynamoComponent tar file..."
943948
tar -xvf /tmp/downloaded.tar
944949
echo "Removing dynamoComponent tar file..."
945950
rm /tmp/downloaded.tar
946-
{{if not .Privileged}}
947-
echo "Changing directory permission..."
948-
chown -R 1000:1000 /workspace
949-
{{end}}
950951
echo "Done"
951952
`)
952953

@@ -961,7 +962,6 @@ echo "Done"
961962
"DynamoComponentDownloadURL": dynamoComponentDownloadURL,
962963
"DynamoComponentRepositoryName": dynamoComponentRepositoryName,
963964
"DynamoComponentVersion": dynamoComponentVersion,
964-
"Privileged": privileged,
965965
})
966966
if err != nil {
967967
err = errors.Wrap(err, "failed to execute download command template")
@@ -1004,6 +1004,38 @@ echo "Done"
10041004
},
10051005
}
10061006

1007+
if dockerConfigJSONSecretName == "" {
1008+
// if no explicit docker config is provided, we need to provide the docker config to the image builder
1009+
dockerRegistry := strings.Split(imageName, "/")[0]
1010+
if isGoogleRegistry(dockerRegistry) {
1011+
// for GCP, we use the google cloud sdk to get the docker config.
1012+
initContainers = append(initContainers, corev1.Container{
1013+
Name: "gcp-init-docker-config",
1014+
Image: "google/cloud-sdk:slim",
1015+
Command: []string{
1016+
"/bin/bash",
1017+
"-c",
1018+
fmt.Sprintf(`
1019+
TOKEN=$(gcloud auth print-access-token)
1020+
cat > %s/config.json <<EOL
1021+
{
1022+
"auths": {
1023+
"%s": {
1024+
"auth": "$(echo -n "oauth2accesstoken:${TOKEN}" | base64 -w 0)"
1025+
}
1026+
}
1027+
}
1028+
EOL
1029+
echo 'Docker config.json created successfully'
1030+
`, consts.DockerConfigVolumeMountPath, dockerRegistry),
1031+
},
1032+
Resources: downloaderContainerResources,
1033+
EnvFrom: downloaderContainerEnvFrom,
1034+
VolumeMounts: volumeMounts,
1035+
})
1036+
}
1037+
}
1038+
10071039
containers := make([]corev1.Container, 0)
10081040

10091041
var globalExtraPodMetadata *dynamoCommon.ExtraPodMetadata
@@ -1111,13 +1143,10 @@ echo "Done"
11111143
Name: "IFS",
11121144
Value: "''",
11131145
},
1114-
}
1115-
1116-
if dockerConfigJSONSecretName != "" {
1117-
builderContainerEnvs = append(builderContainerEnvs, corev1.EnvVar{
1146+
{
11181147
Name: "DOCKER_CONFIG",
1119-
Value: "/kaniko/.docker/",
1120-
})
1148+
Value: consts.DockerConfigVolumeMountPath,
1149+
},
11211150
}
11221151

11231152
kanikoCacheRepo := os.Getenv("KANIKO_CACHE_REPO")
@@ -1174,9 +1203,6 @@ echo "Done"
11741203
if isBuildkit {
11751204
output := fmt.Sprintf("type=image,name=%s,push=true,registry.insecure=%v", imageName, dockerRegistryInsecure)
11761205
buildkitdFlags := []string{}
1177-
if !privileged {
1178-
buildkitdFlags = append(buildkitdFlags, "--oci-worker-no-process-sandbox")
1179-
}
11801206
if isEstargzEnabled() {
11811207
buildkitdFlags = append(buildkitdFlags, "--oci-worker-snapshotter=stargz")
11821208
output += ",oci-mediatypes=true,compression=estargz,force-compression=true"
@@ -1215,23 +1241,6 @@ echo "Done"
12151241
}
12161242
}
12171243

1218-
var builderContainerSecurityContext *corev1.SecurityContext
1219-
1220-
if buildEngine == DynamoComponentImageBuildEngineBuildkit {
1221-
builderContainerSecurityContext = &corev1.SecurityContext{
1222-
Privileged: ptr.To(true),
1223-
}
1224-
} else if buildEngine == DynamoComponentImageBuildEngineBuildkitRootless {
1225-
kubeAnnotations["container.apparmor.security.beta.kubernetes.io/builder"] = "unconfined"
1226-
builderContainerSecurityContext = &corev1.SecurityContext{
1227-
SeccompProfile: &corev1.SeccompProfile{
1228-
Type: corev1.SeccompProfileTypeUnconfined,
1229-
},
1230-
RunAsUser: ptr.To(int64(1000)),
1231-
RunAsGroup: ptr.To(int64(1000)),
1232-
}
1233-
}
1234-
12351244
// add build args to pass via --build-arg
12361245
for _, buildArg := range buildArgs {
12371246
quotedBuildArg := unix.SingleQuote.Quote(buildArg)
@@ -1261,7 +1270,6 @@ echo "Done"
12611270
EnvFrom: builderContainerEnvFrom,
12621271
TTY: true,
12631272
Stdin: true,
1264-
SecurityContext: builderContainerSecurityContext,
12651273
}
12661274

12671275
if globalDefaultImageBuilderContainerResources != nil {
@@ -1284,6 +1292,11 @@ echo "Done"
12841292
Volumes: volumes,
12851293
InitContainers: initContainers,
12861294
Containers: containers,
1295+
SecurityContext: &corev1.PodSecurityContext{
1296+
RunAsUser: ptr.To(int64(1000)),
1297+
RunAsGroup: ptr.To(int64(1000)),
1298+
FSGroup: ptr.To(int64(1000)),
1299+
},
12871300
},
12881301
}
12891302

‎deploy/cloud/operator/internal/controller/dynamocomponentdeployment_controller.go‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1000,21 +1000,23 @@ func (r *DynamoComponentDeploymentReconciler) createOrUpdateOrDeleteServices(ctx
10001000
return
10011001
}
10021002

1003-
func (r *DynamoComponentDeploymentReconciler) createOrUpdateOrDeleteIngress(ctx context.Context, opt generateResourceOption) (modified bool, err error) {
1004-
modified, _, err = commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*networkingv1.Ingress, bool, error) {
1003+
func (r *DynamoComponentDeploymentReconciler) createOrUpdateOrDeleteIngress(ctx context.Context, opt generateResourceOption) (bool, error) {
1004+
modified, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*networkingv1.Ingress, bool, error) {
10051005
return r.generateIngress(ctx, opt)
10061006
})
10071007
if err != nil {
1008-
return
1008+
return false, err
10091009
}
1010-
modified_, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*networkingv1beta1.VirtualService, bool, error) {
1011-
return r.generateVirtualService(ctx, opt)
1012-
})
1013-
if err != nil {
1014-
return
1010+
if r.UseVirtualService {
1011+
modified_, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*networkingv1beta1.VirtualService, bool, error) {
1012+
return r.generateVirtualService(ctx, opt)
1013+
})
1014+
if err != nil {
1015+
return false, err
1016+
}
1017+
return modified || modified_, nil
10151018
}
1016-
modified = modified || modified_
1017-
return
1019+
return modified, nil
10181020
}
10191021

10201022
func (r *DynamoComponentDeploymentReconciler) generateIngress(ctx context.Context, opt generateResourceOption) (*networkingv1.Ingress, bool, error) {

0 commit comments

Comments
 (0)