diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesPersistentVolumeExtensions.cs b/src/Aspire.Hosting.Kubernetes/KubernetesPersistentVolumeExtensions.cs index a9031feb6f5..41d1e878641 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesPersistentVolumeExtensions.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesPersistentVolumeExtensions.cs @@ -236,7 +236,12 @@ public static IResourceBuilder WithVolumeAnn /// /// To bind a workload that does not already have a matching named mount (for /// example a ProjectResource), use the overload that accepts a - /// mountPath instead. + /// mountPath instead. The generated pod uses an Aspire-managed + /// fsGroup of 2000 with an OnRootMismatch change policy so + /// non-root containers can access supported volumes without matching the image's + /// primary group. Use + /// + /// to customize the pod security context when a different group or policy is required. /// /// /// @@ -278,6 +283,13 @@ public static IResourceBuilder WithPersistentVolume( /// When , mounts the volume /// read-only. /// The same builder for chaining. + /// + /// The generated pod uses an Aspire-managed fsGroup of 2000 with + /// an OnRootMismatch change policy so non-root containers can access + /// supported volumes without matching the image's primary group. Use + /// + /// to customize the pod security context when a different group or policy is required. + /// /// /// /// var media = k8s.AddPersistentVolume("media") diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs index f37bd5a8269..ebeccf84d10 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs @@ -21,6 +21,9 @@ namespace Aspire.Hosting.Kubernetes; [AspireExport(ExposeProperties = true)] public partial class KubernetesResource(string name, IResource resource, KubernetesEnvironmentResource kubernetesEnvironmentResource) : Resource(name), IResourceWithParent { + private const long DefaultPersistentVolumeFsGroup = 2000; + private const string DefaultPersistentVolumeFsGroupChangePolicy = "OnRootMismatch"; + /// public KubernetesEnvironmentResource Parent => kubernetesEnvironmentResource; @@ -164,18 +167,31 @@ private void SetLabels() private void CreateApplication() { + var hasPersistentVolumeBinding = resource.HasAnnotationOfType(); + // Promote to a StatefulSet when the workload is bound to a first-class persistent // volume — Kubernetes requires stable identity and ordered rollout for pods that // share named PVCs. The historical IResourceWithConnectionString rule remains so // existing integrations that imply state continue to render as StatefulSets. - if (resource is IResourceWithConnectionString || - resource.HasAnnotationOfType()) + if (resource is IResourceWithConnectionString || hasPersistentVolumeBinding) { Workload = resource.ToStatefulSet(this); - return; + } + else + { + Workload = resource.ToDeployment(this); } - Workload = resource.ToDeployment(this); + if (hasPersistentVolumeBinding) + { + // fsGroup is a supplemental group, not the image's primary GID. A stable + // publisher-owned value lets non-root images access supported volumes without + // coupling the manifest to image-specific identities. Kubernetes customization + // callbacks run after this default is applied and can replace or remove it. + var securityContext = Workload.PodTemplate.Spec.SecurityContext ??= new(); + securityContext.FsGroup ??= DefaultPersistentVolumeFsGroup; + securityContext.FsGroupChangePolicy ??= DefaultPersistentVolumeFsGroupChangePolicy; + } } internal string GetContainerImageName(IResource resourceInstance) diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksPersistentVolumeDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksPersistentVolumeDeploymentTests.cs index 6b129f391cb..864bd3dd727 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksPersistentVolumeDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksPersistentVolumeDeploymentTests.cs @@ -121,6 +121,8 @@ await auto.RunCommandAsync( await WaitForStatefulSetAndManagedDiskAsync(auto, counter); + await VerifyFileSystemGroupAsync(auto, counter, expectedFsGroup: 2000); + await auto.RunCommandAsync( "PVC_UID_BEFORE=$(kubectl get persistentvolumeclaim data --namespace \"$NS\" -o jsonpath='{.metadata.uid}') && " + "POD_UID_BEFORE=$(kubectl get pod apiservice-statefulset-0 --namespace \"$NS\" -o jsonpath='{.metadata.uid}') && " + @@ -150,6 +152,7 @@ await auto.RunCommandAsync( "kubectl wait --for=condition=Ready pod/apiservice-statefulset-0 --namespace \"$NS\" --timeout=5m", counter, TimeSpan.FromMinutes(6)); + await VerifyFileSystemGroupAsync(auto, counter, expectedFsGroup: 3000); await auto.RunCommandAsync( "PVC_UID_AFTER=$(kubectl get persistentvolumeclaim data --namespace \"$NS\" -o jsonpath='{.metadata.uid}') && " + "POD_UID_AFTER=$(kubectl get pod apiservice-statefulset-0 --namespace \"$NS\" -o jsonpath='{.metadata.uid}') && " + @@ -166,6 +169,12 @@ await VerifyApiResponseAsync( apiPort, "?action=read", "PASSED: read aks-pv-marker-42 revision second"); + await VerifyApiResponseAsync( + auto, + counter, + apiPort, + "?action=write-new", + "PASSED: wrote new aks-pv-marker-42 revision second"); await StopPortForwardAsync(auto, counter); await auto.AspireDestroyAsync(counter); @@ -231,17 +240,6 @@ private static void ConfigureAppHost(string appHostPath) builder.AddProject("apiservice") .WithPersistentVolume(data, "/srv/data") .WithEnvironment("DEPLOYMENT_REVISION", "first") - .PublishAsKubernetesService(resource => - { - // .NET containers run as the non-root app user. Apply its group to the - // managed disk so the API can write to the filesystem mounted by kubelet. - var workload = resource.Workload - ?? throw new InvalidOperationException("The API Kubernetes workload was not generated."); - workload.PodTemplate.Spec.SecurityContext = new() - { - FsGroup = 1654 - }; - }) """, appHostPath); @@ -254,7 +252,16 @@ private static void UpdateDeploymentRevision(string appHostPath) content = ReplaceRequired( content, """.WithEnvironment("DEPLOYMENT_REVISION", "first")""", - """.WithEnvironment("DEPLOYMENT_REVISION", "second")""", + """ + .WithEnvironment("DEPLOYMENT_REVISION", "second") + .PublishAsKubernetesService(resource => + { + var podSpec = resource.Workload?.PodTemplate.Spec + ?? throw new InvalidOperationException("The API Kubernetes workload was not generated."); + podSpec.SecurityContext ??= new(); + podSpec.SecurityContext.FsGroup = 3000; + }) + """, appHostPath); File.WriteAllText(appHostPath, content); } @@ -271,6 +278,7 @@ private static void ConfigureApi(string apiProgramPath) var app = builder.Build(); const string markerPath = "/srv/data/marker.txt"; + const string newMarkerPath = "/srv/data/new-marker.txt"; const string markerToken = "aks-pv-marker-42"; var deploymentRevision = app.Configuration["DEPLOYMENT_REVISION"] ?? throw new InvalidOperationException("DEPLOYMENT_REVISION is not configured."); @@ -300,7 +308,13 @@ private static void ConfigureApi(string apiProgramPath) return Results.Ok($"PASSED: read {persistedValue} revision {deploymentRevision}"); } - return Results.BadRequest("FAILED: action must be write or read"); + if (action == "write-new") + { + await File.WriteAllTextAsync(newMarkerPath, markerToken); + return Results.Ok($"PASSED: wrote new {markerToken} revision {deploymentRevision}"); + } + + return Results.BadRequest("FAILED: action must be write, read, or write-new"); }); app.MapDefaultEndpoints(); @@ -333,6 +347,29 @@ await auto.RunCommandAsync( TimeSpan.FromMinutes(6)); } + private static async Task VerifyFileSystemGroupAsync( + Hex1bTerminalAutomator auto, + SequenceCounter counter, + long expectedFsGroup) + { + await auto.RunCommandAsync( + $"FS_GROUP=$(kubectl get statefulset apiservice-statefulset --namespace \"$NS\" -o jsonpath='{{.spec.template.spec.securityContext.fsGroup}}') && " + + $"test \"$FS_GROUP\" = \"{expectedFsGroup}\" && " + + // Verify the Linux identity and mount ownership reported as: + // id -u: 1654 + // id -G: 1654 2000 + // stat -c %g /srv/data: 2000 + // This proves the write succeeds through group access rather than root privileges. + "PROCESS_UID=$(kubectl exec pod/apiservice-statefulset-0 --namespace \"$NS\" -- id -u) && " + + "PROCESS_GROUPS=$(kubectl exec pod/apiservice-statefulset-0 --namespace \"$NS\" -- id -G) && " + + "VOLUME_GROUP=$(kubectl exec pod/apiservice-statefulset-0 --namespace \"$NS\" -- stat -c %g /srv/data) && " + + "test \"$PROCESS_UID\" != \"0\" && " + + $"printf ' %s ' \"$PROCESS_GROUPS\" | grep --fixed-strings --quiet ' {expectedFsGroup} ' && " + + $"test \"$VOLUME_GROUP\" = \"{expectedFsGroup}\" && " + + $"echo \"StatefulSet uses fsGroup {expectedFsGroup}; pod UID is $PROCESS_UID with groups $PROCESS_GROUPS; /srv/data group is $VOLUME_GROUP\"", + counter); + } + private static async Task DeployAsync( Hex1bTerminalAutomator auto, SequenceCounter counter, diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesPublisherTests.cs b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesPublisherTests.cs index e13f8b6aabb..70276c6c08e 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesPublisherTests.cs +++ b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesPublisherTests.cs @@ -1289,6 +1289,71 @@ public async Task PublishAsync_WithFirstClassPersistentVolume_OnProject_BindsVia await settingsTask; } + [Fact] + public async Task PublishAsync_WithFirstClassPersistentVolume_KubernetesCustomizationOverridesDefaultFsGroup() + { + using var workspace = TemporaryWorkspace.Create(outputHelper); + var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish, workspace.Path); + + var k8s = builder.AddKubernetesEnvironment("env"); + var data = k8s.AddPersistentVolume("data"); + + builder.AddProject("api", launchProfileName: null) + .WithPersistentVolume(data, "/srv/data") + .PublishAsKubernetesService(resource => + { + var podSpec = resource.Workload?.PodTemplate.Spec + ?? throw new InvalidOperationException("The Kubernetes workload was not generated."); + podSpec.SecurityContext ??= new(); + podSpec.SecurityContext.FsGroup = 3000; + }); + + var app = builder.Build(); + app.Run(); + + var statefulSetPath = Path.Combine(workspace.Path, "templates", "api", "statefulset.yaml"); + var content = await File.ReadAllTextAsync(statefulSetPath); + + await Verify(content, "yaml"); + } + + [Fact] + public async Task PublishAsync_WithFirstClassPersistentVolume_KubernetesCustomizationCanRemoveDefaultSecurityContext() + { + using var workspace = TemporaryWorkspace.Create(outputHelper); + var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish, workspace.Path); + + var k8s = builder.AddKubernetesEnvironment("env"); + var data = k8s.AddPersistentVolume("data"); + + builder.AddProject("api", launchProfileName: null) + .WithPersistentVolume(data, "/srv/data") + .PublishAsKubernetesService(resource => + { + var podSpec = resource.Workload?.PodTemplate.Spec + ?? throw new InvalidOperationException("The Kubernetes workload was not generated."); + podSpec.SecurityContext = null; + }); + + var app = builder.Build(); + app.Run(); + + var statefulSetPath = Path.Combine(workspace.Path, "templates", "api", "statefulset.yaml"); + var content = await File.ReadAllTextAsync(statefulSetPath); + + var yaml = new YamlStream(); + using (var reader = new StringReader(content)) + { + yaml.Load(reader); + } + + var root = (YamlMappingNode)yaml.Documents[0].RootNode; + var podSpec = (YamlMappingNode)root["spec"]["template"]["spec"]; + Assert.False(podSpec.Children.ContainsKey(new YamlScalarNode("securityContext"))); + + await Verify(content, "yaml"); + } + [Fact] public async Task PublishAsync_WithFirstClassPersistentVolume_FallsThroughForUnboundVolumes() { @@ -1400,6 +1465,10 @@ public async Task PublishAsync_WithFirstClassPersistentVolume_ReadOnlyFlag_Propa var root = (YamlMappingNode)yaml.Documents[0].RootNode; var podSpec = (YamlMappingNode)root["spec"]["template"]["spec"]; + var securityContext = (YamlMappingNode)podSpec["securityContext"]; + Assert.Equal("2000", ((YamlScalarNode)securityContext["fsGroup"]).Value); + Assert.Equal("OnRootMismatch", ((YamlScalarNode)securityContext["fsGroupChangePolicy"]).Value); + // Container mount side: containers[0].volumeMounts[?(@.name == "media")].readOnly == true var container = (YamlMappingNode)((YamlSequenceNode)podSpec["containers"])[0]; var volumeMount = ((YamlSequenceNode)container["volumeMounts"]) diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_BindsByName_PromotesToStatefulSet#00.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_BindsByName_PromotesToStatefulSet#00.verified.yaml index ea51ec46d6c..85d76e84662 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_BindsByName_PromotesToStatefulSet#00.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_BindsByName_PromotesToStatefulSet#00.verified.yaml @@ -26,6 +26,9 @@ spec: - name: "data" persistentVolumeClaim: claimName: "data" + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: "OnRootMismatch" selector: matchLabels: app.kubernetes.io/name: "{{ .Chart.Name }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_FallsThroughForUnboundVolumes#00.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_FallsThroughForUnboundVolumes#00.verified.yaml index f453f912aef..4cb3a03db0a 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_FallsThroughForUnboundVolumes#00.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_FallsThroughForUnboundVolumes#00.verified.yaml @@ -30,6 +30,9 @@ spec: claimName: "data" - name: "scratch" emptyDir: {} + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: "OnRootMismatch" selector: matchLabels: app.kubernetes.io/name: "{{ .Chart.Name }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_KubernetesCustomizationCanRemoveDefaultSecurityContext.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_KubernetesCustomizationCanRemoveDefaultSecurityContext.verified.yaml new file mode 100644 index 00000000000..f1bccdf35ed --- /dev/null +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_KubernetesCustomizationCanRemoveDefaultSecurityContext.verified.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: "api-statefulset" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "{{ .Values.parameters.api.api_image }}" + name: "api" + envFrom: + - configMapRef: + name: "api-config" + volumeMounts: + - name: "data" + mountPath: "/srv/data" + imagePullPolicy: "IfNotPresent" + volumes: + - name: "data" + persistentVolumeClaim: + claimName: "data" + selector: + matchLabels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_KubernetesCustomizationOverridesDefaultFsGroup.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_KubernetesCustomizationOverridesDefaultFsGroup.verified.yaml new file mode 100644 index 00000000000..215b478ca34 --- /dev/null +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_KubernetesCustomizationOverridesDefaultFsGroup.verified.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: "api-statefulset" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "{{ .Values.parameters.api.api_image }}" + name: "api" + envFrom: + - configMapRef: + name: "api-config" + volumeMounts: + - name: "data" + mountPath: "/srv/data" + imagePullPolicy: "IfNotPresent" + volumes: + - name: "data" + persistentVolumeClaim: + claimName: "data" + securityContext: + fsGroup: 3000 + fsGroupChangePolicy: "OnRootMismatch" + selector: + matchLabels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_OnProject_BindsViaMountPathOverload#00.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_OnProject_BindsViaMountPathOverload#00.verified.yaml index 582c3184f0f..f50530f73d7 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_OnProject_BindsViaMountPathOverload#00.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_WithFirstClassPersistentVolume_OnProject_BindsViaMountPathOverload#00.verified.yaml @@ -29,6 +29,9 @@ spec: - name: "media" persistentVolumeClaim: claimName: "media" + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: "OnRootMismatch" selector: matchLabels: app.kubernetes.io/name: "{{ .Chart.Name }}"