Skip to content

Commit a10301e

Browse files
sigvshaun-nx
sigv
and
shaun-nx
committed
Allow configuring IC root filesystem as read-only
The Nginx Ingress Controller has various protections against attacks, such as running the service as non-root to avoid changes to files. An additional industry best practice is having root filesystems set as read-only so that the attack surface is further reduced by limiting changes to binaries and libraries. This commit ensures such a defense in depth is enabled for the Nginx Ingress Controller. To accomplish read-only rootfs, we will create emptyDir Volumes for - `/etc/nginx` - `/var/cache/nginx` - `/var/lib/nginx` - `/var/log/nginx` We populate `/etc` mount with an Init Container, as there are generic configuration files here. If the base image is updated via Helm chart, then Init Container will pull files from the new base image. The other three empty volumes are populated during runtime, as they are not expected to have valuable files as part of base image: - `/var/cache` mount will be used in case caching is enabled. - `/var/lib` is used for some sockets and temporary files. - `/var/log` is used for logging in the default configuration. By default, the emptyDir will be persisted as long as the Pod is not disposed of. Simple Pod crashes will retain the data. There is no maximum size limit implemented currently, but may be added at some point in time as a further protection. For improved performance, cache could be moved to ram tmpfs as well (currently backed by disk). Do note, the initial commit does not set read-only root filesystem as default. Instead, this is an opt-in feature available on the chart. Users of non-Helm YAML manifests need to manually uncomment the aforementioned volumes and initContainers. Co-authored-by: Shaun <[email protected]>
1 parent 0770b96 commit a10301e

File tree

10 files changed

+255
-8
lines changed

10 files changed

+255
-8
lines changed

‎deployments/daemon-set/nginx-ingress.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,19 @@ spec:
2222
securityContext:
2323
seccompProfile:
2424
type: RuntimeDefault
25+
# fsGroup: 101 #nginx
2526
sysctls:
2627
- name: "net.ipv4.ip_unprivileged_port_start"
2728
value: "0"
29+
# volumes:
30+
# - name: nginx-etc
31+
# emptyDir: {}
32+
# - name: nginx-cache
33+
# emptyDir: {}
34+
# - name: nginx-lib
35+
# emptyDir: {}
36+
# - name: nginx-log
37+
# emptyDir: {}
2838
containers:
2939
- image: nginx/nginx-ingress:3.0.2
3040
imagePullPolicy: IfNotPresent
@@ -54,10 +64,20 @@ spec:
5464
# memory: "1Gi"
5565
securityContext:
5666
allowPrivilegeEscalation: false
67+
# readOnlyRootFilesystem: true
5768
runAsUser: 101 #nginx
5869
capabilities:
5970
drop:
6071
- ALL
72+
# volumeMounts:
73+
# - mountPath: /etc/nginx
74+
# name: nginx-etc
75+
# - mountPath: /var/cache/nginx
76+
# name: nginx-cache
77+
# - mountPath: /var/lib/nginx
78+
# name: nginx-lib
79+
# - mountPath: /var/log/nginx
80+
# name: nginx-log
6181
env:
6282
- name: POD_NAMESPACE
6383
valueFrom:
@@ -76,3 +96,19 @@ spec:
7696
#- -external-service=nginx-ingress
7797
#- -enable-prometheus-metrics
7898
#- -global-configuration=$(POD_NAMESPACE)/nginx-configuration
99+
# initContainers:
100+
# - image: nginx/nginx-ingress:3.0.2
101+
# imagePullPolicy: IfNotPresent
102+
# name: init-nginx-ingress
103+
# command: ['cp', '-vdR', '/etc/nginx/.', '/mnt/etc']
104+
# securityContext:
105+
# allowPrivilegeEscalation: false
106+
# readOnlyRootFilesystem: true
107+
# runAsUser: 101 #nginx
108+
# runAsNonRoot: true
109+
# capabilities:
110+
# drop:
111+
# - ALL
112+
# volumeMounts:
113+
# - mountPath: /mnt/etc
114+
# name: nginx-etc

‎deployments/daemon-set/nginx-plus-ingress.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,19 @@ spec:
2222
securityContext:
2323
seccompProfile:
2424
type: RuntimeDefault
25+
# fsGroup: 101 #nginx
2526
sysctls:
2627
- name: "net.ipv4.ip_unprivileged_port_start"
2728
value: "0"
29+
# volumes:
30+
# - name: nginx-etc
31+
# emptyDir: {}
32+
# - name: nginx-cache
33+
# emptyDir: {}
34+
# - name: nginx-lib
35+
# emptyDir: {}
36+
# - name: nginx-log
37+
# emptyDir: {}
2838
containers:
2939
- image: nginx-plus-ingress:3.0.2
3040
imagePullPolicy: IfNotPresent
@@ -54,10 +64,20 @@ spec:
5464
# memory: "1Gi"
5565
securityContext:
5666
allowPrivilegeEscalation: false
67+
# readOnlyRootFilesystem: true
5768
runAsUser: 101 #nginx
5869
capabilities:
5970
drop:
6071
- ALL
72+
# volumeMounts:
73+
# - mountPath: /etc/nginx
74+
# name: nginx-etc
75+
# - mountPath: /var/cache/nginx
76+
# name: nginx-cache
77+
# - mountPath: /var/lib/nginx
78+
# name: nginx-lib
79+
# - mountPath: /var/log/nginx
80+
# name: nginx-log
6181
env:
6282
- name: POD_NAMESPACE
6383
valueFrom:
@@ -79,3 +99,19 @@ spec:
7999
#- -external-service=nginx-ingress
80100
#- -enable-prometheus-metrics
81101
#- -global-configuration=$(POD_NAMESPACE)/nginx-configuration
102+
# initContainers:
103+
# - image: nginx/nginx-ingress:3.0.2
104+
# imagePullPolicy: IfNotPresent
105+
# name: init-nginx-ingress
106+
# command: ['cp', '-vdR', '/etc/nginx/.', '/mnt/etc']
107+
# securityContext:
108+
# allowPrivilegeEscalation: false
109+
# readOnlyRootFilesystem: true
110+
# runAsUser: 101 #nginx
111+
# runAsNonRoot: true
112+
# capabilities:
113+
# drop:
114+
# - ALL
115+
# volumeMounts:
116+
# - mountPath: /mnt/etc
117+
# name: nginx-etc

‎deployments/deployment/nginx-ingress.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,19 @@ spec:
2323
securityContext:
2424
seccompProfile:
2525
type: RuntimeDefault
26+
# fsGroup: 101 #nginx
2627
sysctls:
2728
- name: "net.ipv4.ip_unprivileged_port_start"
2829
value: "0"
30+
# volumes:
31+
# - name: nginx-etc
32+
# emptyDir: {}
33+
# - name: nginx-cache
34+
# emptyDir: {}
35+
# - name: nginx-lib
36+
# emptyDir: {}
37+
# - name: nginx-log
38+
# emptyDir: {}
2939
containers:
3040
- image: nginx/nginx-ingress:3.0.2
3141
imagePullPolicy: IfNotPresent
@@ -53,11 +63,21 @@ spec:
5363
# memory: "1Gi"
5464
securityContext:
5565
allowPrivilegeEscalation: false
66+
# readOnlyRootFilesystem: true
5667
runAsUser: 101 #nginx
5768
runAsNonRoot: true
5869
capabilities:
5970
drop:
6071
- ALL
72+
# volumeMounts:
73+
# - mountPath: /etc/nginx
74+
# name: nginx-etc
75+
# - mountPath: /var/cache/nginx
76+
# name: nginx-cache
77+
# - mountPath: /var/lib/nginx
78+
# name: nginx-lib
79+
# - mountPath: /var/log/nginx
80+
# name: nginx-log
6181
env:
6282
- name: POD_NAMESPACE
6383
valueFrom:
@@ -78,3 +98,19 @@ spec:
7898
#- -external-service=nginx-ingress
7999
#- -enable-prometheus-metrics
80100
#- -global-configuration=$(POD_NAMESPACE)/nginx-configuration
101+
# initContainers:
102+
# - image: nginx/nginx-ingress:3.0.2
103+
# imagePullPolicy: IfNotPresent
104+
# name: init-nginx-ingress
105+
# command: ['cp', '-vdR', '/etc/nginx/.', '/mnt/etc']
106+
# securityContext:
107+
# allowPrivilegeEscalation: false
108+
# readOnlyRootFilesystem: true
109+
# runAsUser: 101 #nginx
110+
# runAsNonRoot: true
111+
# capabilities:
112+
# drop:
113+
# - ALL
114+
# volumeMounts:
115+
# - mountPath: /mnt/etc
116+
# name: nginx-etc

‎deployments/deployment/nginx-plus-ingress.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,19 @@ spec:
2323
securityContext:
2424
seccompProfile:
2525
type: RuntimeDefault
26+
# fsGroup: 101 #nginx
2627
sysctls:
2728
- name: "net.ipv4.ip_unprivileged_port_start"
2829
value: "0"
30+
# volumes:
31+
# - name: nginx-etc
32+
# emptyDir: {}
33+
# - name: nginx-cache
34+
# emptyDir: {}
35+
# - name: nginx-lib
36+
# emptyDir: {}
37+
# - name: nginx-log
38+
# emptyDir: {}
2939
containers:
3040
- image: nginx-plus-ingress:3.0.2
3141
imagePullPolicy: IfNotPresent
@@ -55,11 +65,21 @@ spec:
5565
# memory: "1Gi"
5666
securityContext:
5767
allowPrivilegeEscalation: false
68+
# readOnlyRootFilesystem: true
5869
runAsUser: 101 #nginx
5970
runAsNonRoot: true
6071
capabilities:
6172
drop:
6273
- ALL
74+
# volumeMounts:
75+
# - mountPath: /etc/nginx
76+
# name: nginx-etc
77+
# - mountPath: /var/cache/nginx
78+
# name: nginx-cache
79+
# - mountPath: /var/lib/nginx
80+
# name: nginx-lib
81+
# - mountPath: /var/log/nginx
82+
# name: nginx-log
6383
env:
6484
- name: POD_NAMESPACE
6585
valueFrom:
@@ -84,3 +104,19 @@ spec:
84104
#- -enable-prometheus-metrics
85105
#- -enable-service-insight
86106
#- -global-configuration=$(POD_NAMESPACE)/nginx-configuration
107+
# initContainers:
108+
# - image: nginx/nginx-ingress:3.0.2
109+
# imagePullPolicy: IfNotPresent
110+
# name: init-nginx-ingress
111+
# command: ['cp', '-vdR', '/etc/nginx/.', '/mnt/etc']
112+
# securityContext:
113+
# allowPrivilegeEscalation: false
114+
# readOnlyRootFilesystem: true
115+
# runAsUser: 101 #nginx
116+
# runAsNonRoot: true
117+
# capabilities:
118+
# drop:
119+
# - ALL
120+
# volumeMounts:
121+
# - mountPath: /mnt/etc
122+
# name: nginx-etc

‎deployments/helm-chart/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,7 @@ Parameter | Description | Default
281281
`controller.podDisruptionBudget.maxUnavailable` | The number of Ingress Controller pods that can be unavailable. This is a mutually exclusive setting with "minAvailable". | 0
282282
`controller.strategy` | Specifies the strategy used to replace old Pods with new ones. Docs for [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) and [Daemonset update strategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) | {}
283283
`controller.disableIPV6` | Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack. | false
284+
`controller.readOnlyRootFilesystem` | Configure root filesystem as read-only and add volumes for temporary data. | false
284285
`rbac.create` | Configures RBAC. | true
285286
`prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | true
286287
`prometheus.port` | Configures the port to scrape the metrics. | 9113

‎deployments/helm-chart/templates/controller-daemonset.yaml‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ spec:
4545
securityContext:
4646
seccompProfile:
4747
type: RuntimeDefault
48+
{{- if .Values.controller.readOnlyRootFilesystem }}
49+
fsGroup: 101 #nginx
50+
{{- end }}
4851
sysctls:
4952
- name: "net.ipv4.ip_unprivileged_port_start"
5053
value: "0"
@@ -61,9 +64,19 @@ spec:
6164
affinity:
6265
{{ toYaml .Values.controller.affinity | indent 8 }}
6366
{{- end }}
64-
{{- if or .Values.controller.volumes .Values.nginxServiceMesh.enable }}
67+
{{- if or .Values.controller.readOnlyRootFilesystem .Values.nginxServiceMesh.enable .Values.controller.volumes }}
6568
volumes:
6669
{{- end }}
70+
{{- if .Values.controller.readOnlyRootFilesystem }}
71+
- name: nginx-etc
72+
emptyDir: {}
73+
- name: nginx-cache
74+
emptyDir: {}
75+
- name: nginx-lib
76+
emptyDir: {}
77+
- name: nginx-log
78+
emptyDir: {}
79+
{{- end }}
6780
{{- if .Values.nginxServiceMesh.enable }}
6881
- hostPath:
6982
path: /run/spire/sockets
@@ -116,14 +129,25 @@ spec:
116129
{{- end }}
117130
securityContext:
118131
allowPrivilegeEscalation: false
132+
readOnlyRootFilesystem: {{ .Values.controller.readOnlyRootFilesystem }}
119133
runAsUser: 101 #nginx
120134
runAsNonRoot: true
121135
capabilities:
122136
drop:
123137
- ALL
124-
{{- if or .Values.controller.volumeMounts .Values.nginxServiceMesh.enable }}
138+
{{- if or .Values.controller.readOnlyRootFilesystem .Values.nginxServiceMesh.enable .Values.controller.volumeMounts }}
125139
volumeMounts:
126140
{{- end }}
141+
{{- if .Values.controller.readOnlyRootFilesystem }}
142+
- mountPath: /etc/nginx
143+
name: nginx-etc
144+
- mountPath: /var/cache/nginx
145+
name: nginx-cache
146+
- mountPath: /var/lib/nginx
147+
name: nginx-lib
148+
- mountPath: /var/log/nginx
149+
name: nginx-log
150+
{{- end }}
127151
{{- if .Values.nginxServiceMesh.enable }}
128152
- mountPath: /run/spire/sockets
129153
name: spire-agent-socket
@@ -239,8 +263,28 @@ spec:
239263
{{- if .Values.controller.extraContainers }}
240264
{{ toYaml .Values.controller.extraContainers | nindent 6 }}
241265
{{- end }}
266+
{{- if or .Values.controller.readOnlyRootFilesystem .Values.controller.initContainers }}
267+
initContainers:
268+
{{- end }}
269+
{{- if .Values.controller.readOnlyRootFilesystem }}
270+
- name: init-{{ include "nginx-ingress.name" . }}
271+
image: {{ include "nginx-ingress.image" . }}
272+
imagePullPolicy: "{{ .Values.controller.image.pullPolicy }}"
273+
command: ['cp', '-vdR', '/etc/nginx/.', '/mnt/etc']
274+
securityContext:
275+
allowPrivilegeEscalation: false
276+
readOnlyRootFilesystem: true
277+
runAsUser: 101 #nginx
278+
runAsNonRoot: true
279+
capabilities:
280+
drop:
281+
- ALL
282+
volumeMounts:
283+
- mountPath: /mnt/etc
284+
name: nginx-etc
285+
{{- end }}
242286
{{- if .Values.controller.initContainers }}
243-
initContainers: {{ toYaml .Values.controller.initContainers | nindent 8 }}
287+
{{ toYaml .Values.controller.initContainers | indent 6 }}
244288
{{- end }}
245289
{{- if .Values.controller.strategy }}
246290
updateStrategy:

0 commit comments

Comments
 (0)