switch away from user 0 (root) and delete unnecessary cluster role (#534

) * switch away from user 0 (root) and delete unnecessary cluster role binding * pgsql data dir tweak * avoid chown when already correct * extract initContainers into kustomization (#537) * migrate doc * buildkite checks accommodate kustomize * kubectl version is not working * asas !!!!! * migrate doc specifies kubectl version
sourcegraph · Feb 29, 2020 · 821032e · 821032e
1 parent 7a19888
commit 821032e
Show file tree

Hide file tree

Showing 32 changed files with 276 additions and 56 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -5,3 +5,6 @@ cd $BUILDKITE_BUILD_CHECKOUT_PATH
 
 echo "Installing asdf dependencies as defined in '$BUILDKITE_BUILD_CHECKOUT_PATH/.tool-versions':"
 asdf install
+
+asdf global kubectl 1.17.3
+
diff --git a/.buildkite/verify-label.sh b/.buildkite/verify-label.sh
@@ -6,7 +6,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.."
 
 .buildkite/install-yj.sh
 
-if find base -name "*.yaml" -exec sh -c "cat {} | yj | jq --raw-output '.metadata.labels.deploy'" \; | tee /tmp/deploy-label | grep -v sourcegraph; then
+if find base -name "*.yaml" \( ! -name kustomization.yaml \) -exec sh -c "cat {} | yj | jq --raw-output '.metadata.labels.deploy'" \; | tee /tmp/deploy-label | grep -v sourcegraph; then
     echo "> There exists a yaml file in base/ that does not contain .metadata.labels.deploy == sourcegraph"
     echo "> Run the following command to fix:"
     echo "find base/ -name \"*.yaml\" -exec sh -c \"cat {} | yj | jq '.metadata.labels.deploy = \\\"sourcegraph\\\"' | jy -o {}\" \;"

diff --git a/.buildkite/verify-yaml.sh b/.buildkite/verify-yaml.sh
@@ -5,7 +5,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.."
 
 gcloud container clusters get-credentials dogfood --zone us-central1-a --project sourcegraph-dev
 
-kubectl apply --dry-run --validate --recursive -f base/ --context=gke_sourcegraph-dev_us-central1-a_dogfood
+kubectl apply --dry-run --validate -k base --context=gke_sourcegraph-dev_us-central1-a_dogfood
 kubectl apply --dry-run --validate --recursive -f configure/ --context=gke_sourcegraph-dev_us-central1-a_dogfood
 
 .buildkite/verify-label.sh
diff --git a/base/frontend/sourcegraph-frontend.Deployment.yaml b/base/frontend/sourcegraph-frontend.Deployment.yaml
@@ -82,8 +82,6 @@ spec:
         volumeMounts:
         - mountPath: /mnt/cache
           name: cache-ssd
-      securityContext:
-        runAsUser: 0
       serviceAccountName: sourcegraph-frontend
       volumes:
       - emptyDir: {}

diff --git a/base/github-proxy/github-proxy.Deployment.yaml b/base/github-proxy/github-proxy.Deployment.yaml
@@ -38,5 +38,3 @@ spec:
           requests:
             cpu: 100m
             memory: 250M
-      securityContext:
-        runAsUser: 0
diff --git a/base/gitserver/gitserver.StatefulSet.yaml b/base/gitserver/gitserver.StatefulSet.yaml
@@ -45,12 +45,15 @@ spec:
         volumeMounts:
         - mountPath: /data/repos
           name: repos
+        securityContext:
+          runAsUser: 100
+          runAsGroup: 100
         # See the customization guide (../../../docs/configure.md) for information
         # about configuring gitserver to use an SSH key
         # - mountPath: /root/.ssh
         #   name: ssh
       securityContext:
-        runAsUser: 0
+        fsGroup: 100
       volumes:
       - name: repos
       # See the customization guide (../../../docs/configure.md) for information

diff --git a/base/grafana/grafana.ClusterRole.yaml b/base/grafana/grafana.ClusterRole.yaml
diff --git a/base/grafana/grafana.ClusterRoleBinding.yaml b/base/grafana/grafana.ClusterRoleBinding.yaml
diff --git a/base/grafana/grafana.Deployment.yaml b/base/grafana/grafana.Deployment.yaml
@@ -42,9 +42,12 @@ spec:
           requests:
             cpu: 100m
             memory: 100Mi
+        securityContext:
+          runAsUser: 472
+          runAsGroup: 472
       serviceAccountName: grafana
       securityContext:
-        runAsUser: 0
+        fsGroup: 472
       volumes:
       - name: data
         persistentVolumeClaim:

diff --git a/base/indexed-search/indexed-search.StatefulSet.yaml b/base/indexed-search/indexed-search.StatefulSet.yaml
@@ -43,6 +43,9 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
+        securityContext:
+          runAsUser: 100
+          runAsGroup: 100
       - env:
         image: index.docker.io/sourcegraph/zoekt-indexserver:0.0.20200221095506-2880d98@sha256:91643d83223bb72f4aa2b5031ceb774c8e604a227c58ed54375bd341f25e2ef3
         terminationMessagePolicy: FallbackToLogsOnError
@@ -62,8 +65,11 @@ spec:
         volumeMounts:
         - mountPath: /data
           name: data
+        securityContext:
+          runAsUser: 100
+          runAsGroup: 100
       securityContext:
-        runAsUser: 0
+        fsGroup: 100
       volumes:
       - name: data
   updateStrategy:

diff --git a/base/kustomization.yaml b/base/kustomization.yaml
@@ -0,0 +1,54 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - searcher/searcher.Service.yaml
+  - searcher/searcher.Deployment.yaml
+  - backend.Service.yaml
+  - frontend/sourcegraph-frontend.Role.yaml
+  - frontend/sourcegraph-frontend-internal.Service.yaml
+  - frontend/sourcegraph-frontend.Deployment.yaml
+  - frontend/sourcegraph-frontend.ServiceAccount.yaml
+  - frontend/sourcegraph-frontend.Service.yaml
+  - frontend/sourcegraph-frontend.RoleBinding.yaml
+  - frontend/sourcegraph-frontend.Ingress.yaml
+  - redis/redis-store.Service.yaml
+  - redis/redis-store.Deployment.yaml
+  - redis/redis-cache.Service.yaml
+  - redis/redis-cache.Deployment.yaml
+  - redis/redis-store.PersistentVolumeClaim.yaml
+  - redis/redis-cache.PersistentVolumeClaim.yaml
+  - indexed-search/indexed-search.StatefulSet.yaml
+  - indexed-search/indexed-search.Service.yaml
+  - grafana/grafana.Deployment.yaml
+  - grafana/grafana.ConfigMap.yaml
+  - grafana/grafana.Service.yaml
+  - grafana/grafana.ServiceAccount.yaml
+  - grafana/grafana.PersistentVolumeClaim.yaml
+  - syntect-server/syntect-server.Service.yaml
+  - syntect-server/syntect-server.Deployment.yaml
+  - symbols/symbols.Service.yaml
+  - symbols/symbols.Deployment.yaml
+  - pgsql/pgsql.PersistentVolumeClaim.yaml
+  - pgsql/pgsql.ConfigMap.yaml
+  - pgsql/pgsql.Service.yaml
+  - pgsql/pgsql.Deployment.yaml
+  - prometheus/prometheus.ServiceAccount.yaml
+  - prometheus/prometheus.ClusterRoleBinding.yaml
+  - prometheus/prometheus.ConfigMap.yaml
+  - prometheus/prometheus.Deployment.yaml
+  - prometheus/prometheus.PersistentVolumeClaim.yaml
+  - prometheus/prometheus.ClusterRole.yaml
+  - prometheus/prometheus.Service.yaml
+  - query-runner/query-runner.Service.yaml
+  - query-runner/query-runner.Deployment.yaml
+  - github-proxy/github-proxy.Service.yaml
+  - github-proxy/github-proxy.Deployment.yaml
+  - replacer/replacer.Service.yaml
+  - replacer/replacer.Deployment.yaml
+  - repo-updater/repo-updater.Service.yaml
+  - repo-updater/repo-updater.Deployment.yaml
+  - gitserver/gitserver.Service.yaml
+  - gitserver/gitserver.StatefulSet.yaml
+  - lsif-server/lsif-server.Service.yaml
+  - lsif-server/lsif-server.PersistentVolumeClaim.yaml
+  - lsif-server/lsif-server.Deployment.yaml
diff --git a/base/lsif-server/lsif-server.Deployment.yaml b/base/lsif-server/lsif-server.Deployment.yaml
@@ -63,8 +63,11 @@ spec:
         volumeMounts:
         - mountPath: /lsif-storage
           name: lsif-storage
+        securityContext:
+          runAsUser: 100
+          runAsGroup: 100
       securityContext:
-        runAsUser: 0
+        fsGroup: 100
       volumes:
       - name: lsif-storage
         persistentVolumeClaim:

diff --git a/base/pgsql/pgsql.Deployment.yaml b/base/pgsql/pgsql.Deployment.yaml
@@ -53,6 +53,9 @@ spec:
           name: disk
         - mountPath: /conf
           name: pgsql-conf
+        securityContext:
+          runAsUser: 999
+          runAsGroup: 999
       - env:
         - name: DATA_SOURCE_NAME
           value: postgres://sg:@localhost:5432/?sslmode=disable
@@ -66,8 +69,11 @@ spec:
           requests:
             cpu: 10m
             memory: 50Mi
+        securityContext:
+          runAsUser: 100
+          runAsGroup: 100
       securityContext:
-        runAsUser: 0
+        fsGroup: 999
       volumes:
       - name: disk
         persistentVolumeClaim:

diff --git a/base/prometheus/prometheus.Deployment.yaml b/base/prometheus/prometheus.Deployment.yaml
@@ -56,7 +56,9 @@ spec:
             memory: 500M
       serviceAccountName: prometheus
       securityContext:
-        runAsUser: 0
+        runAsUser: 100
+        runAsGroup: 100
+        fsGroup: 100
       volumes:
       - name: data
         persistentVolumeClaim:

diff --git a/base/query-runner/query-runner.Deployment.yaml b/base/query-runner/query-runner.Deployment.yaml
@@ -38,5 +38,3 @@ spec:
           requests:
             cpu: 500m
             memory: 1G
-      securityContext:
-        runAsUser: 0
diff --git a/base/redis/redis-cache.Deployment.yaml b/base/redis/redis-cache.Deployment.yaml
@@ -49,6 +49,9 @@ spec:
         volumeMounts:
         - mountPath: /redis-data
           name: redis-data
+        securityContext:
+          runAsUser: 999
+          runAsGroup: 999
       - image: index.docker.io/sourcegraph/redis_exporter:18-02-07_bb60087_v0.15.0@sha256:282d59b2692cca68da128a4e28d368ced3d17945cd1d273d3ee7ba719d77b753
         terminationMessagePolicy: FallbackToLogsOnError
         name: redis-exporter
@@ -62,8 +65,11 @@ spec:
           requests:
             cpu: 10m
             memory: 100Mi
+        securityContext:
+          runAsUser: 999
+          runAsGroup: 999
       securityContext:
-        runAsUser: 0
+        fsGroup: 999
       volumes:
       - name: redis-data
         persistentVolumeClaim:

diff --git a/base/redis/redis-store.Deployment.yaml b/base/redis/redis-store.Deployment.yaml
@@ -49,6 +49,9 @@ spec:
         volumeMounts:
         - mountPath: /redis-data
           name: redis-data
+        securityContext:
+          runAsUser: 999
+          runAsGroup: 999
       - image: index.docker.io/sourcegraph/redis_exporter:18-02-07_bb60087_v0.15.0@sha256:282d59b2692cca68da128a4e28d368ced3d17945cd1d273d3ee7ba719d77b753
         terminationMessagePolicy: FallbackToLogsOnError
         name: redis-exporter
@@ -62,8 +65,11 @@ spec:
           requests:
             cpu: 10m
             memory: 100Mi
+        securityContext:
+          runAsUser: 999
+          runAsGroup: 999
       securityContext:
-        runAsUser: 0
+        fsGroup: 999
       volumes:
       - name: redis-data
         persistentVolumeClaim:

diff --git a/base/replacer/replacer.Deployment.yaml b/base/replacer/replacer.Deployment.yaml
@@ -58,8 +58,6 @@ spec:
         volumeMounts:
         - mountPath: /mnt/cache
           name: cache-ssd
-      securityContext:
-        runAsUser: 0
       volumes:
       - emptyDir: {}
         name: cache-ssd
diff --git a/base/repo-updater/repo-updater.Deployment.yaml b/base/repo-updater/repo-updater.Deployment.yaml
@@ -41,5 +41,3 @@ spec:
           requests:
             cpu: 100m
             memory: 500Mi
-      securityContext:
-        runAsUser: 0
diff --git a/base/searcher/searcher.Deployment.yaml b/base/searcher/searcher.Deployment.yaml
@@ -58,8 +58,6 @@ spec:
         volumeMounts:
         - mountPath: /mnt/cache
           name: cache-ssd
-      securityContext:
-        runAsUser: 0
       volumes:
       - emptyDir: {}
         name: cache-ssd
diff --git a/base/symbols/symbols.Deployment.yaml b/base/symbols/symbols.Deployment.yaml
@@ -65,8 +65,6 @@ spec:
         volumeMounts:
         - mountPath: /mnt/cache
           name: cache-ssd
-      securityContext:
-        runAsUser: 0
       volumes:
       - emptyDir: {}
         name: cache-ssd
diff --git a/base/syntect-server/syntect-server.Deployment.yaml b/base/syntect-server/syntect-server.Deployment.yaml
@@ -48,5 +48,3 @@ spec:
           requests:
             cpu: 250m
             memory: 2G
-      securityContext:
-        runAsUser: 0
diff --git a/docs/migrate.md b/docs/migrate.md
@@ -4,6 +4,39 @@ This document records manual migrations that are necessary to apply when upgradi
 Sourcegraph versions. All manual migrations between the version you are upgrading from and the
 version you are upgrading to should be applied (unless otherwise noted).
 
+## 3.14 (Unreleased)
+
+Version 3.14 changes the security context of the installation by switching to a non-root user for all containers.
+This allows running Sourcegraph in clusters with restrictive security policies.
+
+Existing installations that have been run as root before need to migrate their persistent volumes to work in 3.14.
+We are providing a [kustomization](https://kustomize.io/) that needs to be run once to execute the migration:
+
+> NOTE: This needs kubectl client version >= 1.14. If you don't have that you can still install the kustomize
+> binary and generate the yaml file with it and then apply it with -f.
+
+```shell script
+cd overlays/migrate-to-nonroot
+kubectl apply -k .
+```
+
+> NOTE: This needs kubectl client version >= 1.14. If you don't have that you can still install the kustomize
+> binary and generate the yaml file with it and then apply it with -f like so:
+
+```shell script
+cd overlays/migrate-to-nonroot
+kustomize build -o nonroot-migration.yaml
+kubectl apply -f nonroot-migration.yaml
+```
+
+This will inject `initContainers` that do the `chown` command for containers that have persistent volumes and then 
+restart the necessary containers.
+
+> NOTE: The migration still needs the elevated permissions because it needs to run as user root.
+
+New installations do not need this `kustomization` and existing installations can operate from base again after the
+migration.
+
 ## 3.11
 
 In 3.11 we removed the management console. If you make use of `CRITICAL_CONFIG_FILE` or `SITE_CONFIG_FILE`, please refer to the [migration notes for Sourcegraph 3.11+](https://docs.sourcegraph.com/admin/migration/3_11).

diff --git a/overlays/migrate-to-nonroot/gitserver/gitserver.StatefulSet.yaml b/overlays/migrate-to-nonroot/gitserver/gitserver.StatefulSet.yaml
@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: gitserver
+spec:
+  template:
+    spec:
+      initContainers:
+        - name: transfer-file-ownership
+          image: sourcegraph/alpine:3.10@sha256:4d05cd5669726fc38823e92320659a6d1ef7879e62268adec5df658a0bacf65c
+          command: ["sh", "-c", "if [[ \"$(stat -c '%u' /data/repos)\" -ne 100 ]]; then chown -R 100:100 /data/repos; fi"]
+          volumeMounts:
+            - mountPath: /data/repos
+              name: repos
+          securityContext:
+            runAsUser: 0
diff --git a/overlays/migrate-to-nonroot/grafana/grafana.Deployment.yaml b/overlays/migrate-to-nonroot/grafana/grafana.Deployment.yaml
@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+spec:
+  template:
+    spec:
+      initContainers:
+        - name: transfer-file-ownership
+          image: sourcegraph/alpine:3.10@sha256:4d05cd5669726fc38823e92320659a6d1ef7879e62268adec5df658a0bacf65c
+          command: ["sh", "-c", "chown -R 472:472 /var/lib/grafana"]
+          volumeMounts:
+            - mountPath: /var/lib/grafana
+              name: data
+          securityContext:
+            runAsUser: 0