remove non-root overlays (#3404)

* reconcile non-root and non-privileged overlays

.buildkite/integration-test.sh

@@ -22,7 +22,7 @@ cleanup() {
 }
 trap cleanup EXIT
 
-"${DEPLOY_SOURCEGRAPH_ROOT}"/overlay-generate-cluster.sh non-root-create-cluster "${GENERATED_BASE}"
+"${DEPLOY_SOURCEGRAPH_ROOT}"/overlay-generate-cluster.sh non-privileged-create-cluster "${GENERATED_BASE}"
 
 TEST_ARGS=("test" "-timeout" "25m")
 

overlays/migrate-to-nonroot/README.md → overlays/migrate-to-nonprivileged/README.md

@@ -1,8 +1,9 @@
 This kustomization injects initContainers in all pods with persistent volumes to transfer ownership of directories to
-specified non-root users. It is used for migrating existing installations to a non-root environment.
+specified non-root users. It is used for migrating existing installations to a non-root environment. By default this overlay ensures all resources
+stay in the default namespace. If you have used a different namespace, change it in the [kustomization.yaml](./kustomization.yaml)
 
 ```shell script
-./overlay-generate-cluster.sh migrate-to-nonroot generated-cluster
+./overlay-generate-cluster.sh migrate-to-nonprivileged generated-cluster
 ```
 
 After executing the script you can apply the generated manifests from the `generated-cluster` directory:

overlays/migrate-to-nonroot/kustomization.yaml → overlays/migrate-to-nonprivileged/kustomization.yaml

@@ -1,7 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+# If you have used an alternative namespace, please change the default value below before generating your overlays. 
+namespace: default 
 bases:
-  - ../non-root
+  - ../non-privileged
 patchesStrategicMerge:
   - frontend/sourcegraph-frontend.Deployment.yaml
   - gitserver/gitserver.StatefulSet.yaml

overlays/non-privileged-create-cluster/README.md

@@ -1,18 +1,32 @@
-This kustomization is for Sourcegraph installations in clusters with security restrictions.
-It avoids creating `Roles` and does all the rolebinding in a namespace. It configures Prometheus to work in the namespace
-and not require ClusterRole wide privileges when doing service discovery for scraping targets. It also disables cAdvisor.
+This kustomization is for creating fresh Sourcegraph installations that want to run containers as non-root users in clusters with security restrictions.
+It avoids creating Roles and does all the rolebinding in a namespace. It configures Prometheus to work in the namespace and not require ClusterRole wide privileges when doing service discovery for scraping targets. It also disables cAdvisor.
 
-This version and `non-privileged` need to stay in sync. This version is only used for cluster creation.
+This version and non-privileged need to stay in sync. This version is only used for cluster creation.
 
-To use it, execute the following command from the root directory of this repository:
+This kustomization injects a fsGroup security context in each pod so that the volumes are mounted with the
+specified supplemental group id and non-root pod users can write to the mounted volumes.
+
+This is only done once at cluster creation time so this overlay is only referenced by the `create-new-cluster.sh`
+script.
+
+The reason for this approach is the behavior of fsGroup: on every mount it recursively chmod/chown the disk to add
+the group specified by fsGroup and to change permissions to 775 (so group can write). This can take a long time for
+large disks and sometimes times out the whole pod scheduling.
+
+If we only do it at cluster creation time (when the disks are empty) it is fast and since the disks are persistent
+volumes we know that the pod user can write to it even without the fsGroup and subsequent apply operations.
+
+In Kubernetes 1.18 fsGroup gets an additional [feature](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#configure-volume-permission-and-ownership-change-policy-for-pods)
+called `fsGroupChangePolicy` that will allow us to control the chmod/chown better. 
+
+To use it execute the following command from the root directory of this repository:
 
 ```shell script
-./overlay-generate-cluster.sh non-privileged generated-cluster
+./overlay-generate-cluster.sh non-privileged-create-cluster generated-cluster
 ```
 
 After executing the script you can apply the generated manifests from the `generated-cluster` directory:
 
 ```shell script
-kubectl create namespace ns-sourcegraph
-kubectl apply -n ns-sourcegraph --prune -l deploy=sourcegraph -f generated-cluster --recursive
+kubectl apply --prune -l deploy=sourcegraph -f generated-cluster --recursive
 ```

overlays/non-privileged-create-cluster/kustomization.yaml

@@ -2,9 +2,15 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: ns-sourcegraph
 bases:
-  - ../non-root-create-cluster
-resources:
-  - frontend/sourcegraph-frontend.RoleBinding.yaml
-  - prometheus/prometheus.RoleBinding.yaml
+  - ../non-privileged
 patchesStrategicMerge:
-  - prometheus/prometheus.ConfigMap.yaml
+  - codeintel-db/codeintel-db.Deployment.yaml
+  - codeinsights-db/codeinsights-db.Deployment.yaml
+  - gitserver/gitserver.StatefulSet.yaml
+  - grafana/grafana.StatefulSet.yaml
+  - indexed-search/indexed-search.StatefulSet.yaml
+  - minio/minio.Deployment.yaml
+  - pgsql/pgsql.Deployment.yaml
+  - prometheus/prometheus.Deployment.yaml
+  - redis/redis-cache.Deployment.yaml
+  - redis/redis-store.Deployment.yaml

overlays/non-privileged/README.md

@@ -1,13 +1,12 @@
-This kustomization is for Sourcegraph installations in clusters with security restrictions.
-It avoids creating `Roles` and does all the rolebinding in a namespace. It configures Prometheus to work in the namespace
-and not require ClusterRole wide privileges when doing service discovery for scraping targets. It also disables cAdvisor.
+This kustomization is for creating fresh Sourcegraph installations that want to run containers as non-root users in clusters with security restrictions.
+It avoids creating Roles and does all the rolebinding in a namespace. It configures Prometheus to work in the namespace and not require ClusterRole wide privileges when doing service discovery for scraping targets. It also disables cAdvisor.
 
-> Note: To create a fresh installation use `non-privileged-create-cluster` first and then use this overlay.
+> Note: To create a fresh installation use `non-root-privileged-cluster` first and then use this overlay.
 
 To use it, execute the following command from the root directory of this repository:
 
 ```shell script
-./overlay-generate-cluster.sh non-root generated-cluster
+./overlay-generate-cluster.sh non-privileged generated-cluster
 ```
 
 After executing the script you can apply the generated manifests from the `generated-cluster` directory:

overlays/non-privileged/kustomization.yaml

@@ -2,9 +2,30 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: ns-sourcegraph
 bases:
-  - ../non-root
+  - ../bases/deployments
+  - ../bases/pvcs
 resources:
   - frontend/sourcegraph-frontend.RoleBinding.yaml
   - prometheus/prometheus.RoleBinding.yaml
 patchesStrategicMerge:
+  - codeintel-db/codeintel-db.Deployment.yaml
+  - codeinsights-db/codeinsights-db.Deployment.yaml
+  - frontend/sourcegraph-frontend.Deployment.yaml
+  - github-proxy/github-proxy.Deployment.yaml
+  - gitserver/gitserver.StatefulSet.yaml
+  - grafana/grafana.StatefulSet.yaml
+  - minio/minio.Deployment.yaml
+  - indexed-search/indexed-search.StatefulSet.yaml
+  - jaeger/jaeger.Deployment.yaml
+  - pgsql/pgsql.Deployment.yaml
+  - precise-code-intel/worker.Deployment.yaml
   - prometheus/prometheus.ConfigMap.yaml
+  - prometheus/prometheus.Deployment.yaml
+  - query-runner/query-runner.Deployment.yaml
+  - redis/redis-cache.Deployment.yaml
+  - redis/redis-store.Deployment.yaml
+  - worker/worker.Deployment.yaml
+  - repo-updater/repo-updater.Deployment.yaml
+  - searcher/searcher.Deployment.yaml
+  - symbols/symbols.Deployment.yaml
+  - syntect-server/syntect-server.Deployment.yaml

tests/integration/fresh/step1/index.ts

@@ -25,6 +25,16 @@ const storageClass = new k8s.storage.v1.StorageClass(
     { provider: k8sProvider }
 )
 
+const nameSpace = new k8s.core.v1.Namespace(
+    'sourcegraph-namespace',
+    {
+        metadata: {
+            name: 'ns-sourcegraph',
+        },
+    },
+    { provider: k8sProvider }
+)
+
 const globOptions = {
     ignore: ['**/kustomization.yaml'],
 }
@@ -39,7 +49,7 @@ baseFiles.then(
             },
             {
                 providers: { kubernetes: k8sProvider },
-                dependsOn: [storageClass],
+                dependsOn: [storageClass, nameSpace],
             }
         )
 )