OpenShift image and fixed bug on running multiple operators/webhooks

Dockerfile

@@ -16,7 +16,7 @@
 
 ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.4.0
 
-FROM golang:1.11.2-alpine as builder
+FROM golang:1.11.4-alpine as builder
 ARG DEP_VERSION="0.5.0"
 RUN apk add --no-cache bash git
 ADD https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 /usr/bin/dep
@@ -30,8 +30,8 @@ RUN go generate && CGO_ENABLED=0 GOOS=linux go build -o /usr/bin/spark-operator
 
 FROM ${SPARK_IMAGE}
 COPY --from=builder /usr/bin/spark-operator /usr/bin/
-RUN apk add --no-cache openssl curl
+RUN apk add --no-cache openssl curl tini
 COPY hack/gencerts.sh /usr/bin/
 
-ENTRYPOINT ["/usr/bin/spark-operator"]
-
+COPY entrypoint.sh /usr/bin/
+ENTRYPOINT ["/usr/bin/entrypoint.sh"]

Dockerfile.rh

@@ -0,0 +1,53 @@
+# syntax=docker/dockerfile:experimental
+#
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Build an OpenShift image.
+# Before running docker build, make sure
+# 1. Your Docker version is >= 18.09
+# 2. export DOCKER_BUILDKIT=1
+
+ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.4.0
+ARG USER_ID=185
+
+FROM golang:1.11.4-alpine as builder
+ARG DEP_VERSION="0.5.0"
+RUN apk add --no-cache bash git 
+ADD https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 /usr/bin/dep
+RUN chmod +x /usr/bin/dep
+
+WORKDIR ${GOPATH}/src/github.com/GoogleCloudPlatform/spark-on-k8s-operator
+COPY Gopkg.toml Gopkg.lock ./
+RUN dep ensure -vendor-only
+COPY . ./
+RUN go generate && CGO_ENABLED=0 GOOS=linux go build -o /usr/bin/spark-operator
+
+FROM ${SPARK_IMAGE}
+COPY --from=builder /usr/bin/spark-operator /usr/bin/
+USER root
+
+# Comment out the following three lines if you do not have a RedHat subscription.
+COPY install_packages.sh /
+RUN --mount=target=/opt/spark/credentials,type=secret,id=credentials,required /install_packages.sh
+RUN rm /install_packages.sh
+
+RUN chmod -R u+x /tmp
+
+RUN apk add --no-cache openssl curl tini
+COPY hack/gencerts.sh /usr/bin/
+COPY entrypoint.sh /usr/bin/
+USER ${USER_ID}
+ENTRYPOINT ["/usr/bin/entrypoint.sh"]

docs/quick-start-guide.md

@@ -20,10 +20,10 @@ $ helm repo add incubator http://storage.googleapis.com/kubernetes-charts-incuba
 $ helm install incubator/sparkoperator --namespace spark-operator
 ```
 
-Installing the chart will create a namespace `spark-operator` if it doesn't exist, set up RBAC for the operator to run in the namespace. It will also set up RBAC for driver pods of your Spark applications to be able to manipulate executor pods. In addition, the chart will create a Deployment in the namespace `spark-operator`. The chart by default enables a [Mutating Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) for Spark pod customization. A webhook service and a secret storing the x509 certificate called `spark-webhook-certs` are created for that purpose. To install the operator **without** the mutating admission webhook on a Kubernetes cluster, install the chart with the flag `enableWebhook=false`:
+Installing the chart will create a namespace `spark-operator` if it doesn't exist, set up RBAC for the operator to run in the namespace. It will also set up RBAC for driver pods of your Spark applications to be able to manipulate executor pods. In addition, the chart will create a Deployment in the namespace `spark-operator`. The chart by default does not enable [Mutating Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) for Spark pod customization. When enabled, a webhook service and a secret storing the x509 certificate called `spark-webhook-certs` are created for that purpose. To install the operator **with** the mutating admission webhook on a Kubernetes cluster, install the chart with the flag `enableWebhook=true`:
 
 ```bash
-$ helm install incubator/sparkoperator --namespace spark-operator --set enableWebhook=false
+$ helm install incubator/sparkoperator --namespace spark-operator --set enableWebhook=true
 ```
 
 Due to a [known issue](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control#defining_permissions_in_a_role) in GKE, you will need to first grant yourself cluster-admin privileges before you can create custom roles and role bindings on a GKE cluster versioned 1.6 and up. Run the following command before installing the chart on GKE:

entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# echo commands to the terminal output
+set -ex
+
+# Check whether there is a passwd entry for the container UID
+myuid=$(id -u)
+mygid=$(id -g)
+# turn off -e for getent because it will return error code in anonymous uid case
+set +e
+uidentry=$(getent passwd $myuid)
+set -e
+
+echo $myuid
+echo $mygid
+echo $uidentry
+
+# If there is no passwd entry for the container UID, attempt to create one
+if [[ -z "$uidentry" ]] ; then
+    if [[ -w /etc/passwd ]] ; then
+        echo "$myuid:x:$myuid:$mygid:anonymous uid:$SPARK_HOME:/bin/false" >> /etc/passwd
+    else
+        echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
+    fi
+fi
+
+exec /sbin/tini -s -- /usr/bin/spark-operator "$@"

install_packages.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+arg1=$(head -2 /opt/spark/credentials | tail -1)
+arg2=$(head -3 /opt/spark/credentials | tail -1)
+arg3=$(head -1 /opt/spark/credentials | tail -1)
+
+subscription-manager register --username=$arg1 --password=$arg2  --name=docker
+subscription-manager attach --pool=$arg3 && \
+yum install -y openssl
+subscription-manager remove --al 
+subscription-manager unregister
+subscription-manager clean

main.go

@@ -164,10 +164,11 @@ func main() {
 	var hook *webhook.WebHook
 	if *enableWebhook {
 		var err error
-		hook, err = webhook.New(kubeClient, *webhookCertDir, *webhookSvcNamespace, *webhookSvcName, *webhookPort)
+		hook, err = webhook.New(kubeClient, *webhookCertDir, *webhookSvcNamespace, *webhookSvcName, *webhookPort, *namespace)
 		if err != nil {
 			glog.Fatal(err)
 		}
+
 		if err = hook.Start(*webhookConfigName); err != nil {
 			glog.Fatal(err)
 		}

manifest/spark-operator.yaml

@@ -43,6 +43,5 @@ spec:
       - name: sparkoperator
         image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest
         imagePullPolicy: Always
-        command: ["/usr/bin/spark-operator"]
         args:
         - -logtostderr

pkg/webhook/webhook.go

@@ -30,6 +30,7 @@ import (
 
 	admissionv1beta1 "k8s.io/api/admission/v1beta1"
 	"k8s.io/api/admissionregistration/v1beta1"
+	apiv1 "k8s.io/api/core/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -49,18 +50,20 @@ const (
 )
 
 type WebHook struct {
-	clientset  kubernetes.Interface
-	server     *http.Server
-	cert       *certBundle
-	serviceRef *v1beta1.ServiceReference
+	clientset         kubernetes.Interface
+	server            *http.Server
+	cert              *certBundle
+	serviceRef        *v1beta1.ServiceReference
+	sparkJobNamespace string
 }
 
 func New(
 	clientset kubernetes.Interface,
 	certDir string,
 	webhookServiceNamespace string,
 	webhookServiceName string,
-	webhookPort int) (*WebHook, error) {
+	webhookPort int,
+	jobNamespace string) (*WebHook, error) {
 	cert := &certBundle{
 		serverCertFile: filepath.Join(certDir, serverCertFile),
 		serverKeyFile:  filepath.Join(certDir, serverKeyFile),
@@ -72,7 +75,7 @@ func New(
 		Name:      webhookServiceName,
 		Path:      &path,
 	}
-	hook := &WebHook{clientset: clientset, cert: cert, serviceRef: serviceRef}
+	hook := &WebHook{clientset: clientset, cert: cert, serviceRef: serviceRef, sparkJobNamespace: jobNamespace}
 
 	mux := http.NewServeMux()
 	mux.HandleFunc(path, hook.serve)
@@ -114,6 +117,7 @@ func (wh *WebHook) Stop(webhookConfigName string) error {
 }
 
 func (wh *WebHook) serve(w http.ResponseWriter, r *http.Request) {
+	glog.V(2).Info("Serving admission request")
 	var body []byte
 	if r.Body != nil {
 		data, err := ioutil.ReadAll(r.Body)
@@ -145,7 +149,7 @@ func (wh *WebHook) serve(w http.ResponseWriter, r *http.Request) {
 		glog.Error(err)
 		reviewResponse = toAdmissionResponse(err)
 	} else {
-		reviewResponse = mutatePods(review)
+		reviewResponse = mutatePods(review, wh.sparkJobNamespace)
 	}
 
 	response := admissionv1beta1.AdmissionReview{}
@@ -231,7 +235,7 @@ func (wh *WebHook) selfDeregistration(webhookConfigName string) error {
 	return client.Delete(webhookConfigName, metav1.NewDeleteOptions(0))
 }
 
-func mutatePods(review *admissionv1beta1.AdmissionReview) *admissionv1beta1.AdmissionResponse {
+func mutatePods(review *admissionv1beta1.AdmissionReview, sparkJobNs string) *admissionv1beta1.AdmissionResponse {
 	podResource := metav1.GroupVersionResource{
 		Group:    corev1.SchemeGroupVersion.Group,
 		Version:  corev1.SchemeGroupVersion.Version,
@@ -251,7 +255,8 @@ func mutatePods(review *admissionv1beta1.AdmissionReview) *admissionv1beta1.Admi
 
 	response := &admissionv1beta1.AdmissionResponse{Allowed: true}
 
-	if !isSparkPod(pod) {
+	if !isSparkPod(pod) || !inSparkJobNamespace(review.Request.Namespace, sparkJobNs) {
+		glog.V(2).Info(pod.Name, " in namespace ", review.Request.Namespace, " not mutated")
 		return response
 	}
 
@@ -284,6 +289,13 @@ func toAdmissionResponse(err error) *admissionv1beta1.AdmissionResponse {
 	}
 }
 
+func inSparkJobNamespace(podNs string, sparkJobNamespace string) bool {
+	if sparkJobNamespace == apiv1.NamespaceAll {
+		return true
+	}
+	return podNs == sparkJobNamespace
+}
+
 func isSparkPod(pod *corev1.Pod) bool {
 	launchedBySparkOperator, ok := pod.Labels[config.LaunchedBySparkOperatorLabel]
 	if !ok {

pkg/webhook/webhook_test.go

@@ -35,6 +35,8 @@ import (
 )
 
 func TestMutatePod(t *testing.T) {
+	sparkJobNamepace := "default"
+
 	// Testing processing non-Spark pod.
 	pod1 := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
@@ -64,9 +66,10 @@ func TestMutatePod(t *testing.T) {
 			Object: runtime.RawExtension{
 				Raw: podBytes,
 			},
+			Namespace: "default",
 		},
 	}
-	response := mutatePods(review)
+	response := mutatePods(review, sparkJobNamepace)
 	assert.True(t, response.Allowed)
 
 	// Test processing Spark pod without any patch.
@@ -165,7 +168,7 @@ func TestMutatePod(t *testing.T) {
 		t.Error(err)
 	}
 	review.Request.Object.Raw = podBytes
-	response = mutatePods(review)
+	response = mutatePods(review, sparkJobNamepace)
 	assert.True(t, response.Allowed)
 	assert.Equal(t, v1beta1.PatchTypeJSONPatch, *response.PatchType)
 	assert.True(t, len(response.Patch) > 0)