diff --git a/Dockerfile b/Dockerfile index 43fc1cd6b..db3d82d05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.4.0 -FROM golang:1.11.2-alpine as builder +FROM golang:1.11.4-alpine as builder ARG DEP_VERSION="0.5.0" RUN apk add --no-cache bash git ADD https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 /usr/bin/dep @@ -30,8 +30,8 @@ RUN go generate && CGO_ENABLED=0 GOOS=linux go build -o /usr/bin/spark-operator FROM ${SPARK_IMAGE} COPY --from=builder /usr/bin/spark-operator /usr/bin/ -RUN apk add --no-cache openssl curl +RUN apk add --no-cache openssl curl tini COPY hack/gencerts.sh /usr/bin/ -ENTRYPOINT ["/usr/bin/spark-operator"] - +COPY entrypoint.sh /usr/bin/ +ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/Dockerfile.rh b/Dockerfile.rh new file mode 100644 index 000000000..6c6da6d53 --- /dev/null +++ b/Dockerfile.rh @@ -0,0 +1,53 @@ +# syntax=docker/dockerfile:experimental +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Build an OpenShift image. +# Before running docker build, make sure +# 1. Your Docker version is >= 18.09 +# 2. export DOCKER_BUILDKIT=1 + +ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.4.0 +ARG USER_ID=185 + +FROM golang:1.11.4-alpine as builder +ARG DEP_VERSION="0.5.0" +RUN apk add --no-cache bash git +ADD https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 /usr/bin/dep +RUN chmod +x /usr/bin/dep + +WORKDIR ${GOPATH}/src/github.com/GoogleCloudPlatform/spark-on-k8s-operator +COPY Gopkg.toml Gopkg.lock ./ +RUN dep ensure -vendor-only +COPY . ./ +RUN go generate && CGO_ENABLED=0 GOOS=linux go build -o /usr/bin/spark-operator + +FROM ${SPARK_IMAGE} +COPY --from=builder /usr/bin/spark-operator /usr/bin/ +USER root + +# Comment out the following three lines if you do not have a RedHat subscription. +COPY install_packages.sh / +RUN --mount=target=/opt/spark/credentials,type=secret,id=credentials,required /install_packages.sh +RUN rm /install_packages.sh + +RUN chmod -R u+x /tmp + +RUN apk add --no-cache openssl curl tini +COPY hack/gencerts.sh /usr/bin/ +COPY entrypoint.sh /usr/bin/ +USER ${USER_ID} +ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/docs/quick-start-guide.md b/docs/quick-start-guide.md index 9ea29b18a..cda9da9b4 100644 --- a/docs/quick-start-guide.md +++ b/docs/quick-start-guide.md @@ -20,10 +20,10 @@ $ helm repo add incubator http://storage.googleapis.com/kubernetes-charts-incuba $ helm install incubator/sparkoperator --namespace spark-operator ``` -Installing the chart will create a namespace `spark-operator` if it doesn't exist, set up RBAC for the operator to run in the namespace. It will also set up RBAC for driver pods of your Spark applications to be able to manipulate executor pods. In addition, the chart will create a Deployment in the namespace `spark-operator`. The chart by default enables a [Mutating Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) for Spark pod customization. A webhook service and a secret storing the x509 certificate called `spark-webhook-certs` are created for that purpose. To install the operator **without** the mutating admission webhook on a Kubernetes cluster, install the chart with the flag `enableWebhook=false`: +Installing the chart will create a namespace `spark-operator` if it doesn't exist, set up RBAC for the operator to run in the namespace. It will also set up RBAC for driver pods of your Spark applications to be able to manipulate executor pods. In addition, the chart will create a Deployment in the namespace `spark-operator`. The chart by default does not enable [Mutating Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) for Spark pod customization. When enabled, a webhook service and a secret storing the x509 certificate called `spark-webhook-certs` are created for that purpose. To install the operator **with** the mutating admission webhook on a Kubernetes cluster, install the chart with the flag `enableWebhook=true`: ```bash -$ helm install incubator/sparkoperator --namespace spark-operator --set enableWebhook=false +$ helm install incubator/sparkoperator --namespace spark-operator --set enableWebhook=true ``` Due to a [known issue](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control#defining_permissions_in_a_role) in GKE, you will need to first grant yourself cluster-admin privileges before you can create custom roles and role bindings on a GKE cluster versioned 1.6 and up. Run the following command before installing the chart on GKE: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 000000000..a2921e5f9 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# echo commands to the terminal output +set -ex + +# Check whether there is a passwd entry for the container UID +myuid=$(id -u) +mygid=$(id -g) +# turn off -e for getent because it will return error code in anonymous uid case +set +e +uidentry=$(getent passwd $myuid) +set -e + +echo $myuid +echo $mygid +echo $uidentry + +# If there is no passwd entry for the container UID, attempt to create one +if [[ -z "$uidentry" ]] ; then + if [[ -w /etc/passwd ]] ; then + echo "$myuid:x:$myuid:$mygid:anonymous uid:$SPARK_HOME:/bin/false" >> /etc/passwd + else + echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID" + fi +fi + +exec /sbin/tini -s -- /usr/bin/spark-operator "$@" diff --git a/install_packages.sh b/install_packages.sh new file mode 100755 index 000000000..b0a1bf8a5 --- /dev/null +++ b/install_packages.sh @@ -0,0 +1,11 @@ +#!/bin/bash +arg1=$(head -2 /opt/spark/credentials | tail -1) +arg2=$(head -3 /opt/spark/credentials | tail -1) +arg3=$(head -1 /opt/spark/credentials | tail -1) + +subscription-manager register --username=$arg1 --password=$arg2 --name=docker +subscription-manager attach --pool=$arg3 && \ +yum install -y openssl +subscription-manager remove --al +subscription-manager unregister +subscription-manager clean diff --git a/main.go b/main.go index ef719fd2a..c9e767026 100644 --- a/main.go +++ b/main.go @@ -164,10 +164,11 @@ func main() { var hook *webhook.WebHook if *enableWebhook { var err error - hook, err = webhook.New(kubeClient, *webhookCertDir, *webhookSvcNamespace, *webhookSvcName, *webhookPort) + hook, err = webhook.New(kubeClient, *webhookCertDir, *webhookSvcNamespace, *webhookSvcName, *webhookPort, *namespace) if err != nil { glog.Fatal(err) } + if err = hook.Start(*webhookConfigName); err != nil { glog.Fatal(err) } diff --git a/manifest/spark-operator.yaml b/manifest/spark-operator.yaml index ec808684a..7f0563544 100644 --- a/manifest/spark-operator.yaml +++ b/manifest/spark-operator.yaml @@ -43,6 +43,5 @@ spec: - name: sparkoperator image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest imagePullPolicy: Always - command: ["/usr/bin/spark-operator"] args: - -logtostderr diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 4f0aad500..8c912b9ae 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -30,6 +30,7 @@ import ( admissionv1beta1 "k8s.io/api/admission/v1beta1" "k8s.io/api/admissionregistration/v1beta1" + apiv1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -49,10 +50,11 @@ const ( ) type WebHook struct { - clientset kubernetes.Interface - server *http.Server - cert *certBundle - serviceRef *v1beta1.ServiceReference + clientset kubernetes.Interface + server *http.Server + cert *certBundle + serviceRef *v1beta1.ServiceReference + sparkJobNamespace string } func New( @@ -60,7 +62,8 @@ func New( certDir string, webhookServiceNamespace string, webhookServiceName string, - webhookPort int) (*WebHook, error) { + webhookPort int, + jobNamespace string) (*WebHook, error) { cert := &certBundle{ serverCertFile: filepath.Join(certDir, serverCertFile), serverKeyFile: filepath.Join(certDir, serverKeyFile), @@ -72,7 +75,7 @@ func New( Name: webhookServiceName, Path: &path, } - hook := &WebHook{clientset: clientset, cert: cert, serviceRef: serviceRef} + hook := &WebHook{clientset: clientset, cert: cert, serviceRef: serviceRef, sparkJobNamespace: jobNamespace} mux := http.NewServeMux() mux.HandleFunc(path, hook.serve) @@ -114,6 +117,7 @@ func (wh *WebHook) Stop(webhookConfigName string) error { } func (wh *WebHook) serve(w http.ResponseWriter, r *http.Request) { + glog.V(2).Info("Serving admission request") var body []byte if r.Body != nil { data, err := ioutil.ReadAll(r.Body) @@ -145,7 +149,7 @@ func (wh *WebHook) serve(w http.ResponseWriter, r *http.Request) { glog.Error(err) reviewResponse = toAdmissionResponse(err) } else { - reviewResponse = mutatePods(review) + reviewResponse = mutatePods(review, wh.sparkJobNamespace) } response := admissionv1beta1.AdmissionReview{} @@ -231,7 +235,7 @@ func (wh *WebHook) selfDeregistration(webhookConfigName string) error { return client.Delete(webhookConfigName, metav1.NewDeleteOptions(0)) } -func mutatePods(review *admissionv1beta1.AdmissionReview) *admissionv1beta1.AdmissionResponse { +func mutatePods(review *admissionv1beta1.AdmissionReview, sparkJobNs string) *admissionv1beta1.AdmissionResponse { podResource := metav1.GroupVersionResource{ Group: corev1.SchemeGroupVersion.Group, Version: corev1.SchemeGroupVersion.Version, @@ -251,7 +255,8 @@ func mutatePods(review *admissionv1beta1.AdmissionReview) *admissionv1beta1.Admi response := &admissionv1beta1.AdmissionResponse{Allowed: true} - if !isSparkPod(pod) { + if !isSparkPod(pod) || !inSparkJobNamespace(review.Request.Namespace, sparkJobNs) { + glog.V(2).Info(pod.Name, " in namespace ", review.Request.Namespace, " not mutated") return response } @@ -284,6 +289,13 @@ func toAdmissionResponse(err error) *admissionv1beta1.AdmissionResponse { } } +func inSparkJobNamespace(podNs string, sparkJobNamespace string) bool { + if sparkJobNamespace == apiv1.NamespaceAll { + return true + } + return podNs == sparkJobNamespace +} + func isSparkPod(pod *corev1.Pod) bool { launchedBySparkOperator, ok := pod.Labels[config.LaunchedBySparkOperatorLabel] if !ok { diff --git a/pkg/webhook/webhook_test.go b/pkg/webhook/webhook_test.go index 619632e89..52dc9d9dc 100644 --- a/pkg/webhook/webhook_test.go +++ b/pkg/webhook/webhook_test.go @@ -35,6 +35,8 @@ import ( ) func TestMutatePod(t *testing.T) { + sparkJobNamepace := "default" + // Testing processing non-Spark pod. pod1 := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -64,9 +66,10 @@ func TestMutatePod(t *testing.T) { Object: runtime.RawExtension{ Raw: podBytes, }, + Namespace: "default", }, } - response := mutatePods(review) + response := mutatePods(review, sparkJobNamepace) assert.True(t, response.Allowed) // Test processing Spark pod without any patch. @@ -165,7 +168,7 @@ func TestMutatePod(t *testing.T) { t.Error(err) } review.Request.Object.Raw = podBytes - response = mutatePods(review) + response = mutatePods(review, sparkJobNamepace) assert.True(t, response.Allowed) assert.Equal(t, v1beta1.PatchTypeJSONPatch, *response.PatchType) assert.True(t, len(response.Patch) > 0)