Upgrading Spark to 3.0.0, Spark Operator to v1beta2-1.2.0-3.0.0, Konvoy to 1.5

.gitignore

@@ -13,7 +13,7 @@ konvoy-*
 license.*
 
 admin.conf
-cluster.*.yaml
+cluster.yaml
 inventory.yaml
 license.txt
 

Makefile

@@ -7,8 +7,9 @@ SCRIPTS_DIR := $(ROOT_DIR)/scripts
 KUDO_TOOLS_DIR := $(ROOT_DIR)/shared
 SPARK_OPERATOR_DIR := $(ROOT_DIR)/spark-on-k8s-operator
 
-export KONVOY_VERSION ?= v1.3.0
-export WORKER_NODE_INSTANCE_TYPE ?= m5.xlarge
+export KONVOY_VERSION ?= v1.5.0
+export CLUSTER_CONFIG_YAML ?= $(ROOT_DIR)/cluster.template.yaml
+export WORKER_NODE_INSTANCE_TYPE ?= m5.2xlarge
 export WORKER_NODE_COUNT ?= 5
 
 export NAMESPACE ?= spark
@@ -160,7 +161,7 @@ clean-all:
 
 # function for extracting the value of an AWS property passed as an argument
 define get_aws_credential
-$(if $(AWS_PROFILE),$(shell cat ~/.aws/credentials | grep ${AWS_PROFILE} -A3 | tail -n3 | grep $1 | xargs | cut -d' ' -f3),$(error AWS_PROFILE is not set))
+$(if $(AWS_PROFILE),$(shell cat ~/.aws/credentials | grep ${AWS_PROFILE} -A3 | tail -n3 | grep $1 | xargs | cut -d ' ' -f3),$(warning unable to update $1 from AWS credentials file: AWS_PROFILE is not provided. $1 will not be set))
 endef
 
 # function for calculating global checksum of directories and files passed as arguments.

cluster.template.yaml

@@ -0,0 +1,184 @@
+kind: ClusterProvisioner
+apiVersion: konvoy.mesosphere.io/v1beta2
+metadata:
+  name: kudo-spark
+  creationTimestamp: "2020-01-28T23:15:38Z"
+spec:
+  provider: aws
+  aws:
+    region: us-west-2
+    vpc:
+      overrideDefaultRouteTable: true
+      enableInternetGateway: true
+      enableVPCEndpoints: true
+    availabilityZones:
+      - us-west-2c
+    elb:
+      apiServerPort: 6443
+    tags:
+      owner: default
+  nodePools:
+    - name: worker
+      count: 4
+      machine:
+        rootVolumeSize: 80
+        rootVolumeType: gp2
+        imagefsVolumeEnabled: true
+        imagefsVolumeSize: 160
+        imagefsVolumeType: gp2
+        imagefsVolumeDevice: xvdb
+        type: m5.2xlarge
+    - name: control-plane
+      controlPlane: true
+      count: 1
+      machine:
+        rootVolumeSize: 80
+        rootVolumeType: io1
+        rootVolumeIOPS: 1000
+        imagefsVolumeEnabled: true
+        imagefsVolumeSize: 160
+        imagefsVolumeType: gp2
+        imagefsVolumeDevice: xvdb
+        type: m5.xlarge
+    - name: bastion
+      bastion: true
+      count: 0
+      machine:
+        rootVolumeSize: 10
+        rootVolumeType: gp2
+        imagefsVolumeEnabled: false
+        type: m5.large
+  sshCredentials:
+    user: centos
+    publicKeyFile: kudo-spark-ssh.pub
+    privateKeyFile: kudo-spark-ssh.pem
+  version: v1.5.0
+---
+kind: ClusterConfiguration
+apiVersion: konvoy.mesosphere.io/v1beta2
+metadata:
+  name: kudo-spark
+  creationTimestamp: "2020-01-28T23:15:38Z"
+spec:
+  kubernetes:
+    version: 1.17.8
+    controlPlane:
+      controlPlaneEndpointOverride: ""
+      certificate: {}
+    networking:
+      podSubnet: 192.168.0.0/16
+      serviceSubnet: 10.0.0.0/18
+      iptables:
+        addDefaultRules: false
+      httpProxy: ""
+      httpsProxy: ""
+    cloudProvider:
+      provider: aws
+    admissionPlugins:
+      enabled:
+        - AlwaysPullImages
+        - NodeRestriction
+  containerNetworking:
+    calico:
+      version: v3.13.4
+      encapsulation: ipip
+      mtu: 1480
+  containerRuntime:
+    containerd:
+      version: 1.3.4
+  osPackages:
+    enableAdditionalRepositories: true
+  nodePools:
+    - name: worker
+  addons:
+    - configRepository: https://github.com/mesosphere/kubernetes-base-addons
+      configVersion: stable-1.17-2.0.2
+      addonsList:
+        - name: awsebscsiprovisioner
+          enabled: true
+        - name: awsebsprovisioner
+          enabled: false
+          values: |
+            storageclass:
+              isDefault: false
+        - name: cert-manager
+          enabled: true
+        - name: dashboard
+          enabled: true
+        - name: defaultstorageclass-protection
+          enabled: true
+        - name: dex
+          enabled: true
+        - name: dex-k8s-authenticator
+          enabled: true
+        - name: elasticsearch
+          enabled: false
+        - name: elasticsearchexporter
+          enabled: false
+        - name: flagger
+          enabled: false
+        - name: fluentbit
+          enabled: false
+        - name: gatekeeper
+          enabled: true
+        - name: external-dns
+          enabled: false
+        - name: istio
+          enabled: false
+        - name: kibana
+          enabled: false
+        - name: konvoyconfig
+          enabled: true
+        - name: kube-oidc-proxy
+          enabled: true
+        - name: localvolumeprovisioner
+          enabled: false
+          values: |
+            # Multiple storage classes can be defined here. This allows to, e.g.,
+            # distinguish between different disk types.
+            # For each entry a storage class '$name' and
+            # a host folder '/mnt/$dirName' will be created. Volumes mounted to this
+            # folder are made available in the storage class.
+            storageclasses:
+              - name: localvolumeprovisioner
+                dirName: disks
+                isDefault: false
+                reclaimPolicy: Delete
+                volumeBindingMode: WaitForFirstConsumer
+        - name: nvidia
+          enabled: false
+        - name: opsportal
+          enabled: true
+        - name: prometheus
+          enabled: true
+        - name: prometheusadapter
+          enabled: true
+        - name: reloader
+          enabled: false
+        - name: traefik
+          enabled: true
+        - name: traefik-forward-auth
+          enabled: true
+          values: |
+            traefikForwardAuth:
+              allowedUser:
+                valueFrom:
+                  secretKeyRef: null
+        - name: velero
+          enabled: false
+    - configRepository: https://github.com/mesosphere/kubeaddons-conductor
+      configVersion: stable-1.17-1.0.0
+      addonsList:
+        - name: conductor
+          enabled: false
+    - configRepository: https://github.com/mesosphere/kubeaddons-dispatch
+      configVersion: stable-1.17-1.2.2
+      addonsList:
+      - name: dispatch
+        enabled: false
+    - configRepository: https://github.com/mesosphere/kubeaddons-kommander
+      configVersion: stable-1.17-1.1.0
+      addonsList:
+      - name: kommander
+        enabled: false
+  version: v1.5.0

images/builder/Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:1.13.0@sha256:de697ce5ae02f3d9a57b0603fbb648efadfa212727e702ad3a807b43eba7f6d6
 
-ARG KUDO_DOWNLOAD_URL=https://github.com/kudobuilder/kudo/releases/download/v0.13.0/kubectl-kudo_0.13.0_linux_x86_64
-ARG KUBECTL_DOWNLOAD_URL=https://storage.googleapis.com/kubernetes-release/release/v1.16.0/bin/linux/amd64/kubectl
+ARG KUDO_DOWNLOAD_URL=https://github.com/kudobuilder/kudo/releases/download/v0.15.0/kubectl-kudo_0.15.0_linux_x86_64
+ARG KUBECTL_DOWNLOAD_URL=https://storage.googleapis.com/kubernetes-release/release/v1.17.8/bin/linux/amd64/kubectl
 ARG JAVA_URL=https://downloads.mesosphere.com/java/openjdk-8u212b03-hotspot-linux-x64.tar.gz
 ENV JAVA_HOME=/usr/local/java
 ENV PATH=$PATH:${JAVA_HOME}/bin

images/operator/Dockerfile

@@ -1,19 +1,25 @@
 ARG SPARK_IMAGE=mesosphere/spark:spark-2.4.5-hadoop-2.9-k8s
 
-FROM golang:1.12.5-alpine as builder
-LABEL stage=spark-operator-builder
-ARG DEP_VERSION="0.5.3"
-RUN apk add --no-cache bash git
-ADD https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 /usr/bin/dep
-RUN chmod +x /usr/bin/dep
+FROM golang:1.14.1-alpine as builder
 
-COPY . ${GOPATH}/src/github.com/GoogleCloudPlatform/spark-on-k8s-operator
-WORKDIR ${GOPATH}/src/github.com/GoogleCloudPlatform/spark-on-k8s-operator
-RUN dep ensure -vendor-only \
-    && go generate \
-    && CGO_ENABLED=0 GOOS=linux go build -o /usr/bin/spark-operator
+WORKDIR /workspace
+
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# Cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source code
+COPY main.go main.go
+COPY pkg/ pkg/
+
+# Build
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o /usr/bin/spark-operator main.go
 
 FROM ${SPARK_IMAGE}
+USER root
 COPY --from=builder /usr/bin/spark-operator /usr/bin/
 COPY hack/gencerts.sh /usr/bin/
 

images/spark/Dockerfile

@@ -3,8 +3,8 @@ FROM ubuntu:18.04
 ARG DEBIAN_FRONTEND=noninteractive
 ARG TINI_VERSION=v0.18.0
 ARG SPARK_REPO="apache/spark"
-ARG SPARK_TAG="v2.4.5"
-ARG SCALA_VERSION="2.11"
+ARG SPARK_TAG="v3.0.0"
+ARG SCALA_VERSION="2.12"
 ARG HADOOP_VERSION="2.9.2"
 
 ARG SPARK_BUILD_ARGS="\
@@ -30,7 +30,7 @@ ENV PATH $JAVA_HOME/bin:$PATH
 RUN set -ex && \
     apt-get update && \
     ln -s /lib /lib64 && \
-    apt-get install --no-install-recommends -y bash libc6 libpam-modules krb5-user libnss3 git curl openjdk-8-jdk \
+    apt-get install --no-install-recommends -y bash libc6 libpam-modules krb5-user libnss3 git openssl curl openjdk-8-jdk \
         r-base r-base-dev python python-pip python3 python3-pip ngrep && \
     curl -L https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini -o /usr/bin/tini && \
     chmod +x /usr/bin/tini && \

images/spark/conf/entrypoint.sh

@@ -94,20 +94,13 @@ function add_spark_conf_if_non_empty() {
 # This method is required for passing secret data to spark-submit via environment variables,
 # injected by K8s via Secrets.
 function configure_spark_properties() {
-    add_spark_conf_if_non_empty "spark.authenticate.secret" "${SPARK_AUTHENTICATE_SECRET}"
     add_spark_conf_if_non_empty "spark.ssl.keyPassword" "${SPARK_SSL_KEYPASSWORD}"
     add_spark_conf_if_non_empty "spark.ssl.keyStorePassword" "${SPARK_SSL_KEYSTOREPASSWORD}"
     add_spark_conf_if_non_empty "spark.ssl.trustStorePassword" "${SPARK_SSL_TRUSTSTOREPASSWORD}"
 }
 
 configure_spark_properties
 
-# if SPARK_AUTHENTICATE_SECRET is set, enable RPC authetication for executors
-# and provide the auth secret via _SPARK_AUTH_SECRET variable
-if [[ -n "${SPARK_AUTHENTICATE_SECRET}" ]]; then
-  SPARK_EXECUTOR_JAVA_OPTS+=("-Dspark.authenticate=true")
-  export _SPARK_AUTH_SECRET="${SPARK_AUTHENTICATE_SECRET}"
-fi
 
 case "$SPARK_K8S_CMD" in
   driver)

scale-tests/templates/gensort-application.tmpl

@@ -8,7 +8,7 @@ spec:
   image: mesosphere/spark:spark-2.4.5-hadoop-2.9-k8s
   imagePullPolicy: Always
   mainClass: sorting.DatasetGenerator
-  mainApplicationFile: "https://kudo-spark.s3-us-west-2.amazonaws.com/spark-scala-tests-2.4.5-20200225.jar"
+  mainApplicationFile: "https://kudo-spark.s3-us-west-2.amazonaws.com/spark-scala-tests-3.0.0-20200819.jar"
   arguments:
     - "--num-files"
     - "1000"

scale-tests/templates/scale-test-application.tmpl

@@ -8,7 +8,7 @@ spec:
   image: mesosphere/spark:spark-2.4.5-hadoop-2.9-k8s
   imagePullPolicy: Always
   mainClass: MockTaskRunner
-  mainApplicationFile: "https://kudo-spark.s3-us-west-2.amazonaws.com/spark-scala-tests-2.4.5-20200225.jar"
+  mainApplicationFile: "https://kudo-spark.s3-us-west-2.amazonaws.com/spark-scala-tests-3.0.0-20200819.jar"
   arguments:
     - "NUM_TASKS"
     - "TASK_DURATION_SEC"

scale-tests/templates/sort-application.tmpl

@@ -8,7 +8,7 @@ spec:
   image: mesosphere/spark:spark-2.4.5-hadoop-2.9-k8s
   imagePullPolicy: Always
   mainClass: sorting.SortingApp
-  mainApplicationFile: "https://kudo-spark.s3-us-west-2.amazonaws.com/spark-scala-tests-2.4.5-20200225.jar"
+  mainApplicationFile: "https://kudo-spark.s3-us-west-2.amazonaws.com/spark-scala-tests-3.0.0-20200819.jar"
   arguments:
     - "SOURCE_PATH"
     - "TARGET_PATH"

specs/spark-application.yaml

@@ -8,7 +8,7 @@ spec:
   image: mesosphere/spark:spark-2.4.5-hadoop-2.9-k8s
   imagePullPolicy: Always
   mainClass: MockTaskRunner
-  mainApplicationFile: "https://kudo-spark.s3-us-west-2.amazonaws.com/spark-scala-tests-2.4.5-20200225.jar"
+  mainApplicationFile: "https://kudo-spark.s3-us-west-2.amazonaws.com/spark-scala-tests-3.0.0-20200819.jar"
   arguments:
     - "1"
     - "120"

tests/basic_test.go

@@ -66,8 +66,8 @@ func TestJobSubmission(t *testing.T) {
 	}
 
 	job := utils.SparkJob{
-		Name:     "linear-regression",
-		Template: "spark-linear-regression-job.yaml",
+		Name:     "spark-pi",
+		Template: "spark-pi.yaml",
 	}
 
 	err = spark.SubmitJob(&job)
@@ -181,9 +181,9 @@ func TestSparkHistoryServerInstallation(t *testing.T) {
 	}
 
 	job := utils.SparkJob{
-		Name:     "history-server-linear-regression",
+		Name:     "spark-pi-history-server",
 		Params:   sparkAppParams,
-		Template: "spark-linear-regression-history-server-job.yaml",
+		Template: "spark-pi-history-server-job.yaml",
 	}
 
 	// Submit a SparkApplication
@@ -347,7 +347,7 @@ func TestRSupport(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err := spark.WaitForOutput(job, "3   2.997274"); err != nil {
+	if err := spark.WaitForOutput(job, "userId movieId rating prediction"); err != nil {
 		t.Fatal(err)
 	}