Merge pull request #2 from microsoft/master

Extended databricks operator to report metrics into Prometheus (Azure#104)

Author: amrasem

.devcontainer/Dockerfile

@@ -60,7 +60,8 @@ RUN apt-get update \
     && curl -sSL -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
     && chmod +x /usr/local/bin/kubectl \
     # Install Helm
-    && curl -s https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash -
+    && curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash \
+    && helm repo add stable https://kubernetes-charts.storage.googleapis.com/ 
 
 # Enable bash completion
 RUN apt-get update && apt install -y bash-completion && echo "source /etc/bash_completion" >> "/root/.bashrc"
@@ -109,4 +110,4 @@ RUN git clone https://github.com/ahmetb/kubectx.git /root/.kubectx \
 RUN git clone https://github.com/magicmonty/bash-git-prompt.git ~/.bash-git-prompt --depth=1 \
     && echo "if [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh; fi" >> "/root/.bashrc"
 
-ENV PATH="/root/.kubectx:${PATH}"
+ENV PATH="/root/.kubectx:${PATH}"

.gitignore

@@ -21,3 +21,4 @@ cmd/manager/__debug_bin
 .env
 __debug_bin
 settings.json
+operatorsetup.yaml

.golangci.yml renamed to .golangci.yaml

@@ -1,20 +1,24 @@
-
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
+
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
 
+# Prometheus helm installation name
+PROMETHEUS_NAME ?= "prom-azure-databricks-operator"
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
 else
 GOBIN=$(shell go env GOBIN)
 endif
 
+timestamp := $(shell /bin/date "+%Y%m%d-%H%M%S")
 all: manager
 
 # Run tests
-test: generate fmt lint vet manifests
+test: generate fmt vet manifests lint
 	rm -rf cover.* cover
 	mkdir -p cover
 
@@ -27,7 +31,7 @@ test: generate fmt lint vet manifests
 	rm -f cover.out cover.out.tmp cover.json
 
 # Run tests with existing cluster
-test-existing: generate fmt lint vet manifests
+test-existing: generate fmt vet manifests lint
 	rm -rf cover.* cover
 	mkdir -p cover
 
@@ -50,37 +54,35 @@ run: generate fmt lint vet manifests
 # Install CRDs into a cluster
 install: manifests
 	kustomize build config/crd | kubectl apply -f -
+
 # Uninstall CRDs from a cluster
 uninstall: manifests
 	kustomize build config/crd | kubectl delete -f -
 
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
 deploy: manifests
-	cd config/manager && kustomize edit set image controller=${IMG}
-	kustomize build config/default | kubectl apply -f -
-
-deploy-controller:
+ifeq (,$(shell kubectl get namespace azure-databricks-operator-system))
+	@echo "creating azure-databricks-operator-system namespace"
 	kubectl create namespace azure-databricks-operator-system
-	kubectl --namespace azure-databricks-operator-system \
+else
+	@echo "azure-databricks-operator-system namespace exists"
+endif
+ifeq (,$(shell kubectl get secret dbrickssettings --namespace azure-databricks-operator-system))
+	@echo "creating dbrickssettings secret"
+		kubectl --namespace azure-databricks-operator-system \
 		create secret generic dbrickssettings \
 		--from-literal=DatabricksHost="${DATABRICKS_HOST}" \
 		--from-literal=DatabricksToken="${DATABRICKS_TOKEN}"
+else
+	@echo "dbrickssettings secret exists"
+endif
+	cd config/manager && kustomize edit set image controller=${IMG}
+	kustomize build config/default | kubectl apply -f -
+	kustomize build config/default > operatorsetup.yaml
+	
 
-	#create image and load it into cluster
-	IMG="docker.io/controllertest:1" make docker-build
-	kind load docker-image docker.io/controllertest:1 --loglevel "trace"
-	make install
-	make deploy
-	sed -i'' -e 's@image: .*@image: '"IMAGE_URL"'@' ./config/default/manager_image_patch.yaml
 
-timestamp := $(shell /bin/date "+%Y%m%d-%H%M%S")
 
-update-deployed-controller:
-	IMG="docker.io/controllertest:$(timestamp)" make ARGS="${ARGS}" docker-build
-	kind load docker-image docker.io/controllertest:$(timestamp) --loglevel "trace"
-	make install
-	make deploy
-	sed -i'' -e 's@image: .*@image: '"IMAGE_URL"'@' ./config/default/manager_image_patch.yaml
 
 # Generate manifests e.g. CRD, RBAC etc.
 manifests: controller-gen
@@ -103,11 +105,10 @@ generate: controller-gen
 	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
 
 # Build the docker image
-docker-build: test
+docker-build: 
 	docker build . -t ${IMG} ${ARGS}
 	@echo "updating kustomize image patch file for manager resource"
-	sed -i'' -e 's@image: .*@image: '"${IMG}"'@' ./config/default/manager_image_patch.yaml
-
+	cd config/manager && kustomize edit set image controller=${IMG}
 # Push the docker image
 docker-push:
 	docker push ${IMG}
@@ -128,6 +129,7 @@ CONTROLLER_GEN=$(GOBIN)/controller-gen
 else
 CONTROLLER_GEN=$(shell which controller-gen)
 endif
+
 create-kindcluster:
 ifeq (,$(shell kind get clusters))
 	@echo "no kind cluster"
@@ -155,7 +157,21 @@ endif
 	kubectl cluster-info
 
 	@echo "deploying controller to cluster"
-	make deploy-controller
+	make deploy-kindcluster
+	make install
+	make install-prometheus
+
+# Deploy controller
+deploy-kindcluster:
+	#create image and load it into cluster
+	$(eval newimage := "docker.io/controllertest:$(timestamp)")
+	IMG=$(newimage) make docker-build
+	kind load docker-image $(newimage) --loglevel "debug"
+
+	#deploy operator
+	IMG=$(newimage) make deploy
+	#change image name back to orignal image name
+	cd config/manager && kustomize edit set image controller="IMAGE_URL"
 
 install-kind:
 ifeq (,$(shell which kind))
@@ -191,6 +207,12 @@ else
 	@echo "kustomize has been installed"
 endif
 
+install-prometheus:
+	@echo "installing prometheus"
+	# install prometheus (and set to monitor all namespaces in our kind cluster)
+	helm install ${PROMETHEUS_NAME} stable/prometheus-operator --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false 
+	@echo "prometheus has been installed"
+
 install-test-dependency:
 	go get -u github.com/jstemmer/go-junit-report \
 	&& go get github.com/axw/gocov/gocov \

Makefile

@@ -7,12 +7,26 @@
 
 ## Introduction
 
-Kubernetes offers the facility of extending its API through the concept of 'Operators' ([Introducing Operators: Putting Operational Knowledge into Software](https://coreos.com/blog/introducing-operators.html)). This repository contains the resources and code to deploy an Azure Databricks Operator for Kubernetes.
+Kubernetes offers the facility of extending its API through the concept of [Operators](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/). This repository contains the resources and code to deploy an Azure Databricks Operator for Kubernetes.
+
+The Databricks operator is useful in situations where Kubernetes hosted applications wish to launch and use Databricks data engineering and machine learning tasks.
+
+### Key benefits of using Azure Databricks operator
+
+1. Easy to use: Azure Databricks operations can be done by using Kubectl there is no need to learn or install data bricks utils command line and it’s python dependency
+
+2. Security: No need to distribute and use Databricks token, the data bricks token is used by operator 
+
+3. Version control: All the YAML or helm charts which has azure data bricks operations (clusters, jobs, …) can be tracked 
+
+4. Automation: Replicate azure data bricks operations on any data bricks workspace by applying same manifests or helm charts 
+
+
 
 ![alt text](docs/images/azure-databricks-operator-highlevel.jpg "high level architecture")
 
 
-The Databricks operator is useful in situations where Kubernetes hosted applications wish to launch and use Databricks data engineering and machine learning tasks.
+
 
 ![alt text](docs/images/azure-databricks-operator.jpg "high level architecture")
 
@@ -22,12 +36,66 @@ The project was built using
 1. [Kubebuilder](https://book.kubebuilder.io/)
 2. [Golang SDK for DataBricks](https://github.com/xinsnake/databricks-sdk-golang)
 
-## Quick start
+## How to use Azure Databricks operator
+
+1. Download [the latest release manifests](https://github.com/microsoft/azure-databricks-operator/releases):
+
+```sh
+wget https://github.com/microsoft/azure-databricks-operator/releases/latest/download/release.zip
+unzip release.zip
+```
+
+2. Create the `azure-databricks-operator-system` namespace:
+
+```sh
+kubectl create namespace azure-databricks-operator-system
+```
+
+3. Create Kubernetes secrets with values for `DATABRICKS_HOST` and `DATABRICKS_TOKEN`:
+
+```shell
+kubectl --namespace azure-databricks-operator-system \
+    create secret generic dbrickssettings \
+    --from-literal=DatabricksHost="https://xxxx.azuredatabricks.net" \
+    --from-literal=DatabricksToken="xxxxx"
+```
+
+4. Apply the manifests for the Operator and CRDs in `release/config`:
+
+```sh
+kubectl apply -f release/config
+```
+
+
+
+For details deployment guides please see [deploy.md](https://github.com/microsoft/azure-databricks-operator/blob/master/docs/deploy.md)
+
+## Samples 
+
+1. Create a spark cluster on demand and run a databricks notebook.
+
+![alt text](docs/images/sample1.gif "sample1")
+
+2. Create an interactive spark cluster and Run a databricks job on exisiting cluster.
+
+![alt text](docs/images/sample2.gif "sample2")
+
+3. Create azure databricks secret scope by using kuberentese secrets
+
+![alt text](docs/images/sample3.gif "sample3")
 
-For deployment guides please see [deploy.md](https://github.com/microsoft/azure-databricks-operator/blob/master/docs/deploy.md)
 
 For samples and simple use cases on how to use the operator please see [samples.md](docs/samples.md)
 
+## Quick start
+
+On click start by using [vscode](https://code.visualstudio.com/)
+
+![alt text](docs/images/devcontainer.gif "devcontainer")
+
+For more details please see
+[contributing.md](https://github.com/microsoft/azure-databricks-operator/blob/master/docs/contributing.md)
+
 ## Roadmap
 
 Check [roadmap.md](https://github.com/microsoft/azure-databricks-operator/blob/master/docs/roadmap.md) for what has been supported and what's coming.
@@ -36,9 +104,10 @@ Check [roadmap.md](https://github.com/microsoft/azure-databricks-operator/blob/m
 
 Few topics are discussed in the [resources.md](https://github.com/microsoft/azure-databricks-operator/blob/master/docs/resources.md)
 
-- Kubernetes on WSL
-- Build pipelines
 - Dev container
+- Build pipelines
+- Operator metrics
+- Kubernetes on WSL
 
 ## Contributing
 

README.md

@@ -31,10 +31,10 @@ type RunSpec struct {
 	JobName                 string `json:"job_name,omitempty"`
 	*dbmodels.RunParameters `json:",inline"`
 	// dedicated for direct run
-	RunName               string `json:"run_name,omitempty"`
-	*dbmodels.ClusterSpec `json:",inline"`
-	*dbmodels.JobTask     `json:",inline"`
-	TimeoutSeconds        int32 `json:"timeout_seconds,omitempty"`
+	RunName           string `json:"run_name,omitempty"`
+	ClusterSpec       `json:",inline"`
+	*dbmodels.JobTask `json:",inline"`
+	TimeoutSeconds    int32 `json:"timeout_seconds,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1alpha1/run_types.go

@@ -0,0 +1,36 @@
+package v1alpha1
+
+import (
+	dbmodels "github.com/xinsnake/databricks-sdk-golang/azure/models"
+)
+
+// ClusterSpec is similar to dbmodels.ClusterSpec, the reason it
+// exists is because dbmodels.ClusterSpec doesn't support ExistingClusterName
+// ExistingClusterName allows discovering databricks clusters by it's kubernetese object name
+type ClusterSpec struct {
+	ExistingClusterID   string               `json:"existing_cluster_id,omitempty" url:"existing_cluster_id,omitempty"`
+	ExistingClusterName string               `json:"existing_cluster_name,omitempty" url:"existing_cluster_name,omitempty"`
+	NewCluster          *dbmodels.NewCluster `json:"new_cluster,omitempty" url:"new_cluster,omitempty"`
+	Libraries           []dbmodels.Library   `json:"libraries,omitempty" url:"libraries,omitempty"`
+}
+
+// ToK8sClusterSpec converts a databricks ClusterSpec object to k8s ClusterSpec object.
+// It is needed to add ExistingClusterName and follow k8s camleCase naming convention
+func ToK8sClusterSpec(dbjs *dbmodels.ClusterSpec) ClusterSpec {
+	var k8sjs ClusterSpec
+	k8sjs.ExistingClusterID = dbjs.ExistingClusterID
+	k8sjs.NewCluster = dbjs.NewCluster
+	k8sjs.Libraries = dbjs.Libraries
+	return k8sjs
+}
+
+// ToDatabricksClusterSpec converts a k8s ClusterSpec object to a DataBricks ClusterSpec object.
+// It is needed to add ExistingClusterName and follow k8s camleCase naming convention
+func ToDatabricksClusterSpec(k8sjs *ClusterSpec) dbmodels.ClusterSpec {
+
+	var dbjs dbmodels.ClusterSpec
+	dbjs.ExistingClusterID = k8sjs.ExistingClusterID
+	dbjs.NewCluster = k8sjs.NewCluster
+	dbjs.Libraries = k8sjs.Libraries
+	return dbjs
+}

api/v1alpha1/run_types_extra.go

@@ -47,6 +47,8 @@ spec:
           properties:
             existing_cluster_id:
               type: string
+            existing_cluster_name:
+              type: string
             jar_params:
               items:
                 type: string

api/v1alpha1/zz_generated.deepcopy.go

@@ -8,8 +8,7 @@ spec:
     spec:
       containers:
       # Change the value of image field below to your controller image URL
-      - image: IMAGE_URL
-        name: manager
+      - name: manager
         env:
           - name: DATABRICKS_HOST
             valueFrom: