sync between master and forked branch_1712

.devcontainer/Dockerfile

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
 #-------------------------------------------------------------------------------------------------------------
 
-FROM golang:1.12.5
+FROM golang:1.12.5-stretch
 
 # Avoid warnings by switching to noninteractive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -47,7 +47,7 @@ RUN apt-get update \
     # Clean up
     && apt-get autoremove -y \
     && apt-get clean -y \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* 
 
 RUN apt-get update \
     # Install Docker CE CLI
@@ -62,10 +62,16 @@ RUN apt-get update \
     # Install Helm
     && curl -s https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash -
 
+# Enable bash completion
+RUN apt-get update && apt install -y bash-completion && echo "source /etc/bash_completion" >> "/root/.bashrc"
+
 # Verify git, process tools installed
 RUN apt-get -y install git procps wget nano zsh inotify-tools jq
 RUN wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | zsh || true
 
+# Install golangci-linter
+RUN curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.21.0
+
 ENV PATH="/usr/local/kubebuilder/bin:${PATH}"
 
 ENV GO111MODULE=on
@@ -81,3 +87,26 @@ RUN make install-kubebuilder
 RUN make install-kustomize
 RUN make install-test-dependency
 ENV SHELL /bin/bash
+
+# Save command line history 
+RUN echo "export HISTFILE=/root/commandhistory/.bash_history" >> "/root/.bashrc" \
+    && echo "export PROMPT_COMMAND='history -a'" >> "/root/.bashrc" \
+    && mkdir -p /root/commandhistory \
+    && touch /root/commandhistory/.bash_history
+
+# Add useful aliases
+RUN echo "alias k=kubectl" >> "/root/.bashrc"
+# Add autocomplete to kubectl
+RUN echo "source <(kubectl completion bash)" >> "/root/.bashrc"
+RUN echo "source <(kubectl completion bash | sed 's/kubectl/k/g')" >> "/root/.bashrc"
+# Add kubectx 
+RUN git clone https://github.com/ahmetb/kubectx.git /root/.kubectx \
+    && COMPDIR=$(pkg-config --variable=completionsdir bash-completion) \
+    && ln -sf /root/.kubectx/completion/kubens.bash $COMPDIR/kubens \
+    && ln -sf /root/.kubectx/completion/kubectx.bash $COMPDIR/kubectx 
+
+# Git command prompt
+RUN git clone https://github.com/magicmonty/bash-git-prompt.git ~/.bash-git-prompt --depth=1 \
+    && echo "if [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh; fi" >> "/root/.bashrc"
+
+ENV PATH="/root/.kubectx:${PATH}"

.devcontainer/devcontainer.json

@@ -1,17 +1,35 @@
 // If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml.
 {
 	"name": "Go",
-	"dockerComposeFile": "docker-compose.yml",
-    "service": "docker-in-docker",
+	"dockerFile": "./Dockerfile",
 	"workspaceFolder": "/workspace",
-	"postCreateCommand": "make set-kindcluster",
-    "shutdownAction": "stopCompose",
+	"context": "..",
+	"workspaceMount": "src=${localWorkspaceFolder},dst=/workspace,type=bind",
+    "runArgs": [
+		// Mount the env file
+		"--env-file", "${localWorkspaceFolder}/.devcontainer/.env",
+		// Mount go mod cache
+		"-v", "dboperator-gomodcache:/go/pkg",
+		// Cache vscode exentsions installs and homedir
+		"-v", "dboperator-vscodecache:/root/.vscode-server",
+		// Keep command history 
+		"-v", "dboperator-bashhistory:/root/commandhistory",
+		// Enable security needed for docker
+		"--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined",
+		// Mount docker socket for docker builds
+		"-v", "/var/run/docker.sock:/var/run/docker.sock",
+		"--network", "host",
+		"-e GO111MODULE=on",
+		// Mount azure
+		"-v", "${env:HOME}${env:USERPROFILE}/.azure:/home/vscode/.azure"
+		// Optionally mount k8s auth to existing cluster
+		// "-v", "${env:HOME}${env:USERPROFILE}/.kube:/home/vscode/.kube",
+	],
 	"extensions": [
 		"ms-azuretools.vscode-docker",
 		"ms-vscode.go"
 	],
 	"settings": {
-		"terminal.integrated.shell.linux": "zsh",
 		"go.gopath": "/go",
 		"go.inferGopath": true,
 		"go.useLanguageServer": true,

.golangci.yml

@@ -0,0 +1,44 @@
+run:
+  deadline: 5m
+  skip-files: []
+
+linters-settings:
+  linters-settings.govet:
+    check-shadowing: true
+
+  linters-settings.gocyclo:
+    min-complexity: 12.0
+
+  linters-settings.maligned:
+    suggest-new: true
+
+  linters-settings.goconst:
+    min-len: 3.0
+    min-occurrences: 3.0
+
+  linters-settings.misspell:
+    locale: "US"
+    ignore-words: 
+     - listend
+     - analyses
+     - cancelling
+
+linters:
+  enable:
+    - vet
+    - golint
+    - gofmt
+    - deadcode
+    - varcheck
+    - structcheck
+    - misspell
+    - errcheck
+    - gosimple
+    - govet
+    - ineffassign
+
+issues:
+  exclude-use-default: false
+  max-per-linter: 0
+  max-same-issues: 0
+  exclude: []

Makefile

@@ -4,10 +4,17 @@ IMG ?= controller:latest
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
 
+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
 all: manager
 
 # Run tests
-test: generate fmt vet manifests
+test: generate fmt lint vet manifests
 	rm -rf cover.* cover
 	mkdir -p cover
 
@@ -20,7 +27,7 @@ test: generate fmt vet manifests
 	rm -f cover.out cover.out.tmp cover.json
 
 # Run tests with existing cluster
-test-existing: generate fmt vet manifests
+test-existing: generate fmt lint vet manifests
 	rm -rf cover.* cover
 	mkdir -p cover
 
@@ -33,20 +40,23 @@ test-existing: generate fmt vet manifests
 	rm -f cover.out cover.out.tmp cover.json
 
 # Build manager binary
-manager: generate fmt vet
+manager: generate fmt lint vet
 	go build -o bin/manager main.go
 
 # Run against the configured Kubernetes cluster in ~/.kube/config
-run: generate fmt vet
+run: generate fmt lint vet manifests
 	go run ./main.go
 
 # Install CRDs into a cluster
 install: manifests
-	kubectl apply -f config/crd/bases
+	kustomize build config/crd | kubectl apply -f -
+# Uninstall CRDs from a cluster
+uninstall: manifests
+	kustomize build config/crd | kubectl delete -f -
 
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
 deploy: manifests
-	kubectl apply -f config/crd/bases
+	cd config/manager && kustomize edit set image controller=${IMG}
 	kustomize build config/default | kubectl apply -f -
 
 deploy-controller:
@@ -78,18 +88,22 @@ manifests: controller-gen
 
 # Run go fmt against code
 fmt:
-	go fmt ./...
+	find . -name '*.go' | grep -v vendor | xargs gofmt -s -w
 
 # Run go vet against code
 vet:
 	go vet ./...
-
+
+# Run linting
+lint:
+	GO111MODULE=on golangci-lint run
+
 # Generate code
 generate: controller-gen
-	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths=./api/...
+	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
 
 # Build the docker image
-docker-build:
+docker-build: test
 	docker build . -t ${IMG} ${ARGS}
 	@echo "updating kustomize image patch file for manager resource"
 	sed -i'' -e 's@image: .*@image: '"${IMG}"'@' ./config/default/manager_image_patch.yaml
@@ -102,12 +116,18 @@ docker-push:
 # download controller-gen if necessary
 controller-gen:
 ifeq (, $(shell which controller-gen))
-	go get sigs.k8s.io/controller-tools/cmd/[email protected]
-CONTROLLER_GEN="$(shell go env GOPATH)/bin/controller-gen"
+	@{ \
+	set -e ;\
+	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
+	cd $$CONTROLLER_GEN_TMP_DIR ;\
+	go mod init tmp ;\
+	go get sigs.k8s.io/controller-tools/cmd/[email protected] ;\
+	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
+	}
+CONTROLLER_GEN=$(GOBIN)/controller-gen
 else
-CONTROLLER_GEN="$(shell which controller-gen)"
+CONTROLLER_GEN=$(shell which controller-gen)
 endif
-
 create-kindcluster:
 ifeq (,$(shell kind get clusters))
 	@echo "no kind cluster"
@@ -137,7 +157,6 @@ endif
 	@echo "deploying controller to cluster"
 	make deploy-controller
 
-
 install-kind:
 ifeq (,$(shell which kind))
 	@echo "installing kind"
@@ -150,10 +169,10 @@ install-kubebuilder:
 ifeq (,$(shell which kubebuilder))
 	@echo "installing kubebuilder"
 	# download kubebuilder and extract it to tmp
-	curl -sL https://go.kubebuilder.io/dl/2.0.0-rc.0/$(shell go env GOOS)/$(shell go env GOARCH) | tar -xz -C /tmp/
+	curl -sL https://go.kubebuilder.io/dl/2.2.0/$(shell go env GOOS)/$(shell go env GOARCH) | tar -xz -C /tmp/
 	# move to a long-term location and put it on your path
 	# (you'll need to set the KUBEBUILDER_ASSETS env var if you put it somewhere else)
-	mv /tmp/kubebuilder_2.0.0-rc.0_$(shell go env GOOS)_$(shell go env GOARCH) /usr/local/kubebuilder
+	mv /tmp/kubebuilder_2.2.0_$(shell go env GOOS)_$(shell go env GOARCH) /usr/local/kubebuilder
 	export PATH=$PATH:/usr/local/kubebuilder/bin
 else
 	@echo "kubebuilder has been installed"
@@ -163,7 +182,8 @@ install-kustomize:
 ifeq (,$(shell which kustomize))
 	@echo "installing kustomize"
 	# download kustomize
-	curl -o /usr/local/kubebuilder/bin/kustomize -sL "https://go.kubebuilder.io/kustomize/$(shell go env GOOS)/$(shell go env GOARCH)"
+	curl -sL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv3.4.0/kustomize_v3.4.0_$(shell go env GOOS)_$(shell go env GOARCH).tar.gz | tar -xz -C /tmp/
+	mv /tmp/kustomize /usr/local/kubebuilder/bin/kustomize
 	# set permission
 	chmod a+x /usr/local/kubebuilder/bin/kustomize
 	$(shell which kustomize)

README.md

@@ -1,20 +1,22 @@
-
-[![Build Status](https://dev.azure.com/ms/azure-databricks-operator/_apis/build/status/microsoft.azure-databricks-operator?branchName=master)](https://dev.azure.com/ms/azure-databricks-operator/_build/latest?definitionId=254&branchName=master)
-
 # Azure Databricks operator (for Kubernetes)
-
+[![Build Status](https://dev.azure.com/ms/azure-databricks-operator/_apis/build/status/microsoft.azure-databricks-operator?branchName=master)](https://dev.azure.com/ms/azure-databricks-operator/_build/latest?definitionId=254&branchName=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/microsoft/azure-databricks-operator)](https://goreportcard.com/report/github.com/microsoft/azure-databricks-operator)
+[![License: MIT](https://img.shields.io/github/license/microsoft/azure-databricks-operator)](https://github.com/microsoft/azure-databricks-operator/blob/master/LICENSE)
 > This project is experimental. Expect the API to change. It is not recommended for production environments.
 
 
 ## Introduction
 
 Kubernetes offers the facility of extending its API through the concept of 'Operators' ([Introducing Operators: Putting Operational Knowledge into Software](https://coreos.com/blog/introducing-operators.html)). This repository contains the resources and code to deploy an Azure Databricks Operator for Kubernetes.
 
+![alt text](docs/images/azure-databricks-operator-highlevel.jpg "high level architecture")
 
-![alt text](docs/images/azure-databricks-operator.jpg "high level architecture")
 
 The Databricks operator is useful in situations where Kubernetes hosted applications wish to launch and use Databricks data engineering and machine learning tasks.
 
+![alt text](docs/images/azure-databricks-operator.jpg "high level architecture")
+
+
 The project was built using
 
 1. [Kubebuilder](https://book.kubebuilder.io/)

api/v1alpha1/dbfsblock_types.go

@@ -52,10 +52,12 @@ type DbfsBlock struct {
 	Status *DbfsBlockStatus `json:"status,omitempty"`
 }
 
+// IsBeingDeleted returns true if a deletion timestamp is set
 func (dbfsBlock *DbfsBlock) IsBeingDeleted() bool {
 	return !dbfsBlock.ObjectMeta.DeletionTimestamp.IsZero()
 }
 
+// IsSubmitted returns true if the item has been submitted to DataBricks
 func (dbfsBlock *DbfsBlock) IsSubmitted() bool {
 	if dbfsBlock.Status == nil ||
 		dbfsBlock.Status.FileInfo == nil ||
@@ -81,21 +83,28 @@ func (dbfsBlock *DbfsBlock) GetHash() string {
 		return ""
 	}
 	h := sha1.New()
-	h.Write(data)
+	_, err = h.Write(data)
+	if err != nil {
+		return ""
+	}
 	bs := h.Sum(nil)
 	return fmt.Sprintf("%x", bs)
 }
 
+// DbfsBlockFinalizerName is the name of the dbfs block finalizer
 const DbfsBlockFinalizerName = "dbfsBlock.finalizers.databricks.microsoft.com"
 
+// HasFinalizer returns true if the item has the specified finalizer
 func (dbfsBlock *DbfsBlock) HasFinalizer(finalizerName string) bool {
 	return containsString(dbfsBlock.ObjectMeta.Finalizers, finalizerName)
 }
 
+// AddFinalizer adds the specified finalizer
 func (dbfsBlock *DbfsBlock) AddFinalizer(finalizerName string) {
 	dbfsBlock.ObjectMeta.Finalizers = append(dbfsBlock.ObjectMeta.Finalizers, finalizerName)
 }
 
+// RemoveFinalizer removes the specified finalizer
 func (dbfsBlock *DbfsBlock) RemoveFinalizer(finalizerName string) {
 	dbfsBlock.ObjectMeta.Finalizers = removeString(dbfsBlock.ObjectMeta.Finalizers, finalizerName)
 }