Skip to content
This repository was archived by the owner on Jan 28, 2022. It is now read-only.

Commit 21398ff

Browse files
lawrencegripperlawrencegripper
committed
Merge remote-tracking branch 'origin/master' into lg/devcontainerlinux
2 parents 479510d + ef5fe9c commit 21398ff

31 files changed

+772
-95
lines changed

‎README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,14 @@
1010

1111
Kubernetes offers the facility of extending its API through the concept of 'Operators' ([Introducing Operators: Putting Operational Knowledge into Software](https://coreos.com/blog/introducing-operators.html)). This repository contains the resources and code to deploy an Azure Databricks Operator for Kubernetes.
1212

13+
![alt text](docs/images/azure-databricks-operator-highlevel.jpg "high level architecture")
1314

14-
![alt text](docs/images/azure-databricks-operator.jpg "high level architecture")
1515

1616
The Databricks operator is useful in situations where Kubernetes hosted applications wish to launch and use Databricks data engineering and machine learning tasks.
1717

18+
![alt text](docs/images/azure-databricks-operator.jpg "high level architecture")
19+
20+
1821
The project was built using
1922

2023
1. [Kubebuilder](https://book.kubebuilder.io/)

‎api/v1alpha1/dbfsblock_types_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,13 @@ var _ = Describe("DbfsBlock", func() {
5454
It("should create an object successfully", func() {
5555

5656
key = types.NamespacedName{
57-
Name: "foo",
57+
Name: "foo" + RandomString(5),
5858
Namespace: "default",
5959
}
6060
created = &DbfsBlock{
6161
ObjectMeta: metav1.ObjectMeta{
62-
Name: "foo",
63-
Namespace: "default",
62+
Name: key.Name,
63+
Namespace: key.Namespace,
6464
}}
6565

6666
By("creating an API obj")

‎api/v1alpha1/dcluster_types.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ type DclusterStatus struct {
3232
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
3333
// +kubebuilder:printcolumn:name="ClusterID",type="string",JSONPath=".status.cluster_info.cluster_id"
3434
// +kubebuilder:printcolumn:name="State",type="string",JSONPath=".status.cluster_info.state"
35-
// +kubebuilder:printcolumn:name="NumWorkers",type="integer",JSONPath=".status.cluster_info.num_workers"
3635
type Dcluster struct {
3736
metav1.TypeMeta `json:",inline"`
3837
metav1.ObjectMeta `json:"metadata,omitempty"`

‎api/v1alpha1/dcluster_types_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,13 @@ var _ = Describe("Dcluster", func() {
5454
It("should create an object successfully", func() {
5555

5656
key = types.NamespacedName{
57-
Name: "foo",
57+
Name: "foo-" + RandomString(5),
5858
Namespace: "default",
5959
}
6060
created = &Dcluster{
6161
ObjectMeta: metav1.ObjectMeta{
62-
Name: "foo",
63-
Namespace: "default",
62+
Name: key.Name,
63+
Namespace: key.Namespace,
6464
}}
6565

6666
By("creating an API obj")

‎api/v1alpha1/djob_types.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ type Djob struct {
3636
metav1.TypeMeta `json:",inline"`
3737
metav1.ObjectMeta `json:"metadata,omitempty"`
3838

39-
Spec *dbmodels.JobSettings `json:"spec,omitempty"`
40-
Status *DjobStatus `json:"status,omitempty"`
39+
Spec *JobSettings `json:"spec,omitempty"`
40+
Status *DjobStatus `json:"status,omitempty"`
4141
}
4242

4343
// IsBeingDeleted returns true if a deletion timestamp is set

‎api/v1alpha1/djob_types_extra.go‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
Copyright 2019 microsoft.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package v1alpha1
18+
19+
import (
20+
dbmodels "github.com/xinsnake/databricks-sdk-golang/azure/models"
21+
)
22+
23+
// JobSettings is similar to dbmodels.JobSettings, the reason it
24+
// exists is because dbmodels.JobSettings doesn't support ExistingClusterName
25+
// ExistingClusterName allows discovering databricks clusters by it's kubernetese object name
26+
type JobSettings struct {
27+
ExistingClusterID string `json:"existing_cluster_id,omitempty" url:"existing_cluster_id,omitempty"`
28+
ExistingClusterName string `json:"existing_cluster_name,omitempty" url:"existing_cluster_name,omitempty"`
29+
NewCluster *dbmodels.NewCluster `json:"new_cluster,omitempty" url:"new_cluster,omitempty"`
30+
NotebookTask *dbmodels.NotebookTask `json:"notebook_task,omitempty" url:"notebook_task,omitempty"`
31+
SparkJarTask *dbmodels.SparkJarTask `json:"spark_jar_task,omitempty" url:"spark_jar_task,omitempty"`
32+
SparkPythonTask *dbmodels.SparkPythonTask `json:"spark_python_task,omitempty" url:"spark_python_task,omitempty"`
33+
SparkSubmitTask *dbmodels.SparkSubmitTask `json:"spark_submit_task,omitempty" url:"spark_submit_task,omitempty"`
34+
Name string `json:"name,omitempty" url:"name,omitempty"`
35+
Libraries []dbmodels.Library `json:"libraries,omitempty" url:"libraries,omitempty"`
36+
EmailNotifications *dbmodels.JobEmailNotifications `json:"email_notifications,omitempty" url:"email_notifications,omitempty"`
37+
TimeoutSeconds int32 `json:"timeout_seconds,omitempty" url:"timeout_seconds,omitempty"`
38+
MaxRetries int32 `json:"max_retries,omitempty" url:"max_retries,omitempty"`
39+
MinRetryIntervalMillis int32 `json:"min_retry_interval_millis,omitempty" url:"min_retry_interval_millis,omitempty"`
40+
RetryOnTimeout bool `json:"retry_on_timeout,omitempty" url:"retry_on_timeout,omitempty"`
41+
Schedule *dbmodels.CronSchedule `json:"schedule,omitempty" url:"schedule,omitempty"`
42+
MaxConcurrentRuns int32 `json:"max_concurrent_runs,omitempty" url:"max_concurrent_runs,omitempty"`
43+
}
44+
45+
// ToK8sJobSettings converts a databricks JobSettings object to k8s JobSettings object.
46+
// It is needed to add ExistingClusterName and follow k8s camleCase naming convention
47+
func ToK8sJobSettings(dbjs *dbmodels.JobSettings) JobSettings {
48+
var k8sjs JobSettings
49+
k8sjs.ExistingClusterID = dbjs.ExistingClusterID
50+
k8sjs.NewCluster = dbjs.NewCluster
51+
k8sjs.NotebookTask = dbjs.NotebookTask
52+
k8sjs.SparkJarTask = dbjs.SparkJarTask
53+
k8sjs.SparkPythonTask = dbjs.SparkPythonTask
54+
k8sjs.SparkSubmitTask = dbjs.SparkSubmitTask
55+
k8sjs.Name = dbjs.Name
56+
k8sjs.Libraries = dbjs.Libraries
57+
k8sjs.EmailNotifications = dbjs.EmailNotifications
58+
k8sjs.TimeoutSeconds = dbjs.TimeoutSeconds
59+
k8sjs.MaxRetries = dbjs.MaxRetries
60+
k8sjs.MinRetryIntervalMillis = dbjs.MinRetryIntervalMillis
61+
k8sjs.RetryOnTimeout = dbjs.RetryOnTimeout
62+
k8sjs.Schedule = dbjs.Schedule
63+
k8sjs.MaxConcurrentRuns = dbjs.MaxConcurrentRuns
64+
return k8sjs
65+
}
66+
67+
// ToDatabricksJobSettings converts a k8s JobSettings object to a DataBricks JobSettings object.
68+
// It is needed to add ExistingClusterName and follow k8s camleCase naming convention
69+
func ToDatabricksJobSettings(k8sjs *JobSettings) dbmodels.JobSettings {
70+
71+
var dbjs dbmodels.JobSettings
72+
dbjs.ExistingClusterID = k8sjs.ExistingClusterID
73+
dbjs.NewCluster = k8sjs.NewCluster
74+
dbjs.NotebookTask = k8sjs.NotebookTask
75+
dbjs.SparkJarTask = k8sjs.SparkJarTask
76+
dbjs.SparkPythonTask = k8sjs.SparkPythonTask
77+
dbjs.SparkSubmitTask = k8sjs.SparkSubmitTask
78+
dbjs.Name = k8sjs.Name
79+
dbjs.Libraries = k8sjs.Libraries
80+
dbjs.EmailNotifications = k8sjs.EmailNotifications
81+
dbjs.TimeoutSeconds = k8sjs.TimeoutSeconds
82+
dbjs.MaxRetries = k8sjs.MaxRetries
83+
dbjs.MinRetryIntervalMillis = k8sjs.MinRetryIntervalMillis
84+
dbjs.RetryOnTimeout = k8sjs.RetryOnTimeout
85+
dbjs.Schedule = k8sjs.Schedule
86+
dbjs.MaxConcurrentRuns = k8sjs.MaxConcurrentRuns
87+
return dbjs
88+
}

‎api/v1alpha1/djob_types_test.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,14 @@ var _ = Describe("Djob", func() {
5454
It("should create an object successfully", func() {
5555

5656
key = types.NamespacedName{
57-
Name: "foo",
57+
Name: "foo-" + RandomString(5),
5858
Namespace: "default",
5959
}
6060
created = &Djob{
6161
ObjectMeta: metav1.ObjectMeta{
62-
Name: "foo",
63-
Namespace: "default",
62+
Name: key.Name,
63+
Namespace: key.Namespace,
6464
}}
65-
6665
By("creating an API obj")
6766
Expect(k8sClient.Create(context.Background(), created)).To(Succeed())
6867

‎api/v1alpha1/run_types_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,13 @@ var _ = Describe("Run", func() {
5555
It("should create an object successfully", func() {
5656

5757
key = types.NamespacedName{
58-
Name: "foo",
58+
Name: "foo-" + RandomString(5),
5959
Namespace: "default",
6060
}
6161
created = &Run{
6262
ObjectMeta: metav1.ObjectMeta{
63-
Name: "foo",
64-
Namespace: "default",
63+
Name: key.Name,
64+
Namespace: key.Namespace,
6565
}}
6666

6767
By("creating an API obj")

‎api/v1alpha1/secretscope_types.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ type SecretScopeSpec struct {
3737
type SecretScopeStatus struct {
3838
// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
3939
// Important: Run "make" to regenerate code after modifying this file
40-
SecretScope *dbmodels.SecretScope `json:"secretscope,omitempty"`
40+
SecretScope *dbmodels.SecretScope `json:"secretscope,omitempty"`
41+
SecretInClusterAvailable bool `json:"secretinclusteravailable,omitempty"`
4142
}
4243

4344
// +kubebuilder:object:root=true
@@ -51,6 +52,11 @@ type SecretScope struct {
5152
Status SecretScopeStatus `json:"status,omitempty"`
5253
}
5354

55+
// IsSecretAvailable returns true if secret in cluster is available
56+
func (ss *SecretScope) IsSecretAvailable() bool {
57+
return ss.Status.SecretInClusterAvailable
58+
}
59+
5460
// IsSubmitted returns true if the item has been submitted to DataBricks
5561
func (ss *SecretScope) IsSubmitted() bool {
5662
return ss.Status.SecretScope != nil

‎api/v1alpha1/secretscope_types_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,13 @@ var _ = Describe("SecretScope", func() {
5353
It("should create an object successfully", func() {
5454

5555
key = types.NamespacedName{
56-
Name: "foo",
56+
Name: "foo-" + RandomString(5),
5757
Namespace: "default",
5858
}
5959
created = &SecretScope{
6060
ObjectMeta: metav1.ObjectMeta{
61-
Name: "foo",
62-
Namespace: "default",
61+
Name: key.Name,
62+
Namespace: key.Namespace,
6363
}}
6464

6565
By("creating an API obj")

0 commit comments

Comments
 (0)