Skip to content
Closed
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions bundle/config/mutator/process_target_mode.go
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,15 @@ func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUs
}
}

if !isPrincipalUsed && !isRunAsSet(r) {
return diag.Errorf("'run_as' must be set for all jobs when using 'mode: production'")
// We need to verify that there is only a single deployment of the current target.
// A good way to enforce this is to explicitly set root_path or run_as.
if !isExplicitRootSet(b) && !isRunAsSet(r) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We used to require

run_as: alice@company.com

but when considering collaborative deployment, we prefer

root_path: /Users/Alice

This change makes it so either restriction is allowed for mode: production.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we also warn if only run_as is set? It doesn't prevent multiple deployments.

// Only show a warning in case a principal was used for backward compatibility
// with projects from before the DABs GA.
if isPrincipalUsed {
return diag.Warningf("target with 'mode: production' should specify explicit 'workspace.root_path' to make sure only one copy is deployed")

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Service principals still have an exception, like they used to.

}
return diag.Errorf("target with 'mode: production' must specify explicit 'workspace.root_path' to make sure only one copy is deployed")
}
return nil
}
Expand All @@ -172,6 +179,14 @@ func isRunAsSet(r config.Resources) bool {
return true
}

func isExplicitRootSet(b *bundle.Bundle) bool {
targetConfig := b.Config.Targets[b.Config.Bundle.Target]
if targetConfig.Workspace == nil {
return false
}
return targetConfig.Workspace.RootPath != ""
}

func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
switch b.Config.Bundle.Mode {
case config.Development:
Expand Down
22 changes: 20 additions & 2 deletions bundle/config/mutator/process_target_mode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
Branch: "main",
},
},
Targets: map[string]*config.Target{
"": {},
},
Workspace: config.Workspace{
CurrentUser: &config.User{
ShortName: "lennart",
Expand Down Expand Up @@ -213,14 +216,14 @@ func TestProcessTargetModeProduction(t *testing.T) {
b := mockBundle(config.Production)

diags := validateProductionMode(context.Background(), b, false)
require.ErrorContains(t, diags.Error(), "run_as")
require.ErrorContains(t, diags.Error(), "target with 'mode: production' must specify explicit 'workspace.root_path' to make sure only one copy is deployed")

b.Config.Workspace.StatePath = "/Shared/.bundle/x/y/state"
b.Config.Workspace.ArtifactPath = "/Shared/.bundle/x/y/artifacts"
b.Config.Workspace.FilePath = "/Shared/.bundle/x/y/files"

diags = validateProductionMode(context.Background(), b, false)
require.ErrorContains(t, diags.Error(), "production")
require.ErrorContains(t, diags.Error(), "target with 'mode: production' must specify explicit 'workspace.root_path' to make sure only one copy is deployed")

permissions := []resources.Permission{
{
Expand Down Expand Up @@ -262,6 +265,21 @@ func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {
require.NoError(t, diags.Error())
}

func TestProcessTargetModeProductionOkWithRootPath(t *testing.T) {
b := mockBundle(config.Production)

// Our target has all kinds of problems when not using service principals ...
diags := validateProductionMode(context.Background(), b, false)
require.Error(t, diags.Error())

// ... but we're okay if we specify a root path
b.Config.Targets[""].Workspace = &config.Workspace{
RootPath: "some-root-path",
}
diags = validateProductionMode(context.Background(), b, false)
require.NoError(t, diags.Error())
}

// Make sure that we have test coverage for all resource types
func TestAllResourcesMocked(t *testing.T) {
b := mockBundle(config.Development)
Expand Down
1 change: 1 addition & 0 deletions bundle/config/mutator/run_as.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ func validateRunAs(b *bundle.Bundle) error {
}

// DLT pipelines do not support run_as in the API.
// TODO: maybe oly make this check only fail when owner != runas
Comment thread
lennartkats-db marked this conversation as resolved.
Outdated
if len(b.Config.Resources.Pipelines) > 0 {
return errUnsupportedResourceTypeForRunAs{
resourceType: "pipelines",
Expand Down
4 changes: 0 additions & 4 deletions bundle/config/mutator/select_target.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,5 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti
// TODO: remove when Environments section is not supported anymore.
b.Config.Bundle.Environment = b.Config.Bundle.Target

// Clear targets after loading.
b.Config.Targets = nil
b.Config.Environments = nil

Comment thread
lennartkats-db marked this conversation as resolved.
return nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@ import (
"github.com/databricks/databricks-sdk-go/service/workspace"
)

type workspaceRootPermissions struct {
type applyFolderPermissions struct {
}

func ApplyWorkspaceRootPermissions() bundle.Mutator {
return &workspaceRootPermissions{}
func ApplyFolderPermissions() bundle.Mutator {
return &applyFolderPermissions{}
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why the rename? The mutator still applies only to the workspace root path.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The mutator still applies only to the workspace root path.

That seems like a bug :(

In any case I renamed this so apply_folder_permisions to make it a bit shorter and to reflect that it applies permissions to folders, while its sister mutator apply_resource_permissions applies permissions to resources.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Btw I don't think we should change the semantics of this module at this time, but we should include this in the upcoming work on permission warnings.


// Apply implements bundle.Mutator.
func (*workspaceRootPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
func (*applyFolderPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
err := giveAccessForWorkspaceRoot(ctx, b)
if err != nil {
return diag.FromErr(err)
Expand All @@ -26,8 +26,8 @@ func (*workspaceRootPermissions) Apply(ctx context.Context, b *bundle.Bundle) di
return nil
}

func (*workspaceRootPermissions) Name() string {
return "ApplyWorkspaceRootPermissions"
func (*applyFolderPermissions) Name() string {
return "ApplyFolderPermissions"
}

func giveAccessForWorkspaceRoot(ctx context.Context, b *bundle.Bundle) error {
Expand Down Expand Up @@ -67,7 +67,7 @@ func giveAccessForWorkspaceRoot(ctx context.Context, b *bundle.Bundle) error {

func getWorkspaceObjectPermissionLevel(bundlePermission string) (workspace.WorkspaceObjectPermissionLevel, error) {
switch bundlePermission {
case CAN_MANAGE:
case CAN_MANAGE, IS_OWNER:
return workspace.WorkspaceObjectPermissionLevelCanManage, nil
case CAN_RUN:
return workspace.WorkspaceObjectPermissionLevelCanRun, nil
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,6 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) {
WorkspaceObjectType: "directories",
}).Return(nil, nil)

diags := bundle.Apply(context.Background(), b, ApplyWorkspaceRootPermissions())
diags := bundle.Apply(context.Background(), b, ApplyFolderPermissions())
require.NoError(t, diags.Error())
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,47 +7,54 @@ import (
"strings"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/diag"
)

const CAN_MANAGE = "CAN_MANAGE"
const CAN_VIEW = "CAN_VIEW"
const CAN_RUN = "CAN_RUN"
const IS_OWNER = "IS_OWNER"

var allowedLevels = []string{CAN_MANAGE, CAN_VIEW, CAN_RUN}
var allowedLevels = []string{CAN_MANAGE, CAN_VIEW, CAN_RUN, IS_OWNER}
var levelsMap = map[string](map[string]string){
"jobs": {
IS_OWNER: "IS_OWNER",

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Only a few resources, like jobs, actually have an "owner": https://docs.databricks.com/en/security/auth-authz/access-control/index.html. For almost all the others we don't distinguish between the owner and the other can-manage users.

Note that clusters is a special case here. They don't have an owner permission but treat the creator as a kind of owner that has special privileges. There's even a special API for changing that notion of "owner": https://docs.databricks.com/api/workspace/clusters/changeowner. Once we do clusters we should discuss if we want this notion of an owner to affect how clusters are created, or if we perhaps want to show a warning when someone who doesn't have IS_OWNER would be the first creator of a cluster.

CAN_MANAGE: "CAN_MANAGE",
CAN_VIEW: "CAN_VIEW",
CAN_RUN: "CAN_MANAGE_RUN",
},
"pipelines": {
IS_OWNER: "IS_OWNER",
CAN_MANAGE: "CAN_MANAGE",
CAN_VIEW: "CAN_VIEW",
CAN_RUN: "CAN_RUN",
},
"mlflow_experiments": {
IS_OWNER: "CAN_MANAGE",
CAN_MANAGE: "CAN_MANAGE",
CAN_VIEW: "CAN_READ",
},
"mlflow_models": {
IS_OWNER: "CAN_MANAGE",
CAN_MANAGE: "CAN_MANAGE",
CAN_VIEW: "CAN_READ",
},
"model_serving_endpoints": {
IS_OWNER: "CAN_MANAGE",
CAN_MANAGE: "CAN_MANAGE",
CAN_VIEW: "CAN_VIEW",
CAN_RUN: "CAN_QUERY",
},
}

type bundlePermissions struct{}
type applyResourcePermissions struct{}

func ApplyBundlePermissions() bundle.Mutator {
return &bundlePermissions{}
func ApplyResourcePermissions() bundle.Mutator {
return &applyResourcePermissions{}
}

func (m *bundlePermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
func (m *applyResourcePermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
err := validate(b)
if err != nil {
return diag.FromErr(err)
Expand All @@ -74,64 +81,91 @@ func validate(b *bundle.Bundle) error {

func applyForJobs(ctx context.Context, b *bundle.Bundle) {
for key, job := range b.Config.Resources.Jobs {
job.Permissions = append(job.Permissions, convert(
job.Permissions = extendPermissions(job.Permissions, convert(
ctx,
b.Config.Permissions,
job.Permissions,
key,
levelsMap["jobs"],
)...)
))
}
}

func applyForPipelines(ctx context.Context, b *bundle.Bundle) {
for key, pipeline := range b.Config.Resources.Pipelines {
pipeline.Permissions = append(pipeline.Permissions, convert(
pipeline.Permissions = extendPermissions(pipeline.Permissions, convert(
ctx,
b.Config.Permissions,
pipeline.Permissions,
key,
levelsMap["pipelines"],
)...)
))
}
}

func applyForMlExperiments(ctx context.Context, b *bundle.Bundle) {
for key, experiment := range b.Config.Resources.Experiments {
experiment.Permissions = append(experiment.Permissions, convert(
experiment.Permissions = extendPermissions(experiment.Permissions, convert(
ctx,
b.Config.Permissions,
experiment.Permissions,
key,
levelsMap["mlflow_experiments"],
)...)
))
}
}

func applyForMlModels(ctx context.Context, b *bundle.Bundle) {
for key, model := range b.Config.Resources.Models {
model.Permissions = append(model.Permissions, convert(
model.Permissions = extendPermissions(model.Permissions, convert(
ctx,
b.Config.Permissions,
model.Permissions,
key,
levelsMap["mlflow_models"],
)...)
))
}
}

func applyForModelServiceEndpoints(ctx context.Context, b *bundle.Bundle) {
for key, model := range b.Config.Resources.ModelServingEndpoints {
model.Permissions = append(model.Permissions, convert(
model.Permissions = extendPermissions(model.Permissions, convert(
ctx,
b.Config.Permissions,
model.Permissions,
key,
levelsMap["model_serving_endpoints"],
)...)
))
}
}

func (m *bundlePermissions) Name() string {
return "ApplyBundlePermissions"
func extendPermissions(permissions []resources.Permission, newPermissions []resources.Permission) []resources.Permission {
if !includesOwner(permissions) {
// Just merge the permissions, don't need to do anything special
return append(permissions, newPermissions...)
}

// Only apply the owner from top-level permissions if the local resource
// didn't have an owner.
Comment thread
lennartkats-db marked this conversation as resolved.
Outdated
for _, p := range newPermissions {
if p.Level == IS_OWNER {
continue
}
permissions = append(permissions, p)
}

return permissions
}

func includesOwner(permissions []resources.Permission) bool {
for _, p := range permissions {
if p.Level == IS_OWNER {
return true
}
}
return false
}

func (m *applyResourcePermissions) Name() string {
return "ApplyResourcePermissions"
}
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func TestApplyBundlePermissions(t *testing.T) {
},
}

diags := bundle.Apply(context.Background(), b, ApplyBundlePermissions())
diags := bundle.Apply(context.Background(), b, ApplyResourcePermissions())
require.NoError(t, diags.Error())

require.Len(t, b.Config.Resources.Jobs["job_1"].Permissions, 3)
Expand Down Expand Up @@ -138,7 +138,7 @@ func TestWarningOnOverlapPermission(t *testing.T) {
},
}

diags := bundle.Apply(context.Background(), b, ApplyBundlePermissions())
diags := bundle.Apply(context.Background(), b, ApplyResourcePermissions())
require.NoError(t, diags.Error())

require.Contains(t, b.Config.Resources.Jobs["job_1"].Permissions, resources.Permission{Level: "CAN_VIEW", UserName: "TestUser"})
Expand All @@ -148,3 +148,56 @@ func TestWarningOnOverlapPermission(t *testing.T) {
require.Contains(t, b.Config.Resources.Jobs["job_2"].Permissions, resources.Permission{Level: "CAN_VIEW", GroupName: "TestGroup"})

}

func TestOwnerLevel(t *testing.T) {
// Create a bundle with a run_as configuration
b := &bundle.Bundle{
Config: config.Root{
Permissions: []resources.Permission{
{Level: IS_OWNER, UserName: "Alice"},
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job_1": {
Permissions: []resources.Permission{
{Level: CAN_MANAGE, UserName: "Bob"},
},
},
"job_2": {
Permissions: []resources.Permission{
{Level: IS_OWNER, UserName: "Bob"},
},
},
},
Pipelines: map[string]*resources.Pipeline{
"pipeline_1": {},
},
Models: map[string]*resources.MlflowModel{
"model_1": {
Permissions: []resources.Permission{
{Level: CAN_MANAGE, UserName: "Bob"},
},
},
},
Experiments: map[string]*resources.MlflowExperiment{
"experiment_1": {},
},
ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{
"endpoint_1": {},
},
},
},
}

diags := bundle.Apply(context.Background(), b, ApplyResourcePermissions())
require.NoError(t, diags.Error())
require.Contains(t, b.Config.Resources.Jobs["job_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "Bob"})
require.Contains(t, b.Config.Resources.Jobs["job_1"].Permissions, resources.Permission{Level: "IS_OWNER", UserName: "Alice"})
require.Contains(t, b.Config.Resources.Jobs["job_2"].Permissions, resources.Permission{Level: "IS_OWNER", UserName: "Bob"})
require.NotContains(t, b.Config.Resources.Jobs["job_2"].Permissions, resources.Permission{Level: "IS_OWNER", UserName: "Alice"})
require.Contains(t, b.Config.Resources.Pipelines["pipeline_1"].Permissions, resources.Permission{Level: "IS_OWNER", UserName: "Alice"})
require.Contains(t, b.Config.Resources.Experiments["experiment_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "Alice"})
require.Contains(t, b.Config.Resources.Models["model_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "Alice"})
require.Contains(t, b.Config.Resources.Models["model_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "Bob"})
require.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "Alice"})
}
2 changes: 1 addition & 1 deletion bundle/phases/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func Deploy() bundle.Mutator {
files.Upload(),
deploy.StateUpdate(),
deploy.StatePush(),
permissions.ApplyWorkspaceRootPermissions(),
permissions.ApplyFolderPermissions(),
terraform.Interpolate(),
terraform.Write(),
terraform.CheckRunningResource(),
Expand Down
Loading