Fix dolt_clean not respecting dolt_ignore and dolt_nonlocal_tables patterns

go/cmd/dolt/cli/arg_parser_helpers.go

@@ -177,6 +177,7 @@ func CreateRemoteArgParser() *argparser.ArgParser {
 func CreateCleanArgParser() *argparser.ArgParser {
 	ap := argparser.NewArgParserWithVariableArgs("clean")
 	ap.SupportsFlag(DryRunFlag, "", "Tests removing untracked tables without modifying the working set.")
+	ap.SupportsFlag(ExcludeIgnoreRulesFlag, "x", "Do not respect dolt_ignore; remove untracked tables that match dolt_ignore. dolt_nonlocal_tables is always respected.")
 	return ap
 }
 

go/cmd/dolt/cli/flags.go

@@ -17,78 +17,79 @@ package cli
 // Constants for command line flags names. These tend to be used in multiple places, so defining
 // them low in the package dependency tree makes sense.
 const (
-	AbortParam           = "abort"
-	AllFlag              = "all"
-	AllowEmptyFlag       = "allow-empty"
-	AmendFlag            = "amend"
-	AuthorParam          = "author"
-	ArchiveLevelParam    = "archive-level"
-	BranchParam          = "branch"
-	CachedFlag           = "cached"
-	CheckoutCreateBranch = "b"
-	CreateResetBranch    = "B"
-	CommitFlag           = "commit"
-	ContinueFlag         = "continue"
-	CopyFlag             = "copy"
-	DateParam            = "date"
-	DecorateFlag         = "decorate"
-	DeleteFlag           = "delete"
-	DeleteForceFlag      = "D"
-	DepthFlag            = "depth"
-	DryRunFlag           = "dry-run"
-	EmptyParam           = "empty"
-	ForceFlag            = "force"
-	FullFlag             = "full"
-	GraphFlag            = "graph"
-	HardResetParam       = "hard"
-	HostFlag             = "host"
-	IncludeUntrackedFlag = "include-untracked"
-	InteractiveFlag      = "interactive"
-	JobFlag              = "job"
-	ListFlag             = "list"
-	MergesFlag           = "merges"
-	MessageArg           = "message"
-	MinParentsFlag       = "min-parents"
-	MoveFlag             = "move"
-	NoCommitFlag         = "no-commit"
-	NoEditFlag           = "no-edit"
-	NoFFParam            = "no-ff"
-	FFOnlyParam          = "ff-only"
-	NoPrettyFlag         = "no-pretty"
-	NoTLSFlag            = "no-tls"
-	NoJsonMergeFlag      = "dont-merge-json"
-	NotFlag              = "not"
-	NumberFlag           = "number"
-	OneLineFlag          = "oneline"
-	OursFlag             = "ours"
-	OutputOnlyFlag       = "output-only"
-	ParentsFlag          = "parents"
-	PatchFlag            = "patch"
-	PasswordFlag         = "password"
-	PortFlag             = "port"
-	PruneFlag            = "prune"
-	QuietFlag            = "quiet"
-	RemoteParam          = "remote"
-	SetUpstreamFlag      = "set-upstream"
-	SetUpstreamToFlag    = "set-upstream-to"
-	ShallowFlag          = "shallow"
-	ShowIgnoredFlag      = "ignored"
-	ShowSignatureFlag    = "show-signature"
-	SignFlag             = "gpg-sign"
-	SilentFlag           = "silent"
-	SingleBranchFlag     = "single-branch"
-	SkipEmptyFlag        = "skip-empty"
-	SkipVerificationFlag = "skip-verification"
-	SoftResetParam       = "soft"
-	SquashParam          = "squash"
-	StagedFlag           = "staged"
-	StatFlag             = "stat"
-	SystemFlag           = "system"
-	TablesFlag           = "tables"
-	TheirsFlag           = "theirs"
-	TrackFlag            = "track"
-	UpperCaseAllFlag     = "ALL"
-	UserFlag             = "user"
+	AbortParam             = "abort"
+	AllFlag                = "all"
+	AllowEmptyFlag         = "allow-empty"
+	AmendFlag              = "amend"
+	AuthorParam            = "author"
+	ArchiveLevelParam      = "archive-level"
+	BranchParam            = "branch"
+	CachedFlag             = "cached"
+	CheckoutCreateBranch   = "b"
+	CreateResetBranch      = "B"
+	CommitFlag             = "commit"
+	ContinueFlag           = "continue"
+	CopyFlag               = "copy"
+	DateParam              = "date"
+	DecorateFlag           = "decorate"
+	DeleteFlag             = "delete"
+	DeleteForceFlag        = "D"
+	DepthFlag              = "depth"
+	DryRunFlag             = "dry-run"
+	EmptyParam             = "empty"
+	ExcludeIgnoreRulesFlag = "x"
+	ForceFlag              = "force"
+	FullFlag               = "full"
+	GraphFlag              = "graph"
+	HardResetParam         = "hard"
+	HostFlag               = "host"
+	IncludeUntrackedFlag   = "include-untracked"
+	InteractiveFlag        = "interactive"
+	JobFlag                = "job"
+	ListFlag               = "list"
+	MergesFlag             = "merges"
+	MessageArg             = "message"
+	MinParentsFlag         = "min-parents"
+	MoveFlag               = "move"
+	NoCommitFlag           = "no-commit"
+	NoEditFlag             = "no-edit"
+	NoFFParam              = "no-ff"
+	FFOnlyParam            = "ff-only"
+	NoPrettyFlag           = "no-pretty"
+	NoTLSFlag              = "no-tls"
+	NoJsonMergeFlag        = "dont-merge-json"
+	NotFlag                = "not"
+	NumberFlag             = "number"
+	OneLineFlag            = "oneline"
+	OursFlag               = "ours"
+	OutputOnlyFlag         = "output-only"
+	ParentsFlag            = "parents"
+	PatchFlag              = "patch"
+	PasswordFlag           = "password"
+	PortFlag               = "port"
+	PruneFlag              = "prune"
+	QuietFlag              = "quiet"
+	RemoteParam            = "remote"
+	SetUpstreamFlag        = "set-upstream"
+	SetUpstreamToFlag      = "set-upstream-to"
+	ShallowFlag            = "shallow"
+	ShowIgnoredFlag        = "ignored"
+	ShowSignatureFlag      = "show-signature"
+	SignFlag               = "gpg-sign"
+	SilentFlag             = "silent"
+	SingleBranchFlag       = "single-branch"
+	SkipEmptyFlag          = "skip-empty"
+	SkipVerificationFlag   = "skip-verification"
+	SoftResetParam         = "soft"
+	SquashParam            = "squash"
+	StagedFlag             = "staged"
+	StatFlag               = "stat"
+	SystemFlag             = "system"
+	TablesFlag             = "tables"
+	TheirsFlag             = "theirs"
+	TrackFlag              = "track"
+	UpperCaseAllFlag       = "ALL"
+	UserFlag               = "user"
 )
 
 // Flags used by `dolt diff` command and `dolt_diff()` table function.

go/cmd/dolt/commands/clean.go

@@ -32,16 +32,17 @@ const (
 
 var cleanDocContent = cli.CommandDocumentationContent{
 	ShortDesc: "Deletes untracked working tables",
-	LongDesc: "{{.EmphasisLeft}}dolt clean [--dry-run]{{.EmphasisRight}}\n\n" +
+	LongDesc: "{{.EmphasisLeft}}dolt clean [--dry-run] [-x]{{.EmphasisRight}}\n\n" +
 		"The default (parameterless) form clears the values for all untracked working {{.LessThan}}tables{{.GreaterThan}} ." +
-		"This command permanently deletes unstaged or uncommitted tables.\n\n" +
+		"This command permanently deletes unstaged or uncommitted tables. By default, tables matching dolt_ignore or dolt_nonlocal_tables are not removed.\n\n" +
 		"The {{.EmphasisLeft}}--dry-run{{.EmphasisRight}} flag can be used to test whether the clean can succeed without " +
 		"deleting any tables from the current working set.\n\n" +
-		"{{.EmphasisLeft}}dolt clean [--dry-run] {{.LessThan}}tables{{.GreaterThan}}...{{.EmphasisRight}}\n\n" +
+		"The {{.EmphasisLeft}}-x{{.EmphasisRight}} flag causes dolt_ignore to be ignored so that untracked tables matching dolt_ignore are removed; dolt_nonlocal_tables is always respected (similar to git clean -x).\n\n" +
+		"{{.EmphasisLeft}}dolt clean [--dry-run] [-x] {{.LessThan}}tables{{.GreaterThan}}...{{.EmphasisRight}}\n\n" +
 		"If {{.LessThan}}tables{{.GreaterThan}} is specified, only those table names are considered for deleting.\n\n",
 	Synopsis: []string{
-		"[--dry-run]",
-		"[--dry-run] {{.LessThan}}tables{{.GreaterThan}}...",
+		"[--dry-run] [-x]",
+		"[--dry-run] [-x] {{.LessThan}}tables{{.GreaterThan}}...",
 	},
 }
 
@@ -87,6 +88,13 @@ func (cmd CleanCmd) Exec(ctx context.Context, commandStr string, args []string,
 		buffer.WriteString("\"--dry-run\"")
 		firstParamDone = true
 	}
+	if apr.Contains(cli.ExcludeIgnoreRulesFlag) {
+		if firstParamDone {
+			buffer.WriteString(", ")
+		}
+		buffer.WriteString("\"-x\"")
+		firstParamDone = true
+	}
 	if apr.NArg() > 0 {
 		// loop over apr.Args() and add them to the buffer
 		for i := 0; i < apr.NArg(); i++ {

go/libraries/doltcore/doltdb/nonlocal_tables.go

@@ -51,7 +51,8 @@ func getNonlocalTablesRef(_ context.Context, valDesc *val.TupleDesc, valTuple va
 	return result
 }
 
-func GetGlobalTablePatterns(ctx context.Context, root RootValue, schema string, cb func(string)) error {
+// GetNonlocalTablePatterns invokes |cb| once for each table name pattern in dolt_nonlocal_tables on |root| and |schema|.
+func GetNonlocalTablePatterns(ctx context.Context, root RootValue, schema string, cb func(string)) error {
 	table_name := TableName{Name: NonlocalTableName, Schema: schema}
 	table, found, err := root.GetTable(ctx, table_name)
 	if err != nil {

go/libraries/doltcore/doltdb/table_name_patterns.go

@@ -43,6 +43,35 @@ func MatchTablePattern(pattern string, table string) (bool, error) {
 	return re.MatchString(table), nil
 }
 
+// CompiledTablePatterns holds compiled table name patterns for reuse when matching many names without recompiling.
+type CompiledTablePatterns []*regexp.Regexp
+
+// CompileTablePatterns compiles each of |patterns| once and returns them for use with TableMatchesAny. Returns (nil, nil) when |patterns| is empty.
+func CompileTablePatterns(patterns []string) (CompiledTablePatterns, error) {
+	if len(patterns) == 0 {
+		return nil, nil
+	}
+	compiled := make(CompiledTablePatterns, 0, len(patterns))
+	for _, p := range patterns {
+		re, err := compilePattern(p)
+		if err != nil {
+			return nil, err
+		}
+		compiled = append(compiled, re)
+	}
+	return compiled, nil
+}
+
+// TableMatchesAny reports whether |table| matches any of the patterns in |c|.
+func (c CompiledTablePatterns) TableMatchesAny(table string) bool {
+	for _, re := range c {
+		if re.MatchString(table) {
+			return true
+		}
+	}
+	return false
+}
+
 // GetMatchingTables returns all tables that match a pattern
 func GetMatchingTables(ctx *sql.Context, root RootValue, schemaName string, pattern string) (results []string, err error) {
 	// If the pattern doesn't contain any special characters, look up that name.

go/libraries/doltcore/env/actions/checkout.go

@@ -216,7 +216,7 @@ func CleanOldWorkingSet(
 	}
 
 	// we also have to do a clean, because we the ResetHard won't touch any new tables (tables only in the working set)
-	newRoots, err := CleanUntracked(ctx, resetRoots, []string{}, false, true)
+	newRoots, err := CleanUntracked(ctx, resetRoots, []string{}, false, true, false)
 	if err != nil {
 		return err
 	}

go/libraries/doltcore/env/actions/reset.go

@@ -270,60 +270,74 @@ func getUnionedTables(ctx context.Context, tables []doltdb.TableName, stagedRoot
 	return tables, nil
 }
 
-// CleanUntracked deletes untracked tables from the working root.
-// Evaluates untracked tables as: all working tables - all staged tables.
-func CleanUntracked(ctx *sql.Context, roots doltdb.Roots, tables []string, dryrun bool, force bool) (doltdb.Roots, error) {
+// CleanUntracked deletes from the working root the tables that are untracked (in working but not in staged/head). If
+// |tables| is non-empty it uses only those names as candidates; otherwise it uses all working tables. Tables matching
+// dolt_nonlocal_tables are always excluded. When |respectIgnoreRules| is true, tables matching dolt_ignore are also excluded. Does nothing when |dryrun| is true.
+func CleanUntracked(ctx *sql.Context, roots doltdb.Roots, tables []string, dryrun bool, force bool, respectIgnoreRules bool) (doltdb.Roots, error) {
 	untrackedTables := make(map[doltdb.TableName]struct{})
+	for _, name := range tables {
+		resolvedName, tblExists, err := resolve.TableName(ctx, roots.Working, name)
+		if err != nil {
+			return doltdb.Roots{}, err
+		}
+		if !tblExists {
+			return doltdb.Roots{}, fmt.Errorf("%w: '%s'", doltdb.ErrTableNotFound, name)
+		}
+		untrackedTables[resolvedName] = struct{}{}
+	}
 
-	var err error
 	if len(tables) == 0 {
 		allTableNames, err := roots.Working.GetAllTableNames(ctx, true)
 		if err != nil {
-			return doltdb.Roots{}, nil
+			return doltdb.Roots{}, err
 		}
-		for _, tableName := range allTableNames {
-			untrackedTables[tableName] = struct{}{}
-		}
-	} else {
-		for i := range tables {
-			name := tables[i]
-			resolvedName, tblExists, err := resolve.TableName(ctx, roots.Working, name)
+		var candidates []doltdb.TableName
+		if respectIgnoreRules {
+			candidates, err = doltdb.ExcludeIgnoredTables(ctx, roots, allTableNames)
 			if err != nil {
 				return doltdb.Roots{}, err
 			}
-			if !tblExists {
-				return doltdb.Roots{}, fmt.Errorf("%w: '%s'", doltdb.ErrTableNotFound, name)
+		} else {
+			candidates = allTableNames
+		}
+		var nonlocalPatterns []string
+		err = doltdb.GetNonlocalTablePatterns(ctx, roots.Working, doltdb.DefaultSchemaName, func(p string) {
+			nonlocalPatterns = append(nonlocalPatterns, p)
+		})
+		if err != nil {
+			return doltdb.Roots{}, err
+		}
+		compiled, err := doltdb.CompileTablePatterns(nonlocalPatterns)
+		if err != nil {
+			return doltdb.Roots{}, err
+		}
+		for _, tableName := range candidates {
+			if compiled.TableMatchesAny(tableName.Name) {
+				continue
 			}
-			untrackedTables[resolvedName] = struct{}{}
+			untrackedTables[tableName] = struct{}{}
 		}
 	}
 
-	// untracked tables = working tables - staged tables
 	headTblNames := GetAllTableNames(ctx, roots.Staged)
-	if err != nil {
-		return doltdb.Roots{}, err
-	}
-
 	for _, name := range headTblNames {
 		delete(untrackedTables, name)
 	}
 
-	newRoot := roots.Working
-	var toDelete []doltdb.TableName
+	toDelete := make([]doltdb.TableName, 0, len(untrackedTables))
 	for t := range untrackedTables {
 		toDelete = append(toDelete, t)
 	}
 
-	newRoot, err = newRoot.RemoveTables(ctx, force, force, toDelete...)
-	if err != nil {
-		return doltdb.Roots{}, fmt.Errorf("failed to remove tables; %w", err)
-	}
-
 	if dryrun {
 		return roots, nil
 	}
-	roots.Working = newRoot
 
+	newRoot, err := roots.Working.RemoveTables(ctx, force, force, toDelete...)
+	if err != nil {
+		return doltdb.Roots{}, fmt.Errorf("failed to remove tables; %w", err)
+	}
+	roots.Working = newRoot
 	return roots, nil
 }
 

go/libraries/doltcore/sqle/dprocedures/dolt_clean.go

@@ -57,7 +57,8 @@ func doDoltClean(ctx *sql.Context, args []string) (int, error) {
 		return 1, fmt.Errorf("Could not load database %s", dbName)
 	}
 
-	roots, err = actions.CleanUntracked(ctx, roots, apr.Args, apr.ContainsAll(cli.DryRunFlag), false)
+	respectIgnoreRules := !apr.Contains(cli.ExcludeIgnoreRulesFlag)
+	roots, err = actions.CleanUntracked(ctx, roots, apr.Args, apr.ContainsAll(cli.DryRunFlag), false, respectIgnoreRules)
 	if err != nil {
 		return 1, fmt.Errorf("failed to clean; %w", err)
 	}

go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go

@@ -512,7 +512,8 @@ func RunStoredProceduresTest(t *testing.T, h DoltEnginetestHarness) {
 }
 
 func RunDoltStoredProceduresTest(t *testing.T, h DoltEnginetestHarness) {
-	for _, script := range DoltProcedureTests {
+	scripts := append(DoltProcedureTests, DoltCleanProcedureScripts...)
+	for _, script := range scripts {
 		func() {
 			h := h.NewHarness(t)
 			h.UseLocalFileSystem()
@@ -523,7 +524,8 @@ func RunDoltStoredProceduresTest(t *testing.T, h DoltEnginetestHarness) {
 }
 
 func RunDoltStoredProceduresPreparedTest(t *testing.T, h DoltEnginetestHarness) {
-	for _, script := range DoltProcedureTests {
+	scripts := append(DoltProcedureTests, DoltCleanProcedureScripts...)
+	for _, script := range scripts {
 		func() {
 			h := h.NewHarness(t)
 			h.UseLocalFileSystem()