diff --git a/go/cmd/dolt/cli/arg_parser_helpers.go b/go/cmd/dolt/cli/arg_parser_helpers.go index 4b8c234effb..3f345dba404 100644 --- a/go/cmd/dolt/cli/arg_parser_helpers.go +++ b/go/cmd/dolt/cli/arg_parser_helpers.go @@ -177,6 +177,7 @@ func CreateRemoteArgParser() *argparser.ArgParser { func CreateCleanArgParser() *argparser.ArgParser { ap := argparser.NewArgParserWithVariableArgs("clean") ap.SupportsFlag(DryRunFlag, "", "Tests removing untracked tables without modifying the working set.") + ap.SupportsFlag(ExcludeIgnoreRulesFlag, "x", "Do not respect dolt_ignore; remove untracked tables that match dolt_ignore. dolt_nonlocal_tables is always respected.") return ap } diff --git a/go/cmd/dolt/cli/flags.go b/go/cmd/dolt/cli/flags.go index b1ed484d7a4..c3e66ae0c2f 100644 --- a/go/cmd/dolt/cli/flags.go +++ b/go/cmd/dolt/cli/flags.go @@ -17,78 +17,79 @@ package cli // Constants for command line flags names. These tend to be used in multiple places, so defining // them low in the package dependency tree makes sense. const ( - AbortParam = "abort" - AllFlag = "all" - AllowEmptyFlag = "allow-empty" - AmendFlag = "amend" - AuthorParam = "author" - ArchiveLevelParam = "archive-level" - BranchParam = "branch" - CachedFlag = "cached" - CheckoutCreateBranch = "b" - CreateResetBranch = "B" - CommitFlag = "commit" - ContinueFlag = "continue" - CopyFlag = "copy" - DateParam = "date" - DecorateFlag = "decorate" - DeleteFlag = "delete" - DeleteForceFlag = "D" - DepthFlag = "depth" - DryRunFlag = "dry-run" - EmptyParam = "empty" - ForceFlag = "force" - FullFlag = "full" - GraphFlag = "graph" - HardResetParam = "hard" - HostFlag = "host" - IncludeUntrackedFlag = "include-untracked" - InteractiveFlag = "interactive" - JobFlag = "job" - ListFlag = "list" - MergesFlag = "merges" - MessageArg = "message" - MinParentsFlag = "min-parents" - MoveFlag = "move" - NoCommitFlag = "no-commit" - NoEditFlag = "no-edit" - NoFFParam = "no-ff" - FFOnlyParam = "ff-only" - NoPrettyFlag = "no-pretty" - NoTLSFlag = "no-tls" - NoJsonMergeFlag = "dont-merge-json" - NotFlag = "not" - NumberFlag = "number" - OneLineFlag = "oneline" - OursFlag = "ours" - OutputOnlyFlag = "output-only" - ParentsFlag = "parents" - PatchFlag = "patch" - PasswordFlag = "password" - PortFlag = "port" - PruneFlag = "prune" - QuietFlag = "quiet" - RemoteParam = "remote" - SetUpstreamFlag = "set-upstream" - SetUpstreamToFlag = "set-upstream-to" - ShallowFlag = "shallow" - ShowIgnoredFlag = "ignored" - ShowSignatureFlag = "show-signature" - SignFlag = "gpg-sign" - SilentFlag = "silent" - SingleBranchFlag = "single-branch" - SkipEmptyFlag = "skip-empty" - SkipVerificationFlag = "skip-verification" - SoftResetParam = "soft" - SquashParam = "squash" - StagedFlag = "staged" - StatFlag = "stat" - SystemFlag = "system" - TablesFlag = "tables" - TheirsFlag = "theirs" - TrackFlag = "track" - UpperCaseAllFlag = "ALL" - UserFlag = "user" + AbortParam = "abort" + AllFlag = "all" + AllowEmptyFlag = "allow-empty" + AmendFlag = "amend" + AuthorParam = "author" + ArchiveLevelParam = "archive-level" + BranchParam = "branch" + CachedFlag = "cached" + CheckoutCreateBranch = "b" + CreateResetBranch = "B" + CommitFlag = "commit" + ContinueFlag = "continue" + CopyFlag = "copy" + DateParam = "date" + DecorateFlag = "decorate" + DeleteFlag = "delete" + DeleteForceFlag = "D" + DepthFlag = "depth" + DryRunFlag = "dry-run" + EmptyParam = "empty" + ExcludeIgnoreRulesFlag = "x" + ForceFlag = "force" + FullFlag = "full" + GraphFlag = "graph" + HardResetParam = "hard" + HostFlag = "host" + IncludeUntrackedFlag = "include-untracked" + InteractiveFlag = "interactive" + JobFlag = "job" + ListFlag = "list" + MergesFlag = "merges" + MessageArg = "message" + MinParentsFlag = "min-parents" + MoveFlag = "move" + NoCommitFlag = "no-commit" + NoEditFlag = "no-edit" + NoFFParam = "no-ff" + FFOnlyParam = "ff-only" + NoPrettyFlag = "no-pretty" + NoTLSFlag = "no-tls" + NoJsonMergeFlag = "dont-merge-json" + NotFlag = "not" + NumberFlag = "number" + OneLineFlag = "oneline" + OursFlag = "ours" + OutputOnlyFlag = "output-only" + ParentsFlag = "parents" + PatchFlag = "patch" + PasswordFlag = "password" + PortFlag = "port" + PruneFlag = "prune" + QuietFlag = "quiet" + RemoteParam = "remote" + SetUpstreamFlag = "set-upstream" + SetUpstreamToFlag = "set-upstream-to" + ShallowFlag = "shallow" + ShowIgnoredFlag = "ignored" + ShowSignatureFlag = "show-signature" + SignFlag = "gpg-sign" + SilentFlag = "silent" + SingleBranchFlag = "single-branch" + SkipEmptyFlag = "skip-empty" + SkipVerificationFlag = "skip-verification" + SoftResetParam = "soft" + SquashParam = "squash" + StagedFlag = "staged" + StatFlag = "stat" + SystemFlag = "system" + TablesFlag = "tables" + TheirsFlag = "theirs" + TrackFlag = "track" + UpperCaseAllFlag = "ALL" + UserFlag = "user" ) // Flags used by `dolt diff` command and `dolt_diff()` table function. diff --git a/go/cmd/dolt/commands/clean.go b/go/cmd/dolt/commands/clean.go index 73e0fd02918..885e4fe08b6 100644 --- a/go/cmd/dolt/commands/clean.go +++ b/go/cmd/dolt/commands/clean.go @@ -32,16 +32,17 @@ const ( var cleanDocContent = cli.CommandDocumentationContent{ ShortDesc: "Deletes untracked working tables", - LongDesc: "{{.EmphasisLeft}}dolt clean [--dry-run]{{.EmphasisRight}}\n\n" + + LongDesc: "{{.EmphasisLeft}}dolt clean [--dry-run] [-x]{{.EmphasisRight}}\n\n" + "The default (parameterless) form clears the values for all untracked working {{.LessThan}}tables{{.GreaterThan}} ." + - "This command permanently deletes unstaged or uncommitted tables.\n\n" + + "This command permanently deletes unstaged or uncommitted tables. By default, tables matching dolt_ignore or dolt_nonlocal_tables are not removed.\n\n" + "The {{.EmphasisLeft}}--dry-run{{.EmphasisRight}} flag can be used to test whether the clean can succeed without " + "deleting any tables from the current working set.\n\n" + - "{{.EmphasisLeft}}dolt clean [--dry-run] {{.LessThan}}tables{{.GreaterThan}}...{{.EmphasisRight}}\n\n" + + "The {{.EmphasisLeft}}-x{{.EmphasisRight}} flag causes dolt_ignore to be ignored so that untracked tables matching dolt_ignore are removed; dolt_nonlocal_tables is always respected (similar to git clean -x).\n\n" + + "{{.EmphasisLeft}}dolt clean [--dry-run] [-x] {{.LessThan}}tables{{.GreaterThan}}...{{.EmphasisRight}}\n\n" + "If {{.LessThan}}tables{{.GreaterThan}} is specified, only those table names are considered for deleting.\n\n", Synopsis: []string{ - "[--dry-run]", - "[--dry-run] {{.LessThan}}tables{{.GreaterThan}}...", + "[--dry-run] [-x]", + "[--dry-run] [-x] {{.LessThan}}tables{{.GreaterThan}}...", }, } @@ -87,6 +88,13 @@ func (cmd CleanCmd) Exec(ctx context.Context, commandStr string, args []string, buffer.WriteString("\"--dry-run\"") firstParamDone = true } + if apr.Contains(cli.ExcludeIgnoreRulesFlag) { + if firstParamDone { + buffer.WriteString(", ") + } + buffer.WriteString("\"-x\"") + firstParamDone = true + } if apr.NArg() > 0 { // loop over apr.Args() and add them to the buffer for i := 0; i < apr.NArg(); i++ { diff --git a/go/libraries/doltcore/doltdb/nonlocal_tables.go b/go/libraries/doltcore/doltdb/nonlocal_tables.go index dc229b990f8..749838977c6 100644 --- a/go/libraries/doltcore/doltdb/nonlocal_tables.go +++ b/go/libraries/doltcore/doltdb/nonlocal_tables.go @@ -51,7 +51,8 @@ func getNonlocalTablesRef(_ context.Context, valDesc *val.TupleDesc, valTuple va return result } -func GetGlobalTablePatterns(ctx context.Context, root RootValue, schema string, cb func(string)) error { +// GetNonlocalTablePatterns invokes |cb| once for each table name pattern in dolt_nonlocal_tables on |root| and |schema|. +func GetNonlocalTablePatterns(ctx context.Context, root RootValue, schema string, cb func(string)) error { table_name := TableName{Name: NonlocalTableName, Schema: schema} table, found, err := root.GetTable(ctx, table_name) if err != nil { diff --git a/go/libraries/doltcore/doltdb/table_name_patterns.go b/go/libraries/doltcore/doltdb/table_name_patterns.go index 58848c61a69..00ee8b5588c 100644 --- a/go/libraries/doltcore/doltdb/table_name_patterns.go +++ b/go/libraries/doltcore/doltdb/table_name_patterns.go @@ -43,6 +43,35 @@ func MatchTablePattern(pattern string, table string) (bool, error) { return re.MatchString(table), nil } +// CompiledTablePatterns holds compiled table name patterns for reuse when matching many names without recompiling. +type CompiledTablePatterns []*regexp.Regexp + +// CompileTablePatterns compiles each of |patterns| once and returns them for use with TableMatchesAny. Returns (nil, nil) when |patterns| is empty. +func CompileTablePatterns(patterns []string) (CompiledTablePatterns, error) { + if len(patterns) == 0 { + return nil, nil + } + compiled := make(CompiledTablePatterns, 0, len(patterns)) + for _, p := range patterns { + re, err := compilePattern(p) + if err != nil { + return nil, err + } + compiled = append(compiled, re) + } + return compiled, nil +} + +// TableMatchesAny reports whether |table| matches any of the patterns in |c|. +func (c CompiledTablePatterns) TableMatchesAny(table string) bool { + for _, re := range c { + if re.MatchString(table) { + return true + } + } + return false +} + // GetMatchingTables returns all tables that match a pattern func GetMatchingTables(ctx *sql.Context, root RootValue, schemaName string, pattern string) (results []string, err error) { // If the pattern doesn't contain any special characters, look up that name. diff --git a/go/libraries/doltcore/env/actions/checkout.go b/go/libraries/doltcore/env/actions/checkout.go index 83658f13aac..cf5ad58f559 100644 --- a/go/libraries/doltcore/env/actions/checkout.go +++ b/go/libraries/doltcore/env/actions/checkout.go @@ -216,7 +216,7 @@ func CleanOldWorkingSet( } // we also have to do a clean, because we the ResetHard won't touch any new tables (tables only in the working set) - newRoots, err := CleanUntracked(ctx, resetRoots, []string{}, false, true) + newRoots, err := CleanUntracked(ctx, resetRoots, []string{}, false, true, false) if err != nil { return err } diff --git a/go/libraries/doltcore/env/actions/reset.go b/go/libraries/doltcore/env/actions/reset.go index c5ffa8c279c..163cd4451f7 100644 --- a/go/libraries/doltcore/env/actions/reset.go +++ b/go/libraries/doltcore/env/actions/reset.go @@ -270,60 +270,74 @@ func getUnionedTables(ctx context.Context, tables []doltdb.TableName, stagedRoot return tables, nil } -// CleanUntracked deletes untracked tables from the working root. -// Evaluates untracked tables as: all working tables - all staged tables. -func CleanUntracked(ctx *sql.Context, roots doltdb.Roots, tables []string, dryrun bool, force bool) (doltdb.Roots, error) { +// CleanUntracked deletes from the working root the tables that are untracked (in working but not in staged/head). If +// |tables| is non-empty it uses only those names as candidates; otherwise it uses all working tables. Tables matching +// dolt_nonlocal_tables are always excluded. When |respectIgnoreRules| is true, tables matching dolt_ignore are also excluded. Does nothing when |dryrun| is true. +func CleanUntracked(ctx *sql.Context, roots doltdb.Roots, tables []string, dryrun bool, force bool, respectIgnoreRules bool) (doltdb.Roots, error) { untrackedTables := make(map[doltdb.TableName]struct{}) + for _, name := range tables { + resolvedName, tblExists, err := resolve.TableName(ctx, roots.Working, name) + if err != nil { + return doltdb.Roots{}, err + } + if !tblExists { + return doltdb.Roots{}, fmt.Errorf("%w: '%s'", doltdb.ErrTableNotFound, name) + } + untrackedTables[resolvedName] = struct{}{} + } - var err error if len(tables) == 0 { allTableNames, err := roots.Working.GetAllTableNames(ctx, true) if err != nil { - return doltdb.Roots{}, nil + return doltdb.Roots{}, err } - for _, tableName := range allTableNames { - untrackedTables[tableName] = struct{}{} - } - } else { - for i := range tables { - name := tables[i] - resolvedName, tblExists, err := resolve.TableName(ctx, roots.Working, name) + var candidates []doltdb.TableName + if respectIgnoreRules { + candidates, err = doltdb.ExcludeIgnoredTables(ctx, roots, allTableNames) if err != nil { return doltdb.Roots{}, err } - if !tblExists { - return doltdb.Roots{}, fmt.Errorf("%w: '%s'", doltdb.ErrTableNotFound, name) + } else { + candidates = allTableNames + } + var nonlocalPatterns []string + err = doltdb.GetNonlocalTablePatterns(ctx, roots.Working, doltdb.DefaultSchemaName, func(p string) { + nonlocalPatterns = append(nonlocalPatterns, p) + }) + if err != nil { + return doltdb.Roots{}, err + } + compiled, err := doltdb.CompileTablePatterns(nonlocalPatterns) + if err != nil { + return doltdb.Roots{}, err + } + for _, tableName := range candidates { + if compiled.TableMatchesAny(tableName.Name) { + continue } - untrackedTables[resolvedName] = struct{}{} + untrackedTables[tableName] = struct{}{} } } - // untracked tables = working tables - staged tables headTblNames := GetAllTableNames(ctx, roots.Staged) - if err != nil { - return doltdb.Roots{}, err - } - for _, name := range headTblNames { delete(untrackedTables, name) } - newRoot := roots.Working - var toDelete []doltdb.TableName + toDelete := make([]doltdb.TableName, 0, len(untrackedTables)) for t := range untrackedTables { toDelete = append(toDelete, t) } - newRoot, err = newRoot.RemoveTables(ctx, force, force, toDelete...) - if err != nil { - return doltdb.Roots{}, fmt.Errorf("failed to remove tables; %w", err) - } - if dryrun { return roots, nil } - roots.Working = newRoot + newRoot, err := roots.Working.RemoveTables(ctx, force, force, toDelete...) + if err != nil { + return doltdb.Roots{}, fmt.Errorf("failed to remove tables; %w", err) + } + roots.Working = newRoot return roots, nil } diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clean.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clean.go index 27c007c37dc..4e723f83eab 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clean.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clean.go @@ -57,7 +57,8 @@ func doDoltClean(ctx *sql.Context, args []string) (int, error) { return 1, fmt.Errorf("Could not load database %s", dbName) } - roots, err = actions.CleanUntracked(ctx, roots, apr.Args, apr.ContainsAll(cli.DryRunFlag), false) + respectIgnoreRules := !apr.Contains(cli.ExcludeIgnoreRulesFlag) + roots, err = actions.CleanUntracked(ctx, roots, apr.Args, apr.ContainsAll(cli.DryRunFlag), false, respectIgnoreRules) if err != nil { return 1, fmt.Errorf("failed to clean; %w", err) } diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go index 7b94243f280..b257a671194 100755 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go @@ -512,7 +512,8 @@ func RunStoredProceduresTest(t *testing.T, h DoltEnginetestHarness) { } func RunDoltStoredProceduresTest(t *testing.T, h DoltEnginetestHarness) { - for _, script := range DoltProcedureTests { + scripts := append(DoltProcedureTests, DoltCleanProcedureScripts...) + for _, script := range scripts { func() { h := h.NewHarness(t) h.UseLocalFileSystem() @@ -523,7 +524,8 @@ func RunDoltStoredProceduresTest(t *testing.T, h DoltEnginetestHarness) { } func RunDoltStoredProceduresPreparedTest(t *testing.T, h DoltEnginetestHarness) { - for _, script := range DoltProcedureTests { + scripts := append(DoltProcedureTests, DoltCleanProcedureScripts...) + for _, script := range scripts { func() { h := h.NewHarness(t) h.UseLocalFileSystem() diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_clean.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_clean.go new file mode 100644 index 00000000000..d3455770d47 --- /dev/null +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_clean.go @@ -0,0 +1,80 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enginetest + +import ( + "github.com/dolthub/go-mysql-server/enginetest/queries" + "github.com/dolthub/go-mysql-server/sql" +) + +// DoltCleanProcedureScripts are script tests for the dolt_clean procedure. +var DoltCleanProcedureScripts = []queries.ScriptTest{ + { + Name: "dolt_clean does not drop tables matching dolt_ignore", + SetUpScript: []string{ + "CREATE TABLE ignored_foo (id int primary key);", + "INSERT INTO ignored_foo VALUES (1);", + "INSERT INTO dolt_ignore VALUES ('ignored_*', true);", + "CALL dolt_add('dolt_ignore');", + "CALL dolt_commit('-m', 'add dolt_ignore');", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SELECT * FROM ignored_foo;", + Expected: []sql.Row{{1}}, + }, + { + Query: "CALL dolt_clean();", + Expected: []sql.Row{{0}}, + }, + { + Query: "SELECT * FROM ignored_foo;", + Expected: []sql.Row{{1}}, + }, + { + Query: "SHOW TABLES;", + Expected: []sql.Row{{"ignored_foo"}}, + }, + }, + }, + { + Name: "dolt_clean -x drops tables matching dolt_ignore", + SetUpScript: []string{ + "CREATE TABLE ignored_bar (id int primary key);", + "INSERT INTO ignored_bar VALUES (1);", + "INSERT INTO dolt_ignore VALUES ('ignored_*', true);", + "CALL dolt_add('dolt_ignore');", + "CALL dolt_commit('-m', 'add dolt_ignore');", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SELECT * FROM ignored_bar;", + Expected: []sql.Row{{1}}, + }, + { + Query: "CALL dolt_clean('-x');", + Expected: []sql.Row{{0}}, + }, + { + Query: "SELECT * FROM ignored_bar;", + ExpectedErrStr: "table not found: ignored_bar", + }, + { + Query: "SHOW TABLES;", + Expected: []sql.Row{}, + }, + }, + }, +} diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_nonlocal.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_nonlocal.go index 7c0222691ad..eae0562253e 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_nonlocal.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_nonlocal.go @@ -209,4 +209,46 @@ var NonlocalScripts = []queries.ScriptTest{ }, }, }, + { + // https://github.com/dolthub/dolt/issues/10462 + Name: "nonlocal table is not affected by dolt_clean()", + SetUpScript: []string{ + "CREATE TABLE global_test (id int auto_increment primary key, name varchar(100));", + "INSERT INTO global_test (id, name) VALUES (1, 'one');", + "CREATE TABLE foo (id int auto_increment primary key);", + "INSERT INTO dolt_nonlocal_tables (table_name, target_ref, options) VALUES ('global_*', 'main', 'immediate');", + "CALL dolt_add('dolt_nonlocal_tables');", + "CALL dolt_commit('-m', 'set up dolt_nonlocal_tables');", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SELECT * FROM global_test;", + Expected: []sql.Row{{1, "one"}}, + }, + { + Query: "CALL dolt_clean();", + Expected: []sql.Row{{0}}, + }, + { + Query: "SELECT * FROM global_test;", + Expected: []sql.Row{{1, "one"}}, + }, + { + Query: "SHOW TABLES;", + Expected: []sql.Row{{"global_test"}}, + }, + { + Query: "CALL dolt_clean('-x')", + Expected: []sql.Row{{0}}, + }, + { + Query: "SELECT * FROM global_test;", + Expected: []sql.Row{{1, "one"}}, + }, + { + Query: "SHOW TABLES;", + Expected: []sql.Row{{"global_test"}}, + }, + }, + }, } diff --git a/integration-tests/bats/sql-local-remote.bats b/integration-tests/bats/sql-local-remote.bats index 8c1a50bb1ed..5d0417ed42a 100644 --- a/integration-tests/bats/sql-local-remote.bats +++ b/integration-tests/bats/sql-local-remote.bats @@ -1006,7 +1006,7 @@ SQL cd altDB # setup for cherry-pick.bats - dolt clean + dolt clean -x dolt sql -q "CREATE TABLE test(pk BIGINT PRIMARY KEY, v varchar(10), index(v))" dolt add . dolt commit -am "Created table"