feat: support .gitignore files (#19)

Introduces a `--walk` flag which can be used to tell `treefmt` how to traverse the directory specified by `--tree-root`.

By default, it will attempt to use `git ls-files`. If this fails, it falls back to using the filesystem.

You can explicitly traverse the filesystem instead of using git by providing `--walk filesystem`.

Close #1

Reviewed-on: https://git.numtide.com/numtide/treefmt/pulls/19
Reviewed-by: Jonas Chevalier <[email protected]>
Co-authored-by: Brian McGee <[email protected]>
Co-committed-by: Brian McGee <[email protected]>

go.mod

@@ -7,8 +7,9 @@ require (
 	github.com/adrg/xdg v0.4.0
 	github.com/alecthomas/kong v0.8.1
 	github.com/charmbracelet/log v0.3.1
+	github.com/go-git/go-billy/v5 v5.5.0
+	github.com/go-git/go-git/v5 v5.11.0
 	github.com/gobwas/glob v0.2.3
-	github.com/juju/errors v1.0.0
 	github.com/otiai10/copy v1.14.0
 	github.com/stretchr/testify v1.8.4
 	github.com/vmihailenco/msgpack/v5 v5.4.1
@@ -17,19 +18,38 @@ require (
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/charmbracelet/lipgloss v0.9.1 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.15.2 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/skeema/knownhosts v1.2.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	golang.org/x/crypto v0.16.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

gomod2nix.toml

@@ -1,9 +1,18 @@
 schema = 3
 
 [mod]
+  [mod."dario.cat/mergo"]
+    version = "v1.0.0"
+    hash = "sha256-jlpc8dDj+DmiOU4gEawBu8poJJj9My0s9Mvuk9oS8ww="
   [mod."github.com/BurntSushi/toml"]
     version = "v1.3.2"
     hash = "sha256-FIwyH67KryRWI9Bk4R8s1zFP0IgKR4L66wNQJYQZLeg="
+  [mod."github.com/Microsoft/go-winio"]
+    version = "v0.6.1"
+    hash = "sha256-BL0BVaHtmPKQts/711W59AbHXjGKqFS4ZTal0RYnR9I="
+  [mod."github.com/ProtonMail/go-crypto"]
+    version = "v0.0.0-20230828082145-3c4c8a2d2371"
+    hash = "sha256-YxAaQgQoTOhD8hE+aT+T8ytKKxcQW6tgoL2MAU7nTvo="
   [mod."github.com/adrg/xdg"]
     version = "v0.4.0"
     hash = "sha256-zGjkdUQmrVqD6rMO9oDY+TeJCpuqnHyvkPCaXDlac/U="
@@ -19,18 +28,42 @@ schema = 3
   [mod."github.com/charmbracelet/log"]
     version = "v0.3.1"
     hash = "sha256-Er60POPID2eNrRZnBHxoI4yHn0mIKnXYftGKSslbXx0="
+  [mod."github.com/cloudflare/circl"]
+    version = "v1.3.3"
+    hash = "sha256-ItdVkU53Ep01553/tJ4MdAwoTpPljRxiBW9sAd7p0xI="
+  [mod."github.com/cyphar/filepath-securejoin"]
+    version = "v0.2.4"
+    hash = "sha256-heCD0xMxlwnHCHcRBgTjVexHOLyWI2zRW3E8NFKoLzk="
   [mod."github.com/davecgh/go-spew"]
     version = "v1.1.1"
     hash = "sha256-nhzSUrE1fCkN0+RL04N4h8jWmRFPPPWbCuDc7Ss0akI="
+  [mod."github.com/emirpasic/gods"]
+    version = "v1.18.1"
+    hash = "sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4="
+  [mod."github.com/go-git/gcfg"]
+    version = "v1.5.1-0.20230307220236-3a3c6141e376"
+    hash = "sha256-f4k0gSYuo0/q3WOoTxl2eFaj7WZpdz29ih6CKc8Ude8="
+  [mod."github.com/go-git/go-billy/v5"]
+    version = "v5.5.0"
+    hash = "sha256-4XUoD2bOCMCdu83egb/y8kY/Fm0s1rWgPMtiahh38OQ="
+  [mod."github.com/go-git/go-git/v5"]
+    version = "v5.11.0"
+    hash = "sha256-2yUM/FlV+nYxacVynJCnDZeMub4Iu8JL2WBHmlnwOkE="
   [mod."github.com/go-logfmt/logfmt"]
     version = "v0.6.0"
     hash = "sha256-RtIG2qARd5sT10WQ7F3LR8YJhS8exs+KiuUiVf75bWg="
   [mod."github.com/gobwas/glob"]
     version = "v0.2.3"
     hash = "sha256-hYHMUdwxVkMOjSKjR7UWO0D0juHdI4wL8JEy5plu/Jc="
-  [mod."github.com/juju/errors"]
-    version = "v1.0.0"
-    hash = "sha256-9uZ0wNf44ilzLsvXqOsmFUpNOBFAVadj6+ZH8+QMDMk="
+  [mod."github.com/golang/groupcache"]
+    version = "v0.0.0-20210331224755-41bb18bfe9da"
+    hash = "sha256-7Gs7CS9gEYZkbu5P4hqPGBpeGZWC64VDwraSKFF+VR0="
+  [mod."github.com/jbenet/go-context"]
+    version = "v0.0.0-20150711004518-d14ea06fba99"
+    hash = "sha256-VANNCWNNpARH/ILQV9sCQsBWgyL2iFT+4AHZREpxIWE="
+  [mod."github.com/kevinburke/ssh_config"]
+    version = "v1.2.0"
+    hash = "sha256-Ta7ZOmyX8gG5tzWbY2oES70EJPfI90U7CIJS9EAce0s="
   [mod."github.com/lucasb-eyer/go-colorful"]
     version = "v1.2.0"
     hash = "sha256-Gg9dDJFCTaHrKHRR1SrJgZ8fWieJkybljybkI9x0gyE="
@@ -49,12 +82,21 @@ schema = 3
   [mod."github.com/otiai10/copy"]
     version = "v1.14.0"
     hash = "sha256-xsaL1ddkPS544y0Jv7u/INUALBYmYq29ddWvysLXk4A="
+  [mod."github.com/pjbgf/sha1cd"]
+    version = "v0.3.0"
+    hash = "sha256-kX9BdLh2dxtGNaDvc24NORO+C0AZ7JzbrXrtecCdB7w="
   [mod."github.com/pmezard/go-difflib"]
     version = "v1.0.0"
     hash = "sha256-/FtmHnaGjdvEIKAJtrUfEhV7EVo5A/eYrtdnUkuxLDA="
   [mod."github.com/rivo/uniseg"]
     version = "v0.2.0"
     hash = "sha256-GLj0jiGrT03Ept4V6FXCN1yeZ/b6PpS3MEXK6rYQ8Eg="
+  [mod."github.com/sergi/go-diff"]
+    version = "v1.1.0"
+    hash = "sha256-8NJMabldpf40uwQN20T6QXx5KORDibCBJL02KD661xY="
+  [mod."github.com/skeema/knownhosts"]
+    version = "v1.2.1"
+    hash = "sha256-u0jB6ahTdGa+SvcIvPNRLnbSHvgmW9X/ThRq0nWQrJs="
   [mod."github.com/stretchr/testify"]
     version = "v1.8.4"
     hash = "sha256-MoOmRzbz9QgiJ+OOBo5h5/LbilhJfRUryvzHJmXAWjo="
@@ -64,18 +106,36 @@ schema = 3
   [mod."github.com/vmihailenco/tagparser/v2"]
     version = "v2.0.0"
     hash = "sha256-M9QyaKhSmmYwsJk7gkjtqu9PuiqZHSmTkous8VWkWY0="
+  [mod."github.com/xanzy/ssh-agent"]
+    version = "v0.3.3"
+    hash = "sha256-l3pGB6IdzcPA/HLk93sSN6NM2pKPy+bVOoacR5RC2+c="
   [mod."go.etcd.io/bbolt"]
     version = "v1.3.8"
     hash = "sha256-ekKy8198B2GfPldHLYZnvNjID6x07dUPYKgFx84TgVs="
+  [mod."golang.org/x/crypto"]
+    version = "v0.16.0"
+    hash = "sha256-DgSVOnXRK8GF01p5rLtq4qPBcglwEoOk8qhW2EGfJfA="
   [mod."golang.org/x/exp"]
     version = "v0.0.0-20231006140011-7918f672742d"
     hash = "sha256-2SO1etTQ6UCUhADR5sgvDEDLHcj77pJKCIa/8mGDbAo="
+  [mod."golang.org/x/mod"]
+    version = "v0.13.0"
+    hash = "sha256-qh/YmxS0auZEiKzqdn+v84qs31SpkGIJn9rqKLjdKVU="
+  [mod."golang.org/x/net"]
+    version = "v0.19.0"
+    hash = "sha256-3M5rKEvJx4cO/q+06cGjR5sxF5JpnUWY0+fQttrWdT4="
   [mod."golang.org/x/sync"]
     version = "v0.5.0"
     hash = "sha256-EAKeODSsct5HhXPmpWJfulKSCkuUu6kkDttnjyZMNcI="
   [mod."golang.org/x/sys"]
-    version = "v0.13.0"
-    hash = "sha256-/+RDZ0a0oEfJ0k304VqpJpdrl2ZXa3yFlOxy4mjW7w0="
+    version = "v0.15.0"
+    hash = "sha256-n7TlABF6179RzGq3gctPDKDPRtDfnwPdjNCMm8ps2KY="
+  [mod."golang.org/x/tools"]
+    version = "v0.14.0"
+    hash = "sha256-BC/AesMg7LpIg0/e4a7Ab37rfyP2KaWBFrsonIp+JbE="
+  [mod."gopkg.in/warnings.v0"]
+    version = "v0.1.2"
+    hash = "sha256-ATVL9yEmgYbkJ1DkltDGRn/auGAjqGOfjQyBYyUo8s8="
   [mod."gopkg.in/yaml.v3"]
     version = "v3.0.1"
     hash = "sha256-FqL9TKYJ0XkNwJFnq9j0VvJ5ZUU1RvH/52h/f5bkYAU="

internal/cache/cache.go

@@ -7,9 +7,10 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
-	"path/filepath"
 	"time"
 
+	"git.numtide.com/numtide/treefmt/internal/walk"
+
 	"git.numtide.com/numtide/treefmt/internal/format"
 	"github.com/charmbracelet/log"
 
@@ -172,7 +173,7 @@ func putEntry(bucket *bolt.Bucket, path string, entry *Entry) error {
 
 // ChangeSet is used to walk a filesystem, starting at root, and outputting any new or changed paths using pathsCh.
 // It determines if a path is new or has changed by comparing against cache entries.
-func ChangeSet(ctx context.Context, root string, pathsCh chan<- string) error {
+func ChangeSet(ctx context.Context, root string, walkerType walk.Type, pathsCh chan<- string) error {
 	var tx *bolt.Tx
 	var bucket *bolt.Bucket
 	var processed int
@@ -184,14 +185,22 @@ func ChangeSet(ctx context.Context, root string, pathsCh chan<- string) error {
 		}
 	}()
 
-	return filepath.Walk(root, func(path string, info fs.FileInfo, err error) error {
-		if err != nil {
-			return fmt.Errorf("%w: failed to walk path", err)
-		} else if ctx.Err() != nil {
+	w, err := walk.New(walkerType, root)
+	if err != nil {
+		return fmt.Errorf("%w: failed to create walker", err)
+	}
+
+	return w.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
+		select {
+		case <-ctx.Done():
 			return ctx.Err()
-		} else if info.IsDir() {
-			// todo what about symlinks?
-			return nil
+		default:
+			if err != nil {
+				return fmt.Errorf("%w: failed to walk path", err)
+			} else if info.IsDir() {
+				// ignore directories
+				return nil
+			}
 		}
 
 		// ignore symlinks
@@ -222,7 +231,12 @@ func ChangeSet(ctx context.Context, root string, pathsCh chan<- string) error {
 		}
 
 		// pass on the path
-		pathsCh <- path
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			pathsCh <- path
+		}
 
 		// close the current tx if we have reached the batch size
 		processed += 1

internal/cli/cli.go

@@ -1,21 +1,23 @@
 package cli
 
 import (
+	"git.numtide.com/numtide/treefmt/internal/walk"
 	"github.com/alecthomas/kong"
 	"github.com/charmbracelet/log"
 )
 
 var Cli = Options{}
 
 type Options struct {
-	AllowMissingFormatter bool               `default:"false" help:"Do not exit with error if a configured formatter is missing"`
-	WorkingDirectory      kong.ChangeDirFlag `default:"." short:"C" help:"Run as if treefmt was started in the specified working directory instead of the current working directory"`
-	ClearCache            bool               `short:"c" help:"Reset the evaluation cache. Use in case the cache is not precise enough"`
+	AllowMissingFormatter bool               `default:"false" help:"Do not exit with error if a configured formatter is missing."`
+	WorkingDirectory      kong.ChangeDirFlag `default:"." short:"C" help:"Run as if treefmt was started in the specified working directory instead of the current working directory."`
+	ClearCache            bool               `short:"c" help:"Reset the evaluation cache. Use in case the cache is not precise enough."`
 	ConfigFile            string             `type:"existingfile" default:"./treefmt.toml"`
-	FailOnChange          bool               `help:"Exit with error if any changes were made. Useful for CI"`
-	Formatters            []string           `help:"Specify formatters to apply. Defaults to all formatters"`
+	FailOnChange          bool               `help:"Exit with error if any changes were made. Useful for CI."`
+	Formatters            []string           `help:"Specify formatters to apply. Defaults to all formatters."`
 	TreeRoot              string             `type:"existingdir" default:"."`
-	Verbosity             int                `name:"verbose" short:"v" type:"counter" default:"0" env:"LOG_LEVEL" help:"Set the verbosity of logs e.g. -vv"`
+	Walk                  walk.Type          `enum:"auto,git,filesystem" default:"auto" help:"The method used to traverse the files within --tree-root. Currently supports 'auto', 'git' or 'filesystem'."`
+	Verbosity             int                `name:"verbose" short:"v" type:"counter" default:"0" env:"LOG_LEVEL" help:"Set the verbosity of logs e.g. -vv."`
 
 	Format Format `cmd:"" default:"."`
 }

internal/cli/format.go

@@ -190,8 +190,9 @@ func (f *Format) Run() error {
 	})
 
 	eg.Go(func() error {
-		defer close(pathsCh)
-		return cache.ChangeSet(ctx, Cli.TreeRoot, pathsCh)
+		err := cache.ChangeSet(ctx, Cli.TreeRoot, Cli.Walk, pathsCh)
+		close(pathsCh)
+		return err
 	})
 
 	// listen for shutdown and call cancel if required

internal/cli/format_test.go

@@ -4,9 +4,15 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path"
+	"path/filepath"
 	"testing"
 
 	"git.numtide.com/numtide/treefmt/internal/test"
+	"github.com/go-git/go-billy/v5/osfs"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/cache"
+	"github.com/go-git/go-git/v5/storage/filesystem"
 
 	"git.numtide.com/numtide/treefmt/internal/format"
 	"github.com/stretchr/testify/require"
@@ -343,3 +349,58 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
 	as.NoError(err)
 	as.Contains(string(out), "0 files changed")
 }
+
+func TestGitWorktree(t *testing.T) {
+	as := require.New(t)
+
+	tempDir := test.TempExamples(t)
+	configPath := filepath.Join(tempDir, "/treefmt.toml")
+
+	// basic config
+	config := format.Config{
+		Formatters: map[string]*format.Formatter{
+			"echo": {
+				Command:  "echo",
+				Includes: []string{"*"},
+			},
+		},
+	}
+	test.WriteConfig(t, configPath, config)
+
+	// init a git repo
+	repo, err := git.Init(
+		filesystem.NewStorage(
+			osfs.New(path.Join(tempDir, ".git")),
+			cache.NewObjectLRUDefault(),
+		),
+		osfs.New(tempDir),
+	)
+	as.NoError(err, "failed to init git repository")
+
+	// get worktree
+	wt, err := repo.Worktree()
+	as.NoError(err, "failed to get git worktree")
+
+	run := func(changed int) {
+		out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
+		as.NoError(err)
+		as.Contains(string(out), fmt.Sprintf("%d files changed", changed))
+	}
+
+	// run before adding anything to the worktree
+	run(0)
+
+	// add everything to the worktree
+	as.NoError(wt.AddGlob("."))
+	as.NoError(err)
+	run(29)
+
+	// remove python directory
+	as.NoError(wt.RemoveGlob("python/*"))
+	run(26)
+
+	// walk with filesystem instead of git
+	out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem")
+	as.NoError(err)
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 55))
+}

internal/format/glob.go

@@ -11,7 +11,7 @@ func CompileGlobs(patterns []string) ([]glob.Glob, error) {
 	globs := make([]glob.Glob, len(patterns))
 
 	for i, pattern := range patterns {
-		g, err := glob.Compile("**/" + pattern)
+		g, err := glob.Compile(pattern)
 		if err != nil {
 			return nil, fmt.Errorf("%w: failed to compile include pattern '%v'", err, pattern)
 		}

internal/walk/filesystem.go

@@ -0,0 +1,22 @@
+package walk
+
+import (
+	"context"
+	"path/filepath"
+)
+
+type filesystem struct {
+	root string
+}
+
+func (f filesystem) Root() string {
+	return f.root
+}
+
+func (f filesystem) Walk(_ context.Context, fn filepath.WalkFunc) error {
+	return filepath.Walk(f.root, fn)
+}
+
+func NewFilesystem(root string) (Walker, error) {
+	return filesystem{root}, nil
+}