Merge branch 'main' into drop-command-prefixes

commands/command_fetch.go

@@ -18,35 +18,28 @@ import (
 )
 
 var (
-	fetchRecentArg bool
-	fetchAllArg    bool
-	fetchPruneArg  bool
-	fetchDryRunArg bool
-	fetchJsonArg   bool
+	fetchRecentArg  bool
+	fetchAllArg     bool
+	fetchPruneArg   bool
+	fetchRefetchArg bool
+	fetchDryRunArg  bool
+	fetchJsonArg    bool
 )
 
 type fetchWatcher struct {
-	transfers        []*tq.Transfer
-	virtuallyFetched map[string]bool
-}
-
-func newfetchWatcher() *fetchWatcher {
-	ret := &fetchWatcher{}
-	if fetchJsonArg {
-		ret.transfers = make([]*tq.Transfer, 0)
-	}
-	if fetchDryRunArg {
-		ret.virtuallyFetched = make(map[string]bool)
-	}
-	return ret
+	transfers []*tq.Transfer
+	observed  map[string]bool
 }
 
 func (d *fetchWatcher) registerTransfer(t *tq.Transfer) {
-	if d.transfers != nil {
+	if fetchJsonArg {
 		d.transfers = append(d.transfers, t)
 	}
-	if d.virtuallyFetched != nil {
-		d.virtuallyFetched[t.Oid] = true
+	if fetchDryRunArg || fetchRefetchArg {
+		if d.observed == nil {
+			d.observed = make(map[string]bool)
+		}
+		d.observed[t.Oid] = true
 	}
 	if fetchDryRunArg {
 		printProgress("%s %s => %s", tr.Tr.Get("fetch"), t.Oid, t.Name)
@@ -57,15 +50,18 @@ func (d *fetchWatcher) dumpJson() {
 	data := struct {
 		Transfers []*tq.Transfer `json:"transfers"`
 	}{Transfers: d.transfers}
+	if data.Transfers == nil {
+		data.Transfers = []*tq.Transfer{}
+	}
 	encoder := json.NewEncoder(os.Stdout)
 	encoder.SetIndent("", " ")
 	if err := encoder.Encode(data); err != nil {
 		ExitWithError(err)
 	}
 }
 
-func (d *fetchWatcher) hasVirtuallyFetched(oid string) bool {
-	return d.virtuallyFetched[oid]
+func (d *fetchWatcher) hasObserved(oid string) bool {
+	return d.observed[oid]
 }
 
 func hasToPrintTransfers() bool {
@@ -126,7 +122,7 @@ func fetchCommand(cmd *cobra.Command, args []string) {
 	include, exclude := getIncludeExcludeArgs(cmd)
 	fetchPruneCfg := lfs.NewFetchPruneConfig(cfg.Git)
 
-	watcher := newfetchWatcher()
+	watcher := &fetchWatcher{}
 
 	if fetchAllArg {
 		if fetchRecentArg {
@@ -392,8 +388,8 @@ func scanAll() []*lfs.WrappedPointer {
 
 // Fetch
 // Returns true if all completed with no errors, false if errors were written to stderr/log
-func fetch(allpointers []*lfs.WrappedPointer, watcher *fetchWatcher) bool {
-	pointers, meter := missingPointers(allpointers, watcher)
+func fetch(allPointers []*lfs.WrappedPointer, watcher *fetchWatcher) bool {
+	pointersToFetch, meter := pointersToFetch(allPointers, watcher)
 	q := newDownloadQueue(
 		getTransferManifestOperationRemote("download", cfg.Remote()),
 		cfg.Remote(), tq.WithProgress(meter), tq.DryRun(fetchDryRunArg),
@@ -411,7 +407,7 @@ func fetch(allpointers []*lfs.WrappedPointer, watcher *fetchWatcher) bool {
 		}()
 	}
 
-	for _, p := range pointers {
+	for _, p := range pointersToFetch {
 		tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
 
 		q.Add(downloadTransfer(p))
@@ -432,31 +428,38 @@ func fetch(allpointers []*lfs.WrappedPointer, watcher *fetchWatcher) bool {
 	return ok
 }
 
-func missingPointers(allpointers []*lfs.WrappedPointer, watcher *fetchWatcher) ([]*lfs.WrappedPointer, *tq.Meter) {
+func pointersToFetch(allPointers []*lfs.WrappedPointer, watcher *fetchWatcher) ([]*lfs.WrappedPointer, *tq.Meter) {
 	logger := tasklog.NewLogger(os.Stdout,
 		tasklog.ForceProgress(cfg.ForceProgress()),
 	)
 	meter := buildProgressMeter(hasToPrintTransfers(), tq.Download)
 	logger.Enqueue(meter)
 
-	missing := make([]*lfs.WrappedPointer, 0, len(allpointers))
+	pointersToFetch := make([]*lfs.WrappedPointer, 0, len(allPointers))
 
-	for _, p := range allpointers {
-		// no need to download objects that exist locally already
-		lfs.LinkOrCopyFromReference(cfg, p.Oid, p.Size)
-		if cfg.LFSObjectExists(p.Oid, p.Size) {
+	for _, p := range allPointers {
+		// if running with --dry-run or --refetch, skip objects that have already been virtually or
+		// already forcefully fetched
+		if watcher != nil && watcher.hasObserved(p.Oid) {
 			continue
 		}
-		// also if running with --dry-run, skip objects that have already been virtually fetched
-		if watcher != nil && watcher.hasVirtuallyFetched(p.Oid) {
+
+		// empty files are special, we always skip them in upload/download operations
+		if p.Size == 0 {
+			continue
+		}
+
+		// no need to download objects that exist locally already, unless `--refetch` was provided
+		lfs.LinkOrCopyFromReference(cfg, p.Oid, p.Size)
+		if !fetchRefetchArg && cfg.LFSObjectExists(p.Oid, p.Size) {
 			continue
 		}
 
-		missing = append(missing, p)
+		pointersToFetch = append(pointersToFetch, p)
 		meter.Add(p.Size)
 	}
 
-	return missing, meter
+	return pointersToFetch, meter
 }
 
 func init() {
@@ -466,6 +469,7 @@ func init() {
 		cmd.Flags().BoolVarP(&fetchRecentArg, "recent", "r", false, "Fetch recent refs & commits")
 		cmd.Flags().BoolVarP(&fetchAllArg, "all", "a", false, "Fetch all LFS files ever referenced")
 		cmd.Flags().BoolVarP(&fetchPruneArg, "prune", "p", false, "After fetching, prune old data")
+		cmd.Flags().BoolVar(&fetchRefetchArg, "refetch", false, "Also fetch objects that are already present locally")
 		cmd.Flags().BoolVarP(&fetchDryRunArg, "dry-run", "d", false, "Do not fetch, only show what would be fetched")
 		cmd.Flags().BoolVarP(&fetchJsonArg, "json", "j", false, "Give the output in a stable JSON format for scripts")
 	})

commands/command_locks.go

@@ -9,7 +9,6 @@ import (
 	"github.com/git-lfs/git-lfs/v3/errors"
 	"github.com/git-lfs/git-lfs/v3/git"
 	"github.com/git-lfs/git-lfs/v3/locking"
-	"github.com/git-lfs/git-lfs/v3/tools"
 	"github.com/git-lfs/git-lfs/v3/tr"
 	"github.com/spf13/cobra"
 )
@@ -96,9 +95,9 @@ func locksCommand(cmd *cobra.Command, args []string) {
 	for _, lock := range locks {
 		lockPaths = append(lockPaths, lock.Path)
 		locksByPath[lock.Path] = lock
-		maxPathLen = tools.MaxInt(maxPathLen, len(lock.Path))
+		maxPathLen = max(maxPathLen, len(lock.Path))
 		if lock.Owner != nil {
-			maxNameLen = tools.MaxInt(maxNameLen, len(lock.Owner.Name))
+			maxNameLen = max(maxNameLen, len(lock.Owner.Name))
 		}
 	}
 
@@ -110,8 +109,8 @@ func locksCommand(cmd *cobra.Command, args []string) {
 			ownerName = lock.Owner.Name
 		}
 
-		pathPadding := tools.MaxInt(maxPathLen-len(lock.Path), 0)
-		namePadding := tools.MaxInt(maxNameLen-len(ownerName), 0)
+		pathPadding := max(maxPathLen-len(lock.Path), 0)
+		namePadding := max(maxNameLen-len(ownerName), 0)
 		kind := ""
 		if locksOwned != nil {
 			if locksOwned[lock] {

docs/man/git-lfs-checkout.adoc

@@ -69,8 +69,8 @@ $ git merge conflicting-branch
 CONFLICT (content): Merge conflict in path/to/conflicting/file.dat
 
 # Checkout versions of the conflicting file into temp files
-$ git lfs checkout ours.dat --ours path/to/conflicting/file.dat
-$ git lfs checkout theirs.dat --theirs path/to/conflicting/file.dat
+$ git lfs checkout --to ours.dat --ours path/to/conflicting/file.dat
+$ git lfs checkout --to theirs.dat --theirs path/to/conflicting/file.dat
 
 # Compare conflicting versions in ours.dat and theirs.dat,
 # then resolve conflict (e.g., by choosing one version over

docs/man/git-lfs-fetch.adoc

@@ -39,6 +39,8 @@ This does not update the working copy.
 `-p`::
   Prune old and unreferenced objects after fetching, equivalent to running `git
   lfs prune` afterwards. See git-lfs-prune(1) for more details.
+`--refetch`::
+  Also fetch objects that are already present locally.
 `--dry-run`::
 `-d`::
   Print what would be fetched, without actually fetching anything.

git/githistory/ref_updater.go

@@ -8,7 +8,6 @@ import (
 	"github.com/git-lfs/git-lfs/v3/errors"
 	"github.com/git-lfs/git-lfs/v3/git"
 	"github.com/git-lfs/git-lfs/v3/tasklog"
-	"github.com/git-lfs/git-lfs/v3/tools"
 	"github.com/git-lfs/git-lfs/v3/tr"
 	"github.com/git-lfs/gitobj/v2"
 )
@@ -46,7 +45,7 @@ func (r *refUpdater) updateRefs() error {
 
 	var maxNameLen int
 	for _, ref := range r.refs {
-		maxNameLen = tools.MaxInt(maxNameLen, len(ref.Name))
+		maxNameLen = max(maxNameLen, len(ref.Name))
 	}
 
 	seen := make(map[string]struct{})
@@ -144,7 +143,7 @@ func (r *refUpdater) updateOneRef(list *tasklog.ListTask, maxNameLen int, seen m
 		return err
 	}
 
-	namePadding := tools.MaxInt(maxNameLen-len(ref.Name), 0)
+	namePadding := max(maxNameLen-len(ref.Name), 0)
 	list.Entry(fmt.Sprintf("  %s%s\t%s -> %x", ref.Name, strings.Repeat(" ", namePadding), ref.Sha, to))
 	return nil
 }

lfshttp/client.go

@@ -21,7 +21,6 @@ import (
 	"github.com/git-lfs/git-lfs/v3/config"
 	"github.com/git-lfs/git-lfs/v3/creds"
 	"github.com/git-lfs/git-lfs/v3/errors"
-	"github.com/git-lfs/git-lfs/v3/tools"
 	"github.com/git-lfs/git-lfs/v3/tr"
 	"github.com/rubyist/tracerx"
 	"golang.org/x/net/http2"
@@ -215,7 +214,7 @@ func (c *Client) sshResolveWithRetries(e Endpoint, method string) (*sshAuthRespo
 		return nil, errors.New("git-lfs-authenticate has been disabled by request")
 	}
 
-	requests := tools.MaxInt(0, c.sshTries) + 1
+	requests := max(0, c.sshTries) + 1
 	for i := 0; i < requests; i++ {
 		sshRes, err = c.SSH.Resolve(e, method)
 		if err == nil {
@@ -293,7 +292,7 @@ func (c *Client) DoWithRedirect(cli *http.Client, req *http.Request, remote stri
 
 	var res *http.Response
 
-	requests := tools.MaxInt(0, retries) + 1
+	requests := max(0, retries) + 1
 	for i := 0; i < requests; i++ {
 		res, err = cli.Do(req)
 		if err == nil {

lfshttp/stats.go

@@ -11,8 +11,6 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
-
-	"github.com/git-lfs/git-lfs/v3/tools"
 )
 
 type httpTransfer struct {
@@ -140,11 +138,11 @@ func (l *syncLogger) LogResponse(req *http.Request, status int, bodySize int64)
 			fmt.Sprintf(" status=%d body=%d conntime=%d dnstime=%d tlstime=%d restime=%d time=%d",
 				status,
 				bodySize,
-				tools.MaxInt64(t.ConnEnd-t.ConnStart, 0),
-				tools.MaxInt64(t.DNSEnd-t.DNSStart, 0),
-				tools.MaxInt64(t.TLSEnd-t.TLSStart, 0),
-				tools.MaxInt64(now-t.ResponseStart, 0),
-				tools.MaxInt64(now-t.Start, 0),
+				max(t.ConnEnd-t.ConnStart, 0),
+				max(t.DNSEnd-t.DNSStart, 0),
+				max(t.TLSEnd-t.TLSStart, 0),
+				max(now-t.ResponseStart, 0),
+				max(now-t.Start, 0),
 			))
 	}
 }

t/t-fetch.sh

@@ -41,6 +41,11 @@ begin_test "init for fetch tests"
 
   assert_server_object "$reponame" "$contents_oid"
 
+  # Add an empty file
+  touch empty.dat
+  git add empty.dat
+  git commit -m 'empty'
+
   # Add a file in a different branch
   git checkout -b newbranch
   printf "%s" "$b" > b.dat
@@ -118,17 +123,20 @@ EOF
 )
 end_test
 
-begin_test "fetch (empty file)"
+begin_test "fetch --refetch"
 (
   set -e
   cd clone
   rm -rf .git/lfs/objects
 
-  touch empty.dat
-  git add empty.dat
-  git commit -m 'empty'
-
   git lfs fetch
+  assert_local_object "$contents_oid" 1
+
+  corrupt_local_object "$contents_oid"
+  refute_local_object "$contents_oid" 1
+
+  git lfs fetch --refetch
+  assert_local_object "$contents_oid" 1
 
   git lfs fsck 2>&1 | tee fsck.log
   grep "Git LFS fsck OK" fsck.log
@@ -257,6 +265,36 @@ EOF
 )
 end_test
 
+begin_test "fetch --refetch with remote and branches"
+(
+  set -e
+  cd clone
+
+  git checkout newbranch
+  git checkout main
+
+  rm -rf .git/lfs/objects
+
+  git lfs fetch origin main newbranch
+  assert_local_object "$contents_oid" 1
+  assert_local_object "$b_oid" 1
+
+  corrupt_local_object "$contents_oid"
+  corrupt_local_object "$b_oid"
+  refute_local_object "$contents_oid" 1
+  refute_local_object "$b_oid" 1
+
+  git lfs fetch --refetch --json origin main newbranch | tee fetch.json
+  assert_local_object "$contents_oid" 1
+  assert_local_object "$b_oid" 1
+
+  # check that we did not fetch a.dat twice
+  [ 1 -eq $(grep -c '"name": "a.dat"' fetch.json) ]
+  git lfs fsck 2>&1 | tee fsck.log
+  grep "Git LFS fsck OK" fsck.log
+)
+end_test
+
 begin_test "fetch with main commit sha1"
 (
   set -e

tasklog/log.go

@@ -8,7 +8,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/git-lfs/git-lfs/v3/tools"
 	isatty "github.com/mattn/go-isatty"
 	"github.com/olekukonko/ts"
 )
@@ -254,7 +253,7 @@ func (l *Logger) logTask(task Task) {
 // It returns the number of bytes "n" written to the sink and the error "err",
 // if one was encountered.
 func (l *Logger) logLine(str string) (n int, err error) {
-	padding := strings.Repeat(" ", tools.MaxInt(0, l.widthFn()-len(str)))
+	padding := strings.Repeat(" ", max(0, l.widthFn()-len(str)))
 
 	return l.log(str + padding + "\r")
 }