From b8d96278b860e939bb586613c4fcd42013681ac7 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Thu, 27 Jun 2019 19:57:11 +0900 Subject: [PATCH 1/4] test: add PlainClone benchmark Signed-off-by: Nao YONASHIRO --- repository_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/repository_test.go b/repository_test.go index 0148c7845..32fa4fab7 100644 --- a/repository_test.go +++ b/repository_test.go @@ -16,6 +16,7 @@ import ( "golang.org/x/crypto/openpgp" "golang.org/x/crypto/openpgp/armor" openpgperr "golang.org/x/crypto/openpgp/errors" + "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/cache" @@ -2671,3 +2672,22 @@ func BenchmarkObjects(b *testing.B) { }) } } + +func BenchmarkPlainClone(b *testing.B) { + for i := 0; i < b.N; i++ { + t, err := ioutil.TempDir("", "") + if err != nil { + b.Fatal(err) + } + _, err = PlainClone(t, false, &CloneOptions{ + URL: "https://github.com/knqyf263/vuln-list", + Depth: 1, + }) + if err != nil { + b.Error(err) + } + b.StopTimer() + os.RemoveAll(t) + b.StartTimer() + } +} From 5fd82ef76bd3fd3569b3b132884991469a6676c5 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Thu, 27 Jun 2019 05:47:02 +0900 Subject: [PATCH 2/4] feat: improve clone performance on hotspot Signed-off-by: Nao YONASHIRO --- worktree.go | 61 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/worktree.go b/worktree.go index 576ae0abd..17ed9b4c7 100644 --- a/worktree.go +++ b/worktree.go @@ -304,6 +304,7 @@ func (w *Worktree) resetIndex(t *object.Tree) error { if err != nil { return err } + b := newIndexBuilder(idx) changes, err := w.diffTreeWithStaging(t, true) if err != nil { @@ -330,12 +331,12 @@ func (w *Worktree) resetIndex(t *object.Tree) error { name = ch.From.String() } - _, _ = idx.Remove(name) + b.Remove(name) if e == nil { continue } - idx.Entries = append(idx.Entries, &index.Entry{ + b.Add(&index.Entry{ Name: name, Hash: e.Hash, Mode: e.Mode, @@ -343,6 +344,7 @@ func (w *Worktree) resetIndex(t *object.Tree) error { } + b.Write(idx) return w.r.Storer.SetIndex(idx) } @@ -356,17 +358,19 @@ func (w *Worktree) resetWorktree(t *object.Tree) error { if err != nil { return err } + b := newIndexBuilder(idx) for _, ch := range changes { - if err := w.checkoutChange(ch, t, idx); err != nil { + if err := w.checkoutChange(ch, t, b); err != nil { return err } } + b.Write(idx) return w.r.Storer.SetIndex(idx) } -func (w *Worktree) checkoutChange(ch merkletrie.Change, t *object.Tree, idx *index.Index) error { +func (w *Worktree) checkoutChange(ch merkletrie.Change, t *object.Tree, idx *indexBuilder) error { a, err := ch.Action() if err != nil { return err @@ -445,7 +449,7 @@ func (w *Worktree) setHEADCommit(commit plumbing.Hash) error { func (w *Worktree) checkoutChangeSubmodule(name string, a merkletrie.Action, e *object.TreeEntry, - idx *index.Index, + idx *indexBuilder, ) error { switch a { case merkletrie.Modify: @@ -479,11 +483,11 @@ func (w *Worktree) checkoutChangeRegularFile(name string, a merkletrie.Action, t *object.Tree, e *object.TreeEntry, - idx *index.Index, + idx *indexBuilder, ) error { switch a { case merkletrie.Modify: - _, _ = idx.Remove(name) + idx.Remove(name) // to apply perm changes the file is deleted, billy doesn't implement // chmod @@ -569,19 +573,18 @@ func (w *Worktree) checkoutFileSymlink(f *object.File) (err error) { return } -func (w *Worktree) addIndexFromTreeEntry(name string, f *object.TreeEntry, idx *index.Index) error { - _, _ = idx.Remove(name) - idx.Entries = append(idx.Entries, &index.Entry{ +func (w *Worktree) addIndexFromTreeEntry(name string, f *object.TreeEntry, idx *indexBuilder) error { + idx.Remove(name) + idx.Add(&index.Entry{ Hash: f.Hash, Name: name, Mode: filemode.Submodule, }) - return nil } -func (w *Worktree) addIndexFromFile(name string, h plumbing.Hash, idx *index.Index) error { - _, _ = idx.Remove(name) +func (w *Worktree) addIndexFromFile(name string, h plumbing.Hash, idx *indexBuilder) error { + idx.Remove(name) fi, err := w.Filesystem.Lstat(name) if err != nil { return err @@ -605,8 +608,7 @@ func (w *Worktree) addIndexFromFile(name string, h plumbing.Hash, idx *index.Ind if fillSystemInfo != nil { fillSystemInfo(e, fi.Sys()) } - - idx.Entries = append(idx.Entries, e) + idx.Add(e) return nil } @@ -913,3 +915,32 @@ func doCleanDirectories(fs billy.Filesystem, dir string) error { } return nil } + +type indexBuilder struct { + entries map[string]*index.Entry +} + +func newIndexBuilder(idx *index.Index) *indexBuilder { + entries := make(map[string]*index.Entry, len(idx.Entries)) + for _, e := range idx.Entries { + entries[e.Name] = e + } + return &indexBuilder{ + entries: entries, + } +} + +func (b *indexBuilder) Write(idx *index.Index) { + idx.Entries = idx.Entries[:0] + for _, e := range b.entries { + idx.Entries = append(idx.Entries, e) + } +} + +func (b *indexBuilder) Add(e *index.Entry) { + b.entries[e.Name] = e +} + +func (b *indexBuilder) Remove(name string) { + delete(b.entries, filepath.ToSlash(name)) +} \ No newline at end of file From 377253879c75199689b46591234318986743abd9 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Fri, 28 Jun 2019 03:22:35 +0900 Subject: [PATCH 3/4] feat: use CopyBuffer and sync.Pool Signed-off-by: Nao YONASHIRO --- worktree.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/worktree.go b/worktree.go index 17ed9b4c7..4a609e9ea 100644 --- a/worktree.go +++ b/worktree.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "sync" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" @@ -512,6 +513,12 @@ func (w *Worktree) checkoutChangeRegularFile(name string, return nil } +var copyBufferPool = sync.Pool{ + New: func() interface{} { + return make([]byte, 32*1024) + }, +} + func (w *Worktree) checkoutFile(f *object.File) (err error) { mode, err := f.Mode.ToOSFileMode() if err != nil { @@ -535,8 +542,9 @@ func (w *Worktree) checkoutFile(f *object.File) (err error) { } defer ioutil.CheckClose(to, &err) - - _, err = io.Copy(to, from) + buf := copyBufferPool.Get().([]byte) + _, err = io.CopyBuffer(to, from, buf) + copyBufferPool.Put(buf) return } @@ -943,4 +951,4 @@ func (b *indexBuilder) Add(e *index.Entry) { func (b *indexBuilder) Remove(name string) { delete(b.entries, filepath.ToSlash(name)) -} \ No newline at end of file +} From ee7afbf69c24846760ab087bf260a7c29af62a00 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Sat, 29 Jun 2019 02:00:55 +0900 Subject: [PATCH 4/4] feat: use bufio.Writer on SetIndex Signed-off-by: Nao YONASHIRO --- storage/filesystem/index.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/storage/filesystem/index.go b/storage/filesystem/index.go index d04195ccb..be800eff3 100644 --- a/storage/filesystem/index.go +++ b/storage/filesystem/index.go @@ -20,8 +20,14 @@ func (s *IndexStorage) SetIndex(idx *index.Index) (err error) { } defer ioutil.CheckClose(f, &err) + bw := bufio.NewWriter(f) + defer func() { + if e := bw.Flush(); err == nil && e != nil { + err = e + } + }() - e := index.NewEncoder(f) + e := index.NewEncoder(bw) err = e.Encode(idx) return err }