Skip to content

Commit

Permalink
Use 32KB buffer for copyFile() (reduces copy time by 30%) (#1749)
Browse files Browse the repository at this point in the history
* Use 32KB buffer for `copyFile()` (reduces copy time by 30%)

This makes `:paste` comparable in performance to `cp --reflink=never`
for large files.

For large number of small files improvements are less substantial
(compared to `cp -r --reflink=never`), though still noticeable.

In both cases the copy takes about 30% less time than with `buf` size at 4096.

32KB is the same number `io.Copy()` uses internally (when it can), and is about where improvements stop.

Context:
#1685 (comment)

* Apply suggested changes

* Just delete incomplete an file on all errors

* Mention new progress UI update frequency

* Return the old (each 4MB copied) progress update frequency

(progress UI updates don't seem to impact performance much)
MahouShoujoMivutilde authored Jun 14, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 0c72034 commit 2d3cc97
Showing 2 changed files with 25 additions and 21 deletions.
4 changes: 2 additions & 2 deletions app.go
Original file line number Diff line number Diff line change
@@ -332,8 +332,8 @@ func (app *app) loop() {
return
case n := <-app.nav.copyBytesChan:
app.nav.copyBytes += n
// n is usually 4096B so update roughly per 4096B x 1024 = 4MB copied
if app.nav.copyUpdate++; app.nav.copyUpdate >= 1024 {
// n is usually 32*1024B (default io.Copy() buffer) so update roughly per 32KB x 128 = 4MB copied
if app.nav.copyUpdate++; app.nav.copyUpdate >= 128 {
app.nav.copyUpdate = 0
app.ui.draw(app.nav)
}
42 changes: 23 additions & 19 deletions copy.go
Original file line number Diff line number Diff line change
@@ -11,6 +11,24 @@ import (
"github.com/djherbis/times"
)

type ProgressWriter struct {
writer io.Writer
nums chan<- int64
}

func NewProgressWriter(writer io.Writer, nums chan<- int64) *ProgressWriter {
return &ProgressWriter{
writer: writer,
nums: nums,
}
}

func (progressWriter *ProgressWriter) Write(b []byte) (int, error) {
n, err := progressWriter.writer.Write(b)
progressWriter.nums <- int64(n)
return n, err
}

func copySize(srcs []string) (int64, error) {
var total int64

@@ -47,8 +65,6 @@ func copyFile(src, dst string, preserve []string, info os.FileInfo, nums chan in
}
}

buf := make([]byte, 4096)

r, err := os.Open(src)
if err != nil {
return err
@@ -60,23 +76,11 @@ func copyFile(src, dst string, preserve []string, info os.FileInfo, nums chan in
return err
}

for {
n, err := r.Read(buf)
if err != nil && err != io.EOF {
w.Close()
os.Remove(dst)
return err
}

if n == 0 {
break
}

if _, err := w.Write(buf[:n]); err != nil {
return err
}

nums <- int64(n)
_, err = io.Copy(NewProgressWriter(w, nums), r)
if err != nil {
w.Close()
os.Remove(dst)
return err
}

if err := w.Close(); err != nil {

0 comments on commit 2d3cc97

Please sign in to comment.