fix update mtime before close (#3552)

cmd/mdtest.go

@@ -70,7 +70,7 @@ func createFile(jfs *fs.FileSystem, bar *utils.Bar, np int, root string, d int,
 				if bytes < (indx+1)*meta.ChunkSize {
 					size = bytes - indx*meta.ChunkSize
 				}
-				if st := m.Write(ctx, f.Inode(), uint32(indx), 0, meta.Slice{Id: id, Size: uint32(size), Len: uint32(size)}); st != 0 {
+				if st := m.Write(ctx, f.Inode(), uint32(indx), 0, meta.Slice{Id: id, Size: uint32(size), Len: uint32(size)}, time.Now()); st != 0 {
 					return fmt.Errorf("writeend %s: %s", fn, st)
 				}
 			}

pkg/meta/base_test.go

@@ -454,7 +454,7 @@ func testMetaClient(t *testing.T, m Meta) {
 		t.Fatalf("write chunk: %s", st)
 	}
 	var s = Slice{Id: sliceId, Size: 100, Len: 100}
-	if st := m.Write(ctx, inode, 0, 100, s); st != 0 {
+	if st := m.Write(ctx, inode, 0, 100, s, time.Now()); st != 0 {
 		t.Fatalf("write end: %s", st)
 	}
 	var slices []Slice
@@ -1093,11 +1093,11 @@ func testCompaction(t *testing.T, m Meta, trash bool) {
 	// random write
 	var sliceId uint64
 	m.NewSlice(ctx, &sliceId)
-	_ = m.Write(ctx, inode, 1, uint32(0), Slice{Id: sliceId, Size: 64 << 20, Len: 64 << 20})
+	_ = m.Write(ctx, inode, 1, uint32(0), Slice{Id: sliceId, Size: 64 << 20, Len: 64 << 20}, time.Now())
 	m.NewSlice(ctx, &sliceId)
-	_ = m.Write(ctx, inode, 1, uint32(30<<20), Slice{Id: sliceId, Size: 8, Len: 8})
+	_ = m.Write(ctx, inode, 1, uint32(30<<20), Slice{Id: sliceId, Size: 8, Len: 8}, time.Now())
 	m.NewSlice(ctx, &sliceId)
-	_ = m.Write(ctx, inode, 1, uint32(40<<20), Slice{Id: sliceId, Size: 8, Len: 8})
+	_ = m.Write(ctx, inode, 1, uint32(40<<20), Slice{Id: sliceId, Size: 8, Len: 8}, time.Now())
 	var cs1 []Slice
 	_ = m.Read(ctx, inode, 1, &cs1)
 	if len(cs1) != 5 {
@@ -1117,7 +1117,7 @@ func testCompaction(t *testing.T, m Meta, trash bool) {
 	for i := 0; i < 200; i++ {
 		var sliceId uint64
 		m.NewSlice(ctx, &sliceId)
-		if st := m.Write(ctx, inode, 0, uint32(i)*size, Slice{Id: sliceId, Size: size, Len: size}); st != 0 {
+		if st := m.Write(ctx, inode, 0, uint32(i)*size, Slice{Id: sliceId, Size: size, Len: size}, time.Now()); st != 0 {
 			t.Fatalf("write %d: %s", i, st)
 		}
 		time.Sleep(time.Millisecond)
@@ -1200,7 +1200,7 @@ func testConcurrentWrite(t *testing.T, m Meta) {
 				var sliceId uint64
 				m.NewSlice(ctx, &sliceId)
 				var slice = Slice{Id: sliceId, Size: 100, Len: 100}
-				st := m.Write(ctx, inode, indx, 0, slice)
+				st := m.Write(ctx, inode, indx, 0, slice, time.Now())
 				if st != 0 {
 					errno = st
 					break
@@ -1238,7 +1238,7 @@ func testTruncateAndDelete(t *testing.T, m Meta) {
 	if st := m.NewSlice(ctx, &sliceId); st != 0 {
 		t.Fatalf("new chunk: %s", st)
 	}
-	if st := m.Write(ctx, inode, 0, 100, Slice{sliceId, 100, 0, 100}); st != 0 {
+	if st := m.Write(ctx, inode, 0, 100, Slice{sliceId, 100, 0, 100}, time.Now()); st != 0 {
 		t.Fatalf("write file %s", st)
 	}
 	if st := m.Truncate(ctx, inode, 0, 200<<20, attr); st != 0 {
@@ -1297,10 +1297,10 @@ func testCopyFileRange(t *testing.T, m Meta) {
 		t.Fatalf("create file %s", st)
 	}
 	defer m.Unlink(ctx, 1, "fout")
-	m.Write(ctx, iin, 0, 100, Slice{10, 200, 0, 100})
-	m.Write(ctx, iin, 1, 100<<10, Slice{11, 40 << 20, 0, 40 << 20})
-	m.Write(ctx, iin, 3, 0, Slice{12, 63 << 20, 10 << 20, 30 << 20})
-	m.Write(ctx, iout, 2, 10<<20, Slice{13, 50 << 20, 10 << 20, 30 << 20})
+	m.Write(ctx, iin, 0, 100, Slice{10, 200, 0, 100}, time.Now())
+	m.Write(ctx, iin, 1, 100<<10, Slice{11, 40 << 20, 0, 40 << 20}, time.Now())
+	m.Write(ctx, iin, 3, 0, Slice{12, 63 << 20, 10 << 20, 30 << 20}, time.Now())
+	m.Write(ctx, iout, 2, 10<<20, Slice{13, 50 << 20, 10 << 20, 30 << 20}, time.Now())
 	var copied uint64
 	if st := m.CopyFileRange(ctx, iin, 150, iout, 30<<20, 200<<20, 0, &copied); st != 0 {
 		t.Fatalf("copy file range: %s", st)
@@ -2034,7 +2034,7 @@ func testDirStat(t *testing.T, m Meta) {
 	checkResult(0, align4K(0), 1)
 
 	// test dir with file and write
-	if st := m.Write(Background, fileInode, 0, 0, Slice{Id: 1, Size: 1 << 20, Off: 0, Len: 4097}); st != 0 {
+	if st := m.Write(Background, fileInode, 0, 0, Slice{Id: 1, Size: 1 << 20, Off: 0, Len: 4097}, time.Now()); st != 0 {
 		t.Fatalf("write: %s", st)
 	}
 	time.Sleep(1100 * time.Millisecond)
@@ -2130,7 +2130,7 @@ func testClone(t *testing.T, m Meta) {
 	if st := m.NewSlice(Background, &sliceId); st != 0 {
 		t.Fatalf("new chunk: %s", st)
 	}
-	if st := m.Write(Background, file1, 0, 0, Slice{sliceId, 200, 0, 200}); st != 0 {
+	if st := m.Write(Background, file1, 0, 0, Slice{sliceId, 200, 0, 200}, time.Now()); st != 0 {
 		t.Fatalf("write file %s", st)
 	}
 
@@ -2142,7 +2142,7 @@ func testClone(t *testing.T, m Meta) {
 	if st := m.NewSlice(Background, &sliceId2); st != 0 {
 		t.Fatalf("new chunk: %s", st)
 	}
-	if st := m.Write(Background, file2, 0, 0, Slice{sliceId2, 200, 0, 200}); st != 0 {
+	if st := m.Write(Background, file2, 0, 0, Slice{sliceId2, 200, 0, 200}, time.Now()); st != 0 {
 		t.Fatalf("write file %s", st)
 	}
 	var file3 Ino

pkg/meta/benchmarks_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"syscall"
 	"testing"
+	"time"
 
 	"github.com/juicedata/juicefs/pkg/utils"
 	"github.com/sirupsen/logrus"
@@ -536,7 +537,7 @@ func benchWrite(b *testing.B, m Meta) {
 		if err := m.NewSlice(ctx, &sliceId); err != 0 {
 			b.Fatalf("newchunk: %s", err)
 		}
-		if err := m.Write(ctx, inode, 0, offset, Slice{Id: sliceId, Size: step, Len: step}); err != 0 {
+		if err := m.Write(ctx, inode, 0, offset, Slice{Id: sliceId, Size: step, Len: step}, time.Now()); err != 0 {
 			b.Fatalf("write: %s", err)
 		}
 		offset += step
@@ -562,7 +563,7 @@ func benchRead(b *testing.B, m Meta, n int) {
 		if err := m.NewSlice(ctx, &sliceId); err != 0 {
 			b.Fatalf("newchunk: %s", err)
 		}
-		if err := m.Write(ctx, inode, 0, uint32(j)*step, Slice{Id: sliceId, Size: step, Len: step}); err != 0 {
+		if err := m.Write(ctx, inode, 0, uint32(j)*step, Slice{Id: sliceId, Size: step, Len: step}, time.Now()); err != 0 {
 			b.Fatalf("write: %s", err)
 		}
 	}

pkg/meta/interface.go

@@ -382,7 +382,7 @@ type Meta interface {
 	// NewSlice returns an id for new slice.
 	NewSlice(ctx Context, id *uint64) syscall.Errno
 	// Write put a slice of data on top of the given chunk.
-	Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice) syscall.Errno
+	Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time) syscall.Errno
 	// InvalidateChunkCache invalidate chunk cache
 	InvalidateChunkCache(ctx Context, inode Ino, indx uint32) syscall.Errno
 	// CopyFileRange copies part of a file to another one.

pkg/meta/redis.go

@@ -2153,7 +2153,7 @@ func (m *redisMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) s
 	return 0
 }
 
-func (m *redisMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice) syscall.Errno {
+func (m *redisMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time) syscall.Errno {
 	defer m.timeit("Write", time.Now())
 	f := m.of.find(inode)
 	if f != nil {
@@ -2185,8 +2185,8 @@ func (m *redisMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice
 			return err
 		}
 		now := time.Now()
-		attr.Mtime = now.Unix()
-		attr.Mtimensec = uint32(now.Nanosecond())
+		attr.Mtime = mtime.Unix()
+		attr.Mtimensec = uint32(mtime.Nanosecond())
 		attr.Ctime = now.Unix()
 		attr.Ctimensec = uint32(now.Nanosecond())
 

pkg/meta/sql.go

@@ -2093,7 +2093,7 @@ func (m *dbMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) sysc
 	return 0
 }
 
-func (m *dbMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice) syscall.Errno {
+func (m *dbMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time) syscall.Errno {
 	defer m.timeit("Write", time.Now())
 	f := m.of.find(inode)
 	if f != nil {
@@ -2126,9 +2126,8 @@ func (m *dbMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Sl
 		if err := m.checkQuota(ctx, newSpace, 0, m.getParents(s, inode, nodeAttr.Parent)...); err != 0 {
 			return err
 		}
-		now := time.Now().UnixNano() / 1e3
-		nodeAttr.Mtime = now
-		nodeAttr.Ctime = now
+		nodeAttr.Mtime = mtime.UnixNano() / 1e3
+		nodeAttr.Ctime = time.Now().UnixNano() / 1e3
 
 		var ck = chunk{Inode: inode, Indx: indx}
 		ok, err = s.ForUpdate().MustCols("indx").Get(&ck)

pkg/meta/tkv.go

@@ -1843,7 +1843,7 @@ func (m *kvMeta) Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) sysc
 	return 0
 }
 
-func (m *kvMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice) syscall.Errno {
+func (m *kvMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time) syscall.Errno {
 	defer m.timeit("Write", time.Now())
 	f := m.of.find(inode)
 	if f != nil {
@@ -1875,8 +1875,8 @@ func (m *kvMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Sl
 			return err
 		}
 		now := time.Now()
-		attr.Mtime = now.Unix()
-		attr.Mtimensec = uint32(now.Nanosecond())
+		attr.Mtime = mtime.Unix()
+		attr.Mtimensec = uint32(mtime.Nanosecond())
 		attr.Ctime = now.Unix()
 		attr.Ctimensec = uint32(now.Nanosecond())
 		val := tx.append(m.chunkKey(inode, indx), marshalSlice(off, slice.Id, slice.Size, slice.Off, slice.Len))

pkg/vfs/vfs_unix.go

@@ -24,6 +24,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/juicedata/juicefs/pkg/meta"
 	"github.com/juicedata/juicefs/pkg/utils"
@@ -178,6 +179,10 @@ func (v *VFS) SetAttr(ctx Context, ino Ino, set int, opened uint8, mode, uid, gi
 	if set&meta.SetAttrMtime != 0 {
 		attr.Mtime = mtime
 		attr.Mtimensec = mtimensec
+		v.writer.UpdateMtime(ino, time.Unix(mtime, int64(mtimensec)))
+	}
+	if set&meta.SetAttrMtimeNow != 0 {
+		v.writer.UpdateMtime(ino, time.Now())
 	}
 	err = v.Meta.SetAttr(ctx, ino, uint16(set), 0, attr)
 	if err == 0 {

pkg/vfs/writer.go

@@ -45,6 +45,7 @@ type DataWriter interface {
 	Flush(ctx meta.Context, inode Ino) syscall.Errno
 	GetLength(inode Ino) uint64
 	Truncate(inode Ino, length uint64)
+	UpdateMtime(inode Ino, mtime time.Time)
 }
 
 type sliceWriter struct {
@@ -198,7 +199,7 @@ func (c *chunkWriter) commitThread() {
 
 		if err == 0 {
 			var ss = meta.Slice{Id: s.id, Size: s.length, Off: s.soff, Len: s.slen}
-			err = f.w.m.Write(meta.Background, f.inode, c.indx, s.off, ss)
+			err = f.w.m.Write(meta.Background, f.inode, c.indx, s.off, ss, s.lastMod)
 			f.w.reader.Invalidate(f.inode, uint64(c.indx)*meta.ChunkSize+uint64(s.off), uint64(ss.Len))
 		}
 
@@ -336,6 +337,16 @@ func (f *fileWriter) Write(ctx meta.Context, off uint64, data []byte) syscall.Er
 	return f.err
 }
 
+func (f *fileWriter) updateMtime(t time.Time) {
+	f.Lock()
+	defer f.Unlock()
+	for _, c := range f.chunks {
+		for _, s := range c.slices {
+			s.lastMod = t
+		}
+	}
+}
+
 func (f *fileWriter) flush(ctx meta.Context, writeback bool) syscall.Errno {
 	s := time.Now()
 	f.Lock()
@@ -449,7 +460,7 @@ func (w *dataWriter) flushAll() {
 			for i, c := range f.chunks {
 				hs := len(c.slices) / 2
 				for j, s := range c.slices {
-					if !s.freezed && (now.Sub(s.started) > flushDuration || now.Sub(s.lastMod) > time.Second ||
+					if !s.freezed && (now.Sub(s.started) > flushDuration || now.Sub(s.lastMod) > time.Second && now.Sub(s.started) > time.Second ||
 						tooMany && i%2 == lastBit && j <= hs) {
 						s.freezed = true
 						go s.flushData()
@@ -521,3 +532,10 @@ func (w *dataWriter) Truncate(inode Ino, len uint64) {
 		f.Truncate(len)
 	}
 }
+
+func (w *dataWriter) UpdateMtime(inode Ino, mtime time.Time) {
+	f := w.find(inode)
+	if f != nil {
+		f.updateMtime(mtime)
+	}
+}