wal: fsync directory after wal file rename

Fixes etcd-io#6368

Author: Anthony Romano

wal/wal.go

@@ -69,7 +69,11 @@ var (
 // A just opened WAL is in read mode, and ready for reading records.
 // The WAL will be ready for appending after reading out all the previous records.
 type WAL struct {
-	dir      string           // the living directory of the underlay files
+	dir string // the living directory of the underlay files
+
+	// dirFile is a fd for the wal directory for syncing on Rename
+	dirFile *os.File
+
 	metadata []byte           // metadata recorded at the head of each WAL
 	state    raftpb.HardState // hardstate recorded at the head of WAL
 
@@ -131,7 +135,24 @@ func Create(dirpath string, metadata []byte) (*WAL, error) {
 		return nil, err
 	}
 
-	return w.renameWal(tmpdirpath)
+	w, err = w.renameWal(tmpdirpath)
+	if err != nil {
+		return nil, err
+	}
+
+	// directory was renamed; sync parent dir to persist rename
+	pdir, perr := openSyncDir(path.Dir(w.dir))
+	if perr != nil {
+		return nil, perr
+	}
+	if perr = fileutil.Fsync(pdir); perr != nil {
+		return nil, perr
+	}
+	if perr = pdir.Close(); err != nil {
+		return nil, perr
+	}
+
+	return w, nil
 }
 
 // Open opens the WAL at the given snap.
@@ -141,7 +162,14 @@ func Create(dirpath string, metadata []byte) (*WAL, error) {
 // the given snap. The WAL cannot be appended to before reading out all of its
 // previous records.
 func Open(dirpath string, snap walpb.Snapshot) (*WAL, error) {
-	return openAtIndex(dirpath, snap, true)
+	w, err := openAtIndex(dirpath, snap, true)
+	if err != nil {
+		return nil, err
+	}
+	if err = w.openDir(); err != nil {
+		return nil, err
+	}
+	return w, nil
 }
 
 // OpenForRead only opens the wal files for read.
@@ -373,6 +401,10 @@ func (w *WAL) cut() error {
 	if err = os.Rename(newTail.Name(), fpath); err != nil {
 		return err
 	}
+	if err = fileutil.Fsync(w.dirFile); err != nil {
+		return err
+	}
+
 	newTail.Close()
 
 	if newTail, err = fileutil.LockFile(fpath, os.O_WRONLY, fileutil.PrivateFileMode); err != nil {
@@ -467,6 +499,7 @@ func (w *WAL) Close() error {
 			return err
 		}
 	}
+
 	for _, l := range w.locks {
 		if l == nil {
 			continue
@@ -475,6 +508,11 @@ func (w *WAL) Close() error {
 			plog.Errorf("failed to unlock during closing wal: %s", err)
 		}
 	}
+
+	if err := w.dirFile.Close(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -574,6 +612,15 @@ func (w *WAL) seq() uint64 {
 	return seq
 }
 
+func (w *WAL) openDir() error {
+	df, derr := openSyncDir(w.dir)
+	if derr != nil {
+		return derr
+	}
+	w.dirFile = df
+	return nil
+}
+
 func mustSync(st, prevst raftpb.HardState, entsnum int) bool {
 	// Persistent state on all servers:
 	// (Updated on stable storage before responding to RPCs)

wal/wal_unix.go

@@ -34,5 +34,7 @@ func (w *WAL) renameWal(tmpdirpath string) (*WAL, error) {
 	}
 
 	w.fp = newFilePipeline(w.dir, SegmentSizeBytes)
-	return w, nil
+	return w, w.openDir()
 }
+
+func openSyncDir(dir string) (*os.File, error) { return os.Open(dir) }

wal/wal_windows.go

@@ -17,6 +17,7 @@ package wal
 import (
 	"os"
 
+	"github.com/coreos/etcd/pkg/fileutil"
 	"github.com/coreos/etcd/wal/walpb"
 )
 
@@ -37,5 +38,10 @@ func (w *WAL) renameWal(tmpdirpath string) (*WAL, error) {
 		newWAL.Close()
 		return nil, err
 	}
+
 	return newWAL, nil
 }
+
+func openSyncDir(dir string) (*os.File, error) {
+	return os.OpenFile(w.dir, os.O_WRONLY, fileutil.PrivateFileMode)
+}