From 58f1ec06d617f017c5e27470fb802b93e794caea Mon Sep 17 00:00:00 2001 From: rafjaf Date: Sun, 20 Jul 2025 17:04:41 +0200 Subject: [PATCH 1/4] macOS: Fix Unicode normalization issues in forward & reverse mode This commit resolves https://github.com/rfjakob/gocryptfs/issues/850 by addressing Unicode normalization mismatches on macOS between NFC (used by CLI tools) and NFD (used by GUI apps). The solution is inspired by Cryptomator's approach. Forward mode now enforces NFC for storage and adds fallback lookups in NFD. Reverse mode adds fallback lookups without modifying the plaintext FS. --- go.mod | 6 +- go.sum | 2 + internal/fusefrontend/file_dir_ops.go | 4 +- internal/fusefrontend/node.go | 7 + internal/fusefrontend/node_dir_ops.go | 19 +++ internal/fusefrontend/node_open_create.go | 1 + internal/fusefrontend/node_prepare_syscall.go | 122 +++++++++++++++++- internal/fusefrontend_reverse/node_dir_ops.go | 13 ++ internal/fusefrontend_reverse/rpath.go | 31 +++++ 9 files changed, 198 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 0af8ef15..92bc6c1c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/rfjakob/gocryptfs/v2 -go 1.19 +go 1.23.0 + +toolchain go1.24.4 require ( github.com/aperturerobotics/jacobsa-crypto v1.1.0 @@ -14,3 +16,5 @@ require ( golang.org/x/sys v0.30.0 golang.org/x/term v0.29.0 ) + +require golang.org/x/text v0.27.0 // indirect diff --git a/go.sum b/go.sum index e22b76b6..afd28b3b 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/fusefrontend/file_dir_ops.go b/internal/fusefrontend/file_dir_ops.go index b69e7bc4..df14410d 100644 --- a/internal/fusefrontend/file_dir_ops.go +++ b/internal/fusefrontend/file_dir_ops.go @@ -138,6 +138,8 @@ func (f *File) Readdirent(ctx context.Context) (entry *fuse.DirEntry, errno sysc continue } if f.rootNode.args.PlaintextNames { + // Even in plaintext mode, normalize for macOS display + entry.Name = normalizeFilenameForDisplay(cName) return } if !f.rootNode.args.DeterministicNames && cName == nametransform.DirIVFilename { @@ -171,7 +173,7 @@ func (f *File) Readdirent(ctx context.Context) (entry *fuse.DirEntry, errno sysc } // Override the ciphertext name with the plaintext name but reuse the rest // of the structure - entry.Name = name + entry.Name = normalizeFilenameForDisplay(name) return } } diff --git a/internal/fusefrontend/node.go b/internal/fusefrontend/node.go index 95be48dc..58db1f6d 100644 --- a/internal/fusefrontend/node.go +++ b/internal/fusefrontend/node.go @@ -140,6 +140,7 @@ func (n *Node) Access(ctx context.Context, mode uint32) syscall.Errno { // // Symlink-safe through use of Unlinkat(). func (n *Node) Unlink(ctx context.Context, name string) (errno syscall.Errno) { + name = normalizeFilename(name) // Always store as NFC dirfd, cName, errno := n.prepareAtSyscall(name) if errno != 0 { return @@ -274,6 +275,7 @@ func (n *Node) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno { // // Symlink-safe through use of Mknodat(). func (n *Node) Mknod(ctx context.Context, name string, mode, rdev uint32, out *fuse.EntryOut) (inode *fs.Inode, errno syscall.Errno) { + name = normalizeFilename(name) // Always store as NFC dirfd, cName, errno := n.prepareAtSyscall(name) if errno != 0 { return @@ -329,6 +331,7 @@ func (n *Node) Mknod(ctx context.Context, name string, mode, rdev uint32, out *f // // Symlink-safe through use of Linkat(). func (n *Node) Link(ctx context.Context, target fs.InodeEmbedder, name string, out *fuse.EntryOut) (inode *fs.Inode, errno syscall.Errno) { + name = normalizeFilename(name) // Always store as NFC dirfd, cName, errno := n.prepareAtSyscall(name) if errno != 0 { return @@ -379,6 +382,7 @@ func (n *Node) Link(ctx context.Context, target fs.InodeEmbedder, name string, o // // Symlink-safe through use of Symlinkat. func (n *Node) Symlink(ctx context.Context, target, name string, out *fuse.EntryOut) (inode *fs.Inode, errno syscall.Errno) { + name = normalizeFilename(name) // Always store as NFC dirfd, cName, errno := n.prepareAtSyscall(name) if errno != 0 { return @@ -451,6 +455,9 @@ func (n *Node) Rename(ctx context.Context, name string, newParent fs.InodeEmbedd return errno } + name = normalizeFilename(name) // Always store as NFC + newName = normalizeFilename(newName) // Always store as NFC + dirfd, cName, errno := n.prepareAtSyscall(name) if errno != 0 { return diff --git a/internal/fusefrontend/node_dir_ops.go b/internal/fusefrontend/node_dir_ops.go index 11ff83dd..53ce4b21 100644 --- a/internal/fusefrontend/node_dir_ops.go +++ b/internal/fusefrontend/node_dir_ops.go @@ -6,8 +6,10 @@ import ( "io" "runtime" "syscall" + "unicode/utf8" "golang.org/x/sys/unix" + "golang.org/x/text/unicode/norm" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" @@ -20,6 +22,22 @@ import ( const dsStoreName = ".DS_Store" +// normalizeFilename converts filenames to NFC for consistent internal storage +func normalizeFilename(name string) string { + if runtime.GOOS == "darwin" && utf8.ValidString(name) { + return norm.NFC.String(name) + } + return name +} + +// normalizeFilenameForDisplay converts NFC to NFD for macOS GUI compatibility +func normalizeFilenameForDisplay(name string) string { + if runtime.GOOS == "darwin" && utf8.ValidString(name) { + return norm.NFD.String(name) + } + return name +} + // haveDsstore return true if one of the entries in "names" is ".DS_Store". func haveDsstore(entries []fuse.DirEntry) bool { for _, e := range entries { @@ -70,6 +88,7 @@ func (n *Node) mkdirWithIv(dirfd int, cName string, mode uint32, context *fuse.C // // Symlink-safe through use of Mkdirat(). func (n *Node) Mkdir(ctx context.Context, name string, mode uint32, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { + name = normalizeFilename(name) // Always store as NFC dirfd, cName, errno := n.prepareAtSyscall(name) if errno != 0 { return nil, errno diff --git a/internal/fusefrontend/node_open_create.go b/internal/fusefrontend/node_open_create.go index 95985597..24f3e218 100644 --- a/internal/fusefrontend/node_open_create.go +++ b/internal/fusefrontend/node_open_create.go @@ -58,6 +58,7 @@ func (n *Node) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFl // // Symlink-safe through the use of Openat(). func (n *Node) Create(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (inode *fs.Inode, fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) { + name = normalizeFilename(name) // Always store as NFC dirfd, cName, errno := n.prepareAtSyscall(name) if errno != 0 { return diff --git a/internal/fusefrontend/node_prepare_syscall.go b/internal/fusefrontend/node_prepare_syscall.go index 9021350c..98353bd9 100644 --- a/internal/fusefrontend/node_prepare_syscall.go +++ b/internal/fusefrontend/node_prepare_syscall.go @@ -1,7 +1,11 @@ package fusefrontend import ( + "runtime" "syscall" + "unicode/utf8" + + "golang.org/x/text/unicode/norm" "github.com/rfjakob/gocryptfs/v2/internal/tlog" @@ -10,12 +14,10 @@ import ( "github.com/rfjakob/gocryptfs/v2/internal/syscallcompat" ) -// prepareAtSyscall returns a (dirfd, cName) pair that can be used -// with the "___at" family of system calls (openat, fstatat, unlinkat...) to -// access the backing encrypted child file. -func (n *Node) prepareAtSyscall(child string) (dirfd int, cName string, errno syscall.Errno) { +// prepareAtSyscallDirect is the direct version without Unicode normalization fallback +func (n *Node) prepareAtSyscallDirect(child string) (dirfd int, cName string, errno syscall.Errno) { if child == "" { - tlog.Warn.Printf("BUG: prepareAtSyscall: child=%q, should have called prepareAtSyscallMyself", child) + tlog.Warn.Printf("BUG: prepareAtSyscallDirect: child=%q, should have called prepareAtSyscallMyself", child) return n.prepareAtSyscallMyself() } @@ -92,6 +94,116 @@ func (n *Node) prepareAtSyscall(child string) (dirfd int, cName string, errno sy return } +// migrateFilename migrates a filename from NFD to NFC form +func (n *Node) migrateFilename(oldName, newName string) syscall.Errno { + if oldName == newName { + return 0 // Nothing to do + } + + rn := n.rootNode() + + // Get directory file descriptor + dirfd, _, errno := n.prepareAtSyscallMyself() + if errno != 0 { + return errno + } + defer syscall.Close(dirfd) + + // For plaintext names: simple rename + if rn.args.PlaintextNames { + err := syscallcompat.Renameat(dirfd, oldName, dirfd, newName) + return fs.ToErrno(err) + } + + // For encrypted names: encrypt both names and rename + iv, err := rn.nameTransform.ReadDirIVAt(dirfd) + if err != nil { + return fs.ToErrno(err) + } + + var encryptName func(int, string, []byte) (string, error) + if rn.nameTransform.HaveBadnamePatterns() { + encryptName = func(dirfd int, child string, iv []byte) (string, error) { + return rn.nameTransform.EncryptAndHashBadName(child, iv, dirfd) + } + } else { + encryptName = func(dirfd int, child string, iv []byte) (string, error) { + return rn.nameTransform.EncryptAndHashName(child, iv) + } + } + + oldCName, err := encryptName(dirfd, oldName, iv) + if err != nil { + return fs.ToErrno(err) + } + + newCName, err := encryptName(dirfd, newName, iv) + if err != nil { + return fs.ToErrno(err) + } + + err = syscallcompat.Renameat(dirfd, oldCName, dirfd, newCName) + return fs.ToErrno(err) +} + +// prepareAtSyscall returns a (dirfd, cName) pair that can be used +// with the "___at" family of system calls (openat, fstatat, unlinkat...) to +// access the backing encrypted child file. +func (n *Node) prepareAtSyscall(child string) (dirfd int, cName string, errno syscall.Errno) { + if child == "" { + tlog.Warn.Printf("BUG: prepareAtSyscall: child=%q, should have called prepareAtSyscallMyself", child) + return n.prepareAtSyscallMyself() + } + + // On macOS, implement Unicode normalization with fallback and migration + if runtime.GOOS == "darwin" && utf8.ValidString(child) { + // Step 1: Always try NFC first (canonical storage form) + normalizedChild := norm.NFC.String(child) + dirfd, cName, errno = n.prepareAtSyscallDirect(normalizedChild) + if errno == 0 { + return dirfd, cName, 0 // Found NFC version + } + + // Step 2: Try alternate form if input was different + if normalizedChild != child { + // Input was NFD, try original NFD form + dirfdNFD, cNameNFD, errnoNFD := n.prepareAtSyscallDirect(child) + if errnoNFD == 0 { + // Found NFD file - migrate it to NFC + if errno := n.migrateFilename(child, normalizedChild); errno == 0 { + // Migration successful, use NFC + syscall.Close(dirfdNFD) // Close the NFD dirfd + return n.prepareAtSyscallDirect(normalizedChild) + } else { + // Migration failed, use NFD + return dirfdNFD, cNameNFD, 0 + } + } + } + + // Step 3: If input was NFC, also try NFD as fallback + if normalizedChild == child { + nfdChild := norm.NFD.String(child) + if nfdChild != child { + dirfdNFD, cNameNFD, errnoNFD := n.prepareAtSyscallDirect(nfdChild) + if errnoNFD == 0 { + // Found NFD file - migrate it to NFC + if errno := n.migrateFilename(nfdChild, normalizedChild); errno == 0 { + // Migration successful, use NFC + syscall.Close(dirfdNFD) // Close the NFD dirfd + return n.prepareAtSyscallDirect(normalizedChild) + } else { + // Migration failed, use NFD + return dirfdNFD, cNameNFD, 0 + } + } + } + } + } + + return n.prepareAtSyscallDirect(child) // Non-macOS or fallback +} + func (n *Node) prepareAtSyscallMyself() (dirfd int, cName string, errno syscall.Errno) { dirfd = -1 diff --git a/internal/fusefrontend_reverse/node_dir_ops.go b/internal/fusefrontend_reverse/node_dir_ops.go index fdd15ce0..a59716c4 100644 --- a/internal/fusefrontend_reverse/node_dir_ops.go +++ b/internal/fusefrontend_reverse/node_dir_ops.go @@ -3,7 +3,9 @@ package fusefrontend_reverse import ( "context" "fmt" + "runtime" "syscall" + "unicode/utf8" "golang.org/x/sys/unix" @@ -116,3 +118,14 @@ func (n *Node) readdirPlaintextnames(entries []fuse.DirEntry) (stream fs.DirStre } return fs.NewListDirStream(entries), 0 } + +// normalizeFilenameForDisplay converts stored filenames to the form expected by macOS GUI. +// In reverse mode, we present the plaintext files as-is, but ensure proper display normalization. +func normalizeFilenameForDisplay(name string) string { + if runtime.GOOS == "darwin" && utf8.ValidString(name) { + // For reverse mode, we typically want to preserve the original normalization + // of the plaintext files, but ensure they display correctly + return name + } + return name +} diff --git a/internal/fusefrontend_reverse/rpath.go b/internal/fusefrontend_reverse/rpath.go index 9625dc27..eb96f19f 100644 --- a/internal/fusefrontend_reverse/rpath.go +++ b/internal/fusefrontend_reverse/rpath.go @@ -4,8 +4,12 @@ import ( "encoding/base64" "log" "path/filepath" + "runtime" "strings" "syscall" + "unicode/utf8" + + "golang.org/x/text/unicode/norm" "github.com/rfjakob/gocryptfs/v2/internal/nametransform" "github.com/rfjakob/gocryptfs/v2/internal/pathiv" @@ -35,6 +39,33 @@ func (rfs *RootNode) rDecryptName(cName string, dirIV []byte, pDir string) (pNam } return "", err } + + // On macOS, handle Unicode normalization fallback for reverse mode + if runtime.GOOS == "darwin" && utf8.ValidString(pName) { + // Check if the decrypted name actually exists on disk + pPath := filepath.Join(rfs.args.Cipherdir, pDir, pName) + var st syscall.Stat_t + if statErr := syscall.Stat(pPath, &st); statErr != nil { + // Try the alternate Unicode form + var alternateName string + if norm.NFC.String(pName) == pName { + // pName is NFC, try NFD + alternateName = norm.NFD.String(pName) + } else { + // pName is NFD (or mixed), try NFC + alternateName = norm.NFC.String(pName) + } + + if alternateName != pName { + alternatePath := filepath.Join(rfs.args.Cipherdir, pDir, alternateName) + var altSt syscall.Stat_t + if altStatErr := syscall.Stat(alternatePath, &altSt); altStatErr == nil { + // The alternate form exists, use it + return alternateName, nil + } + } + } + } } else if nameType == nametransform.LongNameContent { dirfd, err := syscallcompat.OpenDirNofollow(rfs.args.Cipherdir, filepath.Dir(pDir)) if err != nil { From 5b82f4252f47f976e6a1a8f4078d901402cbe3d8 Mon Sep 17 00:00:00 2001 From: rafjaf Date: Thu, 24 Jul 2025 18:32:14 +0200 Subject: [PATCH 2/4] Added testing for forward mode, and reverted all changes for reverse mode --- internal/fusefrontend/node_prepare_syscall.go | 74 +----------------- internal/fusefrontend_reverse/rpath.go | 31 -------- tests/macos_filename_encoding/nfc_nfd_test.go | 77 +++++++++++++++++++ tests/macos_filename_encoding/test.bash | 4 + 4 files changed, 84 insertions(+), 102 deletions(-) create mode 100644 tests/macos_filename_encoding/nfc_nfd_test.go create mode 100755 tests/macos_filename_encoding/test.bash diff --git a/internal/fusefrontend/node_prepare_syscall.go b/internal/fusefrontend/node_prepare_syscall.go index 98353bd9..26fbb5bb 100644 --- a/internal/fusefrontend/node_prepare_syscall.go +++ b/internal/fusefrontend/node_prepare_syscall.go @@ -94,58 +94,6 @@ func (n *Node) prepareAtSyscallDirect(child string) (dirfd int, cName string, er return } -// migrateFilename migrates a filename from NFD to NFC form -func (n *Node) migrateFilename(oldName, newName string) syscall.Errno { - if oldName == newName { - return 0 // Nothing to do - } - - rn := n.rootNode() - - // Get directory file descriptor - dirfd, _, errno := n.prepareAtSyscallMyself() - if errno != 0 { - return errno - } - defer syscall.Close(dirfd) - - // For plaintext names: simple rename - if rn.args.PlaintextNames { - err := syscallcompat.Renameat(dirfd, oldName, dirfd, newName) - return fs.ToErrno(err) - } - - // For encrypted names: encrypt both names and rename - iv, err := rn.nameTransform.ReadDirIVAt(dirfd) - if err != nil { - return fs.ToErrno(err) - } - - var encryptName func(int, string, []byte) (string, error) - if rn.nameTransform.HaveBadnamePatterns() { - encryptName = func(dirfd int, child string, iv []byte) (string, error) { - return rn.nameTransform.EncryptAndHashBadName(child, iv, dirfd) - } - } else { - encryptName = func(dirfd int, child string, iv []byte) (string, error) { - return rn.nameTransform.EncryptAndHashName(child, iv) - } - } - - oldCName, err := encryptName(dirfd, oldName, iv) - if err != nil { - return fs.ToErrno(err) - } - - newCName, err := encryptName(dirfd, newName, iv) - if err != nil { - return fs.ToErrno(err) - } - - err = syscallcompat.Renameat(dirfd, oldCName, dirfd, newCName) - return fs.ToErrno(err) -} - // prepareAtSyscall returns a (dirfd, cName) pair that can be used // with the "___at" family of system calls (openat, fstatat, unlinkat...) to // access the backing encrypted child file. @@ -155,7 +103,7 @@ func (n *Node) prepareAtSyscall(child string) (dirfd int, cName string, errno sy return n.prepareAtSyscallMyself() } - // On macOS, implement Unicode normalization with fallback and migration + // On macOS, implement Unicode normalization with fallback if runtime.GOOS == "darwin" && utf8.ValidString(child) { // Step 1: Always try NFC first (canonical storage form) normalizedChild := norm.NFC.String(child) @@ -169,15 +117,7 @@ func (n *Node) prepareAtSyscall(child string) (dirfd int, cName string, errno sy // Input was NFD, try original NFD form dirfdNFD, cNameNFD, errnoNFD := n.prepareAtSyscallDirect(child) if errnoNFD == 0 { - // Found NFD file - migrate it to NFC - if errno := n.migrateFilename(child, normalizedChild); errno == 0 { - // Migration successful, use NFC - syscall.Close(dirfdNFD) // Close the NFD dirfd - return n.prepareAtSyscallDirect(normalizedChild) - } else { - // Migration failed, use NFD - return dirfdNFD, cNameNFD, 0 - } + return dirfdNFD, cNameNFD, 0 } } @@ -187,15 +127,7 @@ func (n *Node) prepareAtSyscall(child string) (dirfd int, cName string, errno sy if nfdChild != child { dirfdNFD, cNameNFD, errnoNFD := n.prepareAtSyscallDirect(nfdChild) if errnoNFD == 0 { - // Found NFD file - migrate it to NFC - if errno := n.migrateFilename(nfdChild, normalizedChild); errno == 0 { - // Migration successful, use NFC - syscall.Close(dirfdNFD) // Close the NFD dirfd - return n.prepareAtSyscallDirect(normalizedChild) - } else { - // Migration failed, use NFD - return dirfdNFD, cNameNFD, 0 - } + return dirfdNFD, cNameNFD, 0 } } } diff --git a/internal/fusefrontend_reverse/rpath.go b/internal/fusefrontend_reverse/rpath.go index eb96f19f..9625dc27 100644 --- a/internal/fusefrontend_reverse/rpath.go +++ b/internal/fusefrontend_reverse/rpath.go @@ -4,12 +4,8 @@ import ( "encoding/base64" "log" "path/filepath" - "runtime" "strings" "syscall" - "unicode/utf8" - - "golang.org/x/text/unicode/norm" "github.com/rfjakob/gocryptfs/v2/internal/nametransform" "github.com/rfjakob/gocryptfs/v2/internal/pathiv" @@ -39,33 +35,6 @@ func (rfs *RootNode) rDecryptName(cName string, dirIV []byte, pDir string) (pNam } return "", err } - - // On macOS, handle Unicode normalization fallback for reverse mode - if runtime.GOOS == "darwin" && utf8.ValidString(pName) { - // Check if the decrypted name actually exists on disk - pPath := filepath.Join(rfs.args.Cipherdir, pDir, pName) - var st syscall.Stat_t - if statErr := syscall.Stat(pPath, &st); statErr != nil { - // Try the alternate Unicode form - var alternateName string - if norm.NFC.String(pName) == pName { - // pName is NFC, try NFD - alternateName = norm.NFD.String(pName) - } else { - // pName is NFD (or mixed), try NFC - alternateName = norm.NFC.String(pName) - } - - if alternateName != pName { - alternatePath := filepath.Join(rfs.args.Cipherdir, pDir, alternateName) - var altSt syscall.Stat_t - if altStatErr := syscall.Stat(alternatePath, &altSt); altStatErr == nil { - // The alternate form exists, use it - return alternateName, nil - } - } - } - } } else if nameType == nametransform.LongNameContent { dirfd, err := syscallcompat.OpenDirNofollow(rfs.args.Cipherdir, filepath.Dir(pDir)) if err != nil { diff --git a/tests/macos_filename_encoding/nfc_nfd_test.go b/tests/macos_filename_encoding/nfc_nfd_test.go new file mode 100644 index 00000000..550077fd --- /dev/null +++ b/tests/macos_filename_encoding/nfc_nfd_test.go @@ -0,0 +1,77 @@ +package macos_filename_encoding + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "golang.org/x/text/unicode/norm" + + "github.com/rfjakob/gocryptfs/v2/tests/test_helpers" +) + +var nfcName = norm.NFC.String("e\u0301") // é +var nfdName = norm.NFD.String("e\u0301") // e + combining acute accent + +func TestCreateNFC_AccessNFD(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("macOS only test") + } + if nfcName == nfdName { + t.Fatal("NFC and NFD names should be different") + } + test_helpers.ResetTmpDir(false) + cipherDir := test_helpers.InitFS(t) + mntDir := filepath.Join(test_helpers.TmpDir, "mnt") + test_helpers.MountOrFatal(t, cipherDir, mntDir, "-extpass", "echo test") + defer test_helpers.UnmountPanic(mntDir) + + // Create a file with an NFC name + nfcPath := filepath.Join(mntDir, nfcName) + err := os.WriteFile(nfcPath, []byte("content"), 0600) + if err != nil { + t.Fatalf("Failed to create file with NFC name: %v", err) + } + + // Try to read it with an NFD name + nfdPath := filepath.Join(mntDir, nfdName) + content, err := os.ReadFile(nfdPath) + if err != nil { + t.Fatalf("Failed to read file with NFD name: %v", err) + } + if string(content) != "content" { + t.Errorf("Wrong content: %q", string(content)) + } +} + +func TestCreateNFD_AccessNFC(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("macOS only test") + } + if nfcName == nfdName { + t.Fatal("NFC and NFD names should be different") + } + test_helpers.ResetTmpDir(false) + cipherDir := test_helpers.InitFS(t) + mntDir := filepath.Join(test_helpers.TmpDir, "mnt") + test_helpers.MountOrFatal(t, cipherDir, mntDir, "-extpass", "echo test") + defer test_helpers.UnmountPanic(mntDir) + + // Create a file with an NFD name + nfdPath := filepath.Join(mntDir, nfdName) + err := os.WriteFile(nfdPath, []byte("content"), 0600) + if err != nil { + t.Fatalf("Failed to create file with NFD name: %v", err) + } + + // Try to read it with an NFC name + nfcPath := filepath.Join(mntDir, nfcName) + content, err := os.ReadFile(nfcPath) + if err != nil { + t.Fatalf("Failed to read file with NFC name: %v", err) + } + if string(content) != "content" { + t.Errorf("Wrong content: %q", string(content)) + } +} diff --git a/tests/macos_filename_encoding/test.bash b/tests/macos_filename_encoding/test.bash new file mode 100755 index 00000000..c8461f85 --- /dev/null +++ b/tests/macos_filename_encoding/test.bash @@ -0,0 +1,4 @@ +#!/bin/bash -eu + +cd "$(dirname "$0")/../.." +go test -v ./tests/macos_filename_encoding/... From c72b8d26abe0a9fa44654b87c422ad10398f5131 Mon Sep 17 00:00:00 2001 From: rafjaf Date: Thu, 24 Jul 2025 22:30:10 +0200 Subject: [PATCH 3/4] Reverting node_dir_ops in reverse mode to initial version --- internal/fusefrontend_reverse/node_dir_ops.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/internal/fusefrontend_reverse/node_dir_ops.go b/internal/fusefrontend_reverse/node_dir_ops.go index a59716c4..fdd15ce0 100644 --- a/internal/fusefrontend_reverse/node_dir_ops.go +++ b/internal/fusefrontend_reverse/node_dir_ops.go @@ -3,9 +3,7 @@ package fusefrontend_reverse import ( "context" "fmt" - "runtime" "syscall" - "unicode/utf8" "golang.org/x/sys/unix" @@ -118,14 +116,3 @@ func (n *Node) readdirPlaintextnames(entries []fuse.DirEntry) (stream fs.DirStre } return fs.NewListDirStream(entries), 0 } - -// normalizeFilenameForDisplay converts stored filenames to the form expected by macOS GUI. -// In reverse mode, we present the plaintext files as-is, but ensure proper display normalization. -func normalizeFilenameForDisplay(name string) string { - if runtime.GOOS == "darwin" && utf8.ValidString(name) { - // For reverse mode, we typically want to preserve the original normalization - // of the plaintext files, but ensure they display correctly - return name - } - return name -} From 007129c4cbab36d7e338739fef00711624410c52 Mon Sep 17 00:00:00 2001 From: rafjaf Date: Sat, 26 Jul 2025 12:14:41 +0200 Subject: [PATCH 4/4] Created additional tests for macos filename encoding normalization fix --- tests/macos_filename_encoding/nfc_nfd_test.go | 380 ++++++++++++++++++ 1 file changed, 380 insertions(+) diff --git a/tests/macos_filename_encoding/nfc_nfd_test.go b/tests/macos_filename_encoding/nfc_nfd_test.go index 550077fd..0d020d4d 100644 --- a/tests/macos_filename_encoding/nfc_nfd_test.go +++ b/tests/macos_filename_encoding/nfc_nfd_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "golang.org/x/text/unicode/norm" @@ -14,6 +15,20 @@ import ( var nfcName = norm.NFC.String("e\u0301") // é var nfdName = norm.NFD.String("e\u0301") // e + combining acute accent +// Additional test cases with various Unicode characters +var unicodeTestCases = []struct { + name string + nfc string + nfd string + comment string +}{ + {"acute_e", norm.NFC.String("e\u0301"), norm.NFD.String("e\u0301"), "é (e + combining acute)"}, + {"circumflex_a", norm.NFC.String("a\u0302"), norm.NFD.String("a\u0302"), "â (a + combining circumflex)"}, + {"tilde_n", norm.NFC.String("n\u0303"), norm.NFD.String("n\u0303"), "ñ (n + combining tilde)"}, + {"umlaut_u", norm.NFC.String("u\u0308"), norm.NFD.String("u\u0308"), "ü (u + combining diaeresis)"}, + {"multiple_combining", norm.NFC.String("o\u0302\u0308"), norm.NFD.String("o\u0302\u0308"), "ô̈ (o + circumflex + diaeresis)"}, +} + func TestCreateNFC_AccessNFD(t *testing.T) { if runtime.GOOS != "darwin" { t.Skip("macOS only test") @@ -75,3 +90,368 @@ func TestCreateNFD_AccessNFC(t *testing.T) { t.Errorf("Wrong content: %q", string(content)) } } + +// TestMultipleUnicodeCharacters tests various Unicode characters with different NFC/NFD forms +func TestMultipleUnicodeCharacters(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("macOS only test") + } + + test_helpers.ResetTmpDir(false) + cipherDir := test_helpers.InitFS(t) + mntDir := filepath.Join(test_helpers.TmpDir, "mnt") + test_helpers.MountOrFatal(t, cipherDir, mntDir, "-extpass", "echo test") + defer test_helpers.UnmountPanic(mntDir) + + for _, tc := range unicodeTestCases { + t.Run(tc.name, func(t *testing.T) { + if tc.nfc == tc.nfd { + t.Skipf("NFC and NFD are identical for %s", tc.comment) + } + + // Create with NFC, access with NFD + nfcPath := filepath.Join(mntDir, "nfc_"+tc.nfc+".txt") + err := os.WriteFile(nfcPath, []byte("nfc_content"), 0600) + if err != nil { + t.Fatalf("Failed to create file with NFC name %s: %v", tc.comment, err) + } + + nfdPath := filepath.Join(mntDir, "nfc_"+tc.nfd+".txt") + content, err := os.ReadFile(nfdPath) + if err != nil { + t.Fatalf("Failed to read file with NFD name %s: %v", tc.comment, err) + } + if string(content) != "nfc_content" { + t.Errorf("Wrong content for %s: got %q, want %q", tc.comment, string(content), "nfc_content") + } + + // Create with NFD, access with NFC + nfdPath2 := filepath.Join(mntDir, "nfd_"+tc.nfd+".txt") + err = os.WriteFile(nfdPath2, []byte("nfd_content"), 0600) + if err != nil { + t.Fatalf("Failed to create file with NFD name %s: %v", tc.comment, err) + } + + nfcPath2 := filepath.Join(mntDir, "nfd_"+tc.nfc+".txt") + content, err = os.ReadFile(nfcPath2) + if err != nil { + t.Fatalf("Failed to read file with NFC name %s: %v", tc.comment, err) + } + if string(content) != "nfd_content" { + t.Errorf("Wrong content for %s: got %q, want %q", tc.comment, string(content), "nfd_content") + } + }) + } +} + +// TestDirectoryOperations tests directory creation and listing with NFC/NFD names +func TestDirectoryOperations(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("macOS only test") + } + if nfcName == nfdName { + t.Fatal("NFC and NFD names should be different") + } + + test_helpers.ResetTmpDir(false) + cipherDir := test_helpers.InitFS(t) + mntDir := filepath.Join(test_helpers.TmpDir, "mnt") + test_helpers.MountOrFatal(t, cipherDir, mntDir, "-extpass", "echo test") + defer test_helpers.UnmountPanic(mntDir) + + // Create directory with NFC name + nfcDirPath := filepath.Join(mntDir, "dir_"+nfcName) + err := os.Mkdir(nfcDirPath, 0755) + if err != nil { + t.Fatalf("Failed to create directory with NFC name: %v", err) + } + + // Access with NFD name + nfdDirPath := filepath.Join(mntDir, "dir_"+nfdName) + stat, err := os.Stat(nfdDirPath) + if err != nil { + t.Fatalf("Failed to stat directory with NFD name: %v", err) + } + if !stat.IsDir() { + t.Error("Expected directory, got file") + } + + // Create file inside directory using NFD path + filePath := filepath.Join(nfdDirPath, "test.txt") + err = os.WriteFile(filePath, []byte("dir_content"), 0600) + if err != nil { + t.Fatalf("Failed to create file in directory: %v", err) + } + + // Read file using NFC directory path + filePath2 := filepath.Join(nfcDirPath, "test.txt") + content, err := os.ReadFile(filePath2) + if err != nil { + t.Fatalf("Failed to read file from directory: %v", err) + } + if string(content) != "dir_content" { + t.Errorf("Wrong content: got %q, want %q", string(content), "dir_content") + } + + // Test directory listing + entries, err := os.ReadDir(nfdDirPath) + if err != nil { + t.Fatalf("Failed to list directory: %v", err) + } + if len(entries) != 1 || entries[0].Name() != "test.txt" { + t.Errorf("Unexpected directory contents: %v", entries) + } +} + +// TestFileOperations tests various file operations with NFC/NFD names +func TestFileOperations(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("macOS only test") + } + if nfcName == nfdName { + t.Fatal("NFC and NFD names should be different") + } + + test_helpers.ResetTmpDir(false) + cipherDir := test_helpers.InitFS(t) + mntDir := filepath.Join(test_helpers.TmpDir, "mnt") + test_helpers.MountOrFatal(t, cipherDir, mntDir, "-extpass", "echo test") + defer test_helpers.UnmountPanic(mntDir) + + // Create file with NFC name + nfcPath := filepath.Join(mntDir, "ops_"+nfcName+".txt") + err := os.WriteFile(nfcPath, []byte("original"), 0600) + if err != nil { + t.Fatalf("Failed to create file: %v", err) + } + + // Test stat with NFD name + nfdPath := filepath.Join(mntDir, "ops_"+nfdName+".txt") + stat, err := os.Stat(nfdPath) + if err != nil { + t.Fatalf("Failed to stat file with NFD name: %v", err) + } + if stat.Size() != 8 { + t.Errorf("Wrong file size: got %d, want 8", stat.Size()) + } + + // Test truncate with NFD name + err = os.Truncate(nfdPath, 4) + if err != nil { + t.Fatalf("Failed to truncate file: %v", err) + } + + // Verify truncation with NFC name + content, err := os.ReadFile(nfcPath) + if err != nil { + t.Fatalf("Failed to read truncated file: %v", err) + } + if string(content) != "orig" { + t.Errorf("Wrong content after truncate: got %q, want %q", string(content), "orig") + } + + // Test chmod with NFD name + err = os.Chmod(nfdPath, 0644) + if err != nil { + t.Fatalf("Failed to chmod file: %v", err) + } + + // Verify chmod with NFC name + stat, err = os.Stat(nfcPath) + if err != nil { + t.Fatalf("Failed to stat file after chmod: %v", err) + } + if stat.Mode().Perm() != 0644 { + t.Errorf("Wrong permissions: got %o, want %o", stat.Mode().Perm(), 0644) + } + + // Test removal with NFD name + err = os.Remove(nfdPath) + if err != nil { + t.Fatalf("Failed to remove file with NFD name: %v", err) + } + + // Verify removal with NFC name + _, err = os.Stat(nfcPath) + if !os.IsNotExist(err) { + t.Error("File should not exist after removal") + } +} + +// TestEdgeCases tests edge cases and error conditions +func TestEdgeCases(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("macOS only test") + } + + test_helpers.ResetTmpDir(false) + cipherDir := test_helpers.InitFS(t) + mntDir := filepath.Join(test_helpers.TmpDir, "mnt") + test_helpers.MountOrFatal(t, cipherDir, mntDir, "-extpass", "echo test") + defer test_helpers.UnmountPanic(mntDir) + + // Test filename that is identical in NFC and NFD + identicalName := "regular_ascii.txt" + if norm.NFC.String(identicalName) != norm.NFD.String(identicalName) { + t.Fatal("Test setup error: filename should be identical in NFC and NFD") + } + + identicalPath := filepath.Join(mntDir, identicalName) + err := os.WriteFile(identicalPath, []byte("identical"), 0600) + if err != nil { + t.Fatalf("Failed to create file with identical NFC/NFD name: %v", err) + } + + content, err := os.ReadFile(identicalPath) + if err != nil { + t.Fatalf("Failed to read file with identical name: %v", err) + } + if string(content) != "identical" { + t.Errorf("Wrong content: got %q, want %q", string(content), "identical") + } + + // Test invalid UTF-8 bytes (should fall back to direct access) + invalidUTF8 := string([]byte{0xff, 0xfe, 0xfd}) + invalidPath := filepath.Join(mntDir, invalidUTF8) + err = os.WriteFile(invalidPath, []byte("invalid_utf8"), 0600) + if err != nil { + t.Fatalf("Failed to create file with invalid UTF-8 name: %v", err) + } + + content, err = os.ReadFile(invalidPath) + if err != nil { + t.Fatalf("Failed to read file with invalid UTF-8 name: %v", err) + } + if string(content) != "invalid_utf8" { + t.Errorf("Wrong content: got %q, want %q", string(content), "invalid_utf8") + } + + // Test long filename with Unicode characters + longUnicode := strings.Repeat(nfcName, 50) // 50 repetitions of é + longPath := filepath.Join(mntDir, longUnicode+".txt") + err = os.WriteFile(longPath, []byte("long_unicode"), 0600) + if err != nil { + t.Fatalf("Failed to create file with long Unicode name: %v", err) + } + + // Access with NFD version + longNFD := strings.Repeat(nfdName, 50) + longNFDPath := filepath.Join(mntDir, longNFD+".txt") + content, err = os.ReadFile(longNFDPath) + if err != nil { + t.Fatalf("Failed to read file with long NFD name: %v", err) + } + if string(content) != "long_unicode" { + t.Errorf("Wrong content: got %q, want %q", string(content), "long_unicode") + } +} + +// TestNonExistentFiles tests behavior when files don't exist in either normalization form +func TestNonExistentFiles(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("macOS only test") + } + if nfcName == nfdName { + t.Fatal("NFC and NFD names should be different") + } + + test_helpers.ResetTmpDir(false) + cipherDir := test_helpers.InitFS(t) + mntDir := filepath.Join(test_helpers.TmpDir, "mnt") + test_helpers.MountOrFatal(t, cipherDir, mntDir, "-extpass", "echo test") + defer test_helpers.UnmountPanic(mntDir) + + // Try to access non-existent file with NFC name + nonExistentNFC := filepath.Join(mntDir, "nonexistent_"+nfcName+".txt") + _, err := os.ReadFile(nonExistentNFC) + if !os.IsNotExist(err) { + t.Errorf("Expected ENOENT for non-existent NFC file, got: %v", err) + } + + // Try to access non-existent file with NFD name + nonExistentNFD := filepath.Join(mntDir, "nonexistent_"+nfdName+".txt") + _, err = os.ReadFile(nonExistentNFD) + if !os.IsNotExist(err) { + t.Errorf("Expected ENOENT for non-existent NFD file, got: %v", err) + } + + // Create file with specific normalization and ensure only that form exists initially + specificNFC := filepath.Join(mntDir, "specific_"+nfcName+".txt") + err = os.WriteFile(specificNFC, []byte("content"), 0600) + if err != nil { + t.Fatalf("Failed to create file: %v", err) + } + + // Now both forms should work due to Unicode normalization fallback + specificNFD := filepath.Join(mntDir, "specific_"+nfdName+".txt") + _, err = os.ReadFile(specificNFD) + if err != nil { + t.Errorf("NFD access should work after NFC creation: %v", err) + } +} + +// TestNestedDirectories tests Unicode normalization in nested directory structures +func TestNestedDirectories(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("macOS only test") + } + if nfcName == nfdName { + t.Fatal("NFC and NFD names should be different") + } + + test_helpers.ResetTmpDir(false) + cipherDir := test_helpers.InitFS(t) + mntDir := filepath.Join(test_helpers.TmpDir, "mnt") + test_helpers.MountOrFatal(t, cipherDir, mntDir, "-extpass", "echo test") + defer test_helpers.UnmountPanic(mntDir) + + // Create nested directory structure with mixed NFC/NFD + dir1NFC := filepath.Join(mntDir, "level1_"+nfcName) + err := os.Mkdir(dir1NFC, 0755) + if err != nil { + t.Fatalf("Failed to create level1 directory: %v", err) + } + + // Create subdirectory using NFD path to parent + dir1NFD := filepath.Join(mntDir, "level1_"+nfdName) + dir2Path := filepath.Join(dir1NFD, "level2_"+nfdName) + err = os.Mkdir(dir2Path, 0755) + if err != nil { + t.Fatalf("Failed to create level2 directory: %v", err) + } + + // Create file in nested structure using NFC path + dir2NFC := filepath.Join(dir1NFC, "level2_"+nfcName) + filePath := filepath.Join(dir2NFC, "nested_file.txt") + err = os.WriteFile(filePath, []byte("nested_content"), 0600) + if err != nil { + t.Fatalf("Failed to create nested file: %v", err) + } + + // Access file using different path combinations + filePath2 := filepath.Join(dir2Path, "nested_file.txt") // NFD/NFD path + content, err := os.ReadFile(filePath2) + if err != nil { + t.Fatalf("Failed to read nested file with NFD path: %v", err) + } + if string(content) != "nested_content" { + t.Errorf("Wrong content: got %q, want %q", string(content), "nested_content") + } + + // Test directory traversal with mixed normalization + entries, err := os.ReadDir(dir1NFD) + if err != nil { + t.Fatalf("Failed to list level1 directory: %v", err) + } + if len(entries) != 1 { + t.Errorf("Expected 1 entry in level1, got %d", len(entries)) + } + + entries, err = os.ReadDir(dir2NFC) + if err != nil { + t.Fatalf("Failed to list level2 directory: %v", err) + } + if len(entries) != 1 || entries[0].Name() != "nested_file.txt" { + t.Errorf("Unexpected level2 contents: %v", entries) + } +}