Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sync: fix umask #3784

Merged
merged 2 commits into from
Jun 9, 2023
Merged

sync: fix umask #3784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7915445
sync: fix umask
davies Jun 8, 2023
caf45bb
fix bugs
davies Jun 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/bench.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ func bench(ctx *cli.Context) error {

/* --- Prepare --- */
if _, err := os.Stat(bm.tmpdir); os.IsNotExist(err) {
if err = os.MkdirAll(bm.tmpdir, 0755); err != nil {
if err = os.MkdirAll(bm.tmpdir, 0777); err != nil {
logger.Fatalf("Failed to create %s: %s", bm.tmpdir, err)
}
}
Expand Down
3 changes: 1 addition & 2 deletions cmd/gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,7 @@ func (g *GateWay) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, er
&jfsgateway.Config{
MultiBucket: c.Bool("multi-buckets"),
KeepEtag: c.Bool("keep-etag"),
Mode: uint16(0666 &^ umask),
DirMode: uint16(0777 &^ umask),
Umask: uint16(umask),
},
)
}
Expand Down
9 changes: 5 additions & 4 deletions cmd/mdtest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,10 @@ import (
)

var ctx = meta.NewContext(1, uint32(os.Getuid()), []uint32{uint32(os.Getgid())})
var umask = uint16(utils.GetUmask())

func createDir(jfs *fs.FileSystem, root string, d int, width int) error {
if err := jfs.Mkdir(ctx, root, 0755); err != 0 {
if err := jfs.Mkdir(ctx, root, 0777, umask); err != 0 {
return fmt.Errorf("Mkdir %s: %s", root, err)
}
if d > 0 {
Expand All @@ -56,7 +57,7 @@ func createFile(jfs *fs.FileSystem, bar *utils.Bar, np int, root string, d int,
m := jfs.Meta()
for i := 0; i < files; i++ {
fn := filepath.Join(root, fmt.Sprintf("file.mdtest.%d.%d", np, i))
f, err := jfs.Create(ctx, fn, 0644)
f, err := jfs.Create(ctx, fn, 0666, umask)
if err != 0 {
return fmt.Errorf("create %s: %s", fn, err)
}
Expand Down Expand Up @@ -111,11 +112,11 @@ func runTest(jfs *fs.FileSystem, rootDir string, np, width, depth, files, bytes
logger.Infof("Create %d files in %d dirs", total, dirs)

start := time.Now()
if err := jfs.Mkdir(ctx, rootDir, 0755); err != 0 {
if err := jfs.Mkdir(ctx, rootDir, 0777, umask); err != 0 {
logger.Errorf("mkdir %s: %s", rootDir, err)
}
root := filepath.Join(rootDir, "test-dir.0-0")
if err := jfs.Mkdir(ctx, root, 0755); err != 0 {
if err := jfs.Mkdir(ctx, root, 0777, umask); err != 0 {
logger.Fatalf("Mkdir %s: %s", root, err)
}
root = filepath.Join(root, "mdtest_tree.0")
Expand Down
25 changes: 13 additions & 12 deletions cmd/object.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,9 @@ func toError(eno syscall.Errno) error {

type juiceFS struct {
object.DefaultObjectStorage
name string
jfs *fs.FileSystem
name string
umask uint16
jfs *fs.FileSystem
}

func (j *juiceFS) String() string {
Expand Down Expand Up @@ -120,14 +121,14 @@ var bufPool = sync.Pool{
func (j *juiceFS) Put(key string, in io.Reader) (err error) {
p := j.path(key)
if strings.HasSuffix(p, "/") {
eno := j.jfs.MkdirAll(ctx, p, 0755)
eno := j.jfs.MkdirAll(ctx, p, 0777, j.umask)
return toError(eno)
}
tmp := filepath.Join(filepath.Dir(p), "."+filepath.Base(p)+".tmp"+strconv.Itoa(rand.Int()))
f, eno := j.jfs.Create(ctx, tmp, 0755)
f, eno := j.jfs.Create(ctx, tmp, 0666, j.umask)
if eno == syscall.ENOENT {
_ = j.jfs.MkdirAll(ctx, filepath.Dir(tmp), 0755)
f, eno = j.jfs.Create(ctx, tmp, 0755)
_ = j.jfs.MkdirAll(ctx, filepath.Dir(tmp), 0777, j.umask)
f, eno = j.jfs.Create(ctx, tmp, 0666, j.umask)
}
if eno != 0 {
return toError(eno)
Expand Down Expand Up @@ -313,12 +314,12 @@ func (j *juiceFS) Chown(key string, owner, group string) error {
return toError(f.Chown(ctx, uint32(uid), uint32(gid)))
}

func (d *juiceFS) Symlink(oldName, newName string) error {
p := d.path(newName)
err := d.jfs.Symlink(ctx, oldName, p)
func (j *juiceFS) Symlink(oldName, newName string) error {
p := j.path(newName)
err := j.jfs.Symlink(ctx, oldName, p)
if err == syscall.ENOENT {
_ = d.jfs.MkdirAll(ctx, filepath.Dir(p), 0755)
err = d.jfs.Symlink(ctx, oldName, p)
_ = j.jfs.MkdirAll(ctx, filepath.Dir(p), 0777, j.umask)
err = j.jfs.Symlink(ctx, oldName, p)
}
return toError(err)
}
Expand Down Expand Up @@ -388,7 +389,7 @@ func newJFS(endpoint, accessKey, secretKey, token string) (object.ObjectStorage,
if err != nil {
return nil, fmt.Errorf("Initialize: %s", err)
}
return &juiceFS{object.DefaultObjectStorage{}, format.Name, jfs}, nil
return &juiceFS{object.DefaultObjectStorage{}, format.Name, uint16(utils.GetUmask()), jfs}, nil
}

func init() {
Expand Down
3 changes: 2 additions & 1 deletion cmd/object_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/juicedata/juicefs/pkg/fs"
"github.com/juicedata/juicefs/pkg/meta"
"github.com/juicedata/juicefs/pkg/object"
"github.com/juicedata/juicefs/pkg/utils"
"github.com/juicedata/juicefs/pkg/vfs"
)

Expand Down Expand Up @@ -170,7 +171,7 @@ func TestJFS(t *testing.T) {
t.Fatalf("initialize failed: %s", err)
}

jstore := &juiceFS{object.DefaultObjectStorage{}, "test", jfs}
jstore := &juiceFS{object.DefaultObjectStorage{}, "test", uint16(utils.GetUmask()), jfs}
testFileSystem(t, jstore)
testFileSystem(t, object.WithPrefix(jstore, "unittest/"))
}
2 changes: 1 addition & 1 deletion cmd/umount.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func doUmount(mp string, force bool) error {
}
case "windows":
if !force {
_ = os.Mkdir(filepath.Join(mp, ".UMOUNTIT"), 0755)
_ = os.Mkdir(filepath.Join(mp, ".UMOUNTIT"), 0777)
return nil
} else {
cmd = exec.Command("taskkill", "/IM", "juicefs.exe", "/F")
Expand Down
16 changes: 8 additions & 8 deletions pkg/fs/fs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,7 @@ func parentDir(p string) string {
return path.Dir(strings.TrimRight(p, "/"))
}

func (fs *FileSystem) Mkdir(ctx meta.Context, p string, mode uint16) (err syscall.Errno) {
func (fs *FileSystem) Mkdir(ctx meta.Context, p string, mode uint16, umask uint16) (err syscall.Errno) {
defer trace.StartRegion(context.TODO(), "fs.Mkdir").End()
l := vfs.NewLogContext(ctx)
defer func() { fs.log(l, "Mkdir (%s, %o): %s", p, mode, errstr(err)) }()
Expand All @@ -429,16 +429,16 @@ func (fs *FileSystem) Mkdir(ctx meta.Context, p string, mode uint16) (err syscal
return err
}
var inode Ino
err = fs.m.Mkdir(ctx, fi.inode, path.Base(p), mode, 0, 0, &inode, nil)
err = fs.m.Mkdir(ctx, fi.inode, path.Base(p), mode, umask, 0, &inode, nil)
fs.invalidateEntry(fi.inode, path.Base(p))
return
}

func (fs *FileSystem) MkdirAll(ctx meta.Context, p string, mode uint16) (err syscall.Errno) {
err = fs.Mkdir(ctx, p, mode)
func (fs *FileSystem) MkdirAll(ctx meta.Context, p string, mode uint16, umask uint16) (err syscall.Errno) {
err = fs.Mkdir(ctx, p, mode, umask)
if err == syscall.ENOENT {
_ = fs.MkdirAll(ctx, parentDir(p), mode)
err = fs.Mkdir(ctx, p, mode)
_ = fs.MkdirAll(ctx, parentDir(p), mode, umask)
err = fs.Mkdir(ctx, p, mode, umask)
}
if err == syscall.EEXIST {
err = 0
Expand Down Expand Up @@ -752,7 +752,7 @@ func (fs *FileSystem) resolve(ctx meta.Context, p string, followLastSymlink bool
return fi, 0
}

func (fs *FileSystem) Create(ctx meta.Context, p string, mode uint16) (f *File, err syscall.Errno) {
func (fs *FileSystem) Create(ctx meta.Context, p string, mode uint16, umask uint16) (f *File, err syscall.Errno) {
defer trace.StartRegion(context.TODO(), "fs.Create").End()
l := vfs.NewLogContext(ctx)
defer func() { fs.log(l, "Create (%s,%o): %s", p, mode, errstr(err)) }()
Expand All @@ -766,7 +766,7 @@ func (fs *FileSystem) Create(ctx meta.Context, p string, mode uint16) (f *File,
if err != 0 {
return
}
err = fs.m.Create(ctx, fi.inode, path.Base(p), mode&07777, 0, syscall.O_EXCL, &inode, attr)
err = fs.m.Create(ctx, fi.inode, path.Base(p), mode&07777, umask, syscall.O_EXCL, &inode, attr)
if err == 0 {
fi = AttrToFileInfo(inode, attr)
fi.name = path.Base(p)
Expand Down
10 changes: 5 additions & 5 deletions pkg/fs/fs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func TestFileSystem(t *testing.T) {
if e := fs.Access(ctx, "/", 7); e != 0 {
t.Fatalf("access /: %s", e)
}
f, err := fs.Create(ctx, "/hello", 0644)
f, err := fs.Create(ctx, "/hello", 0666, 022)
if err != 0 {
t.Fatalf("create /hello: %s", err)
}
Expand Down Expand Up @@ -184,7 +184,7 @@ func TestFileSystem(t *testing.T) {
t.Fatalf("open without permission: %s", e)
}

if err := fs.Mkdir(ctx, "/d", 0755); err != 0 {
if err := fs.Mkdir(ctx, "/d", 0777, 022); err != 0 {
t.Fatalf("mkdir /d: %s", err)
}
d, e := fs.Open(ctx, "/", 0)
Expand Down Expand Up @@ -246,13 +246,13 @@ func TestFileSystem(t *testing.T) {
}

// path with trailing /
if err := fs.Mkdir(ctx, "/ddd/", 0777); err != 0 {
if err := fs.Mkdir(ctx, "/ddd/", 0777, 000); err != 0 {
t.Fatalf("mkdir /ddd/: %s", err)
}
if _, err := fs.Create(ctx, "/ddd/ddd", 0777); err != 0 {
if _, err := fs.Create(ctx, "/ddd/ddd", 0777, 000); err != 0 {
t.Fatalf("create /ddd/ddd: %s", err)
}
if _, err := fs.Create(ctx, "/ddd/fff/", 0777); err != syscall.EINVAL {
if _, err := fs.Create(ctx, "/ddd/fff/", 0777, 000); err != syscall.EINVAL {
t.Fatalf("create /ddd/fff/: %s", err)
}
if err := fs.Delete(ctx, "/ddd/"); err != syscall.ENOTEMPTY {
Expand Down
12 changes: 7 additions & 5 deletions pkg/fs/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"syscall"

"github.com/juicedata/juicefs/pkg/meta"
"github.com/juicedata/juicefs/pkg/utils"
"github.com/juicedata/juicefs/pkg/vfs"
"golang.org/x/net/webdav"
)
Expand Down Expand Up @@ -81,12 +82,13 @@ func econv(err error) error {
}

type webdavFS struct {
ctx meta.Context
fs *FileSystem
ctx meta.Context
fs *FileSystem
umask uint16
}

func (hfs *webdavFS) Mkdir(ctx context.Context, name string, perm os.FileMode) error {
return econv(hfs.fs.Mkdir(hfs.ctx, name, uint16(perm)))
return econv(hfs.fs.Mkdir(hfs.ctx, name, uint16(perm), hfs.umask))
}

func (hfs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
Expand All @@ -104,7 +106,7 @@ func (hfs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm o
f, err := hfs.fs.Open(hfs.ctx, name, uint32(mode))
if err != 0 {
if err == syscall.ENOENT && flag&os.O_CREATE != 0 {
f, err = hfs.fs.Create(hfs.ctx, name, uint16(perm))
f, err = hfs.fs.Create(hfs.ctx, name, uint16(perm), hfs.umask)
}
} else if flag&os.O_TRUNC != 0 {
if errno := hfs.fs.Truncate(hfs.ctx, name, 0); errno != 0 {
Expand Down Expand Up @@ -219,7 +221,7 @@ func (h *indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

func StartHTTPServer(fs *FileSystem, config WebdavConfig) {
ctx := meta.NewContext(uint32(os.Getpid()), uint32(os.Getuid()), []uint32{uint32(os.Getgid())})
hfs := &webdavFS{ctx, fs}
hfs := &webdavFS{ctx, fs, uint16(utils.GetUmask())}
srv := &webdav.Handler{
FileSystem: hfs,
LockSystem: webdav.NewMemLS(),
Expand Down
6 changes: 4 additions & 2 deletions pkg/fs/http_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,18 @@ package fs

import (
"context"
"github.com/juicedata/juicefs/pkg/meta"
"io"
"io/fs"
"os"
"testing"

"github.com/juicedata/juicefs/pkg/meta"
"github.com/juicedata/juicefs/pkg/utils"
)

func TestWebdav(t *testing.T) {
jfs := createTestFS(t)
webdavFS := &webdavFS{meta.NewContext(uint32(os.Getpid()), uint32(os.Getuid()), []uint32{uint32(os.Getgid())}), jfs}
webdavFS := &webdavFS{meta.NewContext(uint32(os.Getpid()), uint32(os.Getuid()), []uint32{uint32(os.Getgid())}), jfs, uint16(utils.GetUmask())}
ctx := context.Background()
_, err := webdavFS.Stat(ctx, "/")
if err != nil {
Expand Down
31 changes: 15 additions & 16 deletions pkg/gateway/gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,7 @@ var logger = utils.GetLogger("juicefs")
type Config struct {
MultiBucket bool
KeepEtag bool
Mode uint16
DirMode uint16
Umask uint16
}

func NewJFSGateway(jfs *fs.FileSystem, conf *vfs.Config, gConf *Config) (minio.ObjectLayer, error) {
Expand Down Expand Up @@ -194,7 +193,7 @@ func (n *jfsObjects) MakeBucketWithLocation(ctx context.Context, bucket string,
if !n.gConf.MultiBucket {
return nil
}
eno := n.fs.Mkdir(mctx, n.path(bucket), n.gConf.DirMode)
eno := n.fs.Mkdir(mctx, n.path(bucket), 0777, n.gConf.Umask)
return jfsToObjectErr(ctx, eno, bucket)
}

Expand Down Expand Up @@ -478,8 +477,8 @@ func (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
return n.GetObjectInfo(ctx, srcBucket, srcObject, minio.ObjectOptions{})
}
tmp := n.tpath(dstBucket, "tmp", minio.MustGetUUID())
_ = n.mkdirAll(ctx, path.Dir(tmp), os.FileMode(n.gConf.DirMode))
f, eno := n.fs.Create(mctx, tmp, n.gConf.Mode)
_ = n.mkdirAll(ctx, path.Dir(tmp))
f, eno := n.fs.Create(mctx, tmp, 0666, n.gConf.Umask)
if eno != 0 {
logger.Errorf("create %s: %s", tmp, eno)
return
Expand Down Expand Up @@ -604,19 +603,19 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
}, nil
}

func (n *jfsObjects) mkdirAll(ctx context.Context, p string, mode os.FileMode) error {
func (n *jfsObjects) mkdirAll(ctx context.Context, p string) error {
if fi, eno := n.fs.Stat(mctx, p); eno == 0 {
if !fi.IsDir() {
return fmt.Errorf("%s is not directory", p)
}
return nil
}
eno := n.fs.Mkdir(mctx, p, uint16(mode))
eno := n.fs.Mkdir(mctx, p, 0777, n.gConf.Umask)
if eno != 0 && fs.IsNotExist(eno) {
if err := n.mkdirAll(ctx, path.Dir(p), os.FileMode(n.gConf.DirMode)); err != nil {
if err := n.mkdirAll(ctx, path.Dir(p)); err != nil {
return err
}
eno = n.fs.Mkdir(mctx, p, uint16(mode))
eno = n.fs.Mkdir(mctx, p, 0777, n.gConf.Umask)
}
if eno != 0 && fs.IsExist(eno) {
eno = 0
Expand All @@ -629,8 +628,8 @@ func (n *jfsObjects) mkdirAll(ctx context.Context, p string, mode os.FileMode) e

func (n *jfsObjects) putObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (err error) {
tmpname := n.tpath(bucket, "tmp", minio.MustGetUUID())
_ = n.mkdirAll(ctx, path.Dir(tmpname), os.FileMode(n.gConf.DirMode))
f, eno := n.fs.Create(mctx, tmpname, n.gConf.Mode)
_ = n.mkdirAll(ctx, path.Dir(tmpname))
f, eno := n.fs.Create(mctx, tmpname, 0666, n.gConf.Umask)
if eno != 0 {
logger.Errorf("create %s: %s", tmpname, eno)
err = eno
Expand Down Expand Up @@ -667,7 +666,7 @@ func (n *jfsObjects) putObject(ctx context.Context, bucket, object string, r *mi
}
dir := path.Dir(object)
if dir != "" {
_ = n.mkdirAll(ctx, dir, os.FileMode(n.gConf.DirMode))
_ = n.mkdirAll(ctx, dir)
}
if eno := n.fs.Rename(mctx, tmpname, object, 0); eno != 0 {
err = jfsToObjectErr(ctx, eno, bucket, object)
Expand All @@ -683,7 +682,7 @@ func (n *jfsObjects) PutObject(ctx context.Context, bucket string, object string

p := n.path(bucket, object)
if strings.HasSuffix(object, sep) {
if err = n.mkdirAll(ctx, p, os.FileMode(n.gConf.DirMode)); err != nil {
if err = n.mkdirAll(ctx, p); err != nil {
err = jfsToObjectErr(ctx, err, bucket, object)
return
}
Expand Down Expand Up @@ -728,7 +727,7 @@ func (n *jfsObjects) NewMultipartUpload(ctx context.Context, bucket string, obje
}
uploadID = minio.MustGetUUID()
p := n.upath(bucket, uploadID)
err = n.mkdirAll(ctx, p, os.FileMode(n.gConf.DirMode))
err = n.mkdirAll(ctx, p)
if err == nil {
eno := n.fs.SetXattr(mctx, p, uploadKeyName, []byte(object), 0)
if eno != 0 {
Expand Down Expand Up @@ -882,7 +881,7 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object

tmp := n.ppath(bucket, uploadID, "complete")
_ = n.fs.Delete(mctx, tmp)
f, eno := n.fs.Create(mctx, tmp, n.gConf.Mode)
f, eno := n.fs.Create(mctx, tmp, 0666, n.gConf.Umask)
if eno != 0 {
err = jfsToObjectErr(ctx, eno, bucket, object, uploadID)
logger.Errorf("create complete: %s", err)
Expand All @@ -906,7 +905,7 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
name := n.path(bucket, object)
dir := path.Dir(name)
if dir != "" {
if err = n.mkdirAll(ctx, dir, os.FileMode(n.gConf.DirMode)); err != nil {
if err = n.mkdirAll(ctx, dir); err != nil {
_ = n.fs.Delete(mctx, tmp)
err = jfsToObjectErr(ctx, err, bucket, object, uploadID)
return
Expand Down
Loading