Skip to content

Commit

Permalink
add more test
Browse files Browse the repository at this point in the history
  • Loading branch information
tangyoupeng committed Mar 20, 2024
1 parent 90fbad2 commit 68e98d9
Show file tree
Hide file tree
Showing 9 changed files with 1,347 additions and 5 deletions.
19 changes: 19 additions & 0 deletions pkg/fs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ func (fs *FileStat) Mode() os.FileMode {
if attr.Mode&01000 != 0 {
mode |= os.ModeSticky
}
if attr.AccessACL+attr.DefaultACL > 0 {
mode |= 1 << 18
}
switch attr.Typ {
case meta.TypeDirectory:
mode |= os.ModeDir
Expand Down Expand Up @@ -650,6 +653,22 @@ func (fs *FileSystem) SetFacl(ctx meta.Context, p string, acltype uint8, rule *a
if err != 0 {
return
}
if acltype == acl.TypeDefault && fi.Mode().IsRegular() {
if rule.IsEmpty() {
return
} else {
return syscall.ENOTSUP
}
}
if rule.IsEmpty() {
oldRule := acl.EmptyRule()
if err = fs.m.GetFacl(ctx, fi.inode, acltype, oldRule); err != 0 {
return err
}
rule.Owner = oldRule.Owner
rule.Other = oldRule.Other
rule.Group = oldRule.Group & oldRule.Mask
}
err = fs.m.SetFacl(ctx, fi.inode, acltype, rule)
return
}
Expand Down
8 changes: 4 additions & 4 deletions sdk/java/libjfs/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -889,8 +889,8 @@ func jfs_getfacl(pid int, h int64, path *C.char, acltype int, buf uintptr, blen
if w == nil {
return EINVAL
}
var rule acl.Rule
err := w.GetFacl(w.withPid(pid), C.GoString(path), uint8(acltype), &rule)
rule := acl.EmptyRule()
err := w.GetFacl(w.withPid(pid), C.GoString(path), uint8(acltype), rule)
if err != 0 {
return errno(err)
}
Expand Down Expand Up @@ -925,7 +925,7 @@ func jfs_setfacl(pid int, h int64, path *C.char, acltype int, buf uintptr, alen
if w == nil {
return EINVAL
}
var rule acl.Rule
rule := acl.EmptyRule()
r := utils.NewNativeBuffer(toBuf(buf, alen))
rule.Owner = r.Get16()
rule.Group = r.Get16()
Expand All @@ -945,7 +945,7 @@ func jfs_setfacl(pid int, h int64, path *C.char, acltype int, buf uintptr, alen
rule.NamedGroups = append(rule.NamedGroups, entry)
}
}
return errno(w.SetFacl(w.withPid(pid), C.GoString(path), uint8(acltype), &rule))
return errno(w.SetFacl(w.withPid(pid), C.GoString(path), uint8(acltype), rule))
}

//export jfs_readlink
Expand Down
6 changes: 6 additions & 0 deletions sdk/java/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@
<configuration>
<argLine>${argLine}</argLine>
<trimStackTrace>false</trimStackTrace>
<systemProperties>
<test.cache.data>${project.build.directory}/test-classes</test.cache.data>
</systemProperties>
</configuration>
</plugin>
<plugin>
Expand Down Expand Up @@ -217,6 +220,9 @@
<testResource>
<directory>conf</directory>
</testResource>
<testResource>
<directory>src/test/resources</directory>
</testResource>
</testResources>
</build>
<dependencies>
Expand Down
29 changes: 28 additions & 1 deletion sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ public class JuiceFileSystemImpl extends FileSystem {
* hadoop compatibility
*/
private boolean withStreamCapability;
private Constructor<FileStatus> fileStatusConstructor;

// constructor for BufferedFSOutputStreamWithStreamCapabilities
private Constructor<?> constructor;
private Method setStorageIds;
Expand Down Expand Up @@ -432,6 +434,17 @@ public void initialize(URI uri, Configuration conf) throws IOException {
throw new RuntimeException(e);
}
}
// for hadoop compatibility
boolean hasAclMtd = ReflectionUtil.hasMethod(FileStatus.class.getName(), "hasAcl", (String[]) null);
if (hasAclMtd) {
fileStatusConstructor = ReflectionUtil.getConstructor(FileStatus.class,
long.class, boolean.class, int.class, long.class, long.class,
long.class, FsPermission.class, String.class, String.class, Path.class,
Path.class, boolean.class, boolean.class, boolean.class);
if (fileStatusConstructor == null) {
throw new IOException("incompatible hadoop version");
}
}

uMask = FsPermission.getUMask(conf);
String umaskStr = getConf(conf, "umask", null);
Expand Down Expand Up @@ -1433,14 +1446,25 @@ private FileStatus newFileStatus(Path p, Pointer buf, int size, boolean readlink
int mode = buf.getInt(0);
boolean isdir = ((mode >>> 31) & 1) == 1; // Go
int stickybit = (mode >>> 20) & 1;
boolean hasAcl = (mode >> 18 & 1) == 1;
FsPermission perm = new FsPermission((short) ((mode & 0777) | (stickybit << 9)));
perm = new FsPermissionExtension(perm, hasAcl, false);
long length = buf.getLongLong(4);
long mtime = buf.getLongLong(12);
long atime = buf.getLongLong(20);
String user = buf.getString(28);
String group = buf.getString(28 + user.length() + 1);
assert (30 + user.length() + group.length() == size);
return new FileStatus(length, isdir, 1, blocksize, mtime, atime, perm, user, group, p);

if (fileStatusConstructor == null) {
return new FileStatus(length, isdir, 1, blocksize, mtime, atime, perm, user, group, p);
} else {
try {
return fileStatusConstructor.newInstance(length, isdir, 1, blocksize, mtime, atime, perm, user, group, null, p, hasAcl, false, false);
} catch (Exception e) {
throw new IOException("construct fileStatus failed", e);
}
}
}

@Override
Expand Down Expand Up @@ -1821,6 +1845,9 @@ private void setAclInternal(Path path, AclEntryScope scope, List<AclEntry> aclSp
}
int r = lib.jfs_setfacl(Thread.currentThread().getId(), handle, normalizePath(path), scope.ordinal() + 1, buf,
12 + namedaclsize);
if (r == ENOTSUP) {
throw new IOException("Invalid ACL: only directories may have a default ACL");
}
if (r < 0)
throw error(r, path);
}
Expand Down
63 changes: 63 additions & 0 deletions sdk/java/src/main/java/io/juicefs/utils/FsPermissionExtension.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.juicefs.utils;

import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.FsPermission;

/**
* HDFS permission subclass used to indicate an ACL is present and/or that the
* underlying file/dir is encrypted. The ACL/encrypted bits are not visible
* directly to users of {@link FsPermission} serialization. This is
* done for backwards compatibility in case any existing clients assume the
* value of FsPermission is in a particular range.
*/
@InterfaceAudience.Private
public class FsPermissionExtension extends FsPermission {
private final static short ACL_BIT = 1 << 12;
private final static short ENCRYPTED_BIT = 1 << 13;
private final boolean aclBit;
private final boolean encryptedBit;

/**
* Constructs a new FsPermissionExtension based on the given FsPermission.
*
* @param perm FsPermission containing permission bits
*/
public FsPermissionExtension(FsPermission perm, boolean hasAcl,
boolean isEncrypted) {
super(perm.toShort());
aclBit = hasAcl;
encryptedBit = isEncrypted;
}

@Override
public short toExtendedShort() {
return (short) (toShort() |
(aclBit ? ACL_BIT : 0) | (encryptedBit ? ENCRYPTED_BIT : 0));
}

public boolean getAclBit() {
return aclBit;
}

@Override
public boolean getEncryptedBit() {
return encryptedBit;
}
}
53 changes: 53 additions & 0 deletions sdk/java/src/main/java/io/juicefs/utils/ReflectionUtil.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
* JuiceFS, Copyright 2024 Juicedata, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.juicefs.utils;

import java.lang.reflect.Constructor;

public class ReflectionUtil {
public static boolean hasMethod(String className, String method, String[] params) {
try {
Class<?>[] classes = null;
if (params != null) {
classes = new Class[params.length];
for (int i = 0; i < params.length; i++) {
classes[i] = Class.forName(params[i], false, Thread.currentThread().getContextClassLoader());
}
}
return hasMethod(className, method, classes);
} catch (ClassNotFoundException e) {
return false;
}
}

public static boolean hasMethod(String className, String method, Class<?>[] params) {
try {
Class<?> clazz = Class.forName(className, false, Thread.currentThread().getContextClassLoader());
clazz.getDeclaredMethod(method, params);
} catch (ClassNotFoundException | NoSuchMethodException e) {
return false;
}
return true;
}

public static <T> Constructor<T> getConstructor(Class<T> clazz, Class<?>... params) {
try {
return clazz.getConstructor(params);
} catch (NoSuchMethodException e) {
return null;
}
}
}
20 changes: 20 additions & 0 deletions sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -1100,4 +1100,24 @@ public void testAccessAclNotInherited() throws IOException {
fs.delete(parent, true);
}

public void testFileStatusWithAcl() throws Exception {
List<AclEntry> acls = Lists.newArrayList(
aclEntry(ACCESS, USER, ALL),
aclEntry(ACCESS, USER, "foo", ALL),
aclEntry(ACCESS, OTHER, ALL),
aclEntry(ACCESS, GROUP, ALL)
);
Path p = new Path("/test_acl_status");
fs.delete(p, true);
fs.mkdirs(p);
FileStatus pStatus = fs.getFileStatus(p);
assertFalse(pStatus.hasAcl());

Path f = new Path(p, "f");
fs.create(f).close();
fs.setAcl(f, acls);
FileStatus[] fileStatuses = fs.listStatus(p);
assertTrue(fileStatuses[0].getPermission().getAclBit());
assertTrue(fileStatuses[0].hasAcl());
}
}
79 changes: 79 additions & 0 deletions sdk/java/src/test/java/io/juicefs/acl/TestAclCLI.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.juicefs.acl;

import org.apache.hadoop.cli.CLITestHelperDFS;
import org.apache.hadoop.cli.util.CLICommand;
import org.apache.hadoop.cli.util.CommandExecutor.Result;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

public class TestAclCLI extends CLITestHelperDFS {
private String vol = null;
private String username = null;

protected void initConf() {
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
conf.setBoolean(
DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY, false);
}

@Before
@Override
public void setUp() throws Exception {
super.setUp();
initConf();
vol = "jfs://dev/";
username = System.getProperty("user.name");
}

@After
@Override
public void tearDown() throws Exception {
super.tearDown();
}

@Override
protected String getTestFile() {
return "testAclCLI.xml";
}

@Override
protected String expandCommand(final String cmd) {
String expCmd = cmd;
expCmd = expCmd.replaceAll("NAMENODE", vol);
expCmd = expCmd.replaceAll("USERNAME", username);
expCmd = expCmd.replaceAll("#LF#",
System.getProperty("line.separator"));
expCmd = super.expandCommand(expCmd);
return expCmd;
}

@Override
protected Result execute(CLICommand cmd) throws Exception {
return cmd.getExecutor(vol, conf).executeCommand(cmd.getCmd());
}

@Test
@Override
public void testAll() {
super.testAll();
}
}
Loading

0 comments on commit 68e98d9

Please sign in to comment.