Skip to content

Commit

Permalink
Merge branch 'main' into review_acl
Browse files Browse the repository at this point in the history
  • Loading branch information
davies committed Apr 8, 2024
2 parents 630b112 + c93d7d6 commit a6c8d18
Show file tree
Hide file tree
Showing 72 changed files with 1,269 additions and 1,142 deletions.
214 changes: 214 additions & 0 deletions .github/scripts/command/gateway.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
#!/bin/bash -e
source .github/scripts/common/common.sh

[[ -z "$META" ]] && META=redis
source .github/scripts/start_meta_engine.sh
start_meta_engine $META
META_URL=$(get_meta_url $META)
wget https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z -O mc
chmod +x mc
export MINIO_ROOT_USER=admin
export MINIO_ROOT_PASSWORD=admin123
export MINIO_REFRESH_IAM_INTERVAL=10s

prepare_test()
{
umount_jfs /tmp/jfs $META_URL
lsof -i :9001 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true
lsof -i :9002 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true
python3 .github/scripts/flush_meta.py $META_URL
rm -rf /var/jfs/myjfs || true
rm -rf /var/jfsCache/myjfs || true
}

start_two_gateway()
{
prepare_test
./juicefs format $META_URL myjfs --trash-days 0
./juicefs mount -d $META_URL /tmp/jfs
export MINIO_ROOT_USER=admin
export MINIO_ROOT_PASSWORD=admin123
./juicefs gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag -background
sleep 1
./juicefs gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag -background
sleep 2
./mc alias set gateway1 http://127.0.0.1:9001 admin admin123
./mc alias set gateway2 http://127.0.0.1:9002 admin admin123
}

test_user_management()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
sleep 12
user=$(./mc admin user list gateway2 | grep user1) || true
if [ -z "$user" ]
then
echo "user synchronization error"
exit 1
fi
./mc mb gateway1/test1
./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
if ./mc cp mc gateway1_user1/test1/file1
then
echo "By default, the user has no read and write permission"
exit 1
fi
./mc admin policy set gateway1 readwrite user=user1
if ./mc cp mc gateway1_user1/test1/file1
then
echo "readwrite policy can read and write objects"
else
echo "set readwrite policy fail"
exit 1
fi
./mc cp gateway2/test1/file1 .
compare_md5sum file1 mc
./mc admin user disable gateway1 user1
./mc admin user remove gateway2 user1
sleep 12
user=$(./mc admin user list gateway1 | grep user1) || true
if [ ! -z "$user" ]
then
echo "remove user user1 fail"
echo $user
exit 1
fi
}

test_group_management()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
./mc admin user add gateway1 user2 admin123
./mc admin user add gateway1 user3 admin123
./mc admin group add gateway1 testcents user1 user2 user3
result=$(./mc admin group info gateway1 testcents | grep Members |awk '{print $2}') || true
if [ "$result" != "user1,user2,user3" ]
then
echo "error,result is '$result'"
exit 1
fi
./mc admin policy set gateway1 readwrite group=testcents
sleep 5
./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
./mc mb gateway1/test1
if ./mc cp mc gateway1_user1/test1/file1
then
echo "readwrite policy can read write"
else
echo "the readwrite group has no read and write permission"
exit 1
fi
./mc admin policy set gateway1 readonly group=testcents
sleep 5
if ./mc cp mc gateway1_user1/test1/file1
then
echo "readonly group policy can not write"
exit 1
else
echo "the readonly group has no write permission"
fi

./mc admin group remove gateway1 testcents user1 user2 user3
./mc admin group remove gateway1 testcents
}

test_mult_gateways_set_group()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
./mc admin user add gateway1 user2 admin123
./mc admin user add gateway1 user3 admin123
./mc admin group add gateway1 testcents user1 user2 user3
./mc admin group disable gateway2 testcents
sleep 12
result=$(./mc admin group info gateway2 testcents | grep Members |awk '{print $2}') || true
if [ "$result" != "user1,user2,user3" ]
then
echo "error,result is '$result'"
exit 1
fi
./mc admin group enable gateway1 testcents
./mc admin user add gateway1 user4 admin123
./mc admin group add gateway1 testcents user4
sleep 1
./mc admin group disable gateway2 testcents
sleep 12
result=$(./mc admin group info gateway2 testcents | grep Members |awk '{print $2}') || true
if [ "$result" != "user1,user2,user3,user4" ]
then
echo "error,result is '$result'"
exit 1
fi
}

test_user_svcacct_add()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
./mc admin policy set gateway1 consoleAdmin user=user1
./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
./mc admin user svcacct add gateway1_user1 user1 --access-key 12345678 --secret-key 12345678
./mc admin user svcacct info gateway1_user1 12345678
./mc admin user svcacct set gateway1_user1 12345678 --secret-key 123456789
./mc alias set svcacct1 http://127.0.0.1:9001 12345678 123456789
./mc mb svcacct1/test1
if ./mc cp mc svcacct1/test1/file1
then
echo "svcacct user consoleAdmin policy can read write"
else
echo "the svcacct user has no read and write permission"
exit 1
fi
./mc admin user svcacct disable gateway1_user1 12345678
./mc admin user svcacct rm gateway1_user1 12345678
}

test_user_sts()
{
prepare_test
start_two_gateway
./mc admin user add gateway1 user1 admin123
./mc admin policy set gateway1 consoleAdmin user=user1
./mc alias set gateway1_user1 http://127.0.0.1:9001 user1 admin123
git clone https://github.com/juicedata/minio.git -b gateway-1.1
./mc mb gateway1_user1/test1
./mc cp mc gateway1_user1/test1/mc
cd minio
go run docs/sts/assume-role.go -sts-ep http://127.0.0.1:9001 -u user1 -p admin123 -b test1 -d
go run docs/sts/assume-role.go -sts-ep http://127.0.0.1:9001 -u user1 -p admin123 -b test1
cd -
./mc admin user remove gateway1 user1
}


test_change_credentials()
{
prepare_test
start_two_gateway
./mc mb gateway1/test1
./mc cp mc gateway1/test1/file1
lsof -i :9001 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true
lsof -i :9002 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true
export MINIO_ROOT_USER=newadmin
export MINIO_ROOT_PASSWORD=newadmin123
export MINIO_ROOT_USER_OLD=admin
export MINIO_ROOT_PASSWORD_OLD=admin123
./juicefs gateway $META_URL 127.0.0.1:9001 --multi-buckets --keep-etag --object-tag -background
./juicefs gateway $META_URL 127.0.0.1:9002 --multi-buckets --keep-etag --object-tag -background
sleep 5
./mc alias set gateway1 http://127.0.0.1:9001 newadmin newadmin123
./mc alias set gateway2 http://127.0.0.1:9002 newadmin newadmin123
./mc cp gateway1/test1/file1 file1
./mc cp gateway2/test1/file1 file2
compare_md5sum file1 mc
compare_md5sum file2 mc
}

source .github/scripts/common/run_test.sh && run_test $@

7 changes: 6 additions & 1 deletion .github/workflows/command.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,12 @@ jobs:
timeout-minutes: 30
run: |
sudo META=${{matrix.meta}} .github/scripts/command/fsck.sh
- name: Test Gateway
timeout-minutes: 30
run: |
sudo META=${{matrix.meta}} .github/scripts/command/gateway.sh
- name: Test Info
run: |
sudo META=${{matrix.meta}} .github/scripts/command/info.sh
Expand Down
62 changes: 18 additions & 44 deletions cmd/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,11 @@
package cmd

import (
"fmt"
"github.com/urfave/cli/v2"
"os"
"path"
"runtime"
"strconv"
"strings"
"time"

"github.com/urfave/cli/v2"
)

func globalFlags() []cli.Flag {
Expand Down Expand Up @@ -109,14 +106,14 @@ func storageFlags() []cli.Flag {
Name: "storage-class",
Usage: "the storage class for data written by current client",
},
&cli.IntFlag{
&cli.StringFlag{
Name: "get-timeout",
Value: 60,
Value: "60s",
Usage: "the max number of seconds to download an object",
},
&cli.IntFlag{
&cli.StringFlag{
Name: "put-timeout",
Value: 60,
Value: "60s",
Usage: "the max number of seconds to upload an object",
},
&cli.IntFlag{
Expand Down Expand Up @@ -314,27 +311,27 @@ func shareInfoFlags() []cli.Flag {

func metaCacheFlags(defaultEntryCache float64) []cli.Flag {
return addCategories("META CACHE", []cli.Flag{
&cli.Float64Flag{
&cli.StringFlag{
Name: "attr-cache",
Value: 1.0,
Usage: "attributes cache timeout in seconds",
Value: "1s",
Usage: "attributes cache timeout",
},
&cli.Float64Flag{
&cli.StringFlag{
Name: "entry-cache",
Value: defaultEntryCache,
Usage: "file entry cache timeout in seconds",
Value: fmt.Sprintf("%.1f", defaultEntryCache),
Usage: "file entry cache timeout",
},
&cli.Float64Flag{
&cli.StringFlag{
Name: "dir-entry-cache",
Value: 1.0,
Usage: "dir entry cache timeout in seconds",
Value: "1s",
Usage: "dir entry cache timeout",
},
&cli.Float64Flag{
&cli.StringFlag{
Name: "open-cache",
Value: 0.0,
Value: "0",
Usage: "The seconds to reuse open file without checking update (0 means disable this feature)",
},
&cli.IntFlag{
&cli.Uint64Flag{
Name: "open-cache-limit",
Value: 10000,
Usage: "max number of open files to cache (soft limit, 0 means unlimited)",
Expand All @@ -349,26 +346,3 @@ func expandFlags(compoundFlags ...[]cli.Flag) []cli.Flag {
}
return flags
}

func duration(s string) time.Duration {
v, err := strconv.Atoi(s)
if err == nil {
return time.Second * time.Duration(v)
}

err = nil
var d time.Duration
p := strings.Index(s, "d")
if p >= 0 {
v, err = strconv.Atoi(s[:p])
}
if err == nil && s[p+1:] != "" {
d, err = time.ParseDuration(s[p+1:])
}

if err != nil {
logger.Warnf("Invalid duration value: %s, setting it to 0", s)
return 0
}
return d + time.Hour*time.Duration(v*24)
}
3 changes: 2 additions & 1 deletion cmd/flags_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cmd

import (
"github.com/juicedata/juicefs/pkg/utils"
"testing"
"time"

Expand Down Expand Up @@ -49,7 +50,7 @@ func Test_duration(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, duration(tt.args.s), "duration(%v)", tt.args.s)
assert.Equalf(t, tt.want, utils.Duration(tt.args.s), "duration(%v)", tt.args.s)
})
}
}
5 changes: 4 additions & 1 deletion cmd/format.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,10 @@ func createStorage(format meta.Format) (object.ObjectStorage, error) {
blob = object.WithPrefix(blob, format.Name+"/")
if format.StorageClass != "" {
if os, ok := blob.(object.SupportStorageClass); ok {
os.SetStorageClass(format.StorageClass)
err := os.SetStorageClass(format.StorageClass)
if err != nil {
logger.Warnf("set storage class %q: %v", format.StorageClass, err)
}
} else {
logger.Warnf("Storage class is not supported by %q, will ignore", format.Storage)
}
Expand Down
Loading

0 comments on commit a6c8d18

Please sign in to comment.