Skip to content

Commit

Permalink
pkg/gateway: fix ListMultipartUploads sorting rule and support delimi…
Browse files Browse the repository at this point in the history 
…ter (#4297)
  • Loading branch information
@zhijian-pro
zhijian-pro authored Jan 2, 2024
1 parent 5ecf33a commit d1dcd95
Show file tree
Hide file tree
Showing 2 changed files with 134 additions and 7 deletions.
83 changes: 83 additions & 0 deletions integration/s3gateway_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -732,6 +732,89 @@ function test_multipart_upload() {
fi
fi


for key in "afile" "bfile" "bfile" "documents/report1.pdf" "documents/report2.pdf" "ebook" "photos/2021/a2.png" "photos/2021/a3.png" "photos/2022/a4.png"
do
if [ $rv -eq 0 ]; then
# create multipart
function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${key}"
test_function=${function}
out=$($function)
rv=$?
upload_id=$(echo "$out" | jq -r .UploadId)
fi
done

if [ $rv -eq 0 ]; then
function="${AWS} s3api list-multipart-uploads --bucket ${bucket_name}"
test_function=${function}
out=$($function)
rv=$?
keys=$(echo "$out" | jq 'foreach .Uploads[] as $upload (null; . + $upload.Key)')
if [ $keys != "afilebfilebfiledocuments/report1.pdfdocuments/report2.pdfebookphotos/2021/a2.pngphotos/2021/a3.pngphotos/2022/a4.png" ]; then
rv=1
out="list-multipart-uploads failed"
fi
fi

if [ $rv -eq 0 ]; then
function="${AWS} s3api list-multipart-uploads --bucket ${bucket_name} --key-marker bfile"
test_function=${function}
out=$($function)
rv=$?
keys=$(echo "$out" | jq 'foreach .Uploads[] as $upload (null; . + $upload.Key)')
if [ $keys != "bfilebfiledocuments/report1.pdfdocuments/report2.pdfebookphotos/2021/a2.pngphotos/2021/a3.pngphotos/2022/a4.png" ]; then
rv=1
out="list-multipart-upload failed"
fi
fi

if [ $rv -eq 0 ]; then
function="${AWS} s3api list-multipart-uploads --bucket ${bucket_name} --key-marker bfile --delimiter /"
test_function=${function}
out=$($function)
rv=$?
keys=$(echo "$out" | jq 'foreach .Uploads[] as $upload (null; . + $upload.Key)')
if [ $keys != "afilebfilebfileebook" ]; then
rv=1
out="list-multipart-uploads failed"
fi
keys=$(echo "$out" | jq 'foreach .CommonPrefixes[] as $CommonPrefix (null; . + $CommonPrefix.Prefix)')
if [ $keys != "documents/photos/" ]; then
rv=1
out="list-multipart-uploads failed"
fi
fi

if [ $rv -eq 0 ]; then
function="${AWS} s3api list-multipart-uploads --bucket ${bucket_name} --delimiter / --max-upload 5"
test_function=${function}
out=$($function)
rv=$?
keys=$(echo "$out" | jq 'foreach .Uploads[] as $upload (null; . + $upload.Key)')
if [ $keys != "afilebfilebfileebook" ]; then
rv=1
out="list-multipart-uploads failed"
fi
keys=$(echo "$out" | jq -r '.CommonPrefixes[0].Prefix')
if [ $keys != "documents/" ]; then
rv=1
out="list-multipart-uploads failed"
fi
fi

if [ $rv -eq 0 ]; then
function="${AWS} s3api list-multipart-uploads --bucket ${bucket_name} --prefix documents/"
test_function=${function}
out=$($function)
rv=$?
keys=$(echo "$out" | jq 'foreach .Uploads[] as $upload (null; . + $upload.Key)')
if [ $keys != "documents/report1.pdfdocuments/report2.pdf" ]; then
rv=1
out="list-multipart-upload failed"
fi
fi

if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
Expand Down
58 changes: 51 additions & 7 deletions pkg/gateway/gateway.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -842,12 +842,19 @@ func (n *jfsObjects) ListMultipartUploads(ctx context.Context, bucket string, pr
lmi.KeyMarker = keyMarker
lmi.UploadIDMarker = uploadIDMarker
lmi.MaxUploads = maxUploads
lmi.Delimiter = delimiter
commPrefixSet := make(map[string]struct{})
for _, e := range entries {
uploadID := string(e.Name)
if uploadID > uploadIDMarker {
object_, _ := n.fs.GetXattr(mctx, n.upath(bucket, uploadID), uploadKeyName)
object := string(object_)
if strings.HasPrefix(object, prefix) && object > keyMarker {
// todo: parallel
object_, eno := n.fs.GetXattr(mctx, n.upath(bucket, uploadID), uploadKeyName)
if eno != 0 {
logger.Warnf("get object xattr error %s: %s, ignore this item", n.upath(bucket, uploadID), eno)
continue
}
object := string(object_)
if strings.HasPrefix(object, prefix) {
if keyMarker != "" && object+uploadID > keyMarker+uploadIDMarker || keyMarker == "" {
lmi.Uploads = append(lmi.Uploads, minio.MultipartInfo{
Object: object,
UploadID: uploadID,
Expand All @@ -856,11 +863,48 @@ func (n *jfsObjects) ListMultipartUploads(ctx context.Context, bucket string, pr
}
}
}
if len(lmi.Uploads) > maxUploads {

sort.Slice(lmi.Uploads, func(i, j int) bool {
if lmi.Uploads[i].Object == lmi.Uploads[j].Object {
return lmi.Uploads[i].UploadID < lmi.Uploads[j].UploadID
} else {
return lmi.Uploads[i].Object < lmi.Uploads[j].Object
}
})

if delimiter != "" {
var tmp []minio.MultipartInfo
for _, info := range lmi.Uploads {
if maxUploads == 0 {
lmi.IsTruncated = true
break
}
index := strings.Index(strings.TrimLeft(info.Object, prefix), delimiter)
if index == -1 {
tmp = append(tmp, info)
maxUploads--
} else {
commPrefix := info.Object[:index+1]
if _, ok := commPrefixSet[commPrefix]; ok {
continue
}
commPrefixSet[commPrefix] = struct{}{}
maxUploads--
}
}
lmi.Uploads = tmp
for prefix := range commPrefixSet {
lmi.CommonPrefixes = append(lmi.CommonPrefixes, prefix)
}
sort.Strings(lmi.CommonPrefixes)
} else if len(lmi.Uploads) > maxUploads {
lmi.IsTruncated = true
lmi.Uploads = lmi.Uploads[:maxUploads]
lmi.NextKeyMarker = keyMarker
lmi.NextUploadIDMarker = lmi.Uploads[maxUploads-1].UploadID
}

if len(lmi.Uploads) != 0 {
lmi.NextKeyMarker = lmi.Uploads[len(lmi.Uploads)-1].Object
lmi.NextUploadIDMarker = lmi.Uploads[len(lmi.Uploads)-1].UploadID
}
return lmi, jfsToObjectErr(ctx, err, bucket)
}
Expand Down

0 comments on commit d1dcd95

Please sign in to comment.