From 4b55d4487425f273612d62b74cb967ff66999395 Mon Sep 17 00:00:00 2001
From: zhijian <djx@juicedata.io>
Date: Thu, 21 Sep 2023 20:41:09 +0800
Subject: [PATCH] cmd/gateway: create dst dir path when copy-object (#4058)

Create the parent directory of the target on demand when need
Co-authored-by: Davies Liu <davies@juicedata.io>
---
 integration/s3gateway_test.sh | 13 +++++++++++
 pkg/gateway/gateway.go        | 42 +++++++++++++++++++++++++----------
 2 files changed, 43 insertions(+), 12 deletions(-)

diff --git a/integration/s3gateway_test.sh b/integration/s3gateway_test.sh
index 459e40a6f78a..57c15379c786 100755
--- a/integration/s3gateway_test.sh
+++ b/integration/s3gateway_test.sh
@@ -843,6 +843,19 @@ function test_copy_object() {
         fi
     fi
 
+    if [ $rv -eq 0 ]; then
+        function="${AWS} s3api copy-object --bucket ${bucket_name} --key /not-exist-dir/datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB"
+        test_function=${function}
+        out=$($function)
+        rv=$?
+        hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//')
+        if [ $rv -eq 0 ] && [ "$HASH_1_KB" != "$hash2" ]; then
+            # Verification failed
+            rv=1
+            out="Hash mismatch expected $HASH_1_KB, got $hash2"
+        fi
+    fi
+
     ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
     if [ $rv -eq 0 ]; then
         log_success "$(get_duration "$start_time")" "${test_function}"
diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go
index c2de23af0bde..735e94225151 100644
--- a/pkg/gateway/gateway.go
+++ b/pkg/gateway/gateway.go
@@ -491,8 +491,11 @@ func (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		return n.GetObjectInfo(ctx, srcBucket, srcObject, minio.ObjectOptions{})
 	}
 	tmp := n.tpath(dstBucket, "tmp", minio.MustGetUUID())
-	_ = n.mkdirAll(ctx, path.Dir(tmp))
 	f, eno := n.fs.Create(mctx, tmp, 0666, n.gConf.Umask)
+	if eno == syscall.ENOENT {
+		_ = n.mkdirAll(ctx, path.Dir(tmp))
+		f, eno = n.fs.Create(mctx, tmp, 0666, n.gConf.Umask)
+	}
 	if eno != 0 {
 		logger.Errorf("create %s: %s", tmp, eno)
 		return
@@ -509,6 +512,14 @@ func (n *jfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		return
 	}
 	eno = n.fs.Rename(mctx, tmp, dst, 0)
+	if eno == syscall.ENOENT {
+		if err = n.mkdirAll(ctx, path.Dir(dst)); err != nil {
+			logger.Errorf("mkdirAll %s: %s", path.Dir(dst), err)
+			err = jfsToObjectErr(ctx, err, dstBucket, dstObject)
+			return
+		}
+		eno = n.fs.Rename(mctx, tmp, dst, 0)
+	}
 	if eno != 0 {
 		err = jfsToObjectErr(ctx, eno, srcBucket, srcObject)
 		logger.Errorf("rename %s to %s: %s", tmp, dst, err)
@@ -642,8 +653,11 @@ func (n *jfsObjects) mkdirAll(ctx context.Context, p string) error {
 
 func (n *jfsObjects) putObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (err error) {
 	tmpname := n.tpath(bucket, "tmp", minio.MustGetUUID())
-	_ = n.mkdirAll(ctx, path.Dir(tmpname))
 	f, eno := n.fs.Create(mctx, tmpname, 0666, n.gConf.Umask)
+	if eno == syscall.ENOENT {
+		_ = n.mkdirAll(ctx, path.Dir(tmpname))
+		f, eno = n.fs.Create(mctx, tmpname, 0666, n.gConf.Umask)
+	}
 	if eno != 0 {
 		logger.Errorf("create %s: %s", tmpname, eno)
 		err = eno
@@ -678,13 +692,17 @@ func (n *jfsObjects) putObject(ctx context.Context, bucket, object string, r *mi
 	if err != nil {
 		return
 	}
-	dir := path.Dir(object)
-	if dir != "" {
-		_ = n.mkdirAll(ctx, dir)
+	eno = n.fs.Rename(mctx, tmpname, object, 0)
+	if eno == syscall.ENOENT {
+		if err = n.mkdirAll(ctx, path.Dir(object)); err != nil {
+			logger.Errorf("mkdirAll %s: %s", path.Dir(object), err)
+			err = jfsToObjectErr(ctx, err, bucket, object)
+			return
+		}
+		eno = n.fs.Rename(mctx, tmpname, object, 0)
 	}
-	if eno := n.fs.Rename(mctx, tmpname, object, 0); eno != 0 {
+	if eno != 0 {
 		err = jfsToObjectErr(ctx, eno, bucket, object)
-		return
 	}
 	return
 }
@@ -917,16 +935,16 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
 	}
 
 	name := n.path(bucket, object)
-	dir := path.Dir(name)
-	if dir != "" {
-		if err = n.mkdirAll(ctx, dir); err != nil {
+	eno = n.fs.Rename(mctx, tmp, name, 0)
+	if eno == syscall.ENOENT {
+		if err = n.mkdirAll(ctx, path.Dir(name)); err != nil {
+			logger.Errorf("mkdirAll %s: %s", path.Dir(name), err)
 			_ = n.fs.Delete(mctx, tmp)
 			err = jfsToObjectErr(ctx, err, bucket, object, uploadID)
 			return
 		}
+		eno = n.fs.Rename(mctx, tmp, name, 0)
 	}
-
-	eno = n.fs.Rename(mctx, tmp, name, 0)
 	if eno != 0 {
 		_ = n.fs.Delete(mctx, tmp)
 		err = jfsToObjectErr(ctx, eno, bucket, object, uploadID)