Fix createFile with permission will not work (apache#15556)

HouSong · Feb 3, 2024 · 4d15932 · 4d15932
1 parent 1b42d45
commit 4d15932
Show file tree

Hide file tree

Showing 12 changed files with 72 additions and 125 deletions.
diff --git a/...hinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/...hinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -25,8 +25,6 @@
 import static org.apache.dolphinscheduler.common.constants.Constants.UTF_8;
 import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
 
-import org.apache.dolphinscheduler.common.exception.FileOperateException;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.SystemUtils;
 
@@ -38,21 +36,18 @@
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
-import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.Set;
 import java.util.zip.CRC32;
 import java.util.zip.CheckedInputStream;
 
 import lombok.NonNull;
+import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
-/**
- * file utils
- */
+@UtilityClass
 @Slf4j
 public class FileUtils {
 
@@ -62,14 +57,7 @@ public class FileUtils {
 
     public static final String KUBE_CONFIG_FILE = "config";
 
-    private static final String RWXR_XR_X = "rwxr-xr-x";
-
-    private static final FileAttribute<Set<PosixFilePermission>> PERMISSION_755 =
-            PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(RWXR_XR_X));
-
-    private FileUtils() {
-        throw new UnsupportedOperationException("Construct FileUtils");
-    }
+    private static final Set<PosixFilePermission> PERMISSION_755 = PosixFilePermissions.fromString("rwxr-xr-x");
 
     /**
      * get download file absolute path and name
@@ -161,34 +149,6 @@ public static String getResourceViewSuffixes() {
         return PropertyUtils.getString(RESOURCE_VIEW_SUFFIXES, RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE);
     }
 
-    /**
-     * create directory if absent
-     *
-     * @param execLocalPath execute local path
-     * @throws IOException errors
-     */
-    public static void createWorkDirIfAbsent(String execLocalPath) throws IOException {
-        // if work dir exists, first delete
-        File execLocalPathFile = new File(execLocalPath);
-
-        if (execLocalPathFile.exists()) {
-            try {
-                org.apache.commons.io.FileUtils.forceDelete(execLocalPathFile);
-            } catch (Exception ex) {
-                if (ex instanceof NoSuchFileException || ex.getCause() instanceof NoSuchFileException) {
-                    // this file is already be deleted.
-                } else {
-                    throw ex;
-                }
-            }
-        }
-
-        // create work dir
-        org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile);
-        String mkdirLog = "create dir success " + execLocalPath;
-        log.info(mkdirLog);
-    }
-
     /**
      * write content to file ,if parent path not exists, it will do one's utmost to mkdir
      *
@@ -231,25 +191,6 @@ public static void deleteFile(String filename) {
         org.apache.commons.io.FileUtils.deleteQuietly(new File(filename));
     }
 
-    /**
-     * Gets all the parent subdirectories of the parentDir directory
-     *
-     * @param parentDir parent dir
-     * @return all dirs
-     */
-    public static File[] getAllDir(String parentDir) {
-        if (parentDir == null || "".equals(parentDir)) {
-            throw new RuntimeException("parentDir can not be empty");
-        }
-
-        File file = new File(parentDir);
-        if (!file.exists() || !file.isDirectory()) {
-            throw new RuntimeException("parentDir not exist, or is not a directory:" + parentDir);
-        }
-
-        return file.listFiles(File::isDirectory);
-    }
-
     /**
      * Get Content
      *
@@ -325,59 +266,47 @@ public static String getFileChecksum(String pathName) throws IOException {
         return crcString;
     }
 
-    public static void setFileOwner(Path filePath, String fileOwner) throws FileOperateException {
-        try {
-            // We use linux command to set the file owner, since jdk api will not use sudo.
-            String command = String.format("sudo chown %s %s", fileOwner, filePath.toString());
-            Runtime.getRuntime().exec(command);
-            Process process = Runtime.getRuntime().exec(command);
-            int exitCode = process.waitFor();
-            if (0 != exitCode) {
-                throw new FileOperateException(
-                        "Set file: " + filePath + " to owner: " + fileOwner + " failed, existCode(" + exitCode + ")");
-            }
-        } catch (FileOperateException ex) {
-            throw ex;
-        } catch (Exception ex) {
-            throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner + " failed");
-
+    public static void createFileWith755(@NonNull Path path) throws IOException {
+        if (SystemUtils.IS_OS_WINDOWS) {
+            Files.createFile(path);
+        } else {
+            Files.createFile(path);
+            Files.setPosixFilePermissions(path, PERMISSION_755);
         }
     }
 
-    public static void setDirectoryOwner(Path filePath, String fileOwner) throws FileOperateException {
-        try {
-            // We use linux command to set the file owner, since jdk api will not use sudo.
-            String command = String.format("sudo chown -R %s %s", fileOwner, filePath.toString());
-            Runtime.getRuntime().exec(command);
-            Process process = Runtime.getRuntime().exec(command);
-            int exitCode = process.waitFor();
-            if (0 != exitCode) {
-                throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner
-                        + " failed, existCode(" + exitCode + ")");
+    public static void createDirectoryWith755(@NonNull Path path) throws IOException {
+        if (path.toFile().exists()) {
+            return;
+        }
+        if (OSUtils.isWindows()) {
+            Files.createDirectories(path);
+        } else {
+            Path parent = path.getParent();
+            if (parent != null && !parent.toFile().exists()) {
+                createDirectoryWith755(parent);
             }
-        } catch (FileOperateException ex) {
-            throw ex;
-        } catch (Exception ex) {
-            throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner + " failed");
+
+            Files.createDirectory(path);
+            Files.setPosixFilePermissions(path, PERMISSION_755);
 
         }
     }
 
-    public static void createDirectoryIfNotPresent(Path path) throws IOException {
-        if (Files.exists(path)) {
+    public static void setFileTo755(File file) throws IOException {
+        if (OSUtils.isWindows()) {
             return;
         }
-        Files.createDirectories(path);
-    }
-
-    /**
-     * Create a file with '755'.
-     */
-    public static void createFileWith755(@NonNull Path path) throws IOException {
-        if (SystemUtils.IS_OS_WINDOWS) {
-            Files.createFile(path);
-        } else {
-            Files.createFile(path, PERMISSION_755);
+        if (file.isFile()) {
+            Files.setPosixFilePermissions(file.toPath(), PERMISSION_755);
+            return;
+        }
+        Files.setPosixFilePermissions(file.toPath(), PERMISSION_755);
+        File[] files = file.listFiles();
+        if (files != null) {
+            for (File f : files) {
+                setFileTo755(f);
+            }
         }
     }
 

diff --git a/...cheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/...cheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
@@ -19,8 +19,13 @@
 
 import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
 
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
@@ -55,12 +60,23 @@ public void testGetProcessExecDir() {
     }
 
     @Test
-    public void testCreateWorkDirIfAbsent() {
+    public void createDirectoryWith755() throws IOException {
+        Path path = Paths.get("/tmp/createWorkDirAndUserIfAbsent");
         try {
-            FileUtils.createWorkDirIfAbsent("/tmp/createWorkDirAndUserIfAbsent");
-            Assertions.assertTrue(true);
+            FileUtils.createDirectoryWith755(path);
+            File file = path.toFile();
+            Assertions.assertTrue(file.exists());
+            Assertions.assertTrue(file.isDirectory());
+            Assertions.assertTrue(file.canExecute());
+            Assertions.assertTrue(file.canRead());
+            Assertions.assertTrue(file.canWrite());
+
+            FileUtils.createDirectoryWith755(Paths.get("/"));
         } catch (Exception e) {
-            Assertions.fail();
+            e.printStackTrace();
+            Assertions.fail(e.getMessage());
+        } finally {
+            Files.deleteIfExists(path);
         }
     }
 

diff --git a/...heduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java b/...heduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java
@@ -30,6 +30,7 @@
 
 import java.io.File;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -176,7 +177,7 @@ public static String getPidsStr(int processId) throws Exception {
                                     taskExecutionContext.getProcessInstanceId(),
                                     taskExecutionContext.getTaskInstanceId()));
                 }
-                FileUtils.createWorkDirIfAbsent(taskExecutionContext.getExecutePath());
+                FileUtils.createDirectoryWith755(Paths.get(taskExecutionContext.getExecutePath()));
                 org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils.cancelApplication(taskExecutionContext);
                 return appIds;
             } else {

diff --git a/...-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java b/...-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java
@@ -25,6 +25,7 @@
 
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -170,7 +171,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite)
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            Files.createDirectories(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
         }
 
         BlobClient blobClient = blobContainerClient.getBlobClient(srcFilePath);

diff --git a/...-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java b/...-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java
@@ -25,6 +25,7 @@
 
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -143,7 +144,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite)
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            Files.createDirectories(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
         }
 
         Blob blob = gcsStorage.get(BlobId.of(bucketName, srcFilePath));

diff --git a/...-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java b/...-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java
@@ -24,6 +24,7 @@
 
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -187,7 +188,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite)
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            Files.createDirectories(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
         }
         ObsObject obsObject = obsClient.getObject(bucketName, srcFilePath);
         try (

diff --git a/...-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java b/...-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
@@ -26,6 +26,7 @@
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
 import org.apache.dolphinscheduler.common.factory.OssClientFactory;
 import org.apache.dolphinscheduler.common.model.OssConnection;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -213,7 +214,7 @@ public void download(String srcFilePath, String dstFilePath,
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            Files.createDirectories(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
         }
         OSSObject ossObject = ossClient.getObject(bucketName, srcFilePath);
         try (

diff --git a/...age-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/...age-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java
@@ -25,6 +25,7 @@
 
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -200,7 +201,7 @@ public void download(String srcFilePath, String dstFilePath,
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            Files.createDirectories(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
         }
         S3Object o = s3Client.getObject(bucketName, srcFilePath);
         try (

diff --git a/...a/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java b/...a/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java
@@ -17,8 +17,6 @@
 
 package org.apache.dolphinscheduler.plugin.task.api.shell;
 
-import org.apache.dolphinscheduler.common.constants.TenantConstants;
-import org.apache.dolphinscheduler.common.exception.FileOperateException;
 import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.task.api.utils.AbstractCommandExecutorConstants;
@@ -71,12 +69,8 @@ protected void generateShellScript() throws IOException {
                 "****************************** Script Content *****************************************************************");
     }
 
-    protected List<String> generateBootstrapCommand() throws FileOperateException {
+    protected List<String> generateBootstrapCommand() {
         if (sudoEnable) {
-            if (!TenantConstants.BOOTSTRAPT_SYSTEM_USER.equals(runUser)) {
-                // Set the tenant owner as the working directory
-                FileUtils.setDirectoryOwner(Paths.get(shellDirectory), runUser);
-            }
             return bootstrapCommandInSudoMode();
         }
         return bootstrapCommandInNormalMode();

diff --git a/...er-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java b/...er-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java
@@ -22,6 +22,7 @@
 import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR;
 import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAVA;
 
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@@ -155,7 +156,7 @@ public void coverJavaSourceFileExistException() throws IOException {
             try {
                 Path path = Paths.get(fileName);
                 if (!Files.exists(path)) {
-                    Files.createDirectories(path);
+                    FileUtils.createDirectoryWith755(path);
                 }
                 javaTask.createJavaSourceFileIfNotExists(sourceCode, fileName);
             } finally {