[update][plugin] Refactor: take the shell execute into a single commo…

…n util class

common/pom.xml

@@ -30,6 +30,12 @@
             <version>1.0.6</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-exec</artifactId>
+            <version>1.4.0</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>

common/src/main/java/com/wgzhao/addax/common/base/Key.java

@@ -149,6 +149,10 @@ public class Key
     public static final String EXTENDED_INSERT = "extendedInsert";
     public static final String TABLE_NUMBER= "tableNumber";
 
+    public static final String PRE_SHELL = "preShell";
+    public static final String POST_SHELL = "postShell";
+    public static final String IGNORE_ERROR = "ignoreError";
+
     public Key()
     {
 

common/src/main/java/com/wgzhao/addax/common/util/ShellUtil.java

@@ -0,0 +1,65 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package com.wgzhao.addax.common.util;
+
+import com.wgzhao.addax.common.exception.AddaxException;
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.exec.DefaultExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL;
+import static com.wgzhao.addax.common.spi.ErrorCode.RUNTIME_ERROR;
+
+public class ShellUtil
+{
+    private static final Logger LOG = LoggerFactory.getLogger(ShellUtil.class);
+
+    public static void exec(String command, boolean ignoreError)
+    {
+        CommandLine cmdLine = CommandLine.parse(command);
+        DefaultExecutor executor = DefaultExecutor.builder().get();
+        LOG.info("Running command: {}", command);
+        try {
+            int ret = executor.execute(cmdLine);
+            if (ret != 0) {
+                if (ignoreError) {
+                    LOG.warn("Command failed with return code: {}", ret);
+                }
+                else {
+                    throw AddaxException.asAddaxException(EXECUTE_FAIL, "Command failed with return code: " + ret);
+                }
+            }
+        }
+        catch (Exception e) {
+            if (ignoreError) {
+                LOG.warn("Error running command: {}", command, e);
+            }
+            else {
+                throw AddaxException.asAddaxException(RUNTIME_ERROR, e);
+            }
+        }
+    }
+
+    public static void exec(String command)
+    {
+        exec(command, false);
+    }
+}

plugin/writer/hdfswriter/pom.xml

@@ -64,12 +64,6 @@
             </exclusions>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-exec</artifactId>
-            <version>1.4.0</version>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>

plugin/writer/hdfswriter/src/main/java/com/wgzhao/addax/plugin/writer/hdfswriter/HdfsWriter.java

@@ -25,6 +25,7 @@
 import com.wgzhao.addax.common.plugin.RecordReceiver;
 import com.wgzhao.addax.common.spi.Writer;
 import com.wgzhao.addax.common.util.Configuration;
+import com.wgzhao.addax.common.util.ShellUtil;
 import com.wgzhao.addax.storage.util.FileHelper;
 import org.apache.commons.exec.DefaultExecutor;
 import org.apache.commons.io.Charsets;
@@ -48,6 +49,9 @@
 
 import org.apache.commons.exec.CommandLine;
 
+import static com.wgzhao.addax.common.base.Key.IGNORE_ERROR;
+import static com.wgzhao.addax.common.base.Key.POST_SHELL;
+import static com.wgzhao.addax.common.base.Key.PRE_SHELL;
 import static com.wgzhao.addax.common.spi.ErrorCode.EXECUTE_FAIL;
 import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE;
 import static com.wgzhao.addax.common.spi.ErrorCode.REQUIRED_VALUE;
@@ -209,10 +213,11 @@ private void validateParameter()
         public void prepare()
         {
             // check preShell item
-            List<String> preShells = this.writerSliceConfig.getList("preShell", String.class);
+            List<String> preShells = this.writerSliceConfig.getList(PRE_SHELL, String.class);
+            boolean ignore = this.writerSliceConfig.getBool(IGNORE_ERROR, false);
             if (!preShells.isEmpty()) {
                 for (String preShell : preShells) {
-                    execShell(preShell);
+                    ShellUtil.exec(preShell, ignore);
                 }
             }
 
@@ -263,10 +268,11 @@ public void post()
             hdfsHelper.deleteDir(new Path(tmpStorePath));
 
             //check postShell item
-            List<String> postShells = this.writerSliceConfig.getList("postShell", String.class);
+            List<String> postShells = this.writerSliceConfig.getList(POST_SHELL, String.class);
+            boolean ignore = this.writerSliceConfig.getBool(IGNORE_ERROR, false);
             if (!postShells.isEmpty()) {
                 for (String postShell : postShells) {
-                    execShell(postShell);
+                    ShellUtil.exec(postShell, ignore);
                 }
             }
         }
@@ -385,22 +391,6 @@ private static int getDecimalScale(String type)
                 return Integer.parseInt(type.split(",")[1].replace(")", "").trim());
             }
         }
-
-        private static void execShell(String command)
-        {
-            CommandLine cmdLine = CommandLine.parse(command);
-            DefaultExecutor executor = DefaultExecutor.builder().get();
-            LOG.info("Running command: {}", command);
-            try {
-                int retCode = executor.execute(cmdLine);
-                if (retCode != 0) {
-                    throw AddaxException.asAddaxException(EXECUTE_FAIL, String.format("Command [%s] exited with code %d", command, retCode));
-                }
-            }
-            catch (Exception e) {
-                throw AddaxException.asAddaxException(RUNTIME_ERROR, e);
-            }
-        }
     }
 
     public static class Task

plugin/writer/hdfswriter/src/main/resources/plugin_job_template.json

@@ -7,6 +7,7 @@
     "fileName": "addax",
     "preShell": [],
     "postShell": [],
+    "ignoreError": true,
     "column": [
       {
         "name": "col1",