[Taier_244][Taier-spark] fix sqlProxy class name

Author: vainhope

build/mvn-build.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 echo 'taier Building...'
 
-mvn clean package -DskipTests  -pl \
+mvn clean package -DskipTests -T 1C -pl \
 taier-data-develop,\
 taier-worker/taier-plugins/dummy,\
 taier-worker/taier-plugins/flink/common,\

taier-worker/taier-plugins/spark/yarn2-hdfs2-spark210-core/spark-yarn-client-core/src/main/java/com/dtstack/taier/sparkyarn/sparkext/CarbondataClientExt.java

@@ -36,13 +36,8 @@ public class ClientExtFactory {
     public static ClientExt getClientExt(FilesystemManager filesystemManager,
                                          ClientArguments args,
                                          Configuration hadoopConf,
-                                         SparkConf sparkConf,
-                                         boolean isCarbondata){
+                                         SparkConf sparkConf){
 
-        if(!isCarbondata){
-            return new ClientExt(filesystemManager, args, hadoopConf, sparkConf);
-        }else{
-            return new CarbondataClientExt(filesystemManager, args, hadoopConf, sparkConf);
-        }
+        return new ClientExt(filesystemManager, args, hadoopConf, sparkConf);
     }
 }

taier-worker/taier-plugins/spark/yarn2-hdfs2-spark210-core/spark-yarn-client-core/src/main/java/com/dtstack/taier/sparkyarn/sparkext/ClientExtFactory.java

@@ -23,12 +23,8 @@
 import com.dtstack.taier.base.monitor.AcceptedApplicationMonitor;
 import com.dtstack.taier.base.util.HadoopConfTool;
 import com.dtstack.taier.base.util.KerberosUtils;
-import com.dtstack.taier.pluginapi.CustomThreadFactory;
-import com.dtstack.taier.pluginapi.JarFileInfo;
-import com.dtstack.taier.pluginapi.JobClient;
-import com.dtstack.taier.pluginapi.JobIdentifier;
-import com.dtstack.taier.pluginapi.JobParam;
 import com.dtstack.taier.base.util.Splitter;
+import com.dtstack.taier.pluginapi.*;
 import com.dtstack.taier.pluginapi.client.AbstractClient;
 import com.dtstack.taier.pluginapi.enums.ComputeType;
 import com.dtstack.taier.pluginapi.enums.EJobType;
@@ -65,20 +61,13 @@
 import org.apache.spark.deploy.yarn.ClientArguments;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import sun.misc.BASE64Decoder;
 
 import java.io.File;
 import java.io.IOException;
 import java.net.URLDecoder;
 import java.net.URLEncoder;
 import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -90,8 +79,6 @@ public class SparkYarnClient extends AbstractClient {
 
     private static final Logger logger = LoggerFactory.getLogger(SparkYarnClient.class);
 
-    private static final BASE64Decoder DECODER = new BASE64Decoder();
-
     private static final String HADOOP_USER_NAME = "HADOOP_USER_NAME";
 
     private static final String SPARK_YARN_MODE = "SPARK_YARN_MODE";
@@ -227,9 +214,6 @@ private JobResult submitJobWithJar(JobClient jobClient){
             appArgs = exeArgsStr.split("\\s+");
         }
 
-        Properties confProp = jobClient.getConfProperties();
-        Boolean isCarbonSpark = MathUtil.getBoolean(confProp.get(IS_CARBON_SPARK_KEY), false);
-
         List<String> argList = new ArrayList<>();
         argList.add("--jar");
         argList.add(jarPath);
@@ -254,7 +238,7 @@ private JobResult submitJobWithJar(JobClient jobClient){
         ApplicationId appId = null;
 
         try {
-            ClientExt clientExt = ClientExtFactory.getClientExt(filesystemManager, clientArguments, yarnConf, sparkConf, isCarbonSpark);
+            ClientExt clientExt = ClientExtFactory.getClientExt(filesystemManager, clientArguments, yarnConf, sparkConf);
             clientExt.setSparkYarnConfig(sparkYarnConfig);
             String proxyUserName = sparkYarnConfig.getDtProxyUserName();
             if (StringUtils.isNotBlank(proxyUserName)) {
@@ -419,8 +403,6 @@ private void addEnv2SparkConf(String cmdStr, SparkConf sparkConf) {
     private JobResult submitSparkSqlJobForBatch(JobClient jobClient){
 
         Properties confProp = jobClient.getConfProperties();
-        Boolean isCarbonSpark = MathUtil.getBoolean(confProp.get(IS_CARBON_SPARK_KEY), false);
-
         setHadoopUserName(sparkYarnConfig);
         Map<String, Object> paramsMap = new HashMap<>();
 
@@ -434,9 +416,6 @@ private JobResult submitSparkSqlJobForBatch(JobClient jobClient){
             paramsMap.put("logLevel", logLevel);
         }
 
-        if(isCarbonSpark){
-            paramsMap.put("storePath", sparkYarnConfig.getCarbonStorePath());
-        }
 
         String sqlExeJson = null;
         try{
@@ -448,9 +427,6 @@ private JobResult submitSparkSqlJobForBatch(JobClient jobClient){
         }
 
         String sqlProxyClass = sparkYarnConfig.getSparkSqlProxyMainClass();
-        if(isCarbonSpark){
-            sqlProxyClass = SparkYarnConfig.DEFAULT_CARBON_SQL_PROXY_MAINCLASS;
-        }
 
         List<String> argList = new ArrayList<>();
         argList.add("--jar");
@@ -470,7 +446,7 @@ private JobResult submitSparkSqlJobForBatch(JobClient jobClient){
         ApplicationId appId = null;
 
         try {
-            ClientExt clientExt = ClientExtFactory.getClientExt(filesystemManager, clientArguments, yarnConf, sparkConf, isCarbonSpark);
+            ClientExt clientExt = ClientExtFactory.getClientExt(filesystemManager, clientArguments, yarnConf, sparkConf);
             clientExt.setSparkYarnConfig(sparkYarnConfig);
             String proxyUserName = sparkYarnConfig.getDtProxyUserName();
             if (StringUtils.isNotBlank(proxyUserName)) {

taier-worker/taier-plugins/spark/yarn2-hdfs2-spark210-core/spark-yarn-client-core/src/main/java/com/dtstack/taier/sparkyarn/sparkyarn/SparkYarnClient.java

@@ -37,7 +37,7 @@ public class SparkYarnConfig extends BaseConfig {
 
     private static final String DEFAULT_SPARK_PYTHON_EXTLIBPATH = "%s/pythons/pyspark.zip,/pythons/py4j-0.10.4-src.zip";
 
-    private static final String DEFAULT_SPARK_SQL_PROXY_MAINCLASS = "com.dtstack.sql.main.SqlProxy";
+    private static final String DEFAULT_SPARK_SQL_PROXY_MAINCLASS = "com.dtstack.taier.sql.main.SqlProxy";
 
     public static final String DEFAULT_CARBON_SQL_PROXY_MAINCLASS = "com.dtstack.sql.main.CarbondataSqlProxy";
 

taier-worker/taier-plugins/spark/yarn2-hdfs2-spark210-core/spark-yarn-client-core/src/main/java/com/dtstack/taier/sparkyarn/sparkyarn/SparkYarnConfig.java

@@ -24,7 +24,7 @@ A: 确认Taier目录下 是否有pluginLibs目录
 ### Q: 添加数据源失败
 A: 确认Taier目录下application.properties datasource.plugin.path是否配置正确
 
-### Q: 获取目录失败,清联系管理员
+### Q: 获取目录失败,请联系管理员
 A: 是否正确配置集群，集群是否正确获取到了队列，租户和集群是否绑定成功 参考[快速上手](./quickstart/start.md)
 
 ### Q: 页面访问报错, 无法登陆用户
@@ -33,4 +33,10 @@ A: 确认下前端配置的后端接口是否正确 参考[快速上手](./quick
 ### Q:绑定新租户时初始化时提示数据库已存在
 A: 绑定租户新增schema选择创建 绑定租户已有schema直接选择对接 
 
+### Q:spark sql任务执行提示class not found
+A: 编译完对应的spark210插件 需要将对应的sqlProxy的jar包 放到对应集群spark组件下sparkSqlProxyPath路径下
+
+### Q:spark sql任务执行提示sftp downloadDir error
+A: 确认下sftp组件配置是否正确 sftp组件配置路径 + confPath + 集群名称 是否有对应的xml配置文件
+