add udf yarn-application; (#1131)

change code style

dlink-admin/src/main/java/com/dlink/service/UDFService.java

@@ -19,11 +19,13 @@
 
 package com.dlink.service;
 
+import com.dlink.gateway.GatewayType;
+
 /**
  * @author ZackYoung
  * @since 0.6.8
  */
 public interface UDFService {
 
-    String[] initUDF(String statement);
+    String[] initUDF(String statement, GatewayType gatewayType);
 }

dlink-admin/src/main/java/com/dlink/service/impl/TaskServiceImpl.java

@@ -202,7 +202,7 @@ public class TaskServiceImpl extends SuperServiceImpl<TaskMapper, Task> implemen
                     task.getDatabaseId(), null));
         }
         JobConfig config = buildJobConfig(task);
-        config.setJarFiles(udfService.initUDF(task.getStatement()));
+        config.setJarFiles(udfService.initUDF(task.getStatement(), config.getGatewayConfig().getType()));
         JobManager jobManager = JobManager.build(config);
         if (!config.isJarTask()) {
             return jobManager.executeSql(task.getStatement());

dlink-admin/src/main/java/com/dlink/service/impl/UDFServiceImpl.java

@@ -20,6 +20,7 @@
 package com.dlink.service.impl;
 
 import com.dlink.constant.PathConstant;
+import com.dlink.exception.BusException;
 import com.dlink.gateway.GatewayType;
 import com.dlink.job.JobManager;
 import com.dlink.model.Task;
@@ -53,15 +54,15 @@ public class UDFServiceImpl implements UDFService {
      * 快速获取 session 与 application 等类型，为了减少判断
      */
     private static final Map<String, List<GatewayType>> GATEWAY_TYPE_MAP = MapUtil
-            .builder("session",
-                    Arrays.asList(GatewayType.YARN_SESSION, GatewayType.KUBERNETES_SESSION, GatewayType.STANDALONE))
-            .build();
+        .builder("session",
+            Arrays.asList(GatewayType.YARN_SESSION, GatewayType.KUBERNETES_SESSION, GatewayType.STANDALONE))
+        .build();
 
     @Resource
     TaskService taskService;
 
     @Override
-    public String[] initUDF(String statement) {
+    public String[] initUDF(String statement, GatewayType gatewayType) {
         ProcessEntity process = ProcessContextHolder.getProcess();
         process.info("Initializing Flink UDF...Start");
         Opt<String> udfJarPath = Opt.empty();
@@ -73,10 +74,14 @@ public class UDFServiceImpl implements UDFService {
         }, true);
         if (codeList.size() > 0) {
             udfJarPath = Opt.ofBlankAble(UDFUtil.getUdfNameAndBuildJar(codeList));
+        } else {
+            if (gatewayType == GatewayType.KUBERNETES_APPLICATION) {
+                throw new BusException("udf 暂不支持k8s application");
+            }
         }
         process.info("Initializing Flink UDF...Finish");
         if (udfJarPath.isPresent()) {
-            return new String[]{PathConstant.UDF_PATH + udfJarPath.get()};
+            return new String[] {PathConstant.UDF_PATH + udfJarPath.get()};
         } else {
             return new String[0];
         }

dlink-core/src/main/java/com/dlink/job/JobManager.java

@@ -72,20 +72,15 @@ import org.apache.flink.yarn.configuration.YarnConfigOptions;
 
 import java.time.LocalDateTime;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
-import cn.hutool.core.io.FileUtil;
-import cn.hutool.core.util.URLUtil;
-
 /**
  * JobManager
  *
@@ -166,6 +161,7 @@ public class JobManager {
         JobManager manager = new JobManager(config);
         manager.init();
         manager.executor.initUDF(config.getJarFiles());
+        config.getGatewayConfig().setJarPaths(config.getJarFiles());
         return manager;
     }
 
@@ -398,7 +394,6 @@ public class JobManager {
                             jobGraph.setSavepointRestoreSettings(
                                     SavepointRestoreSettings.forPath(config.getSavePointPath(), true));
                         }
-                        jobGraph.addJars(Arrays.stream(config.getJarFiles()).map(path -> URLUtil.getURL(FileUtil.file(path))).collect(Collectors.toList()));
                         gatewayResult = Gateway.build(config.getGatewayConfig()).submitJobGraph(jobGraph);
                     }
                     job.setResult(InsertResult.success(gatewayResult.getAppId()));
@@ -461,7 +456,6 @@ public class JobManager {
                 jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(config.getSavePointPath(), true));
             }
             // Perjob mode need to submit job graph.
-            jobGraph.addJars(Arrays.stream(config.getJarFiles()).map(path -> URLUtil.getURL(FileUtil.file(path))).collect(Collectors.toList()));
             gatewayResult = Gateway.build(config.getGatewayConfig()).submitJobGraph(jobGraph);
         }
         return gatewayResult;

dlink-gateway/src/main/java/com/dlink/gateway/config/GatewayConfig.java

@@ -42,6 +42,7 @@ import lombok.Setter;
 public class GatewayConfig {
 
     private Integer taskId;
+    private String[] jarPaths;
     private GatewayType type;
     private ClusterConfig clusterConfig;
     private FlinkConfig flinkConfig;

dlink-gateway/src/main/java/com/dlink/gateway/yarn/YarnApplicationGateway.java

@@ -43,9 +43,13 @@ import org.apache.flink.yarn.YarnClusterDescriptor;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
+
+import cn.hutool.core.io.FileUtil;
 
 /**
  * YarnApplicationGateway
@@ -98,6 +102,7 @@ public class YarnApplicationGateway extends YarnGateway {
         if (configuration.contains(TaskManagerOptions.NUM_TASK_SLOTS)) {
             clusterSpecificationBuilder.setSlotsPerTaskManager(configuration.get(TaskManagerOptions.NUM_TASK_SLOTS)).createClusterSpecification();
         }
+        yarnClusterDescriptor.addShipFiles(Arrays.stream(config.getJarPaths()).map(FileUtil::file).collect(Collectors.toList()));
 
         try {
             ClusterClientProvider<ApplicationId> clusterClientProvider = yarnClusterDescriptor.deployApplicationCluster(

dlink-gateway/src/main/java/com/dlink/gateway/yarn/YarnPerJobGateway.java

@@ -40,8 +40,13 @@ import org.apache.flink.yarn.YarnClusterDescriptor;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.stream.Collectors;
+
+import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.util.URLUtil;
 
 /**
  * YarnApplicationGateway
@@ -83,6 +88,8 @@ public class YarnPerJobGateway extends YarnGateway {
             clusterSpecificationBuilder.setSlotsPerTaskManager(configuration.get(TaskManagerOptions.NUM_TASK_SLOTS)).createClusterSpecification();
         }
 
+        jobGraph.addJars(Arrays.stream(config.getJarPaths()).map(path -> URLUtil.getURL(FileUtil.file(path))).collect(Collectors.toList()));
+
         try {
             ClusterClientProvider<ApplicationId> clusterClientProvider = yarnClusterDescriptor.deployJobCluster(
                 clusterSpecificationBuilder.createClusterSpecification(), jobGraph, true);