Unverified Commit f53833af authored by ZackYoung's avatar ZackYoung Committed by GitHub

add udf yarn-application; (#1131)

change code style
parent 6ba6cdb0
...@@ -19,11 +19,13 @@ ...@@ -19,11 +19,13 @@
package com.dlink.service; package com.dlink.service;
import com.dlink.gateway.GatewayType;
/** /**
* @author ZackYoung * @author ZackYoung
* @since 0.6.8 * @since 0.6.8
*/ */
public interface UDFService { public interface UDFService {
String[] initUDF(String statement); String[] initUDF(String statement, GatewayType gatewayType);
} }
...@@ -202,7 +202,7 @@ public class TaskServiceImpl extends SuperServiceImpl<TaskMapper, Task> implemen ...@@ -202,7 +202,7 @@ public class TaskServiceImpl extends SuperServiceImpl<TaskMapper, Task> implemen
task.getDatabaseId(), null)); task.getDatabaseId(), null));
} }
JobConfig config = buildJobConfig(task); JobConfig config = buildJobConfig(task);
config.setJarFiles(udfService.initUDF(task.getStatement())); config.setJarFiles(udfService.initUDF(task.getStatement(), config.getGatewayConfig().getType()));
JobManager jobManager = JobManager.build(config); JobManager jobManager = JobManager.build(config);
if (!config.isJarTask()) { if (!config.isJarTask()) {
return jobManager.executeSql(task.getStatement()); return jobManager.executeSql(task.getStatement());
......
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
package com.dlink.service.impl; package com.dlink.service.impl;
import com.dlink.constant.PathConstant; import com.dlink.constant.PathConstant;
import com.dlink.exception.BusException;
import com.dlink.gateway.GatewayType; import com.dlink.gateway.GatewayType;
import com.dlink.job.JobManager; import com.dlink.job.JobManager;
import com.dlink.model.Task; import com.dlink.model.Task;
...@@ -53,15 +54,15 @@ public class UDFServiceImpl implements UDFService { ...@@ -53,15 +54,15 @@ public class UDFServiceImpl implements UDFService {
* 快速获取 session 与 application 等类型,为了减少判断 * 快速获取 session 与 application 等类型,为了减少判断
*/ */
private static final Map<String, List<GatewayType>> GATEWAY_TYPE_MAP = MapUtil private static final Map<String, List<GatewayType>> GATEWAY_TYPE_MAP = MapUtil
.builder("session", .builder("session",
Arrays.asList(GatewayType.YARN_SESSION, GatewayType.KUBERNETES_SESSION, GatewayType.STANDALONE)) Arrays.asList(GatewayType.YARN_SESSION, GatewayType.KUBERNETES_SESSION, GatewayType.STANDALONE))
.build(); .build();
@Resource @Resource
TaskService taskService; TaskService taskService;
@Override @Override
public String[] initUDF(String statement) { public String[] initUDF(String statement, GatewayType gatewayType) {
ProcessEntity process = ProcessContextHolder.getProcess(); ProcessEntity process = ProcessContextHolder.getProcess();
process.info("Initializing Flink UDF...Start"); process.info("Initializing Flink UDF...Start");
Opt<String> udfJarPath = Opt.empty(); Opt<String> udfJarPath = Opt.empty();
...@@ -73,10 +74,14 @@ public class UDFServiceImpl implements UDFService { ...@@ -73,10 +74,14 @@ public class UDFServiceImpl implements UDFService {
}, true); }, true);
if (codeList.size() > 0) { if (codeList.size() > 0) {
udfJarPath = Opt.ofBlankAble(UDFUtil.getUdfNameAndBuildJar(codeList)); udfJarPath = Opt.ofBlankAble(UDFUtil.getUdfNameAndBuildJar(codeList));
} else {
if (gatewayType == GatewayType.KUBERNETES_APPLICATION) {
throw new BusException("udf 暂不支持k8s application");
}
} }
process.info("Initializing Flink UDF...Finish"); process.info("Initializing Flink UDF...Finish");
if (udfJarPath.isPresent()) { if (udfJarPath.isPresent()) {
return new String[]{PathConstant.UDF_PATH + udfJarPath.get()}; return new String[] {PathConstant.UDF_PATH + udfJarPath.get()};
} else { } else {
return new String[0]; return new String[0];
} }
......
...@@ -72,20 +72,15 @@ import org.apache.flink.yarn.configuration.YarnConfigOptions; ...@@ -72,20 +72,15 @@ import org.apache.flink.yarn.configuration.YarnConfigOptions;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.URLUtil;
/** /**
* JobManager * JobManager
* *
...@@ -166,6 +161,7 @@ public class JobManager { ...@@ -166,6 +161,7 @@ public class JobManager {
JobManager manager = new JobManager(config); JobManager manager = new JobManager(config);
manager.init(); manager.init();
manager.executor.initUDF(config.getJarFiles()); manager.executor.initUDF(config.getJarFiles());
config.getGatewayConfig().setJarPaths(config.getJarFiles());
return manager; return manager;
} }
...@@ -398,7 +394,6 @@ public class JobManager { ...@@ -398,7 +394,6 @@ public class JobManager {
jobGraph.setSavepointRestoreSettings( jobGraph.setSavepointRestoreSettings(
SavepointRestoreSettings.forPath(config.getSavePointPath(), true)); SavepointRestoreSettings.forPath(config.getSavePointPath(), true));
} }
jobGraph.addJars(Arrays.stream(config.getJarFiles()).map(path -> URLUtil.getURL(FileUtil.file(path))).collect(Collectors.toList()));
gatewayResult = Gateway.build(config.getGatewayConfig()).submitJobGraph(jobGraph); gatewayResult = Gateway.build(config.getGatewayConfig()).submitJobGraph(jobGraph);
} }
job.setResult(InsertResult.success(gatewayResult.getAppId())); job.setResult(InsertResult.success(gatewayResult.getAppId()));
...@@ -461,7 +456,6 @@ public class JobManager { ...@@ -461,7 +456,6 @@ public class JobManager {
jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(config.getSavePointPath(), true)); jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(config.getSavePointPath(), true));
} }
// Perjob mode need to submit job graph. // Perjob mode need to submit job graph.
jobGraph.addJars(Arrays.stream(config.getJarFiles()).map(path -> URLUtil.getURL(FileUtil.file(path))).collect(Collectors.toList()));
gatewayResult = Gateway.build(config.getGatewayConfig()).submitJobGraph(jobGraph); gatewayResult = Gateway.build(config.getGatewayConfig()).submitJobGraph(jobGraph);
} }
return gatewayResult; return gatewayResult;
......
...@@ -42,6 +42,7 @@ import lombok.Setter; ...@@ -42,6 +42,7 @@ import lombok.Setter;
public class GatewayConfig { public class GatewayConfig {
private Integer taskId; private Integer taskId;
private String[] jarPaths;
private GatewayType type; private GatewayType type;
private ClusterConfig clusterConfig; private ClusterConfig clusterConfig;
private FlinkConfig flinkConfig; private FlinkConfig flinkConfig;
......
...@@ -43,9 +43,13 @@ import org.apache.flink.yarn.YarnClusterDescriptor; ...@@ -43,9 +43,13 @@ import org.apache.flink.yarn.YarnClusterDescriptor;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
import cn.hutool.core.io.FileUtil;
/** /**
* YarnApplicationGateway * YarnApplicationGateway
...@@ -98,6 +102,7 @@ public class YarnApplicationGateway extends YarnGateway { ...@@ -98,6 +102,7 @@ public class YarnApplicationGateway extends YarnGateway {
if (configuration.contains(TaskManagerOptions.NUM_TASK_SLOTS)) { if (configuration.contains(TaskManagerOptions.NUM_TASK_SLOTS)) {
clusterSpecificationBuilder.setSlotsPerTaskManager(configuration.get(TaskManagerOptions.NUM_TASK_SLOTS)).createClusterSpecification(); clusterSpecificationBuilder.setSlotsPerTaskManager(configuration.get(TaskManagerOptions.NUM_TASK_SLOTS)).createClusterSpecification();
} }
yarnClusterDescriptor.addShipFiles(Arrays.stream(config.getJarPaths()).map(FileUtil::file).collect(Collectors.toList()));
try { try {
ClusterClientProvider<ApplicationId> clusterClientProvider = yarnClusterDescriptor.deployApplicationCluster( ClusterClientProvider<ApplicationId> clusterClientProvider = yarnClusterDescriptor.deployApplicationCluster(
......
...@@ -40,8 +40,13 @@ import org.apache.flink.yarn.YarnClusterDescriptor; ...@@ -40,8 +40,13 @@ import org.apache.flink.yarn.YarnClusterDescriptor;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.URLUtil;
/** /**
* YarnApplicationGateway * YarnApplicationGateway
...@@ -83,6 +88,8 @@ public class YarnPerJobGateway extends YarnGateway { ...@@ -83,6 +88,8 @@ public class YarnPerJobGateway extends YarnGateway {
clusterSpecificationBuilder.setSlotsPerTaskManager(configuration.get(TaskManagerOptions.NUM_TASK_SLOTS)).createClusterSpecification(); clusterSpecificationBuilder.setSlotsPerTaskManager(configuration.get(TaskManagerOptions.NUM_TASK_SLOTS)).createClusterSpecification();
} }
jobGraph.addJars(Arrays.stream(config.getJarPaths()).map(path -> URLUtil.getURL(FileUtil.file(path))).collect(Collectors.toList()));
try { try {
ClusterClientProvider<ApplicationId> clusterClientProvider = yarnClusterDescriptor.deployJobCluster( ClusterClientProvider<ApplicationId> clusterClientProvider = yarnClusterDescriptor.deployJobCluster(
clusterSpecificationBuilder.createClusterSpecification(), jobGraph, true); clusterSpecificationBuilder.createClusterSpecification(), jobGraph, true);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment