Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
D
dlink
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
zhaowei
dlink
Commits
f53833af
Unverified
Commit
f53833af
authored
Oct 21, 2022
by
ZackYoung
Committed by
GitHub
Oct 21, 2022
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
add udf yarn-application; (#1131)
change code style
parent
6ba6cdb0
Changes
8
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
55 additions
and
41 deletions
+55
-41
UDFService.java
dlink-admin/src/main/java/com/dlink/service/UDFService.java
+3
-1
StudioServiceImpl.java
...c/main/java/com/dlink/service/impl/StudioServiceImpl.java
+27
-27
TaskServiceImpl.java
...src/main/java/com/dlink/service/impl/TaskServiceImpl.java
+1
-1
UDFServiceImpl.java
.../src/main/java/com/dlink/service/impl/UDFServiceImpl.java
+10
-5
JobManager.java
dlink-core/src/main/java/com/dlink/job/JobManager.java
+1
-7
GatewayConfig.java
...src/main/java/com/dlink/gateway/config/GatewayConfig.java
+1
-0
YarnApplicationGateway.java
...n/java/com/dlink/gateway/yarn/YarnApplicationGateway.java
+5
-0
YarnPerJobGateway.java
...c/main/java/com/dlink/gateway/yarn/YarnPerJobGateway.java
+7
-0
No files found.
dlink-admin/src/main/java/com/dlink/service/UDFService.java
View file @
f53833af
...
@@ -19,11 +19,13 @@
...
@@ -19,11 +19,13 @@
package
com
.
dlink
.
service
;
package
com
.
dlink
.
service
;
import
com.dlink.gateway.GatewayType
;
/**
/**
* @author ZackYoung
* @author ZackYoung
* @since 0.6.8
* @since 0.6.8
*/
*/
public
interface
UDFService
{
public
interface
UDFService
{
String
[]
initUDF
(
String
statement
);
String
[]
initUDF
(
String
statement
,
GatewayType
gatewayType
);
}
}
dlink-admin/src/main/java/com/dlink/service/impl/StudioServiceImpl.java
View file @
f53833af
This diff is collapsed.
Click to expand it.
dlink-admin/src/main/java/com/dlink/service/impl/TaskServiceImpl.java
View file @
f53833af
...
@@ -202,7 +202,7 @@ public class TaskServiceImpl extends SuperServiceImpl<TaskMapper, Task> implemen
...
@@ -202,7 +202,7 @@ public class TaskServiceImpl extends SuperServiceImpl<TaskMapper, Task> implemen
task
.
getDatabaseId
(),
null
));
task
.
getDatabaseId
(),
null
));
}
}
JobConfig
config
=
buildJobConfig
(
task
);
JobConfig
config
=
buildJobConfig
(
task
);
config
.
setJarFiles
(
udfService
.
initUDF
(
task
.
getStatement
()));
config
.
setJarFiles
(
udfService
.
initUDF
(
task
.
getStatement
()
,
config
.
getGatewayConfig
().
getType
()
));
JobManager
jobManager
=
JobManager
.
build
(
config
);
JobManager
jobManager
=
JobManager
.
build
(
config
);
if
(!
config
.
isJarTask
())
{
if
(!
config
.
isJarTask
())
{
return
jobManager
.
executeSql
(
task
.
getStatement
());
return
jobManager
.
executeSql
(
task
.
getStatement
());
...
...
dlink-admin/src/main/java/com/dlink/service/impl/UDFServiceImpl.java
View file @
f53833af
...
@@ -20,6 +20,7 @@
...
@@ -20,6 +20,7 @@
package
com
.
dlink
.
service
.
impl
;
package
com
.
dlink
.
service
.
impl
;
import
com.dlink.constant.PathConstant
;
import
com.dlink.constant.PathConstant
;
import
com.dlink.exception.BusException
;
import
com.dlink.gateway.GatewayType
;
import
com.dlink.gateway.GatewayType
;
import
com.dlink.job.JobManager
;
import
com.dlink.job.JobManager
;
import
com.dlink.model.Task
;
import
com.dlink.model.Task
;
...
@@ -53,15 +54,15 @@ public class UDFServiceImpl implements UDFService {
...
@@ -53,15 +54,15 @@ public class UDFServiceImpl implements UDFService {
* 快速获取 session 与 application 等类型,为了减少判断
* 快速获取 session 与 application 等类型,为了减少判断
*/
*/
private
static
final
Map
<
String
,
List
<
GatewayType
>>
GATEWAY_TYPE_MAP
=
MapUtil
private
static
final
Map
<
String
,
List
<
GatewayType
>>
GATEWAY_TYPE_MAP
=
MapUtil
.
builder
(
"session"
,
.
builder
(
"session"
,
Arrays
.
asList
(
GatewayType
.
YARN_SESSION
,
GatewayType
.
KUBERNETES_SESSION
,
GatewayType
.
STANDALONE
))
Arrays
.
asList
(
GatewayType
.
YARN_SESSION
,
GatewayType
.
KUBERNETES_SESSION
,
GatewayType
.
STANDALONE
))
.
build
();
.
build
();
@Resource
@Resource
TaskService
taskService
;
TaskService
taskService
;
@Override
@Override
public
String
[]
initUDF
(
String
statement
)
{
public
String
[]
initUDF
(
String
statement
,
GatewayType
gatewayType
)
{
ProcessEntity
process
=
ProcessContextHolder
.
getProcess
();
ProcessEntity
process
=
ProcessContextHolder
.
getProcess
();
process
.
info
(
"Initializing Flink UDF...Start"
);
process
.
info
(
"Initializing Flink UDF...Start"
);
Opt
<
String
>
udfJarPath
=
Opt
.
empty
();
Opt
<
String
>
udfJarPath
=
Opt
.
empty
();
...
@@ -73,10 +74,14 @@ public class UDFServiceImpl implements UDFService {
...
@@ -73,10 +74,14 @@ public class UDFServiceImpl implements UDFService {
},
true
);
},
true
);
if
(
codeList
.
size
()
>
0
)
{
if
(
codeList
.
size
()
>
0
)
{
udfJarPath
=
Opt
.
ofBlankAble
(
UDFUtil
.
getUdfNameAndBuildJar
(
codeList
));
udfJarPath
=
Opt
.
ofBlankAble
(
UDFUtil
.
getUdfNameAndBuildJar
(
codeList
));
}
else
{
if
(
gatewayType
==
GatewayType
.
KUBERNETES_APPLICATION
)
{
throw
new
BusException
(
"udf 暂不支持k8s application"
);
}
}
}
process
.
info
(
"Initializing Flink UDF...Finish"
);
process
.
info
(
"Initializing Flink UDF...Finish"
);
if
(
udfJarPath
.
isPresent
())
{
if
(
udfJarPath
.
isPresent
())
{
return
new
String
[]{
PathConstant
.
UDF_PATH
+
udfJarPath
.
get
()};
return
new
String
[]
{
PathConstant
.
UDF_PATH
+
udfJarPath
.
get
()};
}
else
{
}
else
{
return
new
String
[
0
];
return
new
String
[
0
];
}
}
...
...
dlink-core/src/main/java/com/dlink/job/JobManager.java
View file @
f53833af
...
@@ -72,20 +72,15 @@ import org.apache.flink.yarn.configuration.YarnConfigOptions;
...
@@ -72,20 +72,15 @@ import org.apache.flink.yarn.configuration.YarnConfigOptions;
import
java.time.LocalDateTime
;
import
java.time.LocalDateTime
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Collections
;
import
java.util.Collections
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
java.util.stream.Collectors
;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.slf4j.LoggerFactory
;
import
com.fasterxml.jackson.databind.node.ObjectNode
;
import
com.fasterxml.jackson.databind.node.ObjectNode
;
import
cn.hutool.core.io.FileUtil
;
import
cn.hutool.core.util.URLUtil
;
/**
/**
* JobManager
* JobManager
*
*
...
@@ -166,6 +161,7 @@ public class JobManager {
...
@@ -166,6 +161,7 @@ public class JobManager {
JobManager
manager
=
new
JobManager
(
config
);
JobManager
manager
=
new
JobManager
(
config
);
manager
.
init
();
manager
.
init
();
manager
.
executor
.
initUDF
(
config
.
getJarFiles
());
manager
.
executor
.
initUDF
(
config
.
getJarFiles
());
config
.
getGatewayConfig
().
setJarPaths
(
config
.
getJarFiles
());
return
manager
;
return
manager
;
}
}
...
@@ -398,7 +394,6 @@ public class JobManager {
...
@@ -398,7 +394,6 @@ public class JobManager {
jobGraph
.
setSavepointRestoreSettings
(
jobGraph
.
setSavepointRestoreSettings
(
SavepointRestoreSettings
.
forPath
(
config
.
getSavePointPath
(),
true
));
SavepointRestoreSettings
.
forPath
(
config
.
getSavePointPath
(),
true
));
}
}
jobGraph
.
addJars
(
Arrays
.
stream
(
config
.
getJarFiles
()).
map
(
path
->
URLUtil
.
getURL
(
FileUtil
.
file
(
path
))).
collect
(
Collectors
.
toList
()));
gatewayResult
=
Gateway
.
build
(
config
.
getGatewayConfig
()).
submitJobGraph
(
jobGraph
);
gatewayResult
=
Gateway
.
build
(
config
.
getGatewayConfig
()).
submitJobGraph
(
jobGraph
);
}
}
job
.
setResult
(
InsertResult
.
success
(
gatewayResult
.
getAppId
()));
job
.
setResult
(
InsertResult
.
success
(
gatewayResult
.
getAppId
()));
...
@@ -461,7 +456,6 @@ public class JobManager {
...
@@ -461,7 +456,6 @@ public class JobManager {
jobGraph
.
setSavepointRestoreSettings
(
SavepointRestoreSettings
.
forPath
(
config
.
getSavePointPath
(),
true
));
jobGraph
.
setSavepointRestoreSettings
(
SavepointRestoreSettings
.
forPath
(
config
.
getSavePointPath
(),
true
));
}
}
// Perjob mode need to submit job graph.
// Perjob mode need to submit job graph.
jobGraph
.
addJars
(
Arrays
.
stream
(
config
.
getJarFiles
()).
map
(
path
->
URLUtil
.
getURL
(
FileUtil
.
file
(
path
))).
collect
(
Collectors
.
toList
()));
gatewayResult
=
Gateway
.
build
(
config
.
getGatewayConfig
()).
submitJobGraph
(
jobGraph
);
gatewayResult
=
Gateway
.
build
(
config
.
getGatewayConfig
()).
submitJobGraph
(
jobGraph
);
}
}
return
gatewayResult
;
return
gatewayResult
;
...
...
dlink-gateway/src/main/java/com/dlink/gateway/config/GatewayConfig.java
View file @
f53833af
...
@@ -42,6 +42,7 @@ import lombok.Setter;
...
@@ -42,6 +42,7 @@ import lombok.Setter;
public
class
GatewayConfig
{
public
class
GatewayConfig
{
private
Integer
taskId
;
private
Integer
taskId
;
private
String
[]
jarPaths
;
private
GatewayType
type
;
private
GatewayType
type
;
private
ClusterConfig
clusterConfig
;
private
ClusterConfig
clusterConfig
;
private
FlinkConfig
flinkConfig
;
private
FlinkConfig
flinkConfig
;
...
...
dlink-gateway/src/main/java/com/dlink/gateway/yarn/YarnApplicationGateway.java
View file @
f53833af
...
@@ -43,9 +43,13 @@ import org.apache.flink.yarn.YarnClusterDescriptor;
...
@@ -43,9 +43,13 @@ import org.apache.flink.yarn.YarnClusterDescriptor;
import
org.apache.hadoop.yarn.api.records.ApplicationId
;
import
org.apache.hadoop.yarn.api.records.ApplicationId
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Collection
;
import
java.util.Collection
;
import
java.util.Collections
;
import
java.util.Collections
;
import
java.util.List
;
import
java.util.List
;
import
java.util.stream.Collectors
;
import
cn.hutool.core.io.FileUtil
;
/**
/**
* YarnApplicationGateway
* YarnApplicationGateway
...
@@ -98,6 +102,7 @@ public class YarnApplicationGateway extends YarnGateway {
...
@@ -98,6 +102,7 @@ public class YarnApplicationGateway extends YarnGateway {
if
(
configuration
.
contains
(
TaskManagerOptions
.
NUM_TASK_SLOTS
))
{
if
(
configuration
.
contains
(
TaskManagerOptions
.
NUM_TASK_SLOTS
))
{
clusterSpecificationBuilder
.
setSlotsPerTaskManager
(
configuration
.
get
(
TaskManagerOptions
.
NUM_TASK_SLOTS
)).
createClusterSpecification
();
clusterSpecificationBuilder
.
setSlotsPerTaskManager
(
configuration
.
get
(
TaskManagerOptions
.
NUM_TASK_SLOTS
)).
createClusterSpecification
();
}
}
yarnClusterDescriptor
.
addShipFiles
(
Arrays
.
stream
(
config
.
getJarPaths
()).
map
(
FileUtil:
:
file
).
collect
(
Collectors
.
toList
()));
try
{
try
{
ClusterClientProvider
<
ApplicationId
>
clusterClientProvider
=
yarnClusterDescriptor
.
deployApplicationCluster
(
ClusterClientProvider
<
ApplicationId
>
clusterClientProvider
=
yarnClusterDescriptor
.
deployApplicationCluster
(
...
...
dlink-gateway/src/main/java/com/dlink/gateway/yarn/YarnPerJobGateway.java
View file @
f53833af
...
@@ -40,8 +40,13 @@ import org.apache.flink.yarn.YarnClusterDescriptor;
...
@@ -40,8 +40,13 @@ import org.apache.flink.yarn.YarnClusterDescriptor;
import
org.apache.hadoop.yarn.api.records.ApplicationId
;
import
org.apache.hadoop.yarn.api.records.ApplicationId
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Collection
;
import
java.util.Collection
;
import
java.util.List
;
import
java.util.List
;
import
java.util.stream.Collectors
;
import
cn.hutool.core.io.FileUtil
;
import
cn.hutool.core.util.URLUtil
;
/**
/**
* YarnApplicationGateway
* YarnApplicationGateway
...
@@ -83,6 +88,8 @@ public class YarnPerJobGateway extends YarnGateway {
...
@@ -83,6 +88,8 @@ public class YarnPerJobGateway extends YarnGateway {
clusterSpecificationBuilder
.
setSlotsPerTaskManager
(
configuration
.
get
(
TaskManagerOptions
.
NUM_TASK_SLOTS
)).
createClusterSpecification
();
clusterSpecificationBuilder
.
setSlotsPerTaskManager
(
configuration
.
get
(
TaskManagerOptions
.
NUM_TASK_SLOTS
)).
createClusterSpecification
();
}
}
jobGraph
.
addJars
(
Arrays
.
stream
(
config
.
getJarPaths
()).
map
(
path
->
URLUtil
.
getURL
(
FileUtil
.
file
(
path
))).
collect
(
Collectors
.
toList
()));
try
{
try
{
ClusterClientProvider
<
ApplicationId
>
clusterClientProvider
=
yarnClusterDescriptor
.
deployJobCluster
(
ClusterClientProvider
<
ApplicationId
>
clusterClientProvider
=
yarnClusterDescriptor
.
deployJobCluster
(
clusterSpecificationBuilder
.
createClusterSpecification
(),
jobGraph
,
true
);
clusterSpecificationBuilder
.
createClusterSpecification
(),
jobGraph
,
true
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment