Skip to content

D
dlink
Unverified Commit ce0089f4 authored by aiwenmo's avatar aiwenmo Committed by GitHub
Browse files
Options

[Optimization-932][pom] Optimizate package and auto.sh by loading classpaths (#933) 

Co-authored-by: 's avatarwenmo <32723967+wenmo@users.noreply.github.com>
parent c1b4b556
Show whitespace changes
Inline Side-by-side
Showing with 185 additions and 152 deletions
dlink-admin/pom.xml
View file @ ce0089f4
......@@ -196,83 +196,30 @@
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
</dependencies>
<build>
<finalName>${project.artifactId}</finalName>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot-dependencies.version}</version>
<configuration>
<layout>ZIP</layout>
<mainClass>com.dlink.Dlink</mainClass>
<excludes>
<exclude>
<groupId>org.apache.flink</groupId>
<artifactId>flink-*</artifactId>
</exclude>
</excludes>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<!--<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.1.0</version>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.0.0</version>
<configuration>
&lt;!&ndash;<outputDirectory>${project.build.directory}/app/bin</outputDirectory>&ndash;&gt;
&lt;!&ndash;这些配置将写入到MANIFEST.MF文件中&ndash;&gt;
<archive>
&lt;!&ndash;指定程序入口&ndash;&gt;
<manifest>
&lt;!&ndash;<addClasspath>true</addClasspath>&ndash;&gt;
<classpathPrefix>../lib/</classpathPrefix>
<mainClass>com.dlink.Dlink</mainClass>
</manifest>
&lt;!&ndash; (配置文件外置目录) &ndash;&gt;
<manifestEntries>
<Class-Path>../config/</Class-Path>
</manifestEntries>
</archive>
<excludes>
<exclude>**/*.xml</exclude>
<exclude>**/*.conf</exclude>
<exclude>**/*.properties</exclude>
<exclude>**/*/*.json</exclude>
</excludes>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>src/main/assembly/package.xml</descriptor>
</descriptors>
</configuration>
</plugin>-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>${maven.resource.version}</version>
<executions>
<execution>
<id>copy-static</id>
<phase>validate</phase>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>copy-resources</goal>
<goal>single</goal>
</goals>
<configuration>
<outputDirectory>src/main/resources/static</outputDirectory>
<overwrite>true</overwrite>
<resources>
<resource>
<!-- 因为dlink-web打包目录在项目跟目录，所以从这里复制 -->
<directory>../dlink-web/dist</directory>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<finalName>${project.artifactId}-${project.version}</finalName>
</build>
</project>
dlink-admin/src/main/assembly/package.xml 0 → 100644
View file @ ce0089f4
<assembly>
<id>dist</id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>${basedir}/src/main/bin</directory>
<lineEnding>unix</lineEnding>
<outputDirectory>service</outputDirectory>
<includes>
<include>*.service</include>
</includes>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/bin</directory>
<lineEnding>unix</lineEnding>
<outputDirectory></outputDirectory>
<fileMode>755</fileMode>
<includes>
<include>*.sh</include>
</includes>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/bin</directory>
<lineEnding>windows</lineEnding>
<outputDirectory></outputDirectory>
<includes>
<include>*.bat</include>
</includes>
</fileSet>
<fileSet>
<directory>target/classes</directory>
<outputDirectory>config</outputDirectory>
<includes>
<include>**/*.properties</include>
<include>**/*.yml</include>
<include>**/*.yaml</include>
<include>**/*.xml</include>
</includes>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<unpack>false</unpack>
</dependencySet>
</dependencySets>
</assembly>
\ No newline at end of file
dlink-admin/src/main/java/com/dlink/service/impl/JobInstanceServiceImpl.java
View file @ ce0089f4
......@@ -33,6 +33,7 @@ import com.dlink.model.JobInstance;
import com.dlink.model.JobInstanceCount;
import com.dlink.model.JobInstanceStatus;
import com.dlink.model.JobStatus;
import com.dlink.model.SystemConfiguration;
import com.dlink.service.ClusterConfigurationService;
import com.dlink.service.ClusterService;
import com.dlink.service.HistoryService;
......@@ -184,7 +185,11 @@ public class JobInstanceServiceImpl extends SuperServiceImpl<JobInstanceMapper,
@Override
public LineageResult getLineage(Integer id) {
History history = getJobInfoDetail(id).getHistory();
return LineageBuilder.getLineage(history.getStatement(), history.getConfig().get("useStatementSet").asBoolean());
if (SystemConfiguration.getInstances().isUseLogicalPlan()) {
return LineageBuilder.getColumnLineageByLogicalPlan(history.getStatement());
} else {
return LineageBuilder.getLineage(history.getStatement());
}
}
@Override
......
dlink-assembly/src/main/assembly/package.xml
View file @ ce0089f4
<assembly>
<id>${project.version}</id>
<formats>
<format>tar.gz</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<fileSets>
<!-- 打包dlink-doc中的bin目录到 打包目录下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-doc/bin</directory>
<outputDirectory>./</outputDirectory>
......@@ -18,37 +14,41 @@
<include>auto.sh</include>
</includes>
</fileSet>
<!-- 打包dlink-doc中的配置文件到打包目录/sql下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-doc/sql</directory>
<outputDirectory>sql</outputDirectory>
<directory>${project.parent.basedir}/dlink-admin/target/classes</directory>
<outputDirectory>config</outputDirectory>
<includes>
<include>*.sql</include>
<include>**/*.properties</include>
<include>**/*.yml</include>
<include>**/*.yaml</include>
<include>**/*.xml</include>
</includes>
</fileSet>
<!-- 打包dlink-admin中的配置文件到打包目录/config下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-admin/src/main/resources/
</directory>
<outputDirectory>config</outputDirectory>
<directory>${project.parent.basedir}/dlink-admin/target/dlink-admin/lib</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>application.yml</include>
<include>*.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-admin的jar文件放到打包目录下 -->
<fileSet>
<outputDirectory>plugins</outputDirectory>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-doc/sql</directory>
<outputDirectory>sql</outputDirectory>
<includes>
<include>*.sql</include>
</includes>
</fileSet>
<!--<fileSet>
<directory>${project.parent.basedir}/dlink-admin/target
</directory>
<outputDirectory>./</outputDirectory>
<includes>
<include>dlink-admin-${project.version}.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-client的jar文件放到打包目录/lib下 -->
</fileSet>-->
<fileSet>
<directory>${project.parent.basedir}/dlink-client/dlink-client-1.14/target</directory>
<outputDirectory>lib</outputDirectory>
......@@ -77,6 +77,13 @@
<include>dlink-client-1.13-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-client/dlink-client-1.14/target</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-client-1.14-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-client/dlink-client-1.15/target</directory>
<outputDirectory>extends</outputDirectory>
......@@ -84,7 +91,6 @@
<include>dlink-client-1.15-${project.version}.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-connectors的jar文件放到打包目录/lib下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-connectors/dlink-connector-jdbc-1.14/target</directory>
<outputDirectory>lib</outputDirectory>
......@@ -92,6 +98,27 @@
<include>dlink-connector-jdbc-1.14-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-connectors/dlink-connector-doris-1.13/target</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-connector-doris-1.13-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-connectors/dlink-connector-jdbc-1.11/target</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-connector-jdbc-1.11-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-connectors/dlink-connector-jdbc-1.12/target</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-connector-jdbc-1.12-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-connectors/dlink-connector-jdbc-1.13/target</directory>
<outputDirectory>extends</outputDirectory>
......@@ -99,57 +126,51 @@
<include>dlink-connector-jdbc-1.13-${project.version}.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-function的jar文件放到打包目录/lib下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-function/target
</directory>
<outputDirectory>lib</outputDirectory>
<directory>${project.parent.basedir}/dlink-connectors/dlink-connector-jdbc-1.14/target</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-function-${project.version}.jar</include>
<include>dlink-connector-jdbc-1.14-${project.version}.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-metadata的jar文件放到打包目录/lib下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-clickhouse/target
</directory>
<outputDirectory>lib</outputDirectory>
<directory>${project.parent.basedir}/dlink-connectors/dlink-connector-phoenix-1.13/target</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-metadata-clickhouse-${project.version}.jar</include>
<include>dlink-connector-phoenix-1.13-${project.version}.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-metadata的jar文件放到打包目录/lib下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-mysql/target
<directory>${project.parent.basedir}/dlink-connectors/dlink-connector-phoenix-1.14/target</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-connector-phoenix-1.14-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-function/target
</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>dlink-metadata-mysql-${project.version}.jar</include>
<include>dlink-function-${project.version}.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-metadata的jar文件放到打包目录/lib下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-oracle/target
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-clickhouse/target
</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>dlink-metadata-oracle-${project.version}.jar</include>
<include>dlink-metadata-clickhouse-${project.version}.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-metadata的jar文件放到打包目录/lib下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-postgresql/target
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-doris/target
</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>dlink-metadata-postgresql-${project.version}.jar</include>
<include>dlink-metadata-doris-${project.version}.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-metadata的jar文件放到打包目录/lib下 -->
<fileSet>
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-hive/target
</directory>
......@@ -159,20 +180,19 @@
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-sqlserver/target
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-mysql/target
</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>dlink-metadata-sqlserver-${project.version}.jar</include>
<include>dlink-metadata-mysql-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-doris/target
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-oracle/target
</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>dlink-metadata-doris-${project.version}.jar</include>
<include>dlink-metadata-oracle-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
......@@ -184,27 +204,27 @@
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-alert/dlink-alert-dingtalk/target
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-postgresql/target
</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>dlink-alert-dingtalk-${project.version}.jar</include>
<include>dlink-metadata-postgresql-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-alert/dlink-alert-wechat/target
<directory>${project.parent.basedir}/dlink-metadata/dlink-metadata-sqlserver/target
</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>dlink-alert-wechat-${project.version}.jar</include>
<include>dlink-metadata-sqlserver-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-alert/dlink-alert-feishu/target
<directory>${project.parent.basedir}/dlink-alert/dlink-alert-dingtalk/target
</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>dlink-alert-feishu-${project.version}.jar</include>
<include>dlink-alert-dingtalk-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
......@@ -215,26 +235,22 @@
<include>dlink-alert-email-${project.version}.jar</include>
</includes>
</fileSet>
<!-- 将模块dlink-extends的常用jar文件放到打包目录/plugins下 -->
<!--<fileSet>
<directory>${project.parent.basedir}/dlink-extends/target
<fileSet>
<directory>${project.parent.basedir}/dlink-alert/dlink-alert-feishu/target
</directory>
<outputDirectory>plugins</outputDirectory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>dlink-alert-feishu-${project.version}.jar</include>
</includes>
</fileSet>-->
<!-- 将模块dlink-extends的jar文件放到打包目录/extends下 -->
<!--<fileSet>
<directory>${project.parent.basedir}/dlink-extends/target
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-alert/dlink-alert-wechat/target
</directory>
<outputDirectory>extends</outputDirectory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>flink-sql-connector-*.jar</include>
<include>ojdbc8-*.jar</include>
<include>clickhouse-jdbc-*.jar</include>
<include>dlink-alert-wechat-${project.version}.jar</include>
</includes>
</fileSet>-->
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-app/dlink-app-1.11/target</directory>
<outputDirectory>jar</outputDirectory>
......@@ -271,21 +287,32 @@
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-catalog/dlink-catalog-mysql/dlink-catalog-mysql-1.13/target</directory>
<directory>${project.parent.basedir}/dlink-catalog/dlink-catalog-mysql/dlink-catalog-mysql-1.14/target
</directory>
<outputDirectory>lib</outputDirectory>
<includes>
<include>dlink-catalog-mysql-1.14-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-catalog/dlink-catalog-mysql/dlink-catalog-mysql-1.13/target
</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-catalog-mysql-1.13-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-catalog/dlink-catalog-mysql/dlink-catalog-mysql-1.14/target</directory>
<outputDirectory>lib</outputDirectory>
<directory>${project.parent.basedir}/dlink-catalog/dlink-catalog-mysql/dlink-catalog-mysql-1.14/target
</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-catalog-mysql-1.14-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.parent.basedir}/dlink-catalog/dlink-catalog-mysql/dlink-catalog-mysql-1.15/target</directory>
<directory>${project.parent.basedir}/dlink-catalog/dlink-catalog-mysql/dlink-catalog-mysql-1.15/target
</directory>
<outputDirectory>extends</outputDirectory>
<includes>
<include>dlink-catalog-mysql-1.15-${project.version}.jar</include>
......
dlink-client/dlink-client-1.14/src/main/java/com/dlink/utils/LineageContext.java
View file @ ce0089f4
......@@ -135,7 +135,7 @@ public class LineageContext {
/**
* Calling each program's optimize method in sequence.
*/
public RelNode optimize(RelNode relNode) {
private RelNode optimize(RelNode relNode) {
return flinkChainedProgram.optimize(relNode, new StreamOptimizeContext() {
@Override
public boolean isBatchMode() {
......@@ -149,7 +149,7 @@ public class LineageContext {
@Override
public FunctionCatalog getFunctionCatalog() {
return ((PlannerBase) tableEnv.getPlanner()).getFlinkContext().getFunctionCatalog();
return getPlanner().getFlinkContext().getFunctionCatalog();
}
@Override
......@@ -159,17 +159,18 @@ public class LineageContext {
@Override
public SqlExprToRexConverterFactory getSqlExprToRexConverterFactory() {
return relNode.getCluster().getPlanner().getContext().unwrap(FlinkContext.class).getSqlExprToRexConverterFactory();
return getPlanner().getFlinkContext().getSqlExprToRexConverterFactory();
}
@Override
public <C> C unwrap(Class<C> clazz) {
return StreamOptimizeContext.super.unwrap(clazz);
return getPlanner().getFlinkContext().unwrap(clazz);
}
@Override
public FlinkRelBuilder getFlinkRelBuilder() {
return ((PlannerBase) tableEnv.getPlanner()).getRelBuilder();
return getPlanner().getRelBuilder();
}
@Override
......@@ -186,6 +187,11 @@ public class LineageContext {
public MiniBatchInterval getMiniBatchInterval() {
return MiniBatchInterval.NONE;
}
private PlannerBase getPlanner() {
return (PlannerBase) tableEnv.getPlanner();
}
});
}
......
dlink-doc/bin/auto.sh
View file @ ce0089f4
#!/bin/bash
# 定义变量
# 要运行的jar包路径，加不加引号都行。 注意：等号两边 不能 有空格，否则会提示command找不到
JAR_NAME="./dlink-admin-*.jar"
#java -Djava.ext.dirs=$JAVA_HOME/jre/lib/ext:$JAVA_HOME/jre/lib:./lib -classpath ."/lib/*.jar" -jar dlink-admin-*.jar
# 如果需要将FLINK依赖直接加入启动脚本，在SETTING中增加$FLINK_HOME/lib
SETTING="-Dloader.path=./lib,./plugins -Ddruid.mysql.usePingMethod=false"
# 要运行的jar包路径，加不加引号都行。 注意：等号两边不能有空格，否则会提示command找不到
JAR_NAME="dlink-admin"
# 如果需要将 FLINK 依赖直接加入启动脚本，在 CLASS_PATH 中末尾追加 :$FLINK_HOME/lib/*
CLASS_PATH="./lib/*:config:html:./plugins/*"
#if [ ! -d ${HADOOP_HOME} ]; then
# echo 'WARNING!!!...not find HADOOP_HOME for CLASSPATH.'
#else
......@@ -57,7 +55,7 @@ start() {
pid=$(cat ${PIDPATH}/${PIDFILE})
if [ -z $pid ]; then
nohup java $SETTING -jar -Xms512M -Xmx2048M -XX:PermSize=512M -XX:MaxPermSize=1024M $JAR_NAME >dlink.log 2>&1 &
nohup java -Ddruid.mysql.usePingMethod=false -Xms512M -Xmx2048M -XX:PermSize=512M -XX:MaxPermSize=1024M -XX:+HeapDumpOnOutOfMemoryError -Xverify:none -cp ${CLASS_PATH} com.dlink.Dlink >dlink.log 2>&1 &
echo $! >${PIDPATH}/${PIDFILE}
echo "........................................Start Dlink Successfully........................................"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment