Skip to content

Commit 06a2495

Browse files
zcswl7961zcswl7961
authored
[feat_1192][taier-metrics][add a monitor metrics with taier #1192] (#1195)
Co-authored-by: xingyi <xingyi@dtstack.com> code review check with taier-metircs, @see TpMonitor
1 parent deee65d commit 06a2495

74 files changed

Lines changed: 8325 additions & 11 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
<module>taier-data-develop</module>
4242
<module>taier-worker</module>
4343
<module>taier-datasource</module>
44+
<module>taier-metrics</module>
4445
</modules>
4546

4647
<dependencyManagement>

‎taier-common/src/main/java/com/dtstack/taier/common/client/ClientProxy.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
import com.dtstack.taier.common.exception.LimitResourceException;
2222
import com.dtstack.taier.common.exception.TaierDefineException;
23-
import com.dtstack.taier.pluginapi.CustomThreadFactory;
23+
import com.dtstack.taier.metrics.collect.em.QueueTypeEnum;
2424
import com.dtstack.taier.pluginapi.JobClient;
2525
import com.dtstack.taier.pluginapi.JobIdentifier;
2626
import com.dtstack.taier.pluginapi.callback.CallBack;
@@ -29,9 +29,9 @@
2929
import com.dtstack.taier.pluginapi.enums.TaskStatus;
3030
import com.dtstack.taier.pluginapi.exception.ClientArgumentException;
3131
import com.dtstack.taier.pluginapi.exception.ExceptionUtil;
32+
import com.dtstack.taier.pluginapi.metrics.DynamicMetricsThreadPoolUtil;
3233
import com.dtstack.taier.pluginapi.pojo.CheckResult;
3334
import com.dtstack.taier.pluginapi.pojo.ComponentTestResult;
34-
import com.dtstack.taier.pluginapi.pojo.FileResult;
3535
import com.dtstack.taier.pluginapi.pojo.JobResult;
3636
import com.dtstack.taier.pluginapi.pojo.JudgeResult;
3737

@@ -41,8 +41,6 @@
4141
import java.util.concurrent.CompletableFuture;
4242
import java.util.concurrent.ExecutionException;
4343
import java.util.concurrent.ExecutorService;
44-
import java.util.concurrent.SynchronousQueue;
45-
import java.util.concurrent.ThreadPoolExecutor;
4644
import java.util.concurrent.TimeUnit;
4745
import java.util.concurrent.TimeoutException;
4846

@@ -64,8 +62,12 @@ public class ClientProxy implements IClient {
6462

6563
public ClientProxy(IClient targetClient) {
6664
this.targetClient = targetClient;
67-
executorService = new ThreadPoolExecutor(1, 10, 0L, TimeUnit.MILLISECONDS,
68-
new SynchronousQueue<>(), new CustomThreadFactory(targetClient.getClass().getSimpleName() + "_" + this.getClass().getSimpleName()));
65+
66+
executorService = DynamicMetricsThreadPoolUtil.buildDynamicThreadPool(
67+
1, 10, 0L, TimeUnit.MILLISECONDS,
68+
"ClientProxy", QueueTypeEnum.SYNCHRONOUS_QUEUE.getName(), null, false, this.timeout,
69+
targetClient.getClass().getSimpleName() + "_" + this.getClass().getSimpleName(),
70+
null, false);
6971
}
7072

7173
@Override

‎taier-common/src/main/java/com/dtstack/taier/common/env/EnvironmentContext.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,11 @@
3535
import org.springframework.core.env.Environment;
3636
import org.springframework.stereotype.Component;
3737

38+
import java.util.Arrays;
39+
import java.util.Collections;
40+
import java.util.List;
3841
import java.util.Map;
42+
import java.util.stream.Collectors;
3943

4044

4145
/**
@@ -271,6 +275,12 @@ public class EnvironmentContext implements InitializingBean {
271275
@Value("${schedule.scanning.cycle.day:1}")
272276
private Integer scanningCycleJobDay;
273277

278+
@Value("${engine.monitor.metrics.interval:5}")
279+
private Long monitorMetricsInterval;
280+
281+
@Value("${engine.monitor.metrics.delay:30}")
282+
private Long monitorMetricsDelay;
283+
274284
@Override
275285
public void afterPropertiesSet() throws Exception {
276286
// 读取全局配置并初始化 datasource
@@ -621,4 +631,63 @@ public Integer getTempSelectExpireTime() {
621631
public int getScanningCycleJobDay() {
622632
return scanningCycleJobDay;
623633
}
634+
635+
public long getMonitorMetricsInterval() {
636+
return monitorMetricsInterval;
637+
}
638+
639+
public long getMonitorMetricsDelay() {
640+
return monitorMetricsDelay;
641+
}
642+
643+
/**
644+
* 默认支持的监控指标推送类型
645+
* 平台内置的支持两种: micrometer, logging, output 默认logging
646+
*/
647+
public List<String> getSupportMonitorMetricsType() {
648+
String property = environment.getProperty("taier.monitor.metrics.support.type", "logging");
649+
if (StringUtils.isEmpty(property)) {
650+
return Collections.emptyList();
651+
}
652+
return Arrays.stream(StringUtils.split(property, ",")).map(String::valueOf)
653+
.collect(Collectors.toList());
654+
}
655+
656+
/**
657+
*/
658+
public boolean getIsMonitorMetrics() {
659+
return Boolean.parseBoolean(environment.getProperty("taier.monitor.metrics.enabled", "true"));
660+
}
661+
662+
/**
663+
* push gateway prometheus 默认url
664+
*/
665+
public String getPrometheusPushGatewayUrl() {
666+
return environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.url", "");
667+
}
668+
669+
/**
670+
* push gateway prometheus 默认search参数
671+
*/
672+
public String getPrometheusPushGatewaySearch() {
673+
return environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.search", "");
674+
}
675+
676+
/**
677+
* push gateway 超时时间, 单位s 默认5s
678+
*/
679+
public long getPrometheusPushGatewayTimeout() {
680+
return Long.parseLong(environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.timeout", "5"));
681+
}
682+
683+
/**
684+
* push gateway 默认间隔时间,单位s, 默认60s
685+
*/
686+
public long getPrometheusPushGatewayInterval() {
687+
return Long.parseLong(environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.interval", "60"));
688+
}
689+
690+
691+
692+
624693
}

‎taier-metrics/pom.xml‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
<parent>
7+
<groupId>com.dtstack.taier</groupId>
8+
<artifactId>taier</artifactId>
9+
<version>1.0.0</version>
10+
</parent>
11+
12+
<artifactId>taier-metrics</artifactId>
13+
14+
<properties>
15+
<maven.compiler.source>8</maven.compiler.source>
16+
<maven.compiler.target>8</maven.compiler.target>
17+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
18+
</properties>
19+
20+
<dependencies>
21+
<!-- micrometer 监控 -->
22+
<dependency>
23+
<groupId>io.micrometer</groupId>
24+
<artifactId>micrometer-registry-prometheus</artifactId>
25+
<version>1.10.12</version>
26+
</dependency>
27+
28+
<dependency>
29+
<groupId>io.prometheus</groupId>
30+
<artifactId>simpleclient_pushgateway</artifactId>
31+
<version>0.16.0</version>
32+
</dependency>
33+
34+
<dependency>
35+
<groupId>io.dropwizard.metrics</groupId>
36+
<artifactId>metrics-core</artifactId>
37+
<version>4.2.37</version>
38+
</dependency>
39+
40+
<!-- 不考虑引入spring boot starter actuator-->
41+
<!--<dependency>
42+
<groupId>org.springframework.boot</groupId>
43+
<artifactId>spring-boot-starter-actuator</artifactId>
44+
<exclusions>
45+
<exclusion>
46+
<groupId>org.springframework.boot</groupId>
47+
<artifactId>spring-boot-starter-logging</artifactId>
48+
</exclusion>
49+
</exclusions>
50+
</dependency>-->
51+
52+
<!-- lombok 配置依赖-->
53+
<dependency>
54+
<groupId>org.projectlombok</groupId>
55+
<artifactId>lombok</artifactId>
56+
<version>1.18.36</version>
57+
</dependency>
58+
59+
<!-- 可选依赖,主要包含logging,guava,commons-lang3,commons-logging,commons-collections-->
60+
<dependency>
61+
<groupId>com.google.guava</groupId>
62+
<artifactId>guava</artifactId>
63+
</dependency>
64+
65+
<dependency>
66+
<groupId>org.apache.commons</groupId>
67+
<artifactId>commons-lang3</artifactId>
68+
</dependency>
69+
70+
<dependency>
71+
<groupId>commons-logging</groupId>
72+
<artifactId>commons-logging</artifactId>
73+
<version>1.2</version>
74+
<optional>true</optional>
75+
</dependency>
76+
77+
<dependency>
78+
<groupId>commons-collections</groupId>
79+
<artifactId>commons-collections</artifactId>
80+
</dependency>
81+
82+
</dependencies>
83+
84+
<build>
85+
<plugins>
86+
<plugin>
87+
<groupId>org.apache.maven.plugins</groupId>
88+
<artifactId>maven-compiler-plugin</artifactId>
89+
<version>3.8.1</version>
90+
<configuration>
91+
<source>1.8</source>
92+
<target>1.8</target>
93+
<encoding>UTF-8</encoding>
94+
<annotationProcessorPaths>
95+
<path>
96+
<groupId>org.projectlombok</groupId>
97+
<artifactId>lombok</artifactId>
98+
<version>1.18.36</version>
99+
</path>
100+
</annotationProcessorPaths>
101+
</configuration>
102+
</plugin>
103+
<!-- 要将源码放上去，需要加入这个插件 -->
104+
<plugin>
105+
<artifactId>maven-source-plugin</artifactId>
106+
<version>3.0.1</version>
107+
<configuration>
108+
<attach>true</attach>
109+
</configuration>
110+
<executions>
111+
<execution>
112+
<phase>compile</phase>
113+
<goals>
114+
<goal>jar</goal>
115+
</goals>
116+
</execution>
117+
</executions>
118+
</plugin>
119+
<plugin>
120+
<groupId>org.apache.maven.plugins</groupId>
121+
<artifactId>maven-javadoc-plugin</artifactId>
122+
<version>3.0.1</version>
123+
<configuration>
124+
<reportOutputDirectory>../javadocs</reportOutputDirectory>
125+
<destDir>taier-api-client</destDir>
126+
<additionalJOptions>
127+
<additionalJOption>-Xdoclint:none</additionalJOption>
128+
</additionalJOptions>
129+
</configuration>
130+
</plugin>
131+
</plugins>
132+
</build>
133+
134+
</project>

0 commit comments

Comments
 (0)