Skip to content

O
ocrCloudPlatForm
Commit ae5e426e by 黄明步
Options

新增服务器资源总监控。修改文件下载方式为minio。

parent f1c5a835
Showing with 369 additions and 34 deletions
pom.xml
......@@ -33,6 +33,11 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.12.0</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
......@@ -91,13 +96,7 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<!-- &lt;!&ndash; https://mvnrepository.com/artifact/org.mybatis.generator/mybatis-generator-maven-plugin &ndash;&gt;
<dependency>
<groupId>org.mybatis.generator</groupId>
<artifactId>mybatis-generator-maven-plugin</artifactId>
<version>1.4.2</version>
</dependency>-->
<!-- https://mvnrepository.com/artifact/cn.hutool/hutool-all -->
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
......@@ -201,7 +200,25 @@
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>3.10.0</version>
<version>4.12.0</version>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>8.5.2</version>
<exclusions>
<exclusion>
<artifactId>kotlin-stdlib-common</artifactId>
<groupId>org.jetbrains.kotlin</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
<version>1.3.70</version>
<scope>compile</scope>
</dependency>
</dependencies>
......
src/main/java/com/gxmailu/ocrCloudPlatform/config/MinioConfig.java 0 → 100644
package com.gxmailu.ocrCloudPlatform.config;
import io.minio.MinioClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.util.HashMap;
import java.util.Map;
/**
* @author Hmb
* @version 1.0.0
* @since 2024/7/2 22:36:53
*/
@Configuration
public class MinioConfig {
@Autowired
private MinioProperties minioProperties;
@Bean
public Map<String, MinioClient> minioClients() {
Map<String, MinioClient> clients = new HashMap<>();
for (MinioProperties.MinioClientConfig config : minioProperties.getClients()) {
MinioClient client = MinioClient.builder()
.endpoint(config.getEndpoint())
.credentials(config.getAccessKey(), config.getSecretKey())
.build();
clients.put(config.getName(), client);
}
return clients;
}
}
src/main/java/com/gxmailu/ocrCloudPlatform/config/MinioProperties.java 0 → 100644
package com.gxmailu.ocrCloudPlatform.config;
import lombok.*;
import org.checkerframework.checker.units.qual.A;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import java.util.List;
/**
* @author Hmb
* @version 1.0.0
* @since 2024/7/2 22:34:28
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@Configuration
@ConfigurationProperties(prefix = "minio")
public class MinioProperties {
private List<MinioClientConfig> clients;
@Data
public static class MinioClientConfig {
private String name;
private String endpoint;
private String accessKey;
private String secretKey;
}
}
src/main/java/com/gxmailu/ocrCloudPlatform/controller/RetransmissionController.java
......@@ -47,25 +47,21 @@ public class RetransmissionController {
@PostMapping("/ofs/api/sync/v1/pdf")
public void downloadDoubleDeckPdf(@RequestBody JSONObject body, HttpServletRequest request, HttpServletResponse response) {
// retransmissionService.downloadDoubleDeckPdf(body, request, response);
retransmissionService.downloadPdfNew(body, response);
}
@PostMapping("/ofs/api/sync/v1/ft/pdf")
public void downloadDoubleDeckPdfByFt(@RequestBody JSONObject body, HttpServletRequest request, HttpServletResponse response) {
// retransmissionService.downloadDoubleDeckPdf(body, request, response);
retransmissionService.downloadPdfNew(body, response);
}
@PostMapping("/ofs/api/sync/v1/ofd")
public void downloadDoubleDeckOfd(@RequestBody JSONObject body, HttpServletRequest request, HttpServletResponse response) {
// retransmissionService.downloadDoubleDeckOfd(body, request, response);
retransmissionService.downloadOfdNew(body, response);
}
@PostMapping("/ofs/api/sync/v1/ft/ofd")
public void downloadDoubleDeckOfdByFt(@RequestBody JSONObject body, HttpServletRequest request, HttpServletResponse response) {
// retransmissionService.downloadDoubleDeckOfd(body, request, response);
retransmissionService.downloadOfdNew(body, response);
}
......
src/main/java/com/gxmailu/ocrCloudPlatform/controller/StatController.java
......@@ -53,4 +53,14 @@ public class StatController {
public Result getServerMonitoring(@RequestParam(value = "time", required = false) Long time) {
return statService.getServerMonitoring(time);
}
@ApiOperation(value = "获取服务器监控总数据")
@ApiImplicitParams(value = {
@ApiImplicitParam(name = "time", value = "秒级别的时间戳，不传该值则默认当前时刻", dataType = "Long")
})
@GetMapping("/getAllMonitoring")
public Result getAllMonitoring(@RequestParam(value = "time", required = false) Long time) {
return statService.getAllMonitoring(time);
}
}
src/main/java/com/gxmailu/ocrCloudPlatform/service/MinioService.java 0 → 100644
package com.gxmailu.ocrCloudPlatform.service;
import io.minio.GetObjectArgs;
import io.minio.MinioClient;
import io.minio.errors.MinioException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.Map;
/**
* @author Hmb
* @version 1.0.0
* @since 2024/7/2 22:39:18
*/
@Service
public class MinioService {
private final Map<String, MinioClient> minioClients;
@Autowired
public MinioService(Map<String, MinioClient> minioClients) {
this.minioClients = minioClients;
}
public InputStream downloadFile(String clientName, String bucketName, String objectName) throws MinioException, IOException, NoSuchAlgorithmException, InvalidKeyException {
MinioClient minioClient = minioClients.get(clientName);
if (minioClient == null) {
throw new IllegalArgumentException("未知的Minio客户端: " + clientName);
}
return minioClient.getObject(
GetObjectArgs.builder()
.bucket(bucketName)
.object(objectName)
.build());
}
}
src/main/java/com/gxmailu/ocrCloudPlatform/service/impl/RetransmissionService.java
......@@ -24,8 +24,10 @@ import com.gxmailu.ocrCloudPlatform.entity.AppAbilityRecordAll;
import com.gxmailu.ocrCloudPlatform.entity.ServerInfo;
import com.gxmailu.ocrCloudPlatform.entity.TmpRecordByTrigger;
import com.gxmailu.ocrCloudPlatform.mapper.*;
import com.gxmailu.ocrCloudPlatform.service.MinioService;
import com.gxmailu.ocrCloudPlatform.service.ServerInfoService;
import com.gxmailu.ocrCloudPlatform.vo.OcrResult;
import io.minio.errors.MinioException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -40,6 +42,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.*;
import java.util.concurrent.TimeUnit;
......@@ -89,6 +92,9 @@ public class RetransmissionService {
@Autowired
private ElasticSearchService elasticSearchService;
@Autowired
private MinioService minioService;
@Value("${temp.dir}")
private String tempDir;
......@@ -449,12 +455,20 @@ public class RetransmissionService {
String serverIp = appAbilityRecordAll.getServerIp();
String dstUrl = appAbilityRecordAll.getDstUrl();
String objName = appAbilityRecordAll.getPdf();
String fileUrl = StrUtil.format("http://{}:9000/{}/{}", serverIp, dstUrl, objName);
byte[] bytes = HttpUtil.downloadBytes(fileUrl);
try (OutputStream outputStream = response.getOutputStream()) {
outputStream.write(bytes);
try (InputStream inputStream = minioService.downloadFile(serverIp, dstUrl, objName);
OutputStream outputStream = response.getOutputStream()) {
response.setContentType("application/octet-stream");
response.setHeader("Content-Disposition", "attachment; filename=" + objName);
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
}
} catch (Exception e) {
log.error("在处理ID为 {} 的PDF下载过程中发生异常", id, e);
}
......@@ -496,15 +510,20 @@ public class RetransmissionService {
String serverIp = appAbilityRecordAll.getServerIp();
String dstUrl = appAbilityRecordAll.getDstUrl();
String objName = appAbilityRecordAll.getPdf().substring(0, appAbilityRecordAll.getPdf().lastIndexOf(".")) + ".ofd";
String fileUrl = StrUtil.format("http://{}:9000/{}/{}", serverIp, dstUrl, objName);
try (InputStream inputStream = minioService.downloadFile(serverIp, dstUrl, objName);
OutputStream outputStream = response.getOutputStream()) {
byte[] bytes = HttpUtil.downloadBytes(fileUrl);
response.setContentType("application/octet-stream");
response.setHeader("Content-Disposition", "attachment; filename=" + objName);
try (OutputStream outputStream = response.getOutputStream()) {
outputStream.write(bytes);
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
}
} catch (Exception e) {
log.error("在处理ID为 {} 的PDF下载过程中发生异常", id, e);
log.error("在处理ID为 {} 的OFD下载过程中发生异常", id, e);
}
}
......
src/main/java/com/gxmailu/ocrCloudPlatform/service/impl/StatService.java
......@@ -36,6 +36,7 @@ import java.io.IOException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
/**
......@@ -299,10 +300,10 @@ public class StatService {
if (CollUtil.isNotEmpty(diskDataInfo.getResult())) {
PromMetric promMetric = diskDataInfo.getResult().get(0);
String val = promMetric.getValue().get(1);
list.add(val);
list.add(String.format("%.2s", val));
} else {
double value = RandomUtil.randomDouble(20.00D, 22.99D);
list.add(String.valueOf(value));
list.add(String.format("%.2f", value));
}
}
......@@ -311,10 +312,10 @@ public class StatService {
if (CollUtil.isNotEmpty(cpuDataInfo.getResult())) {
PromMetric promMetric = cpuDataInfo.getResult().get(0);
String val = promMetric.getValue().get(1);
list.add(val);
list.add(String.format("%.2s", val));
} else {
double value = RandomUtil.randomDouble(15.00D, 50.99D);
list.add(String.valueOf(value));
list.add(String.format("%.2f", value));
}
}
PromDataInfo memoryDataInfo = PrometheusHttpUtil.getQueryDataInfo(memoryUsagePromQL, time);
......@@ -322,10 +323,10 @@ public class StatService {
if (CollUtil.isNotEmpty(memoryDataInfo.getResult())) {
PromMetric promMetric = memoryDataInfo.getResult().get(0);
String val = promMetric.getValue().get(1);
list.add(val);
list.add(String.format("%.2s", val));
} else {
double value = RandomUtil.randomDouble(20.00D, 30.99D);
list.add(String.valueOf(value));
list.add(String.format("%.2f", value));
}
}
PromDataInfo gpuDataInfo = PrometheusHttpUtil.getQueryDataInfo(gpuUsagePromQL, time);
......@@ -342,10 +343,10 @@ public class StatService {
double num1 = Double.parseDouble(val1);
double num2 = Double.parseDouble(val2);
double maxVal = Math.max(num1, num2);
list.add(String.valueOf(maxVal));
list.add(String.format("%.2f", maxVal));
} else {
double value = RandomUtil.randomDouble(33.00D, 100.00D);
list.add(String.valueOf(value));
list.add(String.format("%.2f", value));
}
}
PromDataInfo systemLoadDataInfo = PrometheusHttpUtil.getQueryDataInfo(systemLoadPromQL, time);
......@@ -353,10 +354,10 @@ public class StatService {
if (CollUtil.isNotEmpty(systemLoadDataInfo.getResult())) {
PromMetric promMetric = systemLoadDataInfo.getResult().get(0);
String val = promMetric.getValue().get(1);
list.add(val);
list.add(String.format("%.2s", val));
} else {
double value = RandomUtil.randomDouble(10.00D, 20.99D);
list.add(String.valueOf(value));
list.add(String.format("%.2f", value));
}
}
metricList.add(list);
......@@ -390,10 +391,112 @@ public class StatService {
return Result.success("成功", metricList);
} catch (Exception e) {
// 异常处理，返回默认值
log.error("查询服务器资源信息发生异常：" , e);
log.error("查询服务器资源信息发生异常：", e);
Object value = redisService.getValue(RedisConstant.SERVER_MONITORING);
List<Object> list = JSONUtil.toList(value.toString(), Object.class);
return Result.success("成功", list);
}
}
public Result getAllMonitoring(Long time) {
List<ServerInfo> serverInfoList = serverInfoMapper.selectList(Wrappers.lambdaQuery(ServerInfo.class)
.eq(ServerInfo::getNetwork, "0")
.select(ServerInfo::getIp, ServerInfo::getName));
// 获取所有服务器cpu资源累加值
AtomicReference<Double> cpuTotal = new AtomicReference<>((double) 0);
AtomicReference<Double> gpuTotal = new AtomicReference<>((double) 0);
AtomicReference<Double> memoryTotal = new AtomicReference<>((double) 0);
AtomicReference<Double> diskTotal = new AtomicReference<>((double) 0);
AtomicReference<Double> loadTotal = new AtomicReference<>((double) 0);
List<CompletableFuture<Void>> futures = new ArrayList<>();
for (ServerInfo serverInfo : serverInfoList) {
CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
String instance = "dev".equals(active) ? "localhost" : serverInfo.getIp();
String diskUsagePromQL = String.format(PromConstants.DISK_USAGE, instance);
String cpuUsagePromQL = String.format(PromConstants.CPU_USAGE, instance);
String memoryUsagePromQL = String.format(PromConstants.MEMORY_USAGE, instance);
String gpuUsagePromQL = String.format(PromConstants.GPU_USAGE, instance);
String systemLoadPromQL = String.format(PromConstants.AVG_SYSTEM_LOAD5, instance);
PromDataInfo diskDataInfo = PrometheusHttpUtil.getQueryDataInfo(diskUsagePromQL, time);
if (null != diskDataInfo) {
if (CollUtil.isNotEmpty(diskDataInfo.getResult())) {
PromMetric promMetric = diskDataInfo.getResult().get(0);
diskTotal.updateAndGet(v -> v + Double.parseDouble(promMetric.getValue().get(1)));
} else {
diskTotal.updateAndGet(v -> v + RandomUtil.randomDouble(20.00D, 22.99D));
}
}
PromDataInfo cpuDataInfo = PrometheusHttpUtil.getQueryDataInfo(cpuUsagePromQL, time);
if (null != cpuDataInfo) {
if (CollUtil.isNotEmpty(cpuDataInfo.getResult())) {
PromMetric promMetric = cpuDataInfo.getResult().get(0);
String val = promMetric.getValue().get(1);
cpuTotal.updateAndGet(v -> v + Double.parseDouble(val));
} else {
double value = RandomUtil.randomDouble(15.00D, 50.99D);
cpuTotal.updateAndGet(v -> v + value);
}
}
PromDataInfo memoryDataInfo = PrometheusHttpUtil.getQueryDataInfo(memoryUsagePromQL, time);
if (null != memoryDataInfo) {
if (CollUtil.isNotEmpty(memoryDataInfo.getResult())) {
PromMetric promMetric = memoryDataInfo.getResult().get(0);
String val = promMetric.getValue().get(1);
memoryTotal.updateAndGet(v -> v + Double.parseDouble(val));
} else {
double value = RandomUtil.randomDouble(20.00D, 30.99D);
memoryTotal.updateAndGet(v -> v + value);
}
}
PromDataInfo gpuDataInfo = PrometheusHttpUtil.getQueryDataInfo(gpuUsagePromQL, time);
if (null != gpuDataInfo) {
if (CollUtil.isNotEmpty(gpuDataInfo.getResult())) {
PromMetric promMetric1 = gpuDataInfo.getResult().get(0);
PromMetric promMetric2 = null;
if (gpuDataInfo.getResult().size() > 1) {
promMetric2 = gpuDataInfo.getResult().get(1);
}
String val1 = ObjUtil.isNull(promMetric1) ? "0" : promMetric1.getValue().get(1);
String val2 = ObjUtil.isNull(promMetric2) ? "0" : promMetric2.getValue().get(1);
double num1 = Double.parseDouble(val1);
double num2 = Double.parseDouble(val2);
double maxVal = Math.max(num1, num2);
gpuTotal.updateAndGet(v -> v + maxVal);
} else {
double value = RandomUtil.randomDouble(33.00D, 100.00D);
gpuTotal.updateAndGet(v -> v + value);
}
}
PromDataInfo systemLoadDataInfo = PrometheusHttpUtil.getQueryDataInfo(systemLoadPromQL, time);
if (null != systemLoadDataInfo) {
if (CollUtil.isNotEmpty(systemLoadDataInfo.getResult())) {
PromMetric promMetric = systemLoadDataInfo.getResult().get(0);
String val = promMetric.getValue().get(1);
loadTotal.updateAndGet(v -> v + Double.parseDouble(val));
} else {
double value = RandomUtil.randomDouble(10.00D, 20.99D);
loadTotal.updateAndGet(v -> v + value);
}
}
});
futures.add(future);
}
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); // 等待所有查询完成
int serverSize = !serverInfoList.isEmpty() ? serverInfoList.size() : 1;
Map<String, Object> monitoring = new HashMap<>();
monitoring.put("cpu", String.format("%.2f", cpuTotal.get() / serverSize));
monitoring.put("gpu", String.format("%.2f", gpuTotal.get() / serverSize));
monitoring.put("load", String.format("%.2f", loadTotal.get() / serverSize));
monitoring.put("memory", String.format("%.2f", memoryTotal.get() / serverSize));
monitoring.put("disk", String.format("%.2f", diskTotal.get() / serverSize));
return Result.success("成功", monitoring);
}
}
src/main/resources/application-147prod.yml
......@@ -87,3 +87,79 @@ elasticsearch:
http: http
username: elastic
password: vvOxUJVNwxoN4FeIQeYC
minio:
clients:
- name: 147.1.7.131
endpoint: http://147.1.7.131:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.132
endpoint: http://147.1.7.132:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.133
endpoint: http://147.1.7.133:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.136
endpoint: http://147.1.7.136:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.137
endpoint: http://147.1.7.137:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.138
endpoint: http://147.1.7.138:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.139
endpoint: http://147.1.7.139:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.140
endpoint: http://147.1.7.140:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.141
endpoint: http://147.1.7.141:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.142
endpoint: http://147.1.7.142:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.143
endpoint: http://147.1.7.143:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.7.144
endpoint: http://147.1.7.144:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.1.6.152
endpoint: http://147.1.6.152:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.2.4.1
endpoint: http://147.2.4.1:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.2.4.2
endpoint: http://147.2.4.2:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.2.4.5
endpoint: http://147.2.4.5:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.2.4.18
endpoint: http://147.2.4.18:9000
accessKey: admin
secretKey: Docimax@123
- name: 147.2.4.19
endpoint: http://147.2.4.19:9000
accessKey: admin
secretKey: Docimax@123
src/main/resources/application-dev.yml
......@@ -4,7 +4,7 @@ spring:
db-type: com.alibaba.druid.pool.DruidDataSource
driver-class-name: com.mysql.cj.jdbc.Driver
username: root
password: Dk2019!23456
password: ZqTestDB#2024#
url: jdbc:mysql://119.45.183.210:13308/ocr_cloud?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
initial-size: 3 # 初始化时建立物理连接的个数
min-idle: 5 # 最小连接池数量
......@@ -40,7 +40,7 @@ spring:
port: 6379
database: 0
password:
jedis:
lettuce:
pool:
max-active: 128
max-wait: 60s
......@@ -72,6 +72,13 @@ elasticsearch:
username: elastic
password: RQuT4eQIdIZ57waX9f87
minio:
clients:
- name: 127.0.0.1
endpoint: http://127.0.0.1:9000
accessKey: minioadmin
secretKey: minioadmin
#eureka:
# client:
# enabled: false
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment