保存汇聚日志

b181e091 · 梁家彪 · 41dc65d0 · b181e091 · b181e091 · b181e091
Commit b181e091 authored Nov 03, 2023 by 梁家彪
7 changed files
--- a/common-client/src/main/java/com/zq/common/utils/EasyExcelUtil.java
+++ b/common-client/src/main/java/com/zq/common/utils/EasyExcelUtil.java
@@ -3,31 +3,68 @@ package com.zq.common.utils;
 import com.alibaba.excel.EasyExcel;
 import com.alibaba.excel.context.AnalysisContext;
 import com.alibaba.excel.event.AnalysisEventListener;
+import com.zq.common.exception.BusinessException;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.InputStream;
-import java.util.LinkedHashMap;
+import java.util.*;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
 public class EasyExcelUtil {
-    public static List<Map<String, Object>> read(File file){
+    /**
+     * 读取全部数据
+     * @param file
+     * @return
+     */
+    public static List<Map<String, Object>> readAll(File file){
        try {
-            return read(new FileInputStream(file));
+            return readAll(new FileInputStream(file));
        } catch (FileNotFoundException e) {
-            throw new RuntimeException(e);
+            e.printStackTrace();
+            throw new BusinessException("文件读取失败，请检查后重试");
        }
    }
-    public static List<Map<String, Object>> read(InputStream is){
+    /**
-        List<Map<String, Object>> list = new LinkedList<>();
+     * 读取全部数据
-        EasyExcel.read(is, new AnalysisEventListener<Map<Integer, Object>>() {
+     * @param is
-            private Map<Integer, String> headMap;
+     * @return
+     */
+    public static List<Map<String, Object>> readAll(InputStream is){
+        return (List<Map<String, Object>>) read(is, 0).get("rows");
+    }
+    /**
+     * 预览前100条数据
+     * @param file
+     * @return
+     */
+    public static Map<String, Object> preview100(File file){
+        try {
+            return preview100(new FileInputStream(file));
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+            throw new BusinessException("文件解析失败");
+        }
+    }
+    /**
+     * 预览前100条数据
+     * @param is
+     * @return
+     */
+    public static Map<String, Object> preview100(InputStream is){
+        return read(is, 100);
+    }
+    public static Map<String, Object> read(InputStream is, int readSize){
+        Map<String, Object> map = new HashMap();
+        EasyExcel.read(is, new AnalysisEventListener<Map<Integer, Object>>() {
+            Map<Integer, String> headMap;
+            List<Map<String, Object>> list = new LinkedList<>();
+            int total = 0;
            @Override
            public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
                this.headMap = headMap;
@@ -35,16 +72,22 @@ public class EasyExcelUtil {
            @Override
            public void invoke(Map<Integer, Object> valueMap, AnalysisContext analysisContext) {
-                LinkedHashMap map = new LinkedHashMap(headMap.size());
+                total++;
-                for(int i=0; i<headMap.size(); i++){
+                if(readSize<=0 || list.size()<readSize){
-                    map.put(headMap.get(i), valueMap.get(i));
+                    LinkedHashMap<String, Object> map = new LinkedHashMap(headMap.size());
+                    for(int i=0; i<headMap.size(); i++){
+                        map.put(headMap.get(i), valueMap.get(i));
+                    }
+                    list.add(map);
                }
-                list.add(map);
            }
            @Override
-            public void doAfterAllAnalysed(AnalysisContext analysisContext) {}
+            public void doAfterAllAnalysed(AnalysisContext analysisContext) {
+                map.put("rows", list);
+                map.put("total", total);
+            }
        }).sheet().doReadSync();
-        return list;
+        return map;
    }
 }
\ No newline at end of file
--- a/data-collect-server/src/main/java/com/zq/datacollect/controller/DataCollectDbController.java
+++ b/data-collect-server/src/main/java/com/zq/datacollect/controller/DataCollectDbController.java
@@ -12,7 +12,7 @@ import javax.annotation.Resource;
 @RestController
 @Api(tags = "数据库汇聚")
-@RequestMapping(value = "/data/collect")
+@RequestMapping(value = "/data/collect/db")
 public class DataCollectDbController {
    @Resource

--- a/data-collect-server/src/main/java/com/zq/datacollect/entity/DataCollectTaskLog.java
+++ b/data-collect-server/src/main/java/com/zq/datacollect/entity/DataCollectTaskLog.java
@@ -16,7 +16,7 @@ import java.util.Date;
 public class DataCollectTaskLog {
    /**
-     * 数据汇聚表id
+     * 汇聚设置id
     */
    private Integer dataCollectSettingId;
@@ -65,5 +65,14 @@ public class DataCollectTaskLog {
     */
    @TableField(fill = FieldFill.INSERT_UPDATE)
    private Date updateTime;
+    public DataCollectTaskLog(Integer dataCollectSettingId, Date startTime, Date endTime, Integer duration, Integer status, String error) {
+        this.dataCollectSettingId = dataCollectSettingId;
+        this.startTime = startTime;
+        this.endTime = endTime;
+        this.duration = duration;
+        this.status = status;
+        this.error = error;
+    }
 }
--- a/data-collect-server/src/main/java/com/zq/datacollect/service/DataCollectFileService.java
+++ b/data-collect-server/src/main/java/com/zq/datacollect/service/DataCollectFileService.java
@@ -15,7 +15,7 @@ public interface DataCollectFileService {
    void update(DataCollectFileAddReq dataCollectFileAddReq);
-    List<Map<String, Object>> analysisExcel(MultipartFile multipartFile);
+    Map<String, Object> analysisExcel(MultipartFile multipartFile);
    void execute(Integer dataCollectSettingId, MultipartFile file);
 }
\ No newline at end of file
--- a/data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectApiServiceImpl.java
+++ b/data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectApiServiceImpl.java
@@ -12,10 +12,12 @@ import com.zq.common.utils.AssertUtils;
 import com.zq.datacollect.entity.DataCollectSetting;
 import com.zq.datacollect.entity.DataCollectSettingApi;
 import com.zq.datacollect.entity.DataCollectSettingApiPara;
+import com.zq.datacollect.entity.DataCollectTaskLog;
 import com.zq.datacollect.holder.DatabaseHolder;
 import com.zq.datacollect.mapper.DataCollectMapper;
 import com.zq.datacollect.mapper.DataCollectSettingApiMapper;
 import com.zq.datacollect.mapper.DataCollectSettingApiParaMapper;
+import com.zq.datacollect.mapper.DataCollectTaskLogMapper;
 import com.zq.datacollect.service.DataCollectApiService;
 import com.zq.datacollect.util.SqlUtil;
 import com.zq.datacollect.vo.DataCollectApiAddReq;
@@ -29,10 +31,7 @@ import org.springframework.transaction.annotation.Transactional;
 import javax.annotation.Resource;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
-import java.util.LinkedList;
+import java.util.*;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
 @Service
 public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApiMapper, DataCollectSettingApi> implements DataCollectApiService {
@@ -43,6 +42,9 @@ public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApi
    @Resource
    private DataCollectSettingApiParaMapper dataCollectSettingApiParaMapper;
+    @Resource
+    private DataCollectTaskLogMapper dataCollectTaskLogMapper;
    @Override
    public DataCollectSettingApiResp get(Integer dataCollectSettingId) {
        DataCollectSettingApi settingApi = lambdaQuery().eq(DataCollectSettingApi::getDataCollectSettingId, dataCollectSettingId).one();
@@ -73,6 +75,7 @@ public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApi
        AssertUtils.isTrue(updateById(dataCollectApiAddReq.getDataCollectSettingApi()), "修改失败");
        for (DataCollectSettingApiPara dataCollectSettingApiPara : dataCollectApiAddReq.getDataCollectSettingApiParas()) {
            if(dataCollectSettingApiPara.getId() == null){
+                dataCollectSettingApiPara.setDataCollectSettingId(dataCollectApiAddReq.getDataCollectSetting().getId());
                AssertUtils.isTrue(dataCollectSettingApiParaMapper.insert(dataCollectSettingApiPara)==1, "修改失败");
            }else {
                AssertUtils.isTrue(dataCollectSettingApiParaMapper.updateById(dataCollectSettingApiPara)==1, "修改失败");
@@ -84,6 +87,7 @@ public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApi
    @Override
    @Transactional
    public void execute(Integer dataCollectSettingId, DataCollectApiExecuteReq dataCollectApiExecuteReq) {
+        long start = System.currentTimeMillis();
        DataCollectSettingApi settingApi = lambdaQuery().eq(DataCollectSettingApi::getDataCollectSettingId, dataCollectSettingId).one();
        List<DataCollectSettingApiPara> apiParas;
        if(dataCollectApiExecuteReq.getParas() != null && dataCollectApiExecuteReq.getParas().size() > 0){
@@ -120,7 +124,6 @@ public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApi
        }
        String body = request.execute().body();
-        log.error(body);
        LinkedList<Map<String, Object>> list = JSONObject.parseObject(body).getJSONObject("data").getJSONArray("rows").to(LinkedList.class);
        DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectSettingId);
@@ -145,6 +148,8 @@ public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApi
                return list.size();
            }
        });
+        dataCollectTaskLogMapper.insert(new DataCollectTaskLog(dataCollectSettingId, new Date(start), new Date(), Long.valueOf(((System.currentTimeMillis()-start)/1000)).intValue(), 2, null));
    }
    public String urlParams(List<DataCollectSettingApiPara> settingApiParas){

--- a/data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectDbServiceImpl.java
+++ b/data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectDbServiceImpl.java
@@ -6,9 +6,11 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.zq.common.utils.AssertUtils;
 import com.zq.datacollect.entity.DataCollectSetting;
 import com.zq.datacollect.entity.DataCollectSettingDb;
+import com.zq.datacollect.entity.DataCollectTaskLog;
 import com.zq.datacollect.holder.DatabaseHolder;
 import com.zq.datacollect.mapper.DataCollectMapper;
 import com.zq.datacollect.mapper.DataCollectSettingDbMapper;
+import com.zq.datacollect.mapper.DataCollectTaskLogMapper;
 import com.zq.datacollect.service.DataCollectDbService;
 import com.zq.datacollect.util.SqlUtil;
 import com.zq.datacollect.vo.DataCollectDbAddReq;
@@ -20,6 +22,7 @@ import org.springframework.transaction.annotation.Transactional;
 import javax.annotation.Resource;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -30,6 +33,9 @@ public class DataCollectDbServiceImpl extends ServiceImpl<DataCollectSettingDbMa
    @Resource
    private DataCollectMapper dataCollectMapper;
+    @Resource
+    private DataCollectTaskLogMapper dataCollectTaskLogMapper;
    @Override
    public DataCollectSettingDb get(Integer dataCollectSettingId) {
        return lambdaQuery().eq(DataCollectSettingDb::getDataCollectSettingId, dataCollectSettingId).one();
@@ -56,6 +62,7 @@ public class DataCollectDbServiceImpl extends ServiceImpl<DataCollectSettingDbMa
    @Override
    @Transactional
    public void execute(Integer dataCollectSettingId) {
+        long start = System.currentTimeMillis();
        DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectSettingId);
        DataCollectSettingDb settingDb = getOne(new LambdaQueryWrapper<DataCollectSettingDb>().eq(DataCollectSettingDb::getDataCollectSettingId, dataCollectSettingId));
        JdbcTemplate toDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(dataCollectSetting.getToDbId());
@@ -81,5 +88,7 @@ public class DataCollectDbServiceImpl extends ServiceImpl<DataCollectSettingDbMa
                return list.size();
            }
        });
+        dataCollectTaskLogMapper.insert(new DataCollectTaskLog(dataCollectSettingId, new Date(start), new Date(), Long.valueOf(((System.currentTimeMillis()-start)/1000)).intValue(), 2, null));
    }
 }
--- a/data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectFileServiceImpl.java
+++ b/data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectFileServiceImpl.java
 package com.zq.datacollect.service.impl;
 import cn.hutool.core.io.FileUtil;
-import cn.hutool.core.util.StrUtil;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.zq.common.exception.BusinessException;
 import com.zq.common.utils.AssertUtils;
 import com.zq.common.utils.EasyExcelUtil;
 import com.zq.datacollect.entity.DataCollectSetting;
 import com.zq.datacollect.entity.DataCollectSettingFile;
+import com.zq.datacollect.entity.DataCollectTaskLog;
 import com.zq.datacollect.holder.DatabaseHolder;
 import com.zq.datacollect.mapper.DataCollectMapper;
 import com.zq.datacollect.mapper.DataCollectSettingFileMapper;
+import com.zq.datacollect.mapper.DataCollectTaskLogMapper;
 import com.zq.datacollect.service.DataCollectFileService;
 import com.zq.datacollect.util.SqlUtil;
 import com.zq.datacollect.vo.DataCollectFileAddReq;
@@ -21,8 +22,10 @@ import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
 import javax.annotation.Resource;
+import java.io.IOException;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -33,6 +36,9 @@ public class DataCollectFileServiceImpl extends ServiceImpl<DataCollectSettingFi
    @Resource
    private DataCollectMapper dataCollectMapper;
+    @Resource
+    private DataCollectTaskLogMapper dataCollectTaskLogMapper;
    @Override
    public DataCollectSettingFile get(Integer dataCollectSettingId) {
        return lambdaQuery().eq(DataCollectSettingFile::getDataCollectSettingId, dataCollectSettingId).one();
@@ -57,11 +63,11 @@ public class DataCollectFileServiceImpl extends ServiceImpl<DataCollectSettingFi
    }
    @Override
-    public List<Map<String, Object>> analysisExcel(MultipartFile file) {
+    public Map<String, Object> analysisExcel(MultipartFile file) {
        String suffix = FileUtil.extName(file.getOriginalFilename());
        AssertUtils.isTrue(suffix.equals("xls") || suffix.equals("xlsx") || suffix.equals("et"), "不支持的文件类型");
        try {
-            return EasyExcelUtil.read(file.getInputStream());
+            return EasyExcelUtil.preview100(file.getInputStream());
        } catch (Exception e) {
            throw new BusinessException("文件读取失败，请检查后重试");
        }
@@ -70,28 +76,37 @@ public class DataCollectFileServiceImpl extends ServiceImpl<DataCollectSettingFi
    @Override
    @Transactional
    public void execute(Integer dataCollectSettingId, MultipartFile file) {
-        List<Map<String, Object>> list = analysisExcel(file);
+        long start = System.currentTimeMillis();
-        DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectSettingId);
+        String suffix = FileUtil.extName(file.getOriginalFilename());
-        JdbcTemplate toDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(dataCollectSetting.getToDbId());
+        AssertUtils.isTrue(suffix.equals("xls") || suffix.equals("xlsx") || suffix.equals("et"), "不支持的文件类型");
-        Set<String> set = list.get(0).keySet();
+        try {
-        set.remove("id");
+            List<Map<String, Object>> list = EasyExcelUtil.readAll(file.getInputStream());
-        if(dataCollectSetting.getCreateTable()==1 && toDbJdbcTemplate.queryForList("SHOW TABLES LIKE '"+ dataCollectSetting.getToTable() + "'").size()==0){
+            DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectSettingId);
-            toDbJdbcTemplate.execute(SqlUtil.joinCreateTableSql(dataCollectSetting.getToTable(), set));
+            JdbcTemplate toDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(dataCollectSetting.getToDbId());
-        }
+            Set<String> set = list.get(0).keySet();
-        toDbJdbcTemplate.batchUpdate(SqlUtil.jointInsertSql(dataCollectSetting.getToTable(), set), new BatchPreparedStatementSetter() {
+            set.remove("id");
-            @Override
+            if(dataCollectSetting.getCreateTable()==1 && toDbJdbcTemplate.queryForList("SHOW TABLES LIKE '"+ dataCollectSetting.getToTable() + "'").size()==0){
-            public void setValues(PreparedStatement preparedStatement, int i) throws SQLException {
+                toDbJdbcTemplate.execute(SqlUtil.joinCreateTableSql(dataCollectSetting.getToTable(), set));
-                Map<String, Object> map = list.get(i);
-                int index = 1;
-                for (String key : set) {
-                    preparedStatement.setObject(index++, map.get(key));
-                }
            }
+            toDbJdbcTemplate.batchUpdate(SqlUtil.jointInsertSql(dataCollectSetting.getToTable(), set), new BatchPreparedStatementSetter() {
+                @Override
+                public void setValues(PreparedStatement preparedStatement, int i) throws SQLException {
+                    Map<String, Object> map = list.get(i);
+                    int index = 1;
+                    for (String key : set) {
+                        preparedStatement.setObject(index++, map.get(key));
+                    }
+                }
-            @Override
+                @Override
-            public int getBatchSize() {
+                public int getBatchSize() {
-                return list.size();
+                    return list.size();
-            }
+                }
-        });
+            });
+            dataCollectTaskLogMapper.insert(new DataCollectTaskLog(dataCollectSettingId, new Date(start), new Date(), Long.valueOf(((System.currentTimeMillis()-start)/1000)).intValue(), 2, null));
+        } catch (IOException e) {
+            dataCollectTaskLogMapper.insert(new DataCollectTaskLog(dataCollectSettingId, new Date(start), new Date(), Long.valueOf(((System.currentTimeMillis()-start)/1000)).intValue(), 3, e.getMessage()));
+            throw new BusinessException("文件读取失败，请检查后重试");
+        }
    }
 }
\ No newline at end of file