数据连接调整、添加数据清洗接口

data-collect-server/src/main/java/com/zq/datacollect/controller/DataCollectDbController.java

@@ -12,7 +12,7 @@ import javax.annotation.Resource;
 
 @RestController
 @Api(tags = "数据库汇聚")
-@RequestMapping(value = "/data/collect/db")
+@RequestMapping(value = "/data/collect")
 public class DataCollectDbController {
 
     @Resource
@@ -45,4 +45,4 @@ public class DataCollectDbController {
         dataCollectDbService.execute(dataCollectSettingId);
         return ResultVo.success("汇聚执行成功");
     }
-}
\ No newline at end of file
+}

data-collect-server/src/main/java/com/zq/datacollect/holder/DatabaseHolder.java

@@ -29,4 +29,4 @@ public class DatabaseHolder {
             throw new RuntimeException(e);
         }
     }
-}
\ No newline at end of file
+}

data-collect-server/src/main/java/com/zq/datacollect/runner/DatabaseRunner.java

@@ -61,4 +61,4 @@ public class DatabaseRunner implements ApplicationRunner {
         DatabaseHolder.set(jdbcTemplateMap);
         log.info("成功加载数据库：{}个", jdbcTemplateMap.size());
     }
-}
\ No newline at end of file
+}

data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectDbServiceImpl.java

@@ -82,4 +82,4 @@ public class DataCollectDbServiceImpl extends ServiceImpl<DataCollectSettingDbMa
             }
         });
     }
-}
\ No newline at end of file
+}

dataoperation-server/pom.xml

@@ -74,7 +74,7 @@
         <dependency>
             <groupId>org.apache.poi</groupId>
             <artifactId>poi</artifactId>
-            <version>4.0.0</version>
+            <version>4.1.1</version>
         </dependency>
 <!--        <dependency>-->
 <!--            <groupId>com.oracle</groupId>-->
@@ -117,6 +117,12 @@
             <version>1.0.0</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>com.zq</groupId>
+            <artifactId>data-collect-server</artifactId>
+            <version>1.0.0</version>
+            <scope>compile</scope>
+        </dependency>
 
     </dependencies>
     <build>

dataoperation-server/src/main/java/com/zq/dataoperation/DataoperationApplication.java

@@ -5,9 +5,11 @@ import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
 import org.springframework.scheduling.annotation.EnableScheduling;
+import springfox.documentation.swagger2.annotations.EnableSwagger2;
 
+@EnableSwagger2
 @EnableScheduling
-@MapperScan({"com.zq.dataoperation.dao","com.zq.spiderflow.dao"})
+@MapperScan({"com.zq.dataoperation.dao","com.zq.spiderflow.dao","com.zq.datacollect.mapper"})
 @EnableDiscoveryClient
 @SpringBootApplication//(scanBasePackages = {"com.zq.dataoperation", "com.zq.common.config","com.zq.spiderflow"})
 public class DataoperationApplication {

dataoperation-server/src/main/java/com/zq/dataoperation/controller/CommonQueryController.java

@@ -93,12 +93,12 @@ public class CommonQueryController {
         return ResultVo.success();
     }
 
-    @ApiOperation("运行查询")
-    @PostMapping("/run")
-    public ResultVo run(@RequestBody Map<String, Object> body) throws Exception {
-        System.out.println(body);
-        return ResultVo.success(commonQueryService.run(body));
-    }
+//    @ApiOperation("运行查询")
+//    @PostMapping("/run")
+//    public ResultVo run(@RequestBody Map<String, Object> body) throws Exception {
+//        System.out.println(body);
+//        return ResultVo.success(commonQueryService.run(body));
+//    }
 
     @ApiOperation("自定义sql查询")
     @PostMapping("/runSelect")

dataoperation-server/src/main/java/com/zq/dataoperation/controller/ComponentRunController.java

@@ -37,12 +37,12 @@ public class ComponentRunController {
     }
 
     @ApiOperation("记录排序")
-    @PostMapping("/SortRecord")
-    public ResultVo SortRecord(@RequestBody Map<String, Object> body) throws Exception {
-        return ResultVo.success(commonQueryService.SortRecord(body));
+    @PostMapping("/sortRecord")
+    public ResultVo sortRecord(@RequestBody Map<String, Object> body) throws Exception {
+        return ResultVo.success(commonQueryService.sortRecord(body));
     }
 
-    @ApiOperation("记录排序")
+    @ApiOperation("剪切字符串")
     @PostMapping("/cutString")
     public ResultVo CutString(@RequestBody Map<String, Object> body) throws Exception {
         return ResultVo.success(commonQueryService.CutString(body));
@@ -54,10 +54,15 @@ public class ComponentRunController {
         return ResultVo.success(commonQueryService.validateNumber(body));
     }
 
-    @ApiOperation("数据输出")
+    @ApiOperation("excel输出")
     @PostMapping("/excelOutput")
     public ResultVo excelOutput(HttpServletResponse response, @RequestBody Map<String, Object> body) throws Exception {
         return ResultVo.success(commonQueryService.excelOutput(response,body));
     }
 
+    @ApiOperation("数据库输出")
+    @PostMapping("/sqlOutput")
+    public ResultVo sqlOutput(@RequestBody Map<String, Object> body) throws Exception {
+        return ResultVo.success(commonQueryService.sqlOutput(body));
+    }
 }

dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java

@@ -6,7 +6,9 @@ import com.zq.common.utils.AssertUtils;
 import com.zq.common.vo.ResultVo;
 import com.zq.dataoperation.dao.MetaDataDao;
 import com.zq.dataoperation.entity.DataCleanExpressCategory;
+import com.zq.dataoperation.entity.DataCleanRecord;
 import com.zq.dataoperation.entity.DataCleanRule;
+import com.zq.dataoperation.service.DataCleanRecordService;
 import com.zq.dataoperation.service.DataCleanRuleService;
 import com.zq.dataoperation.service.DataCleanService;
 import com.zq.dataoperation.entity.MetaData;
@@ -17,6 +19,7 @@ import org.springframework.web.bind.annotation.*;
 
 import javax.annotation.Resource;
 import java.util.List;
+import java.util.Map;
 
 @Api(tags = "数据清洗")
 @RequiredArgsConstructor
@@ -30,6 +33,8 @@ public class DataCleanController {
     private DataCleanRuleService dataCleanRuleService;
     @Resource
     private MetaDataDao metaDataDao;
+    @Resource
+    private DataCleanRecordService dataCleanRecordService;
 
     /**
      * 新增规则表达式
@@ -93,4 +98,13 @@ public class DataCleanController {
         return null;
     }
 
+    @ApiOperation("新增清洗记录")
+    @PostMapping("/addCleanList")
+    public ResultVo addCleanList(@RequestBody DataCleanRecord dataCleanRecord){
+        AssertUtils.notNull(dataCleanRecord.getQueryDbId(),"缺少连接库ID");
+        AssertUtils.notNull(dataCleanRecord.getDataCollectTaskLogId(),"缺少汇聚任务ID");
+        return ResultVo.success(dataCleanRecordService.runAndSave(dataCleanRecord));
+    }
+
+
 }

dataoperation-server/src/main/java/com/zq/dataoperation/dao/DataCleanRecordDao.java

+package com.zq.dataoperation.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.zq.dataoperation.entity.DataCleanRecord;
+
+public interface DataCleanRecordDao extends BaseMapper<DataCleanRecord> {
+
+}

dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRecord.java

+package com.zq.dataoperation.entity;
+
+import com.baomidou.mybatisplus.annotation.TableName;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Date;
+
+/**
+ * (data_clean_record)实体类
+ *
+ * @author zq
+ * @since 2023-07-04 16:12:53
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+@TableName(value = "data_clean_record")
+public class DataCleanRecord {
+
+   /**
+     * id
+     */
+    private Integer id;
+
+    /**
+     * 连接id
+     */
+    @ApiModelProperty("连接id")
+    private Long queryDbId;
+
+    /**
+     * 表名
+     */
+    @ApiModelProperty("表名")
+    private String tableName;
+
+    /**
+     * 任务汇聚表ID
+     */
+    @ApiModelProperty("任务汇聚表ID")
+    private Integer dataCollectTaskLogId;
+
+    /**
+     * 清洗开始时间
+     */
+    @ApiModelProperty("清洗开始时间")
+    private Date startTime;
+
+    /**
+     * 结束时间
+     */
+    @ApiModelProperty("结束时间")
+    private Date endTime;
+
+    /**
+     * 花费时间
+     */
+    @ApiModelProperty("花费时间")
+    private String duration;
+
+    /**
+     * 1运行中 2正常结束 3异常终止
+     */
+    @ApiModelProperty("状态")
+    private Integer status;
+
+    /**
+     * 花费时间
+     */
+    @ApiModelProperty("花费时间")
+    private String  dataCollectCount;
+
+    /**
+     * 花费时间
+     */
+    @ApiModelProperty("花费时间")
+    private String  dataCleanCount;
+
+    /**
+     * 花费时间
+     */
+    @ApiModelProperty("花费时间")
+    private String  dataDirtyCount;
+
+    /**
+     * 创建时间
+     */
+    @ApiModelProperty("创建时间")
+    private Date createTime;
+
+    /**
+     * 更新时间
+     */
+    @ApiModelProperty("更新时间")
+    private Date updateTime;
+
+}

dataoperation-server/src/main/java/com/zq/dataoperation/holder/DatabaseHolder.java

+package com.zq.dataoperation.holder;
+
+import org.springframework.jdbc.core.JdbcTemplate;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class DatabaseHolder {
+
+    private static Map<Integer, JdbcTemplate> jdbcTemplateMap;
+
+    public static Map<Integer, JdbcTemplate> getJdbcTemplateMap() {
+        return jdbcTemplateMap;
+    }
+
+    public static void set(Map<Integer, JdbcTemplate> map) {
+        jdbcTemplateMap = map;
+    }
+
+    public static void add(Integer id, JdbcTemplate jdbcTemplate) {
+        if (jdbcTemplateMap == null) {
+            jdbcTemplateMap = new HashMap<>();
+        }
+        jdbcTemplateMap.put(id, jdbcTemplate);
+    }
+
+    public static JdbcTemplate getJdbcTemplate(Integer id) {
+        return jdbcTemplateMap.get(id);
+    }
+
+    public static Connection getConnection(Integer id) {
+        try {
+            return jdbcTemplateMap.get(id).getDataSource().getConnection();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

dataoperation-server/src/main/java/com/zq/dataoperation/runner/DatabaseConnectRunner.java

+package com.zq.dataoperation.runner;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.druid.pool.DruidPooledConnection;
+import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
+import com.zq.common.exception.BusinessException;
+import com.zq.dataoperation.entity.QueryDb;
+import com.zq.dataoperation.utils.ConnectionUtil;
+import com.zq.dataoperation.service.CommonQueryService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PreDestroy;
+import javax.annotation.Resource;;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+//@Slf4j
+//@Component
+//public class DatabaseConnectRunner implements ApplicationRunner {
+//
+//    @Resource
+//    private CommonQueryService service;
+//
+//    private Map<Integer, DruidPooledConnection> connectionMap = new HashMap<>();
+//
+//    @Override
+//    public void run(ApplicationArguments args) throws SQLException {
+//        List<QueryDb> datasource = service.getDatasource();
+//        for (QueryDb ds : datasource) {
+//            DruidDataSource druidDataSource = DruidDataSourceBuilder.create().build();
+//            String url = "";
+//            String className = "";
+//            switch (ds.getDbType()) {
+//                case 1:
+//                    className = "com.mysql.cj.jdbc.Driver";
+//                    url = "jdbc:mysql://" + ds.getDbIp() + ":" + ds.getDbPort() + "/" + ds.getDbName() + "?characterEncoding=utf8&useSSL=false&serverTimezone=GMT%2B8&autoReconnect=true";
+//                    break;
+//                case 2:
+//                    className = "oracle.jdbc.driver.OracleDriver";
+//                    url = "jdbc:oracle:thin:@//" + ds.getDbIp() + ":" + ds.getDbPort() + "/" + ds.getDbName();
+//                    break;
+//                case 3:
+//                    className = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
+//                    url = "jdbc:sqlserver://" + ds.getDbIp() + ":" + ds.getDbPort() + "/" + ds.getDbName();
+//                    break;
+//                default:
+//                    throw new BusinessException("不支持的数据库");
+//            }
+//            druidDataSource.setDriverClassName(className);
+//            druidDataSource.setUrl(url);
+//            druidDataSource.setUsername(ds.getUsername());
+//            druidDataSource.setPassword(ds.getPassword());
+//            druidDataSource.init();
+//            connectionMap.put(ds.getId(), druidDataSource.getConnection());
+//        }
+//        ConnectionUtil.set(connectionMap);
+//        log.info("成功加载数据库：{}个", connectionMap.size());
+//    }
+//
+//    @PreDestroy
+//    public void onClose(){
+//        for (DruidPooledConnection connection : connectionMap.values()) {
+//            try {
+//                connection.close();
+//            } catch (SQLException e) {
+//                e.printStackTrace();
+//            }
+//        }
+//    }
+//}

dataoperation-server/src/main/java/com/zq/dataoperation/runner/DatabaseRunner.java

 package com.zq.dataoperation.runner;
 
 import com.alibaba.druid.pool.DruidDataSource;
-import com.alibaba.druid.pool.DruidPooledConnection;
 import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.zq.common.exception.BusinessException;
+import com.zq.dataoperation.dao.QueryDbDao;
 import com.zq.dataoperation.entity.QueryDb;
-import com.zq.dataoperation.utils.ConnectionUtil;
-import com.zq.dataoperation.service.CommonQueryService;
+import com.zq.dataoperation.holder.DatabaseHolder;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
+import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 
-import javax.annotation.PreDestroy;
-import javax.annotation.Resource;;
+import javax.annotation.Resource;
 import java.sql.SQLException;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
+
 
 @Slf4j
 @Component
 public class DatabaseRunner implements ApplicationRunner {
 
     @Resource
-    private CommonQueryService service;
-
-    private Map<Integer, DruidPooledConnection> connectionMap = new HashMap<>();
+    private QueryDbDao queryDbDao;
 
     @Override
     public void run(ApplicationArguments args) throws SQLException {
-        List<QueryDb> datasource = service.getDatasource();
+        List<QueryDb> datasource = queryDbDao.selectList(new QueryWrapper<>());
         for (QueryDb ds : datasource) {
             DruidDataSource druidDataSource = DruidDataSourceBuilder.create().build();
             String url = "";
@@ -56,20 +53,8 @@ public class DatabaseRunner implements ApplicationRunner {
             druidDataSource.setUsername(ds.getUsername());
             druidDataSource.setPassword(ds.getPassword());
             druidDataSource.init();
-            connectionMap.put(ds.getId(), druidDataSource.getConnection());
-        }
-        ConnectionUtil.set(connectionMap);
-        log.info("成功加载数据库：{}个", connectionMap.size());
-    }
-
-    @PreDestroy
-    public void onClose(){
-        for (DruidPooledConnection connection : connectionMap.values()) {
-            try {
-                connection.close();
-            } catch (SQLException e) {
-                e.printStackTrace();
-            }
+            DatabaseHolder.add(ds.getId(), new JdbcTemplate(druidDataSource));
         }
+        log.info("成功加载数据库：{}个", DatabaseHolder.getJdbcTemplateMap().size());
     }
-}
\ No newline at end of file
+}

dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java

 package com.zq.dataoperation.service;
 
+import cn.hutool.json.JSONArray;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
 import com.alibaba.druid.pool.DruidPooledConnection;
@@ -8,19 +9,22 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.zq.common.exception.BusinessException;
 import com.zq.common.vo.PageVo;
 import com.zq.common.vo.ResultVo;
+import com.zq.datacollect.util.SqlUtil;
 import com.zq.dataoperation.dao.CommonQuerySettingDao;
 import com.zq.dataoperation.dao.MetaDataMappingDao;
 import com.zq.dataoperation.dao.QueryDbDao;
 import com.zq.dataoperation.entity.CommonQuerySetting;
 import com.zq.dataoperation.entity.MetaDataMapping;
 import com.zq.dataoperation.entity.QueryDb;
+import com.zq.dataoperation.holder.DatabaseHolder;
 import com.zq.dataoperation.utils.ConnectionUtil;
-import com.zq.dataoperation.utils.FileUtil;
 import com.zq.dataoperation.utils.IdentityUtils;
 import com.zq.dataoperation.utils.SqlUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.jdbc.core.BatchPreparedStatementSetter;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.bind.annotation.RequestBody;
 
 import java.sql.*;
@@ -28,7 +32,6 @@ import java.sql.*;
 import javax.annotation.Resource;
 import javax.servlet.http.HttpServletResponse;
 import java.util.*;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 @Service
@@ -40,7 +43,6 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
     private MetaDataMappingDao metaDataMappingDao;
 
 
-
     private Map<Long, DruidPooledConnection> connectionMap = new HashMap<>();
 
     public List<QueryDb> getDatasource() {
@@ -64,21 +66,22 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         }
     }
 
-    public List<Map<String, Object>> run(Map<String, Object> body) {
-        CommonQuerySetting querySetting = getById(body.get("queryId").toString());
-        QueryDb queryDb = queryDbDao.selectById(querySetting.getQueryDbId());
-        String sql = querySetting.getQuerySql();
-
-        Pattern pattern = Pattern.compile("\\{([^}]+)}");
-        Matcher matcher = pattern.matcher(sql);
-        while (matcher.find()) {
-            String group = matcher.group(1);
-            if (body.containsKey(group)) {
-                sql = sql.replace("{" + group + "}", body.get(group).toString());
-            }
-        }
-        return commonQueryByJdbcTemplate(sql, queryDb.getId());
-    }
+//    public List<Map<String, Object>> run(Map<String, Object> body) {
+//        CommonQuerySetting querySetting = getById(body.get("queryId").toString());
+//        QueryDb queryDb = queryDbDao.selectById(querySetting.getQueryDbId());
+//        String sql = querySetting.getQuerySql();
+//
+//        Pattern pattern = Pattern.compile("\\{([^}]+)}");
+//        Matcher matcher = pattern.matcher(sql);
+//        while (matcher.find()) {
+//            String group = matcher.group(1);
+//            if (body.containsKey(group)) {
+//                sql = sql.replace("{" + group + "}", body.get(group).toString());
+//            }
+//        }
+//        return commonQuery(sql,queryDb.getId());
+//        return commonQueryByJdbcTemplate(sql, queryDb.getId());
+//    }
 
     public Object checkConnect(QueryDb queryDb) {
         QueryDb queryDb1 = queryDbDao.selectById(queryDb.getId());
@@ -104,7 +107,11 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
             queryDbId = mapping.getQueryDbId();
         }
         String sql = "SELECT * FROM " + tableSqlname + " WHERE " + querySql;
-        return commonQuery(sql, queryDbId);
+        List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(queryDbId).queryForList(sql);
+        JSONArray jsonArray = new JSONArray();
+        jsonArray.addAll(mapList);
+        List<Map> objects = jsonArray.toList(Map.class);
+        return objects;
     }
 
     public Object showTables(@RequestBody Map<String, Object> body) throws Exception {
@@ -123,36 +130,45 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         }
         Integer queryId = Integer.valueOf(jsonObject.get("queryId").toString());
         QueryDb queryDb = queryDbDao.selectById(queryId);
-        List<Map> results = null;
-        List<Map> list = null;
+        List<Map<String, Object>> results = null;
+        List<Map<String, Object>> list = null;
+        JdbcTemplate jdbcTemplate = DatabaseHolder.getJdbcTemplate(queryId);
         if (queryDb != null) {
             String sql = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' " + "AND TABLE_NAME LIKE CONCAT('" + likeName + "' ,'%') ORDER BY CREATE_TIME DESC LIMIT " + start + "," + size;
-            results = commonQuery(sql, queryId);
+            results = jdbcTemplate.queryForList(sql);
             String sqlTotal = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' " + "AND TABLE_NAME LIKE CONCAT('" + likeName + "' ,'%')";
-            list = commonQuery(sqlTotal, queryId);
+            list = jdbcTemplate.queryForList(sqlTotal);
         }
+        JSONArray jsonArray = new JSONArray();
+        jsonArray.addAll(results);
+        List<Map> objects = jsonArray.toList(Map.class);
 
         pageVo.setPage(page);
         pageVo.setSize(size);
-        pageVo.setRows(results);
+        pageVo.setRows(objects);
         pageVo.setTotal(list.size());
         return pageVo;
     }
 
     public List<Map> commonQuery(String sql, Integer queryId) throws Exception {
         ResultSet resultSet = null;
+        List<Map> results = new ArrayList<>();
         try {
-            Connection connection = ConnectionUtil.get(queryId);
+            Connection connection = DatabaseHolder.getConnection(queryId);
             connection.prepareStatement(sql).setQueryTimeout(5000);
-            resultSet = connection.prepareStatement(sql).executeQuery();
-            int columnCount = resultSet.getMetaData().getColumnCount();
-            List<Map> results = new ArrayList<>();
-            while (resultSet.next()) {
-                Map<String, String> map = new HashMap<>(columnCount);
-                for (int i = 1; i <= columnCount; i++) {
-                    map.put(resultSet.getMetaData().getColumnName(i), resultSet.getString(i));
+            if (sql.contains("UPDATE") || sql.contains("DELETE") || sql.contains("INSERT")) {
+                boolean b = connection.prepareStatement(sql).execute();
+                if (b) results = null;
+            } else {
+                resultSet = connection.prepareStatement(sql).executeQuery();
+                int columnCount = resultSet.getMetaData().getColumnCount();
+                while (resultSet.next()) {
+                    Map<String, String> map = new HashMap<>(columnCount);
+                    for (int i = 1; i <= columnCount; i++) {
+                        map.put(resultSet.getMetaData().getColumnName(i), resultSet.getString(i));
+                    }
+                    results.add(map);
                 }
-                results.add(map);
             }
             return results;
         } catch (Exception e) {
@@ -164,17 +180,17 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         }
     }
 
-    public List<Map<String, Object>> commonQueryByJdbcTemplate(String sql, Integer queryId) {
-        JdbcTemplate jdbcTemplate = ConnectionUtil.getJdbcTemplate(queryId);
-        return jdbcTemplate.queryForList(sql);
-    }
+//    public List<Map<String, Object>> commonQueryByJdbcTemplate(String sql, Integer queryId) {
+//        JdbcTemplate jdbcTemplate = ConnectionUtil.getJdbcTemplate(queryId);
+//        return jdbcTemplate.queryForList(sql);
+//    }
 
     public Object getTables(Integer id) throws Exception {
         QueryDb queryDb = queryDbDao.selectById(id);
-        List<Map> results = null;
+        List<Map<String, Object>> results = null;
         if (queryDb != null) {
-            String sql = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' ORDER BY CREATE_TIME DESC " ;
-            results = commonQuery(sql, id);
+            String sql = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' ORDER BY CREATE_TIME DESC ";
+            results = DatabaseHolder.getJdbcTemplate(id).queryForList(sql);
         }
         return results;
     }
@@ -183,8 +199,9 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         JSONObject jsonObject = JSONUtil.parseObj(body);
         Integer id = Integer.valueOf(jsonObject.get("id").toString());
         String tableName = jsonObject.get("tableName").toString();
-        String sql="SELECT * FROM "+tableName;
-        return commonQuery(sql,id);
+        String sql = "SELECT * FROM " + tableName;
+        List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(id).queryForList(sql);
+        return mapList;
     }
 
     //----------------------------------组件接口----------------------------------------------------------
@@ -194,19 +211,22 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString());
         String tableName = jsonObject.get("tableName").toString();
         String metaData = jsonObject.get("metaData").toString();
-        String primarySql="SELECT distinct column_name FROM INFORMATION_SCHEMA.`KEY_COLUMN_USAGE` WHERE table_name= '"+tableName+"' AND constraint_name='PRIMARY'";
-        List<Map> primaryList = commonQuery(primarySql, datasourceId);
-        String primaryKey =null;
-        for (Object obj:primaryList) {
+        String primarySql = "SELECT distinct column_name FROM INFORMATION_SCHEMA.`KEY_COLUMN_USAGE` WHERE table_name= '" + tableName + "' AND constraint_name='PRIMARY'";
+        List<Map<String, Object>> primaryList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(primarySql);
+        String primaryKey = null;
+        for (Object obj : primaryList) {
             primaryKey = JSONUtil.parseObj(obj).getStr("COLUMN_NAME");
         }
-        String sql1="SELECT "+metaData+" FROM " + tableName+" GROUP BY "+metaData+" HAVING COUNT("+metaData+")>1;";
-        List<Map> mapList = commonQuery(sql1, datasourceId);
-        if (mapList.size()==0){
-        return "该字段没有重复值";
-        }else{
-        String sql = "SELECT * FROM " + tableName + " WHERE "+primaryKey+" not in (SELECT MIN("+primaryKey+") FROM " + tableName+" GROUP BY "+metaData+" HAVING COUNT("+metaData+")>1);";
-        return commonQuery(sql, datasourceId);
+        String repeat = "SELECT " + metaData + " FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1;";
+        List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(repeat);
+        if (mapList.size() == 0) {
+            return "该字段没有重复值";
+        } else {
+            String sql1 = "SELECT * FROM " + tableName + " WHERE " + primaryKey + " in (SELECT MIN(" + primaryKey + ") FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1) UNION ALL ";
+            String sql2 = "SELECT * FROM " + tableName + " WHERE " + metaData + " not in (SELECT MIN(" + metaData + ") FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1);";
+            String sql = sql1 + sql2;
+
+            return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
         }
     }
 
@@ -216,18 +236,18 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString());
         String tableName = jsonObject.get("tableName").toString();
         String metaData = jsonObject.get("metaData").toString();
-        String sql="SELECT * FROM " + tableName;
+        String sql = "SELECT * FROM " + tableName;
         if (!StringUtils.isNotBlank(metaData)) {
-           return "字段为身份证且不为空！";
+            return "字段为身份证且不为空！";
         }
-        if(IdentityUtils.isLegalIdCard(metaData)!=true){
+        if (IdentityUtils.isLegalIdCard(metaData) != true) {
             return "身份证号不符";
-        }else if(IdentityUtils.is15BitLegalIdCard(metaData)!=true){
+        } else if (IdentityUtils.is15BitLegalIdCard(metaData) != true) {
             return "身份证不满15位！";
-        }else if(IdentityUtils.is18BitLegalIdCard(metaData)!=true){
+        } else if (IdentityUtils.is18BitLegalIdCard(metaData) != true) {
             return "身份证不满18位！";
-        }else {
-            return commonQuery(sql,datasourceId);
+        } else {
+            return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
         }
     }
 
@@ -237,30 +257,35 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString());
         String tableName = jsonObject.get("tableName").toString();
         String metaData = jsonObject.get("metaData").toString();
-        String originStr= jsonObject.get("originStr").toString();
-        String replaceStr= jsonObject.get("replaceStr").toString();
-        String sql ="UPDATE "+tableName+" SET "+metaData+"=REPLACE("+metaData+",'"+originStr+"','"+replaceStr+"');";
-        if (StringUtils.isNotBlank(metaData)&&StringUtils.isNotBlank(tableName)&&StringUtils.isNotBlank(originStr)&&StringUtils.isNotBlank(replaceStr)) {
+        String originStr = jsonObject.get("originStr").toString();
+        String replaceStr = jsonObject.get("replaceStr").toString();
+        String sql = "UPDATE " + tableName + " SET " + metaData + "=REPLACE(" + metaData + ",'" + originStr + "','" + replaceStr + "');";
+        String result = "SELECT * FROM " + tableName;
+        if (!(StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName) && StringUtils.isNotBlank(originStr) && StringUtils.isNotBlank(replaceStr))) {
             return "传入参数不能为空";
-        }else{
-            return commonQuery(sql,datasourceId);
+        } else {
+            List<Map<String, Object>> mapList =DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
+            if (mapList.size() == 0 || mapList == null)
+                return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(result);
         }
+
+        return "";
     }
 
     //记录排序
-    public Object SortRecord(Map<String, Object> body) throws Exception {
+    public Object sortRecord(Map<String, Object> body) throws Exception {
         JSONObject jsonObject = JSONUtil.parseObj(body);
         Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString());
         String tableName = jsonObject.get("tableName").toString();
         String metaData = jsonObject.get("metaData").toString();
-        String sortStr=jsonObject.get("sortStr").toString();
-        if(sortStr.equals("1")){
-            sortStr="ASC";
-        }else{
-            sortStr="DESC";
+        String sortStr = jsonObject.get("sortStr").toString();
+        if (sortStr.equals("1")) {
+            sortStr = "ASC";
+        } else {
+            sortStr = "DESC";
         }
-        String sql="SELECT * FROM "+tableName+" ORDER BY "+metaData+" "+sortStr;
-        return commonQuery(sql,datasourceId);
+        String sql = "SELECT * FROM " + tableName + " ORDER BY " + metaData + " " + sortStr;
+        return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
     }
 
     //剪切字符串
@@ -271,40 +296,40 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         String beginStr = jsonObject.get("beginStr").toString();
         String indexStr = jsonObject.get("indexStr").toString();
         String metaData = jsonObject.get("metaData").toString();
-        String sql="SELECT *,"+" SUBSTR("+metaData+","+beginStr+","+indexStr+") AS "+metaData +"1 FROM "+tableName;
-        if (!StringUtils.isNotBlank(metaData)&&StringUtils.isNotBlank(tableName)&&StringUtils.isNotBlank(beginStr)&&StringUtils.isNotBlank(indexStr)) {
+        String sql = "SELECT *," + " SUBSTR(" + metaData + "," + beginStr + "," + indexStr + ") AS " + metaData + "1 FROM " + tableName;
+        if (!StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName) && StringUtils.isNotBlank(beginStr) && StringUtils.isNotBlank(indexStr)) {
             return "传入参数不能为空";
-        }else{
-            return commonQuery(sql,datasourceId);
+        } else {
+            return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
         }
     }
 
-    //验证电话
+    //验证电话---------------------------------------------------------
     public Object validateNumber(Map<String, Object> body) throws Exception {
         JSONObject jsonObject = JSONUtil.parseObj(body);
         Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString());
         String tableName = jsonObject.get("tableName").toString();
         String metaData = jsonObject.get("metaData").toString();
         HashMap<String, Object> map = new HashMap<>();
-        if (!StringUtils.isNotBlank(metaData)&&StringUtils.isNotBlank(tableName)) {
+        if (!StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName)) {
             return "传入参数不能为空";
         }
-        String phone="SELECT "+metaData+" FROM "+tableName;
-        String sql ="SELECT * FROM "+tableName+" WHERE "+metaData+" REGEXP '^1[3-9][0-9]{9}$'";
-        List<Map> mapList = commonQuery(phone, datasourceId);
+        String phone = "SELECT " + metaData + " FROM " + tableName;
+        String sql = "SELECT * FROM " + tableName + " WHERE " + metaData + " REGEXP '^1[3-9][0-9]{9}$'";
+        List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(phone);
         String regex = "/^1\\d{10}$|^(0\\d{2,3}-?|\\(0\\d{2,3}\\))?[1-9]\\d{4,7}(-\\d{1,8})?$/";
         Pattern pattern = Pattern.compile(regex);
-        for (Object list:mapList) {
+        for (Object list : mapList) {
             String meta = list.toString();
-            if(!pattern.matcher(meta).matches()){
+            if (!pattern.matcher(meta).matches()) {
                 return "此字段不是电话号码";
             }
         }
-        List<Map> dataList = commonQuery(sql, datasourceId);
-        String dirty="SELECT * FROM "+tableName+" WHERE "+metaData+" NOT REGEXP '^1[3-9][0-9]{9}$'";
-        List<Map> dirtyList = commonQuery(dirty, datasourceId);
-        map.put("dataList",dataList);
-        map.put("dirtyList",dirtyList);
+        List<Map<String, Object>> dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
+        String dirty = "SELECT * FROM " + tableName + " WHERE " + metaData + " NOT REGEXP '^1[3-9][0-9]{9}$'";
+        List<Map<String, Object>> dirtyList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dirty);
+        map.put("dataList", dataList);
+        map.put("dirtyList", dirtyList);
         return map;
     }
 
@@ -313,11 +338,75 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString());
         String tableName = jsonObject.get("tableName").toString();
         String filePath = jsonObject.get("filePath").toString();
-        String sql="SELECT * FROM "+tableName;
-        List<Map> list = commonQuery(sql, datasourceId);
-        FileUtil.outputExcel(list,response,filePath,tableName);
+        String sql = "SELECT * FROM " + tableName;
+        List<Map<String, Object>> list = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
+//        for (Object object:list) {
+//            String s = JSONUtil.parse(object).toString();
+//            String []data= sql.substring(1,s.length()-1).split(",");
+//            if (filePath.equals("csv")){
+//                String path="/tmp/tmpfiles/"+tableName+"."+filePath;
+//                File file = new File(path);
+//                if(file.exists())
+//                    file.delete();
+//                CsvWriter csvWriter=new CsvWriter(path, Charset.forName("UTF-8"));
+//                csvWriter.write(data);
+//                csvWriter.flush();
+//                csvWriter.close();
+//            }else if(filePath.equals("xlsx")){
+//                FileUtil.downloadExcel(list,response,tableName);
+//            }
+//
+//        }
         return list;
     }
 
 
+    @Transactional
+    public Object sqlOutput(Map<String, Object> body) throws Exception {
+        long startTime = System.currentTimeMillis();
+
+        JSONObject jsonObject = JSONUtil.parseObj(body);
+        Integer oldId = Integer.valueOf(jsonObject.get("oldId").toString());
+        Integer newId = Integer.valueOf(jsonObject.get("newId").toString());
+        String newTableName = jsonObject.get("newTableName").toString();
+        String oldTableName = jsonObject.get("oldTableName").toString();
+
+        JdbcTemplate toDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(newId);
+        JdbcTemplate fromDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(oldId);
+        List<Map<String, Object>> list = fromDbJdbcTemplate.queryForList("SELECT * FROM " + oldTableName);
+        Set<String> set = list.get(0).keySet();
+        set.remove("id");
+
+        String sqlQuery = "SHOW TABLES LIKE '" + newTableName + "';";
+        List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(newId).queryForList(sqlQuery);
+        if (mapList.size() == 0) {
+            toDbJdbcTemplate.execute(SqlUtil.joinCreateTableSql(newTableName, set));
+        }
+        toDbJdbcTemplate.batchUpdate(SqlUtil.jointInsertSql(newTableName, set), new BatchPreparedStatementSetter() {
+            @Override
+            public void setValues(PreparedStatement preparedStatement, int i) throws SQLException {
+                Map<String, Object> map = list.get(i);
+                int index = 1;
+                for (String key : set) {
+                    preparedStatement.setObject(index++, map.get(key));
+                }
+            }
+
+            @Override
+            public int getBatchSize() {
+                return list.size();
+            }
+        });
+
+        List<Map<String, Object>> newCount = fromDbJdbcTemplate.queryForList("SELECT COUNT(1) FROM " + newTableName);
+        List<Map<String, Object>> oldCount = fromDbJdbcTemplate.queryForList("SELECT COUNT(1) FROM " + oldTableName);
+        List<Map<String, Object>> newTable = fromDbJdbcTemplate.queryForList("SELECT * FROM " + newTableName);
+        long times = System.currentTimeMillis() - startTime;
+        Map<String, Object> map = new HashMap<>();
+        map.put("executeTime",times);
+        map.put("oldCount",oldCount);
+        map.put("newCount",newCount);
+        map.put("newTable",newTable);
+        return map;
+    }
 }

dataoperation-server/src/main/java/com/zq/dataoperation/service/DataCleanRecordService.java

+package com.zq.dataoperation.service;
+
+import cn.hutool.core.date.DateUtil;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.zq.common.utils.AssertUtils;
+import com.zq.common.vo.ResultVo;
+import com.zq.datacollect.entity.DataCollectTaskLog;
+import com.zq.datacollect.mapper.DataCollectTaskLogMapper;
+import com.zq.dataoperation.dao.DataCleanRecordDao;
+import com.zq.dataoperation.dao.QueryDbDao;
+import com.zq.dataoperation.entity.DataCleanRecord;
+import com.zq.dataoperation.entity.QueryDb;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+
+@Slf4j
+@RequiredArgsConstructor
+@Service
+public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, DataCleanRecord> {
+
+    @Resource
+    private DataCleanRecordDao dataCleanRecordDao;
+    @Resource
+    private QueryDbDao queryDbDao;
+    @Resource
+    private DataCollectTaskLogMapper dataCollectTaskLogMapper;
+
+
+    public Object runAndSave(DataCleanRecord dataCleanRecord) {
+        QueryDb queryDb = queryDbDao.selectById(dataCleanRecord.getQueryDbId());
+        AssertUtils.notNull(queryDb, "无连接表ID");
+        DataCollectTaskLog dataCollectTaskLog = dataCollectTaskLogMapper.selectById(dataCleanRecord.getDataCollectTaskLogId());
+        AssertUtils.notNull(dataCollectTaskLog, "无连接表ID");
+        DataCleanRecord build = DataCleanRecord.builder()
+                .queryDbId(dataCleanRecord.getQueryDbId())
+                .dataCollectTaskLogId(dataCollectTaskLog.getDataCollectSettingId())
+                .tableName(dataCleanRecord.getTableName())
+                .endTime(DateUtil.date())
+                .status(dataCollectTaskLog.getStatus())
+                .dataCollectCount(dataCleanRecord.getDataCollectCount())
+                .dataCleanCount(dataCleanRecord.getDataCleanCount())
+                .dataDirtyCount(dataCleanRecord.getDataDirtyCount())
+                .createTime(DateUtil.date())
+                .build();
+        DataCleanRecord record = dataCleanRecordDao.selectById(dataCleanRecord.getId());
+        int i = 0;
+        if (record == null) {
+            i = dataCleanRecordDao.insert(build);
+        } else {
+            i = dataCleanRecordDao.updateById(build);
+        }
+        return i>0 ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败");
+    }
+}

dataoperation-server/src/main/java/com/zq/dataoperation/utils/ConnectionUtil.java

@@ -16,8 +16,4 @@ public class ConnectionUtil {
     public static Connection get(Integer id){
         return connectionMap.get(id).getConnection();
     }
-
-    public static Connection get(Integer id) throws SQLException {
-        return dataSourceMap.get(id).getConnection();
-    }
 }

dataoperation-server/src/main/java/com/zq/dataoperation/utils/FileUtil.java

@@ -140,7 +140,7 @@ public class FileUtil extends cn.hutool.core.io.FileUtil {
     /**
      * 导出excel
      */
-    public static void downloadExcel(List<Map<String, Object>> list, HttpServletResponse response, String fileName) throws IOException {
+    public static void downloadExcel(List<Map> list, HttpServletResponse response, String fileName) throws IOException {
         String tempPath = fileName + ".xlsx";
         File file = new File(tempPath);
         BigExcelWriter writer = ExcelUtil.getBigWriter(file);
@@ -158,26 +158,6 @@ public class FileUtil extends cn.hutool.core.io.FileUtil {
         IoUtil.close(out);
     }
 
-    /**
-     * 导出excel
-     */
-    public static void outputExcel(List<Map> list, HttpServletResponse response, String filePath,String fileName) throws IOException {
-//        String tempPath = fileName + ".xlsx";
-        File file = new File(fileName+"."+filePath);
-        BigExcelWriter writer = ExcelUtil.getBigWriter(file);
-        // 一次性写出内容，使用默认样式，强制输出标题
-        writer.write(list, true);
-        //response为HttpServletResponse对象
-        response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;charset=utf-8");
-        //test.xls是弹出下载对话框的文件名，不能为中文，中文请自行编码
-        response.setHeader("Content-Disposition", "attachment;filename=file.xlsx");
-        ServletOutputStream out = response.getOutputStream();
-        // 终止后删除临时文件
-        file.deleteOnExit();
-        writer.flush(out, true);
-        //此处记得关闭输出Servlet流
-        IoUtil.close(out);
-    }
 
     public static String getFileType(String type) {
         String documents = "txt doc pdf ppt pps xlsx xls docx";

dataoperation-server/src/main/java/com/zq/dataoperation/utils/TableUtil.java

 package com.zq.dataoperation.utils;
 
+import com.zq.dataoperation.holder.DatabaseHolder;
 import com.zq.dataoperation.vo.Column;
 import lombok.SneakyThrows;
+import org.springframework.jdbc.core.JdbcTemplate;
 
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -18,7 +20,7 @@ public class TableUtil {
     public static Object getColumns(Integer queryId, String tableName) {
         String driver = "com.mysql.cj.jdbc.Driver";
         Class.forName(driver);
-        Connection connection = ConnectionUtil.get(queryId);
+        Connection connection = DatabaseHolder.getConnection(queryId);
         DatabaseMetaData metaData = connection.getMetaData();
         String primaryColumn = "";
         ResultSet idxRs = metaData.getIndexInfo(connection.getCatalog(), connection.getCatalog(), tableName, false, false);

spider-flow-core/src/main/java/com/zq/spiderflow/core/mapper/SpiderFlowMapper.java

@@ -34,8 +34,8 @@ public interface SpiderFlowMapper extends BaseMapper<SpiderFlow>{
 	})
 	IPage<SpiderFlow> selectSpiderPage(Page<SpiderFlow> page,@Param("name") String name);
 
-	@Insert("insert into sp_flow(id,name,xml,enabled) values(#{id},#{name},#{xml},'0')")
-	int insertSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml);
+	@Insert("insert into sp_flow(id,name,xml,flow_type,enabled) values(#{id},#{name},#{xml},#{flowType},'0')")
+	int insertSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml, @Param("flowType") String flowType);
 
 	@Update("update sp_flow set name = #{name},xml = #{xml} where id = #{id}")
 	int updateSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml);

spider-flow-core/src/main/java/com/zq/spiderflow/core/model/SpiderFlow.java

@@ -39,6 +39,11 @@ public class SpiderFlow {
 
     private String enabled;
 
+    /**
+     * 1模型2汇聚3清洗
+     */
+    private String flowType;
+
     private Date createDate;
 
     private Date lastExecuteTime;

spider-flow-core/src/main/java/com/zq/spiderflow/core/service/SpiderFlowService.java

@@ -128,7 +128,7 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
 			}
 		}else{//insert 任务
 			String id = UUID.randomUUID().toString().replace("-", "");
-			sfMapper.insertSpiderFlow(id, spiderFlow.getName(), spiderFlow.getXml());
+			sfMapper.insertSpiderFlow(id, spiderFlow.getName(), spiderFlow.getXml(),spiderFlow.getFlowType());
 			spiderFlow.setId(id);
 		}
 		File file = new File(workspace,spiderFlow.getId() + File.separator + "xmls" + File.separator + System.currentTimeMillis() + ".xml");
@@ -259,7 +259,7 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
 
 	public PageVo<SpiderFlow> getPageList(SpiderFlowVo vo) {
 		LambdaQueryWrapper<SpiderFlow> queryWrapper = new LambdaQueryWrapper<>();
-		queryWrapper.orderByDesc(SpiderFlow::getCreateDate);
+		queryWrapper.eq(SpiderFlow::getFlowType,vo.getFlowType()).orderByDesc(SpiderFlow::getCreateDate);
 		return PagingUtils.paging(vo, sfMapper, queryWrapper, SpiderFlow.class);
 	}
 }

spider-flow-core/src/main/java/com/zq/spiderflow/core/vo/SpiderFlowVo.java

@@ -52,5 +52,10 @@ public class SpiderFlowVo extends PageReqVo{
     @TableField(exist = false)
     private Integer running;
 
+    /**
+     * 1模型2汇聚3清洗
+     */
+    private String flowType;
+
 
 }