解决数据库连接无响应问题

dataoperation-server/pom.xml

@@ -111,20 +111,15 @@
             <groupId>org.springframework</groupId>
             <artifactId>spring-test</artifactId>
         </dependency>
+
         <dependency>
             <groupId>com.zq</groupId>
             <artifactId>sys-server</artifactId>
             <version>1.0.0</version>
             <scope>compile</scope>
         </dependency>
-        <dependency>
-            <groupId>com.zq</groupId>
-            <artifactId>data-collect-server</artifactId>
-            <version>1.0.0</version>
-            <scope>compile</scope>
-        </dependency>
-
     </dependencies>
+
     <build>
         <plugins>
             <plugin>

dataoperation-server/src/main/java/com/zq/dataoperation/controller/CommonQueryController.java

@@ -93,12 +93,11 @@ public class CommonQueryController {
         return ResultVo.success();
     }
 
-//    @ApiOperation("运行查询")
-//    @PostMapping("/run")
-//    public ResultVo run(@RequestBody Map<String, Object> body) throws Exception {
-//        System.out.println(body);
-//        return ResultVo.success(commonQueryService.run(body));
-//    }
+    @ApiOperation("运行查询")
+    @PostMapping("/run")
+    public ResultVo run(@RequestBody Map<String, Object> body) {
+        return ResultVo.success(commonQueryService.run(body));
+    }
 
     @ApiOperation("自定义sql查询")
     @PostMapping("/runSelect")

dataoperation-server/src/main/java/com/zq/dataoperation/dao/DataCollectTaskLogMapper.java

+package com.zq.dataoperation.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.zq.dataoperation.entity.DataCollectTaskLog;
+
+public interface DataCollectTaskLogMapper extends BaseMapper<DataCollectTaskLog> {
+}
\ No newline at end of file

dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCollectTaskLog.java

+package com.zq.dataoperation.entity;
+
+import com.baomidou.mybatisplus.annotation.*;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Date;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@TableName(value = "data_collect_task_log")
+public class DataCollectTaskLog {
+
+    /**
+     * 数据汇聚表id
+     */
+    private Integer dataCollectSettingId;
+
+    /**
+     * 开始时间
+     */
+    private Date startTime;
+
+    /**
+     * 结束时间
+     */
+    private Date endTime;
+
+    /**
+     * 花费多少秒
+     */
+    private Integer duration;
+
+    /**
+     * 状态
+     * 1运行中 2正常结束 3异常终止
+     */
+    private Integer status;
+
+    /**
+     * 错误信息
+     * 异常终止时记录异常信息
+     */
+    private String error;
+
+    /**
+     * 使用序列
+     * id
+     */
+    @TableId(type = IdType.AUTO)
+    private Integer id;
+
+    /**
+     * createTime
+     */
+    @TableField(fill = FieldFill.INSERT)
+    private Date createTime;
+
+    /**
+     * updateTime
+     */
+    @TableField(fill = FieldFill.INSERT_UPDATE)
+    private Date updateTime;
+}
+

dataoperation-server/src/main/java/com/zq/dataoperation/holder/DatabaseHolder.java

@@ -11,26 +11,19 @@ public class DatabaseHolder {
 
     private static Map<Integer, JdbcTemplate> jdbcTemplateMap;
 
-    public static Map<Integer, JdbcTemplate> getJdbcTemplateMap() {
-        return jdbcTemplateMap;
-    }
-
     public static void set(Map<Integer, JdbcTemplate> map) {
         jdbcTemplateMap = map;
     }
 
-    public static void add(Integer id, JdbcTemplate jdbcTemplate) {
-        if (jdbcTemplateMap == null) {
-            jdbcTemplateMap = new HashMap<>();
-        }
+    public static void add(Integer id, JdbcTemplate jdbcTemplate){
         jdbcTemplateMap.put(id, jdbcTemplate);
     }
 
-    public static JdbcTemplate getJdbcTemplate(Integer id) {
+    public static JdbcTemplate getJdbcTemplate(Integer id){
         return jdbcTemplateMap.get(id);
     }
 
-    public static Connection getConnection(Integer id) {
+    public static Connection getConnection(Integer id){
         try {
             return jdbcTemplateMap.get(id).getDataSource().getConnection();
         } catch (SQLException e) {

dataoperation-server/src/main/java/com/zq/dataoperation/runner/DatabaseRunner.java

@@ -2,32 +2,34 @@ package com.zq.dataoperation.runner;
 
 import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.zq.common.exception.BusinessException;
-import com.zq.dataoperation.dao.QueryDbDao;
 import com.zq.dataoperation.entity.QueryDb;
 import com.zq.dataoperation.holder.DatabaseHolder;
+import com.zq.dataoperation.service.CommonQueryService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 
-import javax.annotation.Resource;
+import javax.annotation.Resource;;
 import java.sql.SQLException;
+import java.util.HashMap;
 import java.util.List;
-
+import java.util.Map;
 
 @Slf4j
 @Component
 public class DatabaseRunner implements ApplicationRunner {
 
     @Resource
-    private QueryDbDao queryDbDao;
+    private CommonQueryService service;
+
+    private Map<Integer, JdbcTemplate> jdbcTemplateMap = new HashMap<>();
 
     @Override
     public void run(ApplicationArguments args) throws SQLException {
-        List<QueryDb> datasource = queryDbDao.selectList(new QueryWrapper<>());
+        List<QueryDb> datasource = service.getDatasource();
         for (QueryDb ds : datasource) {
             DruidDataSource druidDataSource = DruidDataSourceBuilder.create().build();
             String url = "";
@@ -53,8 +55,9 @@ public class DatabaseRunner implements ApplicationRunner {
             druidDataSource.setUsername(ds.getUsername());
             druidDataSource.setPassword(ds.getPassword());
             druidDataSource.init();
-            DatabaseHolder.add(ds.getId(), new JdbcTemplate(druidDataSource));
+            jdbcTemplateMap.put(ds.getId(), new JdbcTemplate(druidDataSource));
         }
-        log.info("成功加载数据库：{}个", DatabaseHolder.getJdbcTemplateMap().size());
+        DatabaseHolder.set(jdbcTemplateMap);
+        log.info("成功加载数据库：{}个", jdbcTemplateMap.size());
     }
 }
\ No newline at end of file

dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java

 package com.zq.dataoperation.service;
 
-import cn.hutool.json.JSONArray;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
 import com.alibaba.druid.pool.DruidPooledConnection;
@@ -9,7 +8,6 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.zq.common.exception.BusinessException;
 import com.zq.common.vo.PageVo;
 import com.zq.common.vo.ResultVo;
-import com.zq.datacollect.util.SqlUtil;
 import com.zq.dataoperation.dao.CommonQuerySettingDao;
 import com.zq.dataoperation.dao.MetaDataMappingDao;
 import com.zq.dataoperation.dao.QueryDbDao;
@@ -17,7 +15,6 @@ import com.zq.dataoperation.entity.CommonQuerySetting;
 import com.zq.dataoperation.entity.MetaDataMapping;
 import com.zq.dataoperation.entity.QueryDb;
 import com.zq.dataoperation.holder.DatabaseHolder;
-import com.zq.dataoperation.utils.ConnectionUtil;
 import com.zq.dataoperation.utils.IdentityUtils;
 import com.zq.dataoperation.utils.SqlUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -32,6 +29,7 @@ import java.sql.*;
 import javax.annotation.Resource;
 import javax.servlet.http.HttpServletResponse;
 import java.util.*;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 @Service
@@ -43,6 +41,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
     private MetaDataMappingDao metaDataMappingDao;
 
 
+
     private Map<Long, DruidPooledConnection> connectionMap = new HashMap<>();
 
     public List<QueryDb> getDatasource() {
@@ -66,22 +65,21 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         }
     }
 
-//    public List<Map<String, Object>> run(Map<String, Object> body) {
-//        CommonQuerySetting querySetting = getById(body.get("queryId").toString());
-//        QueryDb queryDb = queryDbDao.selectById(querySetting.getQueryDbId());
-//        String sql = querySetting.getQuerySql();
-//
-//        Pattern pattern = Pattern.compile("\\{([^}]+)}");
-//        Matcher matcher = pattern.matcher(sql);
-//        while (matcher.find()) {
-//            String group = matcher.group(1);
-//            if (body.containsKey(group)) {
-//                sql = sql.replace("{" + group + "}", body.get(group).toString());
-//            }
-//        }
-//        return commonQuery(sql,queryDb.getId());
-//        return commonQueryByJdbcTemplate(sql, queryDb.getId());
-//    }
+    public List<Map<String, Object>> run(Map<String, Object> body) {
+        CommonQuerySetting querySetting = getById(body.get("queryId").toString());
+        QueryDb queryDb = queryDbDao.selectById(querySetting.getQueryDbId());
+        String sql = querySetting.getQuerySql();
+
+        Pattern pattern = Pattern.compile("\\{([^}]+)}");
+        Matcher matcher = pattern.matcher(sql);
+        while (matcher.find()) {
+            String group = matcher.group(1);
+            if (body.containsKey(group)) {
+                sql = sql.replace("{" + group + "}", body.get(group).toString());
+            }
+        }
+        return commonQuery(sql, queryDb.getId());
+    }
 
     public Object checkConnect(QueryDb queryDb) {
         QueryDb queryDb1 = queryDbDao.selectById(queryDb.getId());
@@ -95,7 +93,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         }
     }
 
-    public List<Map> runSelect(Map<String, Object> body) throws Exception {
+    public List<Map<String, Object>> runSelect(Map<String, Object> body) throws Exception {
         JSONObject jsonObject = JSONUtil.parseObj(body);
         String querySql = jsonObject.get("querySql").toString();
         Long id = Long.valueOf(jsonObject.get("id").toString());
@@ -107,11 +105,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
             queryDbId = mapping.getQueryDbId();
         }
         String sql = "SELECT * FROM " + tableSqlname + " WHERE " + querySql;
-        List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(queryDbId).queryForList(sql);
-        JSONArray jsonArray = new JSONArray();
-        jsonArray.addAll(mapList);
-        List<Map> objects = jsonArray.toList(Map.class);
-        return objects;
+        return commonQuery(sql, queryDbId);
     }
 
     public Object showTables(@RequestBody Map<String, Object> body) throws Exception {
@@ -120,7 +114,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         int size = (int) jsonObject.get("size");
         String likeName = jsonObject.get("likeName").toString();
         int start = 0;
-        PageVo<Map> pageVo = new PageVo<>();
+        PageVo<Map<String, Object>> pageVo = new PageVo<>();
         if (page - 1 == 0) {
             pageVo.setStart(0);
             start = 0;
@@ -132,65 +126,31 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         QueryDb queryDb = queryDbDao.selectById(queryId);
         List<Map<String, Object>> results = null;
         List<Map<String, Object>> list = null;
-        JdbcTemplate jdbcTemplate = DatabaseHolder.getJdbcTemplate(queryId);
         if (queryDb != null) {
             String sql = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' " + "AND TABLE_NAME LIKE CONCAT('" + likeName + "' ,'%') ORDER BY CREATE_TIME DESC LIMIT " + start + "," + size;
-            results = jdbcTemplate.queryForList(sql);
+            results = commonQuery(sql, queryId);
             String sqlTotal = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' " + "AND TABLE_NAME LIKE CONCAT('" + likeName + "' ,'%')";
-            list = jdbcTemplate.queryForList(sqlTotal);
+            list = commonQuery(sqlTotal, queryId);
         }
-        JSONArray jsonArray = new JSONArray();
-        jsonArray.addAll(results);
-        List<Map> objects = jsonArray.toList(Map.class);
 
         pageVo.setPage(page);
         pageVo.setSize(size);
-        pageVo.setRows(objects);
+        pageVo.setRows(results);
         pageVo.setTotal(list.size());
         return pageVo;
     }
 
-    public List<Map> commonQuery(String sql, Integer queryId) throws Exception {
-        ResultSet resultSet = null;
-        List<Map> results = new ArrayList<>();
-        try {
-            Connection connection = DatabaseHolder.getConnection(queryId);
-            connection.prepareStatement(sql).setQueryTimeout(5000);
-            if (sql.contains("UPDATE") || sql.contains("DELETE") || sql.contains("INSERT")) {
-                boolean b = connection.prepareStatement(sql).execute();
-                if (b) results = null;
-            } else {
-                resultSet = connection.prepareStatement(sql).executeQuery();
-                int columnCount = resultSet.getMetaData().getColumnCount();
-                while (resultSet.next()) {
-                    Map<String, String> map = new HashMap<>(columnCount);
-                    for (int i = 1; i <= columnCount; i++) {
-                        map.put(resultSet.getMetaData().getColumnName(i), resultSet.getString(i));
-                    }
-                    results.add(map);
-                }
-            }
-            return results;
-        } catch (Exception e) {
-            throw new Exception(e);
-        } finally {
-            if (resultSet != null) {
-                resultSet.close();
-            }
-        }
+    public List<Map<String, Object>> commonQuery(String sql, Integer queryId) {
+        JdbcTemplate jdbcTemplate = DatabaseHolder.getJdbcTemplate(queryId);
+        return jdbcTemplate.queryForList(sql);
     }
 
-//    public List<Map<String, Object>> commonQueryByJdbcTemplate(String sql, Integer queryId) {
-//        JdbcTemplate jdbcTemplate = ConnectionUtil.getJdbcTemplate(queryId);
-//        return jdbcTemplate.queryForList(sql);
-//    }
-
     public Object getTables(Integer id) throws Exception {
         QueryDb queryDb = queryDbDao.selectById(id);
         List<Map<String, Object>> results = null;
         if (queryDb != null) {
-            String sql = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' ORDER BY CREATE_TIME DESC ";
-            results = DatabaseHolder.getJdbcTemplate(id).queryForList(sql);
+            String sql = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' ORDER BY CREATE_TIME DESC " ;
+            results = commonQuery(sql, id);
         }
         return results;
     }
@@ -199,9 +159,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         JSONObject jsonObject = JSONUtil.parseObj(body);
         Integer id = Integer.valueOf(jsonObject.get("id").toString());
         String tableName = jsonObject.get("tableName").toString();
-        String sql = "SELECT * FROM " + tableName;
-        List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(id).queryForList(sql);
-        return mapList;
+        String sql="SELECT * FROM "+tableName;
+        return commonQuery(sql,id);
     }
 
     //----------------------------------组件接口----------------------------------------------------------
@@ -380,9 +339,9 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
         String sqlQuery = "SHOW TABLES LIKE '" + newTableName + "';";
         List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(newId).queryForList(sqlQuery);
         if (mapList.size() == 0) {
-            toDbJdbcTemplate.execute(SqlUtil.joinCreateTableSql(newTableName, set));
+            toDbJdbcTemplate.execute(SqlUtils.joinCreateTableSql(newTableName, set));
         }
-        toDbJdbcTemplate.batchUpdate(SqlUtil.jointInsertSql(newTableName, set), new BatchPreparedStatementSetter() {
+        toDbJdbcTemplate.batchUpdate(SqlUtils.jointInsertSql(newTableName, set), new BatchPreparedStatementSetter() {
             @Override
             public void setValues(PreparedStatement preparedStatement, int i) throws SQLException {
                 Map<String, Object> map = list.get(i);

dataoperation-server/src/main/java/com/zq/dataoperation/service/DataCleanRecordService.java

@@ -4,11 +4,11 @@ import cn.hutool.core.date.DateUtil;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.zq.common.utils.AssertUtils;
 import com.zq.common.vo.ResultVo;
-import com.zq.datacollect.entity.DataCollectTaskLog;
-import com.zq.datacollect.mapper.DataCollectTaskLogMapper;
 import com.zq.dataoperation.dao.DataCleanRecordDao;
+import com.zq.dataoperation.dao.DataCollectTaskLogMapper;
 import com.zq.dataoperation.dao.QueryDbDao;
 import com.zq.dataoperation.entity.DataCleanRecord;
+import com.zq.dataoperation.entity.DataCollectTaskLog;
 import com.zq.dataoperation.entity.QueryDb;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

dataoperation-server/src/main/java/com/zq/dataoperation/utils/ConnectionUtil.java

-package com.zq.dataoperation.utils;
-
-import com.alibaba.druid.pool.DruidPooledConnection;
-
-import java.sql.Connection;
-import java.util.Map;
-
-public class ConnectionUtil {
-
-    private static Map<Integer, DruidPooledConnection> connectionMap;
-
-    public static void set(Map<Integer, DruidPooledConnection> map) {
-        connectionMap = map;
-    }
-
-    public static Connection get(Integer id){
-        return connectionMap.get(id).getConnection();
-    }
-}

dataoperation-server/src/main/java/com/zq/dataoperation/utils/SqlUtils.java

@@ -31,6 +31,7 @@ import java.sql.*;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * @author /
@@ -234,4 +235,30 @@ public class SqlUtils {
         return sqlList;
     }
 
+    public static String jointInsertSql(String tableName, Set<String> cloums){
+        StringBuilder fileds = new StringBuilder();
+        for (String cloum : cloums) {
+            fileds.append(cloum.replaceAll("\\p{Punct}", "")).append(",");
+        }
+        fileds.deleteCharAt(fileds.length()-1);
+        StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(fileds).append(") VALUES (");
+        for (int i = 0; i < cloums.size(); i++) {
+            if(i==0){
+                sql.append("?");
+            }else {
+                sql.append(",?");
+            }
+        }
+        sql.append(")");
+        return sql.toString();
+    }
+
+    public static String joinCreateTableSql(String tableName, Set<String> cloums){
+        StringBuilder sql = new StringBuilder("CREATE TABLE ").append(tableName).append("(");
+        for (String cloum : cloums) {
+            sql.append(cloum.replaceAll("\\p{Punct}", "")).append(" VARCHAR(255),");
+        }
+        sql.deleteCharAt(sql.length()-1).append(")");
+        return sql.toString();
+    }
 }
\ No newline at end of file

dataoperation-server/src/main/java/com/zq/dataoperation/utils/TableUtil.java

@@ -3,7 +3,6 @@ package com.zq.dataoperation.utils;
 import com.zq.dataoperation.holder.DatabaseHolder;
 import com.zq.dataoperation.vo.Column;
 import lombok.SneakyThrows;
-import org.springframework.jdbc.core.JdbcTemplate;
 
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -18,12 +17,17 @@ public class TableUtil {
 
     @SneakyThrows
     public static Object getColumns(Integer queryId, String tableName) {
+        List<Column> columns = new ArrayList<>();
+        Connection connection = null;
+        ResultSet idxRs = null;
+        ResultSet columnRs = null;
+        try {
             String driver = "com.mysql.cj.jdbc.Driver";
             Class.forName(driver);
-        Connection connection = DatabaseHolder.getConnection(queryId);
+            connection = DatabaseHolder.getConnection(queryId);
             DatabaseMetaData metaData = connection.getMetaData();
             String primaryColumn = "";
-        ResultSet idxRs = metaData.getIndexInfo(connection.getCatalog(), connection.getCatalog(), tableName, false, false);
+            idxRs = metaData.getIndexInfo(connection.getCatalog(), connection.getCatalog(), tableName, false, false);
             while (idxRs.next()) {
                 String indexName = idxRs.getString("INDEX_NAME");
                 if ("PRIMARY".equals(indexName)) {
@@ -32,8 +36,7 @@ public class TableUtil {
                 }
             }
 
-        List<Column> columns = new ArrayList<>();
-        ResultSet columnRs = metaData.getColumns(connection.getCatalog(), connection.getCatalog(), tableName, "%");
+            columnRs = metaData.getColumns(connection.getCatalog(), connection.getCatalog(), tableName, "%");
             while (columnRs.next()) {
                 Column column = new Column();
                 column.setName(columnRs.getString("COLUMN_NAME"));
@@ -46,7 +49,17 @@ public class TableUtil {
                 column.setQueryId(queryId.toString());
                 columns.add(column);
             }
-
+        }finally {
+            if(columnRs!=null){
+                columnRs.close();
+            }
+            if(idxRs!=null){
+                idxRs.close();
+            }
+            if(connection!=null){
+                connection.close();
+            }
+        }
         return columns;
     }
 }