Commit 41dc65d0 by wqc

组件调整、通用接口修改

parent 7d2fadcb
...@@ -5,6 +5,7 @@ import cn.hutool.json.JSONUtil; ...@@ -5,6 +5,7 @@ import cn.hutool.json.JSONUtil;
import com.alibaba.druid.pool.DruidPooledConnection; import com.alibaba.druid.pool.DruidPooledConnection;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.common.context.ThreadContext;
import com.zq.common.exception.BusinessException; import com.zq.common.exception.BusinessException;
import com.zq.common.vo.PageVo; import com.zq.common.vo.PageVo;
import com.zq.common.vo.ResultVo; import com.zq.common.vo.ResultVo;
...@@ -18,6 +19,8 @@ import com.zq.dataoperation.holder.DatabaseHolder; ...@@ -18,6 +19,8 @@ import com.zq.dataoperation.holder.DatabaseHolder;
import com.zq.dataoperation.utils.IdentityUtils; import com.zq.dataoperation.utils.IdentityUtils;
import com.zq.dataoperation.utils.SqlUtils; import com.zq.dataoperation.utils.SqlUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
...@@ -40,9 +43,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -40,9 +43,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
@Resource @Resource
private MetaDataMappingDao metaDataMappingDao; private MetaDataMappingDao metaDataMappingDao;
private static final Logger log = LoggerFactory.getLogger(ThreadContext.class);
private Map<Long, DruidPooledConnection> connectionMap = new HashMap<>();
public List<QueryDb> getDatasource() { public List<QueryDb> getDatasource() {
return queryDbDao.selectList(new QueryWrapper<>()); return queryDbDao.selectList(new QueryWrapper<>());
...@@ -97,14 +98,13 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -97,14 +98,13 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
JSONObject jsonObject = JSONUtil.parseObj(body); JSONObject jsonObject = JSONUtil.parseObj(body);
String querySql = jsonObject.get("querySql").toString(); String querySql = jsonObject.get("querySql").toString();
Long id = Long.valueOf(jsonObject.get("id").toString()); Long id = Long.valueOf(jsonObject.get("id").toString());
String tableName = jsonObject.get("tableName").toString();
List<MetaDataMapping> list = metaDataMappingDao.selectByCateId(id); List<MetaDataMapping> list = metaDataMappingDao.selectByCateId(id);
String tableSqlname = null;
Integer queryDbId = null; Integer queryDbId = null;
for (MetaDataMapping mapping : list) { for (MetaDataMapping mapping : list) {
tableSqlname = mapping.getTableSqlname();
queryDbId = mapping.getQueryDbId(); queryDbId = mapping.getQueryDbId();
} }
String sql = "SELECT * FROM " + tableSqlname + " WHERE " + querySql; String sql = "SELECT * FROM " + tableName + " WHERE " + querySql;
return commonQuery(sql, queryDbId); return commonQuery(sql, queryDbId);
} }
...@@ -149,7 +149,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -149,7 +149,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
QueryDb queryDb = queryDbDao.selectById(id); QueryDb queryDb = queryDbDao.selectById(id);
List<Map<String, Object>> results = null; List<Map<String, Object>> results = null;
if (queryDb != null) { if (queryDb != null) {
String sql = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' ORDER BY CREATE_TIME DESC " ; String sql = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' ORDER BY CREATE_TIME DESC ";
results = commonQuery(sql, id); results = commonQuery(sql, id);
} }
return results; return results;
...@@ -159,8 +159,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -159,8 +159,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
JSONObject jsonObject = JSONUtil.parseObj(body); JSONObject jsonObject = JSONUtil.parseObj(body);
Integer id = Integer.valueOf(jsonObject.get("id").toString()); Integer id = Integer.valueOf(jsonObject.get("id").toString());
String tableName = jsonObject.get("tableName").toString(); String tableName = jsonObject.get("tableName").toString();
String sql="SELECT * FROM "+tableName; String sql = "SELECT * FROM " + tableName;
return commonQuery(sql,id); return commonQuery(sql, id);
} }
//----------------------------------组件接口---------------------------------------------------------- //----------------------------------组件接口----------------------------------------------------------
...@@ -178,15 +178,21 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -178,15 +178,21 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
} }
String repeat = "SELECT " + metaData + " FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1;"; String repeat = "SELECT " + metaData + " FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1;";
List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(repeat); List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(repeat);
List<Map<String, Object>> dataList = null;
if (mapList.size() == 0) { if (mapList.size() == 0) {
return "该字段没有重复值"; return "该字段没有重复值";
} else { } else {
String sql1 = "SELECT * FROM " + tableName + " WHERE " + primaryKey + " in (SELECT MIN(" + primaryKey + ") FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1) UNION ALL "; String sql1 = "SELECT * FROM " + tableName + " WHERE " + primaryKey + " in (SELECT MIN(" + primaryKey + ") FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1) UNION ALL ";
String sql2 = "SELECT * FROM " + tableName + " WHERE " + metaData + " not in (SELECT MIN(" + metaData + ") FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1);"; String sql2 = "SELECT * FROM " + tableName + " WHERE " + metaData + " not in (SELECT MIN(" + metaData + ") FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1);";
String sql = sql1 + sql2; String sql = sql1 + sql2;
dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
} }
String dirtysql = "SELECT * FROM " + tableName + " WHERE " + metaData + " in (SELECT MIN(" + metaData + ") FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1);";
List<Map<String, Object>> dirtyList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dirtysql);
Map<String, Object> map = new HashMap<>();
map.put("dataList", dataList);
map.put("dirtyList", dirtyList);
return map;
} }
//身份证清洗 //身份证清洗
...@@ -195,20 +201,42 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -195,20 +201,42 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString()); Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString());
String tableName = jsonObject.get("tableName").toString(); String tableName = jsonObject.get("tableName").toString();
String metaData = jsonObject.get("metaData").toString(); String metaData = jsonObject.get("metaData").toString();
Map<String, Object> map = new HashMap<>();
String sql = "SELECT * FROM " + tableName; String sql = "SELECT * FROM " + tableName;
if (!StringUtils.isNotBlank(metaData)) { String idCardSql = "SELECT " + metaData + " FROM " + tableName;
List<Map<String, Object>> mapList = commonQuery(idCardSql, datasourceId);
for (Map<String, Object> idCard : mapList) {
if (!StringUtils.isNotBlank(idCard.toString())) {
return "字段为身份证且不为空!"; return "字段为身份证且不为空!";
} // }
if (IdentityUtils.isLegalIdCard(metaData) != true) { // if (IdentityUtils.isLegalIdCard(idCard.toString()) != true) {
return "身份证号不符"; // continue;
} else if (IdentityUtils.is15BitLegalIdCard(metaData) != true) { // } else if (IdentityUtils.is15BitLegalIdCard(idCard.toString()) != true) {
return "身份证不满15位!"; // return "身份证不满15位!";
} else if (IdentityUtils.is18BitLegalIdCard(metaData) != true) { // } else if (IdentityUtils.is18BitLegalIdCard(idCard.toString()) != true) {
return "身份证不满18位!"; // return "身份证不满18位!";
} else {
/** 身份证正则 */
String idCardRegex = "[1-9]\\d{5}(18|19|20)\\d{2}((0[1-9])|(1[0-2]))(([0-2][1-9])|10|20|30|31)\\d{3}[0-9Xx]";
Pattern r = Pattern.compile(idCardRegex);
String card = String.valueOf(idCard.get("id_card"));
boolean matches = r.matcher(card).matches();
List<Map<String, Object>> dataList = null;
List<Map<String, Object>> dirtyList = null;
if (matches) {
String dataSql="SELECT * FROM " + tableName + " WHERE " + metaData + " REGEXP '" + idCardRegex + "'"+" OR LENGTH("+metaData+")=18";
dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dataSql);
map.put("dataList", dataList);
} else { } else {
return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql); String dirtysql = sql + " WHERE " + metaData + " NOT REGEXP '" + idCardRegex + "'"+" AND LENGTH("+metaData+")<18";
dirtyList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dirtysql);
map.put("dirtyList", dirtyList);
}
} }
} }
return map;
}
//字符串替换 //字符串替换
public Object replaceString(Map<String, Object> body) throws Exception { public Object replaceString(Map<String, Object> body) throws Exception {
...@@ -220,15 +248,12 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -220,15 +248,12 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
String replaceStr = jsonObject.get("replaceStr").toString(); String replaceStr = jsonObject.get("replaceStr").toString();
String sql = "UPDATE " + tableName + " SET " + metaData + "=REPLACE(" + metaData + ",'" + originStr + "','" + replaceStr + "');"; String sql = "UPDATE " + tableName + " SET " + metaData + "=REPLACE(" + metaData + ",'" + originStr + "','" + replaceStr + "');";
String result = "SELECT * FROM " + tableName; String result = "SELECT * FROM " + tableName;
if (!(StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName) && StringUtils.isNotBlank(originStr) && StringUtils.isNotBlank(replaceStr))) { List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
return "传入参数不能为空"; List<Map<String, Object>> dataList = null;
} else { if (mapList.size() == 0 || mapList == null) {
List<Map<String, Object>> mapList =DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql); dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(result);
if (mapList.size() == 0 || mapList == null)
return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(result);
} }
return dataList;
return "";
} }
//记录排序 //记录排序
...@@ -244,7 +269,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -244,7 +269,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
sortStr = "DESC"; sortStr = "DESC";
} }
String sql = "SELECT * FROM " + tableName + " ORDER BY " + metaData + " " + sortStr; String sql = "SELECT * FROM " + tableName + " ORDER BY " + metaData + " " + sortStr;
return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql); List<Map<String, Object>> dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
return dataList;
} }
//剪切字符串 //剪切字符串
...@@ -256,11 +282,13 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -256,11 +282,13 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
String indexStr = jsonObject.get("indexStr").toString(); String indexStr = jsonObject.get("indexStr").toString();
String metaData = jsonObject.get("metaData").toString(); String metaData = jsonObject.get("metaData").toString();
String sql = "SELECT *," + " SUBSTR(" + metaData + "," + beginStr + "," + indexStr + ") AS " + metaData + "1 FROM " + tableName; String sql = "SELECT *," + " SUBSTR(" + metaData + "," + beginStr + "," + indexStr + ") AS " + metaData + "1 FROM " + tableName;
List<Map<String, Object>> dataList = null;
if (!StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName) && StringUtils.isNotBlank(beginStr) && StringUtils.isNotBlank(indexStr)) { if (!StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName) && StringUtils.isNotBlank(beginStr) && StringUtils.isNotBlank(indexStr)) {
return "传入参数不能为空"; return "传入参数不能为空";
} else { } else {
return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql); dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
} }
return dataList;
} }
//验证电话--------------------------------------------------------- //验证电话---------------------------------------------------------
...@@ -269,7 +297,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -269,7 +297,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString()); Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString());
String tableName = jsonObject.get("tableName").toString(); String tableName = jsonObject.get("tableName").toString();
String metaData = jsonObject.get("metaData").toString(); String metaData = jsonObject.get("metaData").toString();
HashMap<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
if (!StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName)) { if (!StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName)) {
return "传入参数不能为空"; return "传入参数不能为空";
} }
...@@ -280,7 +308,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -280,7 +308,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Pattern pattern = Pattern.compile(regex); Pattern pattern = Pattern.compile(regex);
for (Object list : mapList) { for (Object list : mapList) {
String meta = list.toString(); String meta = list.toString();
if (!pattern.matcher(meta).matches()) { if (pattern.matcher(meta).matches()) {
return "此字段不是电话号码"; return "此字段不是电话号码";
} }
} }
...@@ -329,43 +357,58 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -329,43 +357,58 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Integer newId = Integer.valueOf(jsonObject.get("newId").toString()); Integer newId = Integer.valueOf(jsonObject.get("newId").toString());
String newTableName = jsonObject.get("newTableName").toString(); String newTableName = jsonObject.get("newTableName").toString();
String oldTableName = jsonObject.get("oldTableName").toString(); String oldTableName = jsonObject.get("oldTableName").toString();
List<Map<String, Object>> list = (List) jsonObject.getObj("dataList");
List<Map<String, Object>> listMap = null;
Set<String> set = null;
JdbcTemplate toDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(newId); JdbcTemplate toDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(newId);
if (list.size() == 0 || list == null) {
JdbcTemplate fromDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(oldId); JdbcTemplate fromDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(oldId);
List<Map<String, Object>> list = fromDbJdbcTemplate.queryForList("SELECT * FROM " + oldTableName); listMap = fromDbJdbcTemplate.queryForList("SELECT * FROM " + oldTableName);
Set<String> set = list.get(0).keySet(); } else {
listMap = list;
}
set = listMap.get(0).keySet();
set.remove("id"); set.remove("id");
String sqlQuery = "SHOW TABLES LIKE '" + newTableName + "';"; String sqlQuery = "SHOW TABLES LIKE '" + newTableName + "';";
List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(newId).queryForList(sqlQuery); List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(newId).queryForList(sqlQuery);
if (mapList.size() == 0) { if (mapList.size() == 0) {
toDbJdbcTemplate.execute(SqlUtils.joinCreateTableSql(newTableName, set)); toDbJdbcTemplate.execute(SqlUtils.joinCreateTableSql(newTableName, set));
} else {
log.warn("已创有此数据表!!!");
} }
Set<String> finalSet = set;
List<Map<String, Object>> finalListMap = listMap;
toDbJdbcTemplate.batchUpdate(SqlUtils.jointInsertSql(newTableName, set), new BatchPreparedStatementSetter() { toDbJdbcTemplate.batchUpdate(SqlUtils.jointInsertSql(newTableName, set), new BatchPreparedStatementSetter() {
@Override @Override
public void setValues(PreparedStatement preparedStatement, int i) throws SQLException { public void setValues(PreparedStatement preparedStatement, int i) throws SQLException {
Map<String, Object> map = list.get(i); Map<String, Object> map = finalListMap.get(i);
int index = 1; int index = 1;
for (String key : set) { for (String key : finalSet) {
preparedStatement.setObject(index++, map.get(key)); try {
String keyStr =String.valueOf(map.get(key));
preparedStatement.setObject(index++, keyStr);
} catch (NullPointerException e) {
e.printStackTrace();
}
} }
preparedStatement.close();
} }
@Override @Override
public int getBatchSize() { public int getBatchSize() {
return list.size(); return finalListMap.size();
} }
}); });
List<Map<String, Object>> newCount = fromDbJdbcTemplate.queryForList("SELECT COUNT(1) AS newCount FROM " + newTableName); List<Map<String, Object>> newCount = toDbJdbcTemplate.queryForList("SELECT * FROM " + newTableName);
List<Map<String, Object>> oldCount = fromDbJdbcTemplate.queryForList("SELECT COUNT(1) AS oldCount FROM " + oldTableName);
List<Map<String, Object>> newTable = fromDbJdbcTemplate.queryForList("SELECT * FROM " + newTableName);
long times = System.currentTimeMillis() - startTime; long times = System.currentTimeMillis() - startTime;
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
map.put("executeTime",times); map.put("executeTime", times);
map.put("oldCount",oldCount); map.put("oldCount", listMap.size());
map.put("newCount",newCount); map.put("newCount", newCount.size());
map.put("newTable",newTable);
return map; return map;
} }
} }
...@@ -238,7 +238,8 @@ public class SqlUtils { ...@@ -238,7 +238,8 @@ public class SqlUtils {
public static String jointInsertSql(String tableName, Set<String> cloums){ public static String jointInsertSql(String tableName, Set<String> cloums){
StringBuilder fileds = new StringBuilder(); StringBuilder fileds = new StringBuilder();
for (String cloum : cloums) { for (String cloum : cloums) {
fileds.append(cloum.replaceAll("\\p{Punct}", "")).append(","); // fileds.append(cloum.replaceAll("\\p{Punct}", "")).append(",");
fileds.append(cloum).append(",");
} }
fileds.deleteCharAt(fileds.length()-1); fileds.deleteCharAt(fileds.length()-1);
StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(fileds).append(") VALUES ("); StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(fileds).append(") VALUES (");
...@@ -256,7 +257,8 @@ public class SqlUtils { ...@@ -256,7 +257,8 @@ public class SqlUtils {
public static String joinCreateTableSql(String tableName, Set<String> cloums){ public static String joinCreateTableSql(String tableName, Set<String> cloums){
StringBuilder sql = new StringBuilder("CREATE TABLE ").append(tableName).append("("); StringBuilder sql = new StringBuilder("CREATE TABLE ").append(tableName).append("(");
for (String cloum : cloums) { for (String cloum : cloums) {
sql.append(cloum.replaceAll("\\p{Punct}", "")).append(" VARCHAR(255),"); // sql.append(cloum.replaceAll("\\p{Punct}", "")).append(" VARCHAR(255),");
sql.append(cloum).append(" VARCHAR(255),");
} }
sql.deleteCharAt(sql.length()-1).append(")"); sql.deleteCharAt(sql.length()-1).append(")");
return sql.toString(); return sql.toString();
......
...@@ -41,7 +41,7 @@ ...@@ -41,7 +41,7 @@
meta_data_mapping t meta_data_mapping t
LEFT JOIN meta_data m ON t.metadata_id = m.id LEFT JOIN meta_data m ON t.metadata_id = m.id
LEFT JOIN meta_data_category y ON m.category_id = y.id LEFT JOIN meta_data_category y ON m.category_id = y.id
WHERE y.id=#{id} WHERE t.table_sqlname=#{tableName} OR y.id=#{id}
</select> </select>
<select id="getMetaByTable" resultType="com.zq.dataoperation.vo.MetaDataReq"> <select id="getMetaByTable" resultType="com.zq.dataoperation.vo.MetaDataReq">
SELECT SELECT
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment