Skip to content

N
nnjcy-data-model
Commit 41dc65d0 by wqc
Options

组件调整、通用接口修改

parent 7d2fadcb
Showing with 93 additions and 48 deletions
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
......@@ -5,6 +5,7 @@ import cn.hutool.json.JSONUtil;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.common.context.ThreadContext;
import com.zq.common.exception.BusinessException;
import com.zq.common.vo.PageVo;
import com.zq.common.vo.ResultVo;
......@@ -18,6 +19,8 @@ import com.zq.dataoperation.holder.DatabaseHolder;
import com.zq.dataoperation.utils.IdentityUtils;
import com.zq.dataoperation.utils.SqlUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
......@@ -40,9 +43,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
@Resource
private MetaDataMappingDao metaDataMappingDao;
private Map<Long, DruidPooledConnection> connectionMap = new HashMap<>();
private static final Logger log = LoggerFactory.getLogger(ThreadContext.class);
public List<QueryDb> getDatasource() {
return queryDbDao.selectList(new QueryWrapper<>());
......@@ -97,14 +98,13 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
JSONObject jsonObject = JSONUtil.parseObj(body);
String querySql = jsonObject.get("querySql").toString();
Long id = Long.valueOf(jsonObject.get("id").toString());
String tableName = jsonObject.get("tableName").toString();
List<MetaDataMapping> list = metaDataMappingDao.selectByCateId(id);
String tableSqlname = null;
Integer queryDbId = null;
for (MetaDataMapping mapping : list) {
tableSqlname = mapping.getTableSqlname();
queryDbId = mapping.getQueryDbId();
}
String sql = "SELECT * FROM " + tableSqlname + " WHERE " + querySql;
String sql = "SELECT * FROM " + tableName + " WHERE " + querySql;
return commonQuery(sql, queryDbId);
}
......@@ -149,7 +149,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
QueryDb queryDb = queryDbDao.selectById(id);
List<Map<String, Object>> results = null;
if (queryDb != null) {
String sql = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' ORDER BY CREATE_TIME DESC " ;
String sql = "SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='" + queryDb.getDbName() + "' ORDER BY CREATE_TIME DESC ";
results = commonQuery(sql, id);
}
return results;
......@@ -159,8 +159,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
JSONObject jsonObject = JSONUtil.parseObj(body);
Integer id = Integer.valueOf(jsonObject.get("id").toString());
String tableName = jsonObject.get("tableName").toString();
String sql="SELECT * FROM "+tableName;
return commonQuery(sql,id);
String sql = "SELECT * FROM " + tableName;
return commonQuery(sql, id);
}
//----------------------------------组件接口----------------------------------------------------------
......@@ -178,15 +178,21 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
String repeat = "SELECT " + metaData + " FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1;";
List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(repeat);
List<Map<String, Object>> dataList = null;
if (mapList.size() == 0) {
return "该字段没有重复值";
} else {
String sql1 = "SELECT * FROM " + tableName + " WHERE " + primaryKey + " in (SELECT MIN(" + primaryKey + ") FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1) UNION ALL ";
String sql2 = "SELECT * FROM " + tableName + " WHERE " + metaData + " not in (SELECT MIN(" + metaData + ") FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1);";
String sql = sql1 + sql2;
return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
}
String dirtysql = "SELECT * FROM " + tableName + " WHERE " + metaData + " in (SELECT MIN(" + metaData + ") FROM " + tableName + " GROUP BY " + metaData + " HAVING COUNT(" + metaData + ")>1);";
List<Map<String, Object>> dirtyList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dirtysql);
Map<String, Object> map = new HashMap<>();
map.put("dataList", dataList);
map.put("dirtyList", dirtyList);
return map;
}
//身份证清洗
......@@ -195,20 +201,42 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString());
String tableName = jsonObject.get("tableName").toString();
String metaData = jsonObject.get("metaData").toString();
Map<String, Object> map = new HashMap<>();
String sql = "SELECT * FROM " + tableName;
if (!StringUtils.isNotBlank(metaData)) {
String idCardSql = "SELECT " + metaData + " FROM " + tableName;
List<Map<String, Object>> mapList = commonQuery(idCardSql, datasourceId);
for (Map<String, Object> idCard : mapList) {
if (!StringUtils.isNotBlank(idCard.toString())) {
return "字段为身份证且不为空！";
}
if (IdentityUtils.isLegalIdCard(metaData) != true) {
return "身份证号不符";
} else if (IdentityUtils.is15BitLegalIdCard(metaData) != true) {
return "身份证不满15位！";
} else if (IdentityUtils.is18BitLegalIdCard(metaData) != true) {
return "身份证不满18位！";
// }
// if (IdentityUtils.isLegalIdCard(idCard.toString()) != true) {
// continue;
// } else if (IdentityUtils.is15BitLegalIdCard(idCard.toString()) != true) {
// return "身份证不满15位！";
// } else if (IdentityUtils.is18BitLegalIdCard(idCard.toString()) != true) {
// return "身份证不满18位！";
} else {
/** 身份证正则 */
String idCardRegex = "[1-9]\\d{5}(18|19|20)\\d{2}((0[1-9])|(1[0-2]))(([0-2][1-9])|10|20|30|31)\\d{3}[0-9Xx]";
Pattern r = Pattern.compile(idCardRegex);
String card = String.valueOf(idCard.get("id_card"));
boolean matches = r.matcher(card).matches();
List<Map<String, Object>> dataList = null;
List<Map<String, Object>> dirtyList = null;
if (matches) {
String dataSql="SELECT * FROM " + tableName + " WHERE " + metaData + " REGEXP '" + idCardRegex + "'"+" OR LENGTH("+metaData+")=18";
dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dataSql);
map.put("dataList", dataList);
} else {
return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
String dirtysql = sql + " WHERE " + metaData + " NOT REGEXP '" + idCardRegex + "'"+" AND LENGTH("+metaData+")<18";
dirtyList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dirtysql);
map.put("dirtyList", dirtyList);
}
}
}
return map;
}
//字符串替换
public Object replaceString(Map<String, Object> body) throws Exception {
......@@ -220,15 +248,12 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
String replaceStr = jsonObject.get("replaceStr").toString();
String sql = "UPDATE " + tableName + " SET " + metaData + "=REPLACE(" + metaData + ",'" + originStr + "','" + replaceStr + "');";
String result = "SELECT * FROM " + tableName;
if (!(StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName) && StringUtils.isNotBlank(originStr) && StringUtils.isNotBlank(replaceStr))) {
return "传入参数不能为空";
} else {
List<Map<String, Object>> mapList =DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
if (mapList.size() == 0 || mapList == null)
return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(result);
List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
List<Map<String, Object>> dataList = null;
if (mapList.size() == 0 || mapList == null) {
dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(result);
}
return "";
return dataList;
}
//记录排序
......@@ -244,7 +269,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
sortStr = "DESC";
}
String sql = "SELECT * FROM " + tableName + " ORDER BY " + metaData + " " + sortStr;
return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
List<Map<String, Object>> dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
return dataList;
}
//剪切字符串
......@@ -256,11 +282,13 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
String indexStr = jsonObject.get("indexStr").toString();
String metaData = jsonObject.get("metaData").toString();
String sql = "SELECT *," + " SUBSTR(" + metaData + "," + beginStr + "," + indexStr + ") AS " + metaData + "1 FROM " + tableName;
List<Map<String, Object>> dataList = null;
if (!StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName) && StringUtils.isNotBlank(beginStr) && StringUtils.isNotBlank(indexStr)) {
return "传入参数不能为空";
} else {
return DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(sql);
}
return dataList;
}
//验证电话---------------------------------------------------------
......@@ -269,7 +297,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Integer datasourceId = Integer.valueOf(jsonObject.get("datasourceId").toString());
String tableName = jsonObject.get("tableName").toString();
String metaData = jsonObject.get("metaData").toString();
HashMap<String, Object> map = new HashMap<>();
Map<String, Object> map = new HashMap<>();
if (!StringUtils.isNotBlank(metaData) && StringUtils.isNotBlank(tableName)) {
return "传入参数不能为空";
}
......@@ -280,7 +308,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Pattern pattern = Pattern.compile(regex);
for (Object list : mapList) {
String meta = list.toString();
if (!pattern.matcher(meta).matches()) {
if (pattern.matcher(meta).matches()) {
return "此字段不是电话号码";
}
}
......@@ -329,43 +357,58 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Integer newId = Integer.valueOf(jsonObject.get("newId").toString());
String newTableName = jsonObject.get("newTableName").toString();
String oldTableName = jsonObject.get("oldTableName").toString();
List<Map<String, Object>> list = (List) jsonObject.getObj("dataList");
List<Map<String, Object>> listMap = null;
Set<String> set = null;
JdbcTemplate toDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(newId);
if (list.size() == 0 || list == null) {
JdbcTemplate fromDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(oldId);
List<Map<String, Object>> list = fromDbJdbcTemplate.queryForList("SELECT * FROM " + oldTableName);
Set<String> set = list.get(0).keySet();
listMap = fromDbJdbcTemplate.queryForList("SELECT * FROM " + oldTableName);
} else {
listMap = list;
}
set = listMap.get(0).keySet();
set.remove("id");
String sqlQuery = "SHOW TABLES LIKE '" + newTableName + "';";
List<Map<String, Object>> mapList = DatabaseHolder.getJdbcTemplate(newId).queryForList(sqlQuery);
if (mapList.size() == 0) {
toDbJdbcTemplate.execute(SqlUtils.joinCreateTableSql(newTableName, set));
} else {
log.warn("已创有此数据表!!!");
}
Set<String> finalSet = set;
List<Map<String, Object>> finalListMap = listMap;
toDbJdbcTemplate.batchUpdate(SqlUtils.jointInsertSql(newTableName, set), new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement preparedStatement, int i) throws SQLException {
Map<String, Object> map = list.get(i);
Map<String, Object> map = finalListMap.get(i);
int index = 1;
for (String key : set) {
preparedStatement.setObject(index++, map.get(key));
for (String key : finalSet) {
try {
String keyStr =String.valueOf(map.get(key));
preparedStatement.setObject(index++, keyStr);
} catch (NullPointerException e) {
e.printStackTrace();
}
}
preparedStatement.close();
}
@Override
public int getBatchSize() {
return list.size();
return finalListMap.size();
}
});
List<Map<String, Object>> newCount = fromDbJdbcTemplate.queryForList("SELECT COUNT(1) AS newCount FROM " + newTableName);
List<Map<String, Object>> oldCount = fromDbJdbcTemplate.queryForList("SELECT COUNT(1) AS oldCount FROM " + oldTableName);
List<Map<String, Object>> newTable = fromDbJdbcTemplate.queryForList("SELECT * FROM " + newTableName);
List<Map<String, Object>> newCount = toDbJdbcTemplate.queryForList("SELECT * FROM " + newTableName);
long times = System.currentTimeMillis() - startTime;
Map<String, Object> map = new HashMap<>();
map.put("executeTime",times);
map.put("oldCount",oldCount);
map.put("newCount",newCount);
map.put("newTable",newTable);
map.put("executeTime", times);
map.put("oldCount", listMap.size());
map.put("newCount", newCount.size());
return map;
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/utils/SqlUtils.java
......@@ -238,7 +238,8 @@ public class SqlUtils {
public static String jointInsertSql(String tableName, Set<String> cloums){
StringBuilder fileds = new StringBuilder();
for (String cloum : cloums) {
fileds.append(cloum.replaceAll("\\p{Punct}", "")).append(",");
// fileds.append(cloum.replaceAll("\\p{Punct}", "")).append(",");
fileds.append(cloum).append(",");
}
fileds.deleteCharAt(fileds.length()-1);
StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(fileds).append(") VALUES (");
......@@ -256,7 +257,8 @@ public class SqlUtils {
public static String joinCreateTableSql(String tableName, Set<String> cloums){
StringBuilder sql = new StringBuilder("CREATE TABLE ").append(tableName).append("(");
for (String cloum : cloums) {
sql.append(cloum.replaceAll("\\p{Punct}", "")).append(" VARCHAR(255),");
// sql.append(cloum.replaceAll("\\p{Punct}", "")).append(" VARCHAR(255),");
sql.append(cloum).append(" VARCHAR(255),");
}
sql.deleteCharAt(sql.length()-1).append(")");
return sql.toString();
......
dataoperation-server/src/main/resources/mapper/MetaDataMappingMapper.xml
......@@ -41,7 +41,7 @@
meta_data_mapping t
LEFT JOIN meta_data m ON t.metadata_id = m.id
LEFT JOIN meta_data_category y ON m.category_id = y.id
WHERE y.id=#{id}
WHERE t.table_sqlname=#{tableName} OR y.id=#{id}
</select>
<select id="getMetaByTable" resultType="com.zq.dataoperation.vo.MetaDataReq">
SELECT
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment