Skip to content

N
nnjcy-data-model
Commit fb4b529e by 梁家彪
Options

Merge remote-tracking branch 'origin/master'

parents 71a88eef 62e37181
Showing with 145 additions and 57 deletions
dataoperation-server/src/main/java/com/zq/dataoperation/entity/QueryDb.java
......@@ -25,6 +25,11 @@ public class QueryDb {
private String dbName;
/**
* 数据库中文名称
*/
private String chDbName;
/**
* 数据库ip
*/
private String dbIp;
......
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
......@@ -5,7 +5,6 @@ import cn.hutool.json.JSONUtil;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.jcraft.jsch.Logger;
import com.zq.common.exception.BusinessException;
import com.zq.common.vo.PageVo;
import com.zq.common.vo.ResultVo;
......@@ -17,10 +16,8 @@ import com.zq.dataoperation.entity.MetaDataMapping;
import com.zq.dataoperation.entity.QueryDb;
import com.zq.dataoperation.utils.ConnectionUtil;
import com.zq.dataoperation.utils.IdentityUtils;
import com.zq.dataoperation.utils.MobileUtil;
import com.zq.dataoperation.utils.SqlUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
......@@ -191,12 +188,18 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Long datasourceId = Long.valueOf(jsonObject.get("datasourceId").toString());
String tableName = jsonObject.get("tableName").toString();
String metaData = jsonObject.get("metaData").toString();
String primarySql="SELECT distinct column_name FROM INFORMATION_SCHEMA.`KEY_COLUMN_USAGE` WHERE table_name= '"+tableName+"' AND constraint_name='PRIMARY'";
List<Map> primaryList = commonQuery(primarySql, datasourceId);
String primaryKey =null;
for (Object obj:primaryList) {
primaryKey = JSONUtil.parseObj(obj).getStr("COLUMN_NAME");
}
String sql1="SELECT "+metaData+" FROM " + tableName+" GROUP BY "+metaData+" HAVING COUNT("+metaData+")>1;";
List<Map> mapList = commonQuery(sql1, datasourceId);
if (mapList.size()==0){
return "该字段没有重复值";
}else{
String sql = "SELECT * FROM " + tableName + " WHERE id not in (SELECT MIN(id) FROM " + tableName+" GROUP BY "+metaData+" HAVING COUNT("+metaData+")>1);";
String sql = "SELECT * FROM " + tableName + " WHERE "+primaryKey+" not in (SELECT MIN("+primaryKey+") FROM " + tableName+" GROUP BY "+metaData+" HAVING COUNT("+metaData+")>1);";
return commonQuery(sql, datasourceId);
}
}
......@@ -279,16 +282,15 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
String phone="SELECT "+metaData+" FROM "+tableName;
String sql ="SELECT * FROM "+tableName+" WHERE "+metaData+" REGEXP '^1[3-9][0-9]{9}$'";
// List<Map> mapList = commonQuery(phone, datasourceId);
// for (Object list:mapList) {
// String meta = list.toString();
// if(MobileUtil.checkPhone(meta)!=true) {
// log.error("不是国内手机号");
// continue;
// }else{
// dataList = commonQuery(sql, datasourceId);
// }
// }
List<Map> mapList = commonQuery(phone, datasourceId);
String regex = "/^1\\d{10}$|^(0\\d{2,3}-?|\\(0\\d{2,3}\\))?[1-9]\\d{4,7}(-\\d{1,8})?$/";
Pattern pattern = Pattern.compile(regex);
for (Object list:mapList) {
String meta = list.toString();
if(!pattern.matcher(meta).matches()){
return "此字段不是电话号码";
}
}
List<Map> dataList = commonQuery(sql, datasourceId);
String dirty="SELECT * FROM "+tableName+" WHERE "+metaData+" NOT REGEXP '^1[3-9][0-9]{9}$'";
List<Map> dirtyList = commonQuery(dirty, datasourceId);
......
dataoperation-server/src/main/java/com/zq/dataoperation/utils/TableUtil.java
......@@ -41,6 +41,7 @@ public class TableUtil {
column.setNotNull(!columnRs.getBoolean("NULLABLE"));
column.setPrimary(column.getName().equals(primaryColumn));
column.setComment(columnRs.getString("REMARKS"));
column.setQueryId(queryId.toString());
columns.add(column);
}
......
dataoperation-server/src/main/java/com/zq/dataoperation/vo/Column.java
......@@ -5,6 +5,8 @@ import lombok.Data;
@Data
public class Column {
String queryId;
String name;
String type;
......
spider-flow-web/src/main/java/com/zq/spiderflow/controller/MetaDataController.java
package com.zq.spiderflow.controller;
import cn.hutool.core.date.DateUtil;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.zq.common.vo.ResultVo;
import com.zq.spiderflow.entity.MetaData;
......@@ -83,24 +84,30 @@ public class MetaDataController {
/**
* 新增元数据
* @param metaData
* @param vo
* @return
*/
@ApiOperation("新增元数据")
@PostMapping("/add")
public ResultVo addMetaData(@RequestBody MetaData metaData) {
AssertUtils.hasText(metaData.getMetaName(), "缺少元数据名称");
AssertUtils.hasText(metaData.getDataType(), "缺少元数据类型");
AssertUtils.hasText(metaData.getMutiMetaExpress(), "缺少元数据表达式");
AssertUtils.hasText(metaData.getEnglishName(), "缺少元数据英文名称");
AssertUtils.hasText(metaData.getDataCode(), "缺少元数据代码");
MetaData temp = metaDataService.getOne(
Wrappers.lambdaQuery(MetaData.builder().dataCode(metaData.getDataCode()).build())
);
if (temp != null) {
ResultVo.fail("元数据代码不能重复！");
}
boolean b = metaDataService.save(metaData);
public ResultVo addMetaData(@RequestBody MetaDataReq vo) {
AssertUtils.hasText(vo.getMetaName(), "缺少元数据名称");
AssertUtils.hasText(vo.getDataType(), "缺少元数据类型");
AssertUtils.hasText(vo.getMutiMetaExpress(), "缺少元数据表达式");
AssertUtils.hasText(vo.getEnglishName(), "缺少元数据英文名称");
MetaData builder = MetaData.builder()
.metaName(vo.getMetaName())
.dataType(vo.getDataType())
.dataLength(vo.getDataLength())
.fieldPrecision(vo.getFieldPrecision())
.dataCode(vo.getTableName() + "_" + vo.getMetaName())
.englishName(vo.getEnglishName())
.dataLevel(vo.getDataLevel())
.notNull(vo.getNotNull())
.categoryId(vo.getCategoryId())
.mutiMetaFlag(vo.getMutiMetaFlag())
.mutiMetaExpress(vo.getMutiMetaExpress())
.createTime(DateUtil.date()).build();
boolean b = metaDataService.save(builder);
return b ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败");
}
......
spider-flow-web/src/main/java/com/zq/spiderflow/controller/MetaDataMappingController.java
......@@ -7,6 +7,7 @@ import com.zq.spiderflow.service.MetaDataMappingService;
import com.zq.spiderflow.service.MetaDataService;
import com.zq.spiderflow.util.AssertUtils;
import com.zq.spiderflow.vo.MetaDataMappingReq;
import com.zq.spiderflow.vo.MetaDataReq;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*;
......@@ -60,6 +61,16 @@ public class MetaDataMappingController {
}
/**
* 配置元数据映射
* @return
*/
@ApiOperation("配置元数据映射")
@PostMapping("/setCofingMapping")
public ResultVo setCofingMapping(@RequestBody MetaDataReq metaDataReq){
return ResultVo.success(metaDataMappingService.setCofingMapping(metaDataReq));
}
/**
* 新增或修改元数据映射
* @param metaDataMapping
* @return
......
spider-flow-web/src/main/java/com/zq/spiderflow/dao/MetaDataMappingDao.java
......@@ -19,4 +19,6 @@ public interface MetaDataMappingDao extends BaseMapper<MetaDataMapping> {
List<MetaDataMapping> selectByCateId(Long id);
List<MetaDataReq> getMetaByTable(String tableSqlName);
int selectByMetadataId(Long id);
}
spider-flow-web/src/main/java/com/zq/spiderflow/service/MetaDataMappingService.java
package com.zq.spiderflow.service;
import cn.hutool.core.date.DateUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.common.utils.PagingUtils;
import com.zq.common.vo.PageVo;
import com.zq.common.vo.ResultVo;
import com.zq.spiderflow.dao.MetaDataMappingDao;
import com.zq.spiderflow.entity.MetaData;
import com.zq.spiderflow.entity.MetaDataMapping;
import com.zq.spiderflow.util.AssertUtils;
import com.zq.spiderflow.vo.MetaDataReq;
import org.apache.commons.lang3.StringUtils;
import com.zq.spiderflow.vo.MetaDataMappingReq;
import org.springframework.stereotype.Service;
......@@ -29,7 +32,7 @@ public class MetaDataMappingService extends ServiceImpl<MetaDataMappingDao, Meta
if (StringUtils.isNotBlank(req.getFieldName())) {
wrapper.like(MetaDataMapping::getFieldName, req.getFieldName());
}
return PagingUtils.paging(req, metaDataMappingDao, wrapper, MetaDataMapping.class);
return PagingUtils.paging(req, metaDataMappingDao, wrapper.orderByDesc(MetaDataMapping::getCreateTime), MetaDataMapping.class);
}
public MetaDataMapping getByMetaDataId(Long metaDataId) {
......@@ -37,8 +40,36 @@ public class MetaDataMappingService extends ServiceImpl<MetaDataMappingDao, Meta
}
public ResultVo getSqlName() {
List<MetaDataMapping> list =metaDataMappingDao.getSqlName();
AssertUtils.notNull(list,"还未映射数据库表名");
return ResultVo.success(list);
List<MetaDataMapping> list = metaDataMappingDao.getSqlName();
AssertUtils.notNull(list, "还未映射数据库表名");
return ResultVo.success(list);
}
public ResultVo setCofingMapping(MetaDataReq metaDataReq) {
// MetaDataMapping byMetaDataId = getByMetaDataId(metaDataReq.getId());
// if (byMetaDataId!=null){
// return ResultVo.success("已映射有元数据");
// }else{
//关联映射元数据表插入数据
MetaDataMapping mapping = new MetaDataMapping();
String tableSqlName = metaDataReq.getTableSqlName();
mapping.setTableSqlname(metaDataReq.getTableSqlName());
mapping.setTableName(metaDataReq.getTableName());
mapping.setFieldName(metaDataReq.getMetaName());
mapping.setFieldType(metaDataReq.getDataType());
mapping.setLength(metaDataReq.getDataLength());
mapping.setPrecision(metaDataReq.getFieldPrecision());
mapping.setIsPk(0);
mapping.setMetadataId(metaDataReq.getId());
mapping.setCreateTime(DateUtil.date());
mapping.setQueryDbId(metaDataReq.getQueryDbId());
List<MetaDataMapping> mappings = metaDataMappingDao.selectRepeat(tableSqlName);
if (mappings != null && mappings.size() > 0) {
for (MetaDataMapping one : mappings) {
metaDataMappingDao.deleteById(one.getId());
}
}
int i = metaDataMappingDao.insert(mapping);
return i>0 ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败");
}
}
spider-flow-web/src/main/java/com/zq/spiderflow/service/MetaDataService.java
......@@ -141,28 +141,28 @@ public class MetaDataService extends ServiceImpl<MetaDataDao, MetaData> {
}
metaDataDao.insert(metaData);
//关联映射元数据表插入数据
MetaDataMapping mapping = new MetaDataMapping();
tableSqlname=Convert.toStr(CollUtil.get(objectList, 6));
tableName =Convert.toStr(CollUtil.get(objectList, 7));
mapping.setTableSqlname(tableSqlname);
mapping.setTableName(tableName);
mapping.setFieldName(metaData.getEnglishName());
mapping.setFieldType(metaData.getDataType());
mapping.setLength(metaData.getDataLength());
mapping.setPrecision(metaData.getFieldPrecision());
mapping.setIsPk(0);
mapping.setMetadataId(metaData.getId());
mapping.setCreateTime(DateUtil.date());
mapping.setQueryDbId(queryDbId);
List<MetaDataMapping> mappings= metaDataMappingDao.selectRepeat(tableSqlname);
if (mappings != null && mappings.size() > 0) {
for (MetaDataMapping one : mappings) {
metaDataMappingDao.deleteById(one.getId());
}
}
metaDataMappingDao.insert(mapping);
// //关联映射元数据表插入数据
// MetaDataMapping mapping = new MetaDataMapping();
// tableSqlname=Convert.toStr(CollUtil.get(objectList, 6));
// tableName =Convert.toStr(CollUtil.get(objectList, 7));
// mapping.setTableSqlname(tableSqlname);
// mapping.setTableName(tableName);
// mapping.setFieldName(metaData.getEnglishName());
// mapping.setFieldType(metaData.getDataType());
// mapping.setLength(metaData.getDataLength());
// mapping.setPrecision(metaData.getFieldPrecision());
// mapping.setIsPk(0);
// mapping.setMetadataId(metaData.getId());
// mapping.setCreateTime(DateUtil.date());
// mapping.setQueryDbId(queryDbId);
//
// List<MetaDataMapping> mappings= metaDataMappingDao.selectRepeat(tableSqlname);
// if (mappings != null && mappings.size() > 0) {
// for (MetaDataMapping one : mappings) {
// metaDataMappingDao.deleteById(one.getId());
// }
// }
// metaDataMappingDao.insert(mapping);
}
return "导入成功";
......
spider-flow-web/src/main/java/com/zq/spiderflow/vo/MetaDataReq.java
package com.zq.spiderflow.vo;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
......@@ -18,7 +16,7 @@ public class MetaDataReq extends PageReqVo {
* 使用序列
* id
*/
@TableId(type = IdType.INPUT)
private Long id;
/**
......@@ -31,6 +29,10 @@ public class MetaDataReq extends PageReqVo {
*/
private String dataType;
/**
* 数据元长度
*/
private String dataLength;
/**
* 精度
......@@ -53,6 +55,21 @@ public class MetaDataReq extends PageReqVo {
private String dataLevel;
/**
* 是否非空
*/
private String notNull;
/**
* 是否复合元数据0否1是
*/
private Integer mutiMetaFlag;
/**
* 复合元数据表达式
*/
private String mutiMetaExpress;
/**
* 所属领域id
*/
private Long categoryId;
......@@ -60,4 +77,6 @@ public class MetaDataReq extends PageReqVo {
private Long queryDbId;
private String tableSqlName;
private String tableName;
}
spider-flow-web/src/main/java/com/zq/spiderflow/vo/MetaDataVo.java
......@@ -18,6 +18,11 @@ public class MetaDataVo {
/**
* 数据元名称
*/
private Long id;
/**
* 数据元名称
*/
private String metaName;
/**
......
spider-flow-web/src/main/resources/mapper/MetaDataMappingMapper.xml
......@@ -53,4 +53,7 @@
WHERE
p.table_sqlname = #{tableSqlName}
</select>
<select id="selectByMetadataId" resultType="java.lang.Integer">
</select>
</mapper>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment