Skip to content

N
nnjcy-data-model
Commit 62e37181 by wqc
Options

元数据管理修改，去重组件修改

parent 7fa12f91
Showing with 145 additions and 57 deletions
dataoperation-server/src/main/java/com/zq/dataoperation/entity/QueryDb.java
...@@ -25,6 +25,11 @@ public class QueryDb { ...@@ -25,6 +25,11 @@ public class QueryDb {
private String dbName; private String dbName;
/** /**
* 数据库中文名称
*/
private String chDbName;
/**
* 数据库ip * 数据库ip
*/ */
private String dbIp; private String dbIp;
......
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
...@@ -5,7 +5,6 @@ import cn.hutool.json.JSONUtil; ...@@ -5,7 +5,6 @@ import cn.hutool.json.JSONUtil;
import com.alibaba.druid.pool.DruidPooledConnection; import com.alibaba.druid.pool.DruidPooledConnection;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.jcraft.jsch.Logger;
import com.zq.common.exception.BusinessException; import com.zq.common.exception.BusinessException;
import com.zq.common.vo.PageVo; import com.zq.common.vo.PageVo;
import com.zq.common.vo.ResultVo; import com.zq.common.vo.ResultVo;
...@@ -17,10 +16,8 @@ import com.zq.dataoperation.entity.MetaDataMapping; ...@@ -17,10 +16,8 @@ import com.zq.dataoperation.entity.MetaDataMapping;
import com.zq.dataoperation.entity.QueryDb; import com.zq.dataoperation.entity.QueryDb;
import com.zq.dataoperation.utils.ConnectionUtil; import com.zq.dataoperation.utils.ConnectionUtil;
import com.zq.dataoperation.utils.IdentityUtils; import com.zq.dataoperation.utils.IdentityUtils;
import com.zq.dataoperation.utils.MobileUtil;
import com.zq.dataoperation.utils.SqlUtils; import com.zq.dataoperation.utils.SqlUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
...@@ -191,12 +188,18 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -191,12 +188,18 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Long datasourceId = Long.valueOf(jsonObject.get("datasourceId").toString()); Long datasourceId = Long.valueOf(jsonObject.get("datasourceId").toString());
String tableName = jsonObject.get("tableName").toString(); String tableName = jsonObject.get("tableName").toString();
String metaData = jsonObject.get("metaData").toString(); String metaData = jsonObject.get("metaData").toString();
String primarySql="SELECT distinct column_name FROM INFORMATION_SCHEMA.`KEY_COLUMN_USAGE` WHERE table_name= '"+tableName+"' AND constraint_name='PRIMARY'";
List<Map> primaryList = commonQuery(primarySql, datasourceId);
String primaryKey =null;
for (Object obj:primaryList) {
primaryKey = JSONUtil.parseObj(obj).getStr("COLUMN_NAME");
}
String sql1="SELECT "+metaData+" FROM " + tableName+" GROUP BY "+metaData+" HAVING COUNT("+metaData+")>1;"; String sql1="SELECT "+metaData+" FROM " + tableName+" GROUP BY "+metaData+" HAVING COUNT("+metaData+")>1;";
List<Map> mapList = commonQuery(sql1, datasourceId); List<Map> mapList = commonQuery(sql1, datasourceId);
if (mapList.size()==0){ if (mapList.size()==0){
return "该字段没有重复值"; return "该字段没有重复值";
}else{ }else{
String sql = "SELECT * FROM " + tableName + " WHERE id not in (SELECT MIN(id) FROM " + tableName+" GROUP BY "+metaData+" HAVING COUNT("+metaData+")>1);"; String sql = "SELECT * FROM " + tableName + " WHERE "+primaryKey+" not in (SELECT MIN("+primaryKey+") FROM " + tableName+" GROUP BY "+metaData+" HAVING COUNT("+metaData+")>1);";
return commonQuery(sql, datasourceId); return commonQuery(sql, datasourceId);
} }
} }
...@@ -279,16 +282,15 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -279,16 +282,15 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
} }
String phone="SELECT "+metaData+" FROM "+tableName; String phone="SELECT "+metaData+" FROM "+tableName;
String sql ="SELECT * FROM "+tableName+" WHERE "+metaData+" REGEXP '^1[3-9][0-9]{9}$'"; String sql ="SELECT * FROM "+tableName+" WHERE "+metaData+" REGEXP '^1[3-9][0-9]{9}$'";
// List<Map> mapList = commonQuery(phone, datasourceId); List<Map> mapList = commonQuery(phone, datasourceId);
// for (Object list:mapList) { String regex = "/^1\\d{10}$|^(0\\d{2,3}-?|\\(0\\d{2,3}\\))?[1-9]\\d{4,7}(-\\d{1,8})?$/";
// String meta = list.toString(); Pattern pattern = Pattern.compile(regex);
// if(MobileUtil.checkPhone(meta)!=true) { for (Object list:mapList) {
// log.error("不是国内手机号"); String meta = list.toString();
// continue; if(!pattern.matcher(meta).matches()){
// }else{ return "此字段不是电话号码";
// dataList = commonQuery(sql, datasourceId); }
// } }
// }
List<Map> dataList = commonQuery(sql, datasourceId); List<Map> dataList = commonQuery(sql, datasourceId);
String dirty="SELECT * FROM "+tableName+" WHERE "+metaData+" NOT REGEXP '^1[3-9][0-9]{9}$'"; String dirty="SELECT * FROM "+tableName+" WHERE "+metaData+" NOT REGEXP '^1[3-9][0-9]{9}$'";
List<Map> dirtyList = commonQuery(dirty, datasourceId); List<Map> dirtyList = commonQuery(dirty, datasourceId);
......
dataoperation-server/src/main/java/com/zq/dataoperation/utils/TableUtil.java
...@@ -41,6 +41,7 @@ public class TableUtil { ...@@ -41,6 +41,7 @@ public class TableUtil {
column.setNotNull(!columnRs.getBoolean("NULLABLE")); column.setNotNull(!columnRs.getBoolean("NULLABLE"));
column.setPrimary(column.getName().equals(primaryColumn)); column.setPrimary(column.getName().equals(primaryColumn));
column.setComment(columnRs.getString("REMARKS")); column.setComment(columnRs.getString("REMARKS"));
column.setQueryId(queryId.toString());
columns.add(column); columns.add(column);
} }
......
dataoperation-server/src/main/java/com/zq/dataoperation/vo/Column.java
...@@ -5,6 +5,8 @@ import lombok.Data; ...@@ -5,6 +5,8 @@ import lombok.Data;
@Data @Data
public class Column { public class Column {
String queryId;
String name; String name;
String type; String type;
......
spider-flow-web/src/main/java/com/zq/spiderflow/controller/MetaDataController.java
package com.zq.spiderflow.controller; package com.zq.spiderflow.controller;
import cn.hutool.core.date.DateUtil;
import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.zq.common.vo.ResultVo; import com.zq.common.vo.ResultVo;
import com.zq.spiderflow.entity.MetaData; import com.zq.spiderflow.entity.MetaData;
...@@ -83,24 +84,30 @@ public class MetaDataController { ...@@ -83,24 +84,30 @@ public class MetaDataController {
/** /**
* 新增元数据 * 新增元数据
* @param metaData * @param vo
* @return * @return
*/ */
@ApiOperation("新增元数据") @ApiOperation("新增元数据")
@PostMapping("/add") @PostMapping("/add")
public ResultVo addMetaData(@RequestBody MetaData metaData) { public ResultVo addMetaData(@RequestBody MetaDataReq vo) {
AssertUtils.hasText(metaData.getMetaName(), "缺少元数据名称"); AssertUtils.hasText(vo.getMetaName(), "缺少元数据名称");
AssertUtils.hasText(metaData.getDataType(), "缺少元数据类型"); AssertUtils.hasText(vo.getDataType(), "缺少元数据类型");
AssertUtils.hasText(metaData.getMutiMetaExpress(), "缺少元数据表达式"); AssertUtils.hasText(vo.getMutiMetaExpress(), "缺少元数据表达式");
AssertUtils.hasText(metaData.getEnglishName(), "缺少元数据英文名称"); AssertUtils.hasText(vo.getEnglishName(), "缺少元数据英文名称");
AssertUtils.hasText(metaData.getDataCode(), "缺少元数据代码"); MetaData builder = MetaData.builder()
MetaData temp = metaDataService.getOne( .metaName(vo.getMetaName())
Wrappers.lambdaQuery(MetaData.builder().dataCode(metaData.getDataCode()).build()) .dataType(vo.getDataType())
); .dataLength(vo.getDataLength())
if (temp != null) { .fieldPrecision(vo.getFieldPrecision())
ResultVo.fail("元数据代码不能重复！"); .dataCode(vo.getTableName() + "_" + vo.getMetaName())
} .englishName(vo.getEnglishName())
boolean b = metaDataService.save(metaData); .dataLevel(vo.getDataLevel())
.notNull(vo.getNotNull())
.categoryId(vo.getCategoryId())
.mutiMetaFlag(vo.getMutiMetaFlag())
.mutiMetaExpress(vo.getMutiMetaExpress())
.createTime(DateUtil.date()).build();
boolean b = metaDataService.save(builder);
return b ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败"); return b ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败");
} }
......
spider-flow-web/src/main/java/com/zq/spiderflow/controller/MetaDataMappingController.java
...@@ -7,6 +7,7 @@ import com.zq.spiderflow.service.MetaDataMappingService; ...@@ -7,6 +7,7 @@ import com.zq.spiderflow.service.MetaDataMappingService;
import com.zq.spiderflow.service.MetaDataService; import com.zq.spiderflow.service.MetaDataService;
import com.zq.spiderflow.util.AssertUtils; import com.zq.spiderflow.util.AssertUtils;
import com.zq.spiderflow.vo.MetaDataMappingReq; import com.zq.spiderflow.vo.MetaDataMappingReq;
import com.zq.spiderflow.vo.MetaDataReq;
import io.swagger.annotations.Api; import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
...@@ -60,6 +61,16 @@ public class MetaDataMappingController { ...@@ -60,6 +61,16 @@ public class MetaDataMappingController {
} }
/** /**
* 配置元数据映射
* @return
*/
@ApiOperation("配置元数据映射")
@PostMapping("/setCofingMapping")
public ResultVo setCofingMapping(@RequestBody MetaDataReq metaDataReq){
return ResultVo.success(metaDataMappingService.setCofingMapping(metaDataReq));
}
/**
* 新增或修改元数据映射 * 新增或修改元数据映射
* @param metaDataMapping * @param metaDataMapping
* @return * @return
......
spider-flow-web/src/main/java/com/zq/spiderflow/dao/MetaDataMappingDao.java
...@@ -19,4 +19,6 @@ public interface MetaDataMappingDao extends BaseMapper<MetaDataMapping> { ...@@ -19,4 +19,6 @@ public interface MetaDataMappingDao extends BaseMapper<MetaDataMapping> {
List<MetaDataMapping> selectByCateId(Long id); List<MetaDataMapping> selectByCateId(Long id);
List<MetaDataReq> getMetaByTable(String tableSqlName); List<MetaDataReq> getMetaByTable(String tableSqlName);
int selectByMetadataId(Long id);
} }
spider-flow-web/src/main/java/com/zq/spiderflow/service/MetaDataMappingService.java
package com.zq.spiderflow.service; package com.zq.spiderflow.service;
import cn.hutool.core.date.DateUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.common.utils.PagingUtils; import com.zq.common.utils.PagingUtils;
import com.zq.common.vo.PageVo; import com.zq.common.vo.PageVo;
import com.zq.common.vo.ResultVo; import com.zq.common.vo.ResultVo;
import com.zq.spiderflow.dao.MetaDataMappingDao; import com.zq.spiderflow.dao.MetaDataMappingDao;
import com.zq.spiderflow.entity.MetaData;
import com.zq.spiderflow.entity.MetaDataMapping; import com.zq.spiderflow.entity.MetaDataMapping;
import com.zq.spiderflow.util.AssertUtils; import com.zq.spiderflow.util.AssertUtils;
import com.zq.spiderflow.vo.MetaDataReq;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import com.zq.spiderflow.vo.MetaDataMappingReq; import com.zq.spiderflow.vo.MetaDataMappingReq;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
...@@ -29,7 +32,7 @@ public class MetaDataMappingService extends ServiceImpl<MetaDataMappingDao, Meta ...@@ -29,7 +32,7 @@ public class MetaDataMappingService extends ServiceImpl<MetaDataMappingDao, Meta
if (StringUtils.isNotBlank(req.getFieldName())) { if (StringUtils.isNotBlank(req.getFieldName())) {
wrapper.like(MetaDataMapping::getFieldName, req.getFieldName()); wrapper.like(MetaDataMapping::getFieldName, req.getFieldName());
} }
return PagingUtils.paging(req, metaDataMappingDao, wrapper, MetaDataMapping.class); return PagingUtils.paging(req, metaDataMappingDao, wrapper.orderByDesc(MetaDataMapping::getCreateTime), MetaDataMapping.class);
} }
public MetaDataMapping getByMetaDataId(Long metaDataId) { public MetaDataMapping getByMetaDataId(Long metaDataId) {
...@@ -37,8 +40,36 @@ public class MetaDataMappingService extends ServiceImpl<MetaDataMappingDao, Meta ...@@ -37,8 +40,36 @@ public class MetaDataMappingService extends ServiceImpl<MetaDataMappingDao, Meta
} }
public ResultVo getSqlName() { public ResultVo getSqlName() {
List<MetaDataMapping> list =metaDataMappingDao.getSqlName(); List<MetaDataMapping> list = metaDataMappingDao.getSqlName();
AssertUtils.notNull(list,"还未映射数据库表名"); AssertUtils.notNull(list, "还未映射数据库表名");
return ResultVo.success(list); return ResultVo.success(list);
}
public ResultVo setCofingMapping(MetaDataReq metaDataReq) {
// MetaDataMapping byMetaDataId = getByMetaDataId(metaDataReq.getId());
// if (byMetaDataId!=null){
// return ResultVo.success("已映射有元数据");
// }else{
//关联映射元数据表插入数据
MetaDataMapping mapping = new MetaDataMapping();
String tableSqlName = metaDataReq.getTableSqlName();
mapping.setTableSqlname(metaDataReq.getTableSqlName());
mapping.setTableName(metaDataReq.getTableName());
mapping.setFieldName(metaDataReq.getMetaName());
mapping.setFieldType(metaDataReq.getDataType());
mapping.setLength(metaDataReq.getDataLength());
mapping.setPrecision(metaDataReq.getFieldPrecision());
mapping.setIsPk(0);
mapping.setMetadataId(metaDataReq.getId());
mapping.setCreateTime(DateUtil.date());
mapping.setQueryDbId(metaDataReq.getQueryDbId());
List<MetaDataMapping> mappings = metaDataMappingDao.selectRepeat(tableSqlName);
if (mappings != null && mappings.size() > 0) {
for (MetaDataMapping one : mappings) {
metaDataMappingDao.deleteById(one.getId());
}
}
int i = metaDataMappingDao.insert(mapping);
return i>0 ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败");
} }
} }
spider-flow-web/src/main/java/com/zq/spiderflow/service/MetaDataService.java
...@@ -141,28 +141,28 @@ public class MetaDataService extends ServiceImpl<MetaDataDao, MetaData> { ...@@ -141,28 +141,28 @@ public class MetaDataService extends ServiceImpl<MetaDataDao, MetaData> {
} }
metaDataDao.insert(metaData); metaDataDao.insert(metaData);
//关联映射元数据表插入数据 // //关联映射元数据表插入数据
MetaDataMapping mapping = new MetaDataMapping(); // MetaDataMapping mapping = new MetaDataMapping();
tableSqlname=Convert.toStr(CollUtil.get(objectList, 6)); // tableSqlname=Convert.toStr(CollUtil.get(objectList, 6));
tableName =Convert.toStr(CollUtil.get(objectList, 7)); // tableName =Convert.toStr(CollUtil.get(objectList, 7));
mapping.setTableSqlname(tableSqlname); // mapping.setTableSqlname(tableSqlname);
mapping.setTableName(tableName); // mapping.setTableName(tableName);
mapping.setFieldName(metaData.getEnglishName()); // mapping.setFieldName(metaData.getEnglishName());
mapping.setFieldType(metaData.getDataType()); // mapping.setFieldType(metaData.getDataType());
mapping.setLength(metaData.getDataLength()); // mapping.setLength(metaData.getDataLength());
mapping.setPrecision(metaData.getFieldPrecision()); // mapping.setPrecision(metaData.getFieldPrecision());
mapping.setIsPk(0); // mapping.setIsPk(0);
mapping.setMetadataId(metaData.getId()); // mapping.setMetadataId(metaData.getId());
mapping.setCreateTime(DateUtil.date()); // mapping.setCreateTime(DateUtil.date());
mapping.setQueryDbId(queryDbId); // mapping.setQueryDbId(queryDbId);
//
List<MetaDataMapping> mappings= metaDataMappingDao.selectRepeat(tableSqlname); // List<MetaDataMapping> mappings= metaDataMappingDao.selectRepeat(tableSqlname);
if (mappings != null && mappings.size() > 0) { // if (mappings != null && mappings.size() > 0) {
for (MetaDataMapping one : mappings) { // for (MetaDataMapping one : mappings) {
metaDataMappingDao.deleteById(one.getId()); // metaDataMappingDao.deleteById(one.getId());
} // }
} // }
metaDataMappingDao.insert(mapping); // metaDataMappingDao.insert(mapping);
} }
return "导入成功"; return "导入成功";
......
spider-flow-web/src/main/java/com/zq/spiderflow/vo/MetaDataReq.java
package com.zq.spiderflow.vo; package com.zq.spiderflow.vo;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
...@@ -18,7 +16,7 @@ public class MetaDataReq extends PageReqVo { ...@@ -18,7 +16,7 @@ public class MetaDataReq extends PageReqVo {
* 使用序列 * 使用序列
* id * id
*/ */
@TableId(type = IdType.INPUT)
private Long id; private Long id;
/** /**
...@@ -31,6 +29,10 @@ public class MetaDataReq extends PageReqVo { ...@@ -31,6 +29,10 @@ public class MetaDataReq extends PageReqVo {
*/ */
private String dataType; private String dataType;
/**
* 数据元长度
*/
private String dataLength;
/** /**
* 精度 * 精度
...@@ -53,6 +55,21 @@ public class MetaDataReq extends PageReqVo { ...@@ -53,6 +55,21 @@ public class MetaDataReq extends PageReqVo {
private String dataLevel; private String dataLevel;
/** /**
* 是否非空
*/
private String notNull;
/**
* 是否复合元数据0否1是
*/
private Integer mutiMetaFlag;
/**
* 复合元数据表达式
*/
private String mutiMetaExpress;
/**
* 所属领域id * 所属领域id
*/ */
private Long categoryId; private Long categoryId;
...@@ -60,4 +77,6 @@ public class MetaDataReq extends PageReqVo { ...@@ -60,4 +77,6 @@ public class MetaDataReq extends PageReqVo {
private Long queryDbId; private Long queryDbId;
private String tableSqlName; private String tableSqlName;
private String tableName;
} }
spider-flow-web/src/main/java/com/zq/spiderflow/vo/MetaDataVo.java
...@@ -18,6 +18,11 @@ public class MetaDataVo { ...@@ -18,6 +18,11 @@ public class MetaDataVo {
/** /**
* 数据元名称 * 数据元名称
*/ */
private Long id;
/**
* 数据元名称
*/
private String metaName; private String metaName;
/** /**
......
spider-flow-web/src/main/resources/mapper/MetaDataMappingMapper.xml
...@@ -53,4 +53,7 @@ ...@@ -53,4 +53,7 @@
WHERE WHERE
p.table_sqlname = #{tableSqlName} p.table_sqlname = #{tableSqlName}
</select> </select>
<select id="selectByMetadataId" resultType="java.lang.Integer">
</select>
</mapper> </mapper>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment