Skip to content

N
nnjcy-data-model
Commit b4e75904 by wqc
Options

元数据关联优化

parent 47109b33
Showing with 201 additions and 62 deletions
dataoperation-server/pom.xml
......@@ -111,6 +111,12 @@
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
</dependency>
<dependency>
<groupId>com.zq</groupId>
<artifactId>spider-flow-web</artifactId>
<version>1.0.0</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
......
dataoperation-server/src/main/java/com/zq/dataoperation/controller/CommonQueryController.java
......@@ -97,4 +97,10 @@ public class CommonQueryController{
System.out.println(body);
return ResultVo.success(commonQueryService.run( body));
}
@ApiOperation("自定义sql查询")
@PostMapping("/runSelect")
public ResultVo runSelect(@RequestBody Map<String, Object> body) throws Exception {
return ResultVo.success(commonQueryService.runSelect(body));
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
......@@ -2,10 +2,10 @@ package com.zq.dataoperation.service;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.alibaba.druid.pool.DruidConnectionHolder;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.common.exception.BusinessException;
......@@ -15,11 +15,14 @@ import com.zq.dataoperation.dao.QueryDbDao;
import com.zq.dataoperation.entity.CommonQuerySetting;
import com.zq.dataoperation.entity.QueryDb;
import com.zq.dataoperation.utils.ConnectionUtil;
import org.spiderflow.dao.MetaDataMappingDao;
import org.spiderflow.entity.MetaDataMapping;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Wrapper;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......@@ -32,25 +35,27 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
@Resource
private QueryDbDao queryDbDao;
@Resource
private MetaDataMappingDao metaDataMappingDao;
private Map<Long, DruidPooledConnection> connectionMap = new HashMap<>();
public List<QueryDb> getDatasource(){
public List<QueryDb> getDatasource() {
return queryDbDao.selectList(new QueryWrapper<>());
}
public ResultVo addDatasource(QueryDb queryDb){
if(queryDbDao.insert(queryDb) != 1){
public ResultVo addDatasource(QueryDb queryDb) {
if (queryDbDao.insert(queryDb) != 1) {
throw new BusinessException("添加失败");
}
return ResultVo.success();
}
public List<CommonQuerySetting> get(){
public List<CommonQuerySetting> get() {
return query().list();
}
public void add(CommonQuerySetting commonQuerySetting){
public void add(CommonQuerySetting commonQuerySetting) {
if (!save(commonQuerySetting)) {
throw new BusinessException("添加失败");
}
......@@ -64,9 +69,9 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Pattern pattern = Pattern.compile("\\{([^}]+)}");
Matcher matcher = pattern.matcher(sql);
while (matcher.find()){
while (matcher.find()) {
String group = matcher.group(1);
if(jsonObject.containsKey(group)){
if (jsonObject.containsKey(group)) {
sql = sql.replace("{" + group + "}", jsonObject.getStr(group));
}
}
......@@ -74,7 +79,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
ResultSet resultSet = null;
try {
Connection connection = ConnectionUtil.get(queryDb.getId());
resultSet = connection.prepareStatement(sql).executeQuery();
resultSet = connection.prepareStatement(sql).executeQuery();
int columnCount = resultSet.getMetaData().getColumnCount();
List<Map> results = new ArrayList<>();
while (resultSet.next()) {
......@@ -120,11 +125,15 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
druidDataSource.setUsername(queryDb.getUsername());
druidDataSource.setPassword(queryDb.getPassword());
druidDataSource.init();
// if (druidDataSource.isInited()==true) {
// return ResultVo.success("连接成功");
// }else {
// return ResultVo.success("连接失败");
// }
// connectionMap.put(queryDb.getId(), connection);
if (druidDataSource.getConnection().isValid(10000)) {
return ResultVo.success("连接成功");
}else if(druidDataSource.getConnection().isClosed()){
} else if (druidDataSource.getConnection().isClosed()) {
return ResultVo.success("连接失败");
}
} catch (Exception e) {
......@@ -132,4 +141,41 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
return ResultVo.success();
}
public List<Map> runSelect(Map<String, Object> body) throws Exception {
JSONObject jsonObject = JSONUtil.parseObj(body);
String querySql = jsonObject.get("querySql").toString();
Long id = Long.valueOf(jsonObject.get("id").toString());
LambdaQueryWrapper<MetaDataMapping> wrapper = new LambdaQueryWrapper<>();
wrapper.eq(MetaDataMapping::getMetadataId, id);
List<MetaDataMapping> list = metaDataMappingDao.selectList(wrapper);
ResultSet resultSet = null;
String tableSqlname = null;
Long queryDbId = null;
for (MetaDataMapping mapping : list) {
tableSqlname = mapping.getTableSqlname();
queryDbId = mapping.getQueryDbId();
}
String sql = "SELECT * FROM " + tableSqlname + " WHERE " + querySql;
try {
Connection connection = ConnectionUtil.get(queryDbId);
resultSet = connection.prepareStatement(sql).executeQuery();
int columnCount = resultSet.getMetaData().getColumnCount();
List<Map> results = new ArrayList<>();
while (resultSet.next()) {
Map<String, String> map = new HashMap<>(columnCount);
for (int i = 1; i <= columnCount; i++) {
map.put(resultSet.getMetaData().getColumnName(i), resultSet.getString(i));
}
results.add(map);
}
return results;
} catch (Exception e) {
throw new Exception(e);
} finally {
if (resultSet != null) {
resultSet.close();
}
}
}
}
spider-flow-web/src/main/java/org/spiderflow/controller/MetaDataController.java
......@@ -42,9 +42,9 @@ public class MetaDataController {
* @return
*/
@ApiOperation("获取全部元数据")
@GetMapping("/getAll")
public ResultVo<List<MetaData>> getAll() {
return ResultVo.success(metaDataService.list(null));
@PostMapping("/getAllMeta")
public ResultVo<PageVo<MetaData>> getAll(@RequestBody MetaDataReq req) {
return ResultVo.success(metaDataService.getAll(req));
}
/**
......
spider-flow-web/src/main/java/org/spiderflow/controller/MetaDataMappingController.java
......@@ -37,7 +37,15 @@ public class MetaDataMappingController {
}
/**
* 获取映射数据库表名
* @return
*/
@ApiOperation("获取映射数据库表名")
@GetMapping("/getSqlName")
public ResultVo getSqlName(){
return ResultVo.success(metaDataMappingService.getSqlName());
}
/**
* 根据id获取元数据映射
......@@ -57,12 +65,16 @@ public class MetaDataMappingController {
* @return
*/
@ApiOperation("新增或修改元数据映射")
@PostMapping("/update")
@PostMapping("/updateMapping")
public ResultVo update(@RequestBody MetaDataMapping metaDataMapping) {
AssertUtils.hasText(metaDataMapping.getFieldName(), "缺少列名");
AssertUtils.hasText(metaDataMapping.getFieldType(), "缺少列类型");
AssertUtils.hasText(metaDataMapping.getTableSqlname(), "缺少数据库表名");
AssertUtils.hasText(metaDataMapping.getTableName(), "缺少数据库表中文名");
AssertUtils.hasText(metaDataMapping.getLength(),"缺少列长度");
AssertUtils.hasText(metaDataMapping.getPrecision(),"缺少列精度");
AssertUtils.notNull(metaDataMapping.getIsPk(),"是否主键");
AssertUtils.notNull(metaDataMapping.getQueryDbId(),"缺少数据源");
boolean b = metaDataMappingService.saveOrUpdate(metaDataMapping);
return b ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败");
}
......
spider-flow-web/src/main/java/org/spiderflow/dao/MetaDataMappingDao.java
package org.spiderflow.dao;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
import org.spiderflow.entity.MetaDataMapping;
import org.springframework.stereotype.Repository;
import org.spiderflow.vo.MetaDataMappingReq;
import java.util.List;
public interface MetaDataMappingDao extends BaseMapper<MetaDataMapping> {
List<MetaDataMapping> selectRepeat(String tableSqlname);
List<MetaDataMapping> getSqlName();
}
spider-flow-web/src/main/java/org/spiderflow/service/MetaDataMappingService.java
......@@ -6,12 +6,15 @@ import org.apache.commons.lang3.StringUtils;
import org.spiderflow.dao.MetaDataMappingDao;
import org.spiderflow.entity.MetaData;
import org.spiderflow.entity.MetaDataMapping;
import org.spiderflow.util.AssertUtils;
import org.spiderflow.util.PagingUtils;
import org.spiderflow.vo.MetaDataMappingReq;
import org.spiderflow.vo.PageVo;
import org.spiderflow.vo.ResultVo;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
@Service
public class MetaDataMappingService extends ServiceImpl<MetaDataMappingDao, MetaDataMapping> {
......@@ -21,8 +24,8 @@ public class MetaDataMappingService extends ServiceImpl<MetaDataMappingDao, Meta
public PageVo<MetaDataMapping> getPageList(MetaDataMappingReq req) {
LambdaQueryWrapper<MetaDataMapping> wrapper = new LambdaQueryWrapper<>();
if (StringUtils.isNotBlank(req.getTableSqlName())) {
wrapper.like(MetaDataMapping::getTableSqlname, req.getTableSqlName());
if (StringUtils.isNotBlank(req.getTableSqlname())) {
wrapper.like(MetaDataMapping::getTableSqlname, req.getTableSqlname());
}
if (StringUtils.isNotBlank(req.getFieldName())) {
wrapper.like(MetaDataMapping::getFieldName, req.getFieldName());
......@@ -33,4 +36,10 @@ public class MetaDataMappingService extends ServiceImpl<MetaDataMappingDao, Meta
public MetaDataMapping getByMetaDataId(Long metaDataId) {
return metaDataMappingDao.selectOne(new LambdaQueryWrapper<MetaDataMapping>().eq(MetaDataMapping::getMetadataId, metaDataId));
}
public ResultVo getSqlName() {
List<MetaDataMapping> list =metaDataMappingDao.getSqlName();
AssertUtils.notNull(list,"还未映射数据库表名");
return ResultVo.success(list);
}
}
spider-flow-web/src/main/java/org/spiderflow/service/MetaDataService.java
......@@ -14,6 +14,7 @@ import org.spiderflow.core.exception.BusinessException;
import org.spiderflow.dao.MetaDataDao;
import org.spiderflow.dao.MetaDataMappingDao;
import org.spiderflow.entity.MetaData;
import org.spiderflow.entity.MetaDataCategory;
import org.spiderflow.entity.MetaDataMapping;
import org.spiderflow.util.PagingUtils;
import org.spiderflow.vo.MetaDataReq;
......@@ -38,6 +39,8 @@ public class MetaDataService extends ServiceImpl<MetaDataDao, MetaData> {
private MetaDataDao metaDataDao;
@Resource
private MetaDataMappingDao metaDataMappingDao;
@Resource
private MetaDataCategoryService metaDataCategoryService;
public PageVo<MetaDataVo> getPageList(MetaDataReq req) {
List<MetaDataVo> list = metaDataDao.getPage(req);
......@@ -78,7 +81,7 @@ public class MetaDataService extends ServiceImpl<MetaDataDao, MetaData> {
}
public Object addExcleData(MultipartFile file, String dataLevel,Long categoryId,Long queryDbId) {
public Object addExcleData(MultipartFile file, String dataLevel, Long categoryId, Long queryDbId) {
ExcelReader reader;
try {
reader = ExcelUtil.getReader(file.getInputStream());
......@@ -87,49 +90,90 @@ public class MetaDataService extends ServiceImpl<MetaDataDao, MetaData> {
throw new BusinessException("文件解析失败");
}
List<List<Object>> readList = reader.read(1);
String englishName = null;
String dataType = null;
String notNull = null;
String dataLength = null;
String fieldPrecision = null;
String metaName = null;
String dataCode = null;
String tableSqlname = null;
String tableName = null;
for (List<Object> objectList : readList) {
int index = 0;
MetaData metaData = new MetaData();
metaData.setEnglishName(Convert.toStr(CollUtil.get(objectList, 0)));
metaData.setDataType(Convert.toStr(CollUtil.get(objectList, 1)));
metaData.setNotNull(Convert.toStr(CollUtil.get(objectList, 2)));
metaData.setDataLength(Convert.toStr(CollUtil.get(objectList, 3)));
metaData.setFieldPrecision(Convert.toStr(CollUtil.get(objectList, 4)));
metaData.setMetaName(Convert.toStr(CollUtil.get(objectList, 5)));
String dataCode = Convert.toStr(CollUtil.get(objectList, 6)) + "_" + Convert.toStr(CollUtil.get(objectList, 0));
metaData.setDataCode(dataCode);
metaData.setCreateTime(DateUtil.date());
metaData.setDataLevel(dataLevel);
metaData.setCategoryId(categoryId);
metaData.setMutiMetaFlag(0);
List<MetaData> list = metaDataDao.selectList(Wrappers.lambdaQuery(MetaData.builder().dataCode(dataCode).build()));
if (list != null && list.size() > 0) {
for (MetaData one : list) {
metaDataDao.deleteById(one.getId());
}
englishName = Convert.toStr(CollUtil.get(objectList, 0));
dataType = Convert.toStr(CollUtil.get(objectList, 1));
notNull = Convert.toStr(CollUtil.get(objectList, 2));
dataLength = Convert.toStr(CollUtil.get(objectList, 3));
fieldPrecision = Convert.toStr(CollUtil.get(objectList, 4));
metaName = Convert.toStr(CollUtil.get(objectList, 5));
tableSqlname = Convert.toStr(CollUtil.get(objectList, 6));
tableName = Convert.toStr(CollUtil.get(objectList, 7));
dataCode = Convert.toStr(CollUtil.get(objectList, 6)) + "_" + Convert.toStr(CollUtil.get(objectList, 0));
}
//关联目录表插入数据
MetaDataCategory metaDataCategory = new MetaDataCategory();
MetaDataCategory parent = metaDataCategoryService.getOne(new LambdaQueryWrapper<MetaDataCategory>()
.eq(MetaDataCategory::getId, categoryId));
// 如果存在记录，获取其id，并设置为新记录的parentId
if (parent != null) {
metaDataCategory.setCategoryCode(parent.getId());
// 更新其subCount字段为原来的值加一
parent.setSubCount(parent.getSubCount() + 1);
parent.setStandardLevel(parent.getStandardLevel() + 1);
}
BeanUtil.copyProperties(parent, metaDataCategory);
metaDataCategory.setCategoryName(tableName);
metaDataCategoryService.save(metaDataCategory);
//元数据表关联插入数据
MetaData metaData = new MetaData();
metaData.setEnglishName(englishName);
metaData.setDataType(dataType);
metaData.setNotNull(notNull);
metaData.setDataLength(dataLength);
metaData.setFieldPrecision(fieldPrecision);
metaData.setMetaName(metaName);
metaData.setDataCode(dataCode);
metaData.setCreateTime(DateUtil.date());
metaData.setDataLevel(dataLevel);
metaData.setCategoryId(metaDataCategory.getId());
metaData.setMutiMetaFlag(0);
List<MetaData> list = metaDataDao.selectList(Wrappers.lambdaQuery(MetaData.builder().dataCode(dataCode).build()));
if (list != null && list.size() > 0) {
for (MetaData one : list) {
metaDataDao.deleteById(one.getId());
}
metaDataDao.insert(metaData);
MetaDataMapping mapping = new MetaDataMapping();
String tableSqlname=Convert.toStr(CollUtil.get(objectList, 6));
mapping.setTableSqlname(tableSqlname);
mapping.setTableName(Convert.toStr(CollUtil.get(objectList, 7)));
mapping.setFieldName(metaData.getEnglishName());
mapping.setFieldType(metaData.getDataType());
mapping.setLength(metaData.getDataLength());
mapping.setPrecision(metaData.getFieldPrecision());
mapping.setIsPk(0);
mapping.setMetadataId(metaData.getId());
mapping.setCreateTime(DateUtil.date());
mapping.setQueryDbId(queryDbId);
}
metaDataDao.insert(metaData);
List<MetaDataMapping> mappings= metaDataMappingDao.selectRepeat(tableSqlname);
if (mappings != null && mappings.size() > 0) {
for (MetaDataMapping one : mappings) {
metaDataMappingDao.deleteById(one.getId());
}
//关联映射元数据表插入数据
MetaDataMapping mapping = new MetaDataMapping();
mapping.setTableSqlname(tableSqlname);
mapping.setTableName(tableName);
mapping.setFieldName(metaData.getEnglishName());
mapping.setFieldType(metaData.getDataType());
mapping.setLength(metaData.getDataLength());
mapping.setPrecision(metaData.getFieldPrecision());
mapping.setIsPk(0);
mapping.setMetadataId(metaData.getId());
mapping.setCreateTime(DateUtil.date());
mapping.setQueryDbId(queryDbId);
List<MetaDataMapping> mappings = metaDataMappingDao.selectRepeat(tableSqlname);
if (mappings != null && mappings.size() > 0) {
for (MetaDataMapping one : mappings) {
metaDataMappingDao.deleteById(one.getId());
}
metaDataMappingDao.insert(mapping);
}
metaDataMappingDao.insert(mapping);
return "导入成功";
}
public PageVo<MetaData> getAll(MetaDataReq req) {
LambdaQueryWrapper<MetaData> queryWrapper = new LambdaQueryWrapper<>();
if (StringUtils.isNotBlank(req.getMetaName())) {
queryWrapper.like(MetaData::getMetaName, req.getMetaName());
}
return PagingUtils.paging(req, metaDataDao, queryWrapper, MetaData.class);
}
}
spider-flow-web/src/main/java/org/spiderflow/vo/MetaDataMappingReq.java
......@@ -15,7 +15,7 @@ public class MetaDataMappingReq extends PageReqVo {
/**
* 数据库表名
*/
private String tableSqlName;
private String tableSqlname;
/**
* 列名
......@@ -46,4 +46,4 @@ public class MetaDataMappingReq extends PageReqVo {
* 数据元id
*/
private Long metadataId;
}
\ No newline at end of file
}
spider-flow-web/src/main/resources/mapper/MetaDataMappingMapper.xml
......@@ -16,4 +16,20 @@
table_sqlname = #{tableSqlname}
AND field_name IN(SELECT field_name FROM meta_data_mapping GROUP BY field_name HAVING COUNT(field_name) > 1)
</select>
<select id="getSqlName" resultType="org.spiderflow.entity.MetaDataMapping">
SELECT DISTINCT
table_sqlname
FROM
meta_data_mapping
WHERE
table_sqlname IN (
SELECT
table_sqlname
FROM
meta_data_mapping
GROUP BY
table_sqlname
HAVING
COUNT( table_sqlname ) > 1)
</select>
</mapper>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment