Skip to content

N
nnjcy-data-model
Commit bb8bd40d by wqc
Options

数据查询、数据库资源、元数据映射更改

parent 9ac8eefd
Showing with 299 additions and 40 deletions
dataoperation-server/src/main/java/com/zq/dataoperation/controller/CommonQueryController.java
package com.zq.dataoperation.controller;
import com.zq.common.annotation.AnonymousAccess;
import cn.hutool.core.date.DateUtil;
import com.alibaba.druid.pool.DruidConnectionHolder;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
import com.zq.common.exception.BusinessException;
import com.zq.common.utils.AssertUtils;
import com.zq.common.vo.ResultVo;
import com.zq.dataoperation.entity.CommonQuerySetting;
import com.zq.dataoperation.entity.QueryDb;
import com.zq.dataoperation.service.CommonQueryService;
import com.zq.dataoperation.service.QueryDbService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
@Api(tags = "通用查询")
@RestController
@RequestMapping(value = "/data/query")
public class CommonQueryController {
public class CommonQueryController{
@Resource
private CommonQueryService commonQueryService;
@Resource
private QueryDbService queryDbService;
@ApiOperation("获取全部数据源")
@GetMapping("/datasource")
public ResultVo getDatasource(){
......@@ -28,12 +38,46 @@ public class CommonQueryController {
}
@ApiOperation("添加数据源")
@PostMapping("/add/datasource")
@PostMapping("/addDatasource")
public ResultVo addDatasource(@RequestBody QueryDb queryDb){
commonQueryService.addDatasource(queryDb);
return ResultVo.success();
}
@ApiOperation("测试连接数据源")
@PostMapping("/checkConnect")
public ResultVo checkConnect(@RequestBody QueryDb queryDb){
return ResultVo.success(commonQueryService.checkConnect(queryDb));
}
@ApiOperation("修改数据源")
@PostMapping("/updateDatasource")
public ResultVo updateDatasource(@RequestBody QueryDb queryDb){
AssertUtils.notNull(queryDb.getDbName(), "缺少数据库名");
AssertUtils.hasText(queryDb.getDbIp(), "缺少IP");
AssertUtils.notNull(queryDb.getDbPort(), "缺少端口");
AssertUtils.notNull(queryDb.getDbType(), "缺少数据库类型");
AssertUtils.hasText(queryDb.getUsername(), "缺少连接用户名");
AssertUtils.hasText(queryDb.getPassword(),"缺少连接密码");
queryDb.setUpdateTime(DateUtil.date());
boolean b = queryDbService.updateById(queryDb);
return b ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败");
}
/**
* 根据ID删除数据源
* @param id
* @return
*/
@ApiOperation("根据ID删除数据源")
@DeleteMapping("/deleteDatasource/{id}")
public ResultVo deleteDatasource(@PathVariable Long id) {
AssertUtils.notNull(id, "删除至少需要一个ID！");
boolean b = queryDbService.removeById(id);
return b ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败！");
}
@ApiOperation("获取全部查询")
@GetMapping
public ResultVo get(){
......@@ -48,8 +92,9 @@ public class CommonQueryController {
}
@ApiOperation("运行查询")
@PostMapping("/run/{queryId}")
public ResultVo run(@PathVariable("queryId") Long queryId, @RequestBody Map<String, String> body) throws SQLException {
return ResultVo.success(commonQueryService.run(queryId, body));
@PostMapping("/run")
public ResultVo run(@RequestBody Map<String, Object> body) throws Exception {
System.out.println(body);
return ResultVo.success(commonQueryService.run( body));
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
package com.zq.dataoperation.service;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.alibaba.druid.pool.DruidConnectionHolder;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.common.exception.BusinessException;
import com.zq.common.vo.ResultVo;
import com.zq.dataoperation.dao.CommonQuerySettingDao;
import com.zq.dataoperation.dao.QueryDbDao;
import com.zq.dataoperation.entity.CommonQuerySetting;
......@@ -13,7 +20,6 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......@@ -27,14 +33,17 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
@Resource
private QueryDbDao queryDbDao;
private Map<Long, DruidPooledConnection> connectionMap = new HashMap<>();
public List<QueryDb> getDatasource(){
return queryDbDao.selectList(new QueryWrapper<>());
}
public void addDatasource(QueryDb queryDb){
public ResultVo addDatasource(QueryDb queryDb){
if(queryDbDao.insert(queryDb) != 1){
throw new BusinessException("添加失败");
}
return ResultVo.success();
}
public List<CommonQuerySetting> get(){
......@@ -47,8 +56,9 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
}
public List<Map> run(Long queryId, Map<String, String> body) throws SQLException {
CommonQuerySetting querySetting = getById(queryId);
public List<Map> run(Map<String, Object> body) throws Exception {
JSONObject jsonObject = JSONUtil.parseObj(body);
CommonQuerySetting querySetting = getById(jsonObject.get("queryId").toString());
QueryDb queryDb = queryDbDao.selectById(querySetting.getQueryDbId());
String sql = querySetting.getQuerySql();
......@@ -56,16 +66,18 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
Matcher matcher = pattern.matcher(sql);
while (matcher.find()){
String group = matcher.group(1);
if(body.containsKey(group)){
sql = sql.replace("{" + group + "}", body.get(group));
if(jsonObject.containsKey(group)){
sql = sql.replace("{" + group + "}", jsonObject.getStr(group));
}
}
ResultSet resultSet = null;
try {
Connection connection = ConnectionUtil.get(queryDb.getId());
ResultSet resultSet = connection.prepareStatement(sql).executeQuery();
resultSet = connection.prepareStatement(sql).executeQuery();
int columnCount = resultSet.getMetaData().getColumnCount();
List<Map> results = new ArrayList<>();
while (resultSet.next()){
while (resultSet.next()) {
Map<String, String> map = new HashMap<>(columnCount);
for (int i = 1; i <= columnCount; i++) {
map.put(resultSet.getMetaData().getColumnName(i), resultSet.getString(i));
......@@ -73,5 +85,51 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
results.add(map);
}
return results;
} catch (Exception e) {
throw new Exception(e);
} finally {
if (resultSet != null) {
resultSet.close();
}
}
}
public Object checkConnect(QueryDb queryDb) {
DruidDataSource druidDataSource = DruidDataSourceBuilder.create().build();
String url = "";
String className = "";
switch (queryDb.getDbType()) {
case 1:
className = "com.mysql.cj.jdbc.Driver";
url = "jdbc:mysql://" + queryDb.getDbIp() + ":" + queryDb.getDbPort() + "/" + queryDb.getDbName() + "?characterEncoding=utf8&useSSL=false&serverTimezone=GMT%2B8&autoReconnect=true";
break;
case 2:
className = "oracle.jdbc.driver.OracleDriver";
url = "jdbc:oracle:thin:@//" + queryDb.getDbIp() + ":" + queryDb.getDbPort() + "/" + queryDb.getDbName();
break;
case 3:
className = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
url = "jdbc:sqlserver://" + queryDb.getDbIp() + ":" + queryDb.getDbPort() + "/" + queryDb.getDbName();
break;
default:
throw new BusinessException("不支持的数据库");
}
try {
druidDataSource.setDriverClassName(className);
druidDataSource.setUrl(url);
druidDataSource.setUsername(queryDb.getUsername());
druidDataSource.setPassword(queryDb.getPassword());
druidDataSource.init();
// connectionMap.put(queryDb.getId(), connection);
if (druidDataSource.getConnection().isValid(10000)) {
return ResultVo.success("连接成功");
}else if(druidDataSource.getConnection().isClosed()){
return ResultVo.success("连接失败");
}
} catch (Exception e) {
return ResultVo.fail(e.getMessage());
}
return ResultVo.success();
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/service/QueryDbService.java 0 → 100644
package com.zq.dataoperation.service;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.dataoperation.dao.QueryDbDao;
import com.zq.dataoperation.entity.QueryDb;
import org.springframework.stereotype.Service;
@Service
public class QueryDbService extends ServiceImpl<QueryDbDao, QueryDb> {
}
spider-flow-web/pom.xml
......@@ -23,6 +23,23 @@
<version>1.0.0</version>
</dependency>
<!-- excel工具 -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>4.1.2</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>4.1.2</version>
</dependency>
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
<version>2.11.0</version>
</dependency>
<!-- 引入selenium插件 -->
<dependency>
<groupId>com.zq</groupId>
......
spider-flow-web/src/main/java/org/spiderflow/controller/MetaDataController.java
......@@ -13,6 +13,7 @@ import org.spiderflow.vo.MetaDataVo;
import org.spiderflow.vo.PageVo;
import org.spiderflow.vo.ResultVo;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.util.List;
......@@ -66,15 +67,8 @@ public class MetaDataController {
*/
@ApiOperation("根据类别目录id获取分页元数据列表")
@PostMapping("/getPageByCategoryId")
public ResultVo<PageVo<MetaData>> getByCategoryId(@RequestBody MetaDataReq req) {
// 调用service的getPageByCategoryId方法，传入请求参数，获取分页结果
PageVo<MetaData> pageResult = metaDataService.getPageByCategoryId(req);
// 判断分页结果是否为空
if (pageResult == null) {
return ResultVo.fail("无数据");
}
// 返回分页结果
return ResultVo.success(pageResult);
public ResultVo getByCategoryId(@RequestBody MetaDataReq req) {
return ResultVo.success(metaDataService.getPageByCategoryId(req));
}
......@@ -115,6 +109,8 @@ public class MetaDataController {
AssertUtils.hasText(metaData.getDataType(), "缺少元数据类型");
AssertUtils.hasText(metaData.getMutiMetaExpress(), "缺少元数据表达式");
AssertUtils.hasText(metaData.getEnglishName(), "缺少元数据英文名称");
AssertUtils.hasText(metaData.getMutiMetaFlag().toString(),"是否复合元数据");
boolean b = metaDataService.updateById(metaData);
return b ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败");
}
......@@ -144,4 +140,16 @@ public class MetaDataController {
boolean b = metaDataService.removeById(id);
return b ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败！");
}
/**
* 根据excle解析
* @param file
* @return
*/
@ApiOperation("根据excle解析")
@PostMapping("/addExcleData")
public ResultVo addExcleData(MultipartFile file,String dataLevel,Long categoryId,Long queryDbId) {
AssertUtils.isTrue(file != null && file.getSize() > 0, "请选择文件");
return ResultVo.success(metaDataService.addExcleData(file,dataLevel,categoryId,queryDbId));
}
}
spider-flow-web/src/main/java/org/spiderflow/controller/MetaDataMappingController.java
......@@ -2,9 +2,9 @@ package org.spiderflow.controller;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.RequiredArgsConstructor;
import org.spiderflow.entity.MetaDataMapping;
import org.spiderflow.service.MetaDataMappingService;
import org.spiderflow.service.MetaDataService;
import org.spiderflow.util.AssertUtils;
import org.spiderflow.vo.MetaDataMappingReq;
import org.spiderflow.vo.PageVo;
......@@ -22,6 +22,9 @@ public class MetaDataMappingController {
@Resource
private MetaDataMappingService metaDataMappingService;
@Resource
private MetaDataService metaDataService;
/**
* 获取元数据映射分页列表
* @param req
......@@ -34,6 +37,8 @@ public class MetaDataMappingController {
}
/**
* 根据id获取元数据映射
* @param id
......
spider-flow-web/src/main/java/org/spiderflow/dao/MetaDataMappingDao.java
......@@ -5,5 +5,8 @@ import org.apache.ibatis.annotations.Mapper;
import org.spiderflow.entity.MetaDataMapping;
import org.springframework.stereotype.Repository;
import java.util.List;
public interface MetaDataMappingDao extends BaseMapper<MetaDataMapping> {
List<MetaDataMapping> selectRepeat(String tableSqlname);
}
spider-flow-web/src/main/java/org/spiderflow/entity/MetaDataMapping.java
......@@ -47,13 +47,13 @@ public class MetaDataMapping {
/**
* 列长度
*/
private Integer length;
private String length;
/**
* 列精度
*/
@TableField("`precision`")
private Integer precision;
private String precision;
/**
* 是否主键
......@@ -66,6 +66,11 @@ public class MetaDataMapping {
private Long metadataId;
/**
* 查询数据源id
*/
private Long queryDbId;
/**
* createTime
*/
@TableField(fill = FieldFill.INSERT)
......
spider-flow-web/src/main/java/org/spiderflow/service/MetaDataMappingService.java
......@@ -2,7 +2,9 @@ package org.spiderflow.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import org.apache.commons.lang3.StringUtils;
import org.spiderflow.dao.MetaDataMappingDao;
import org.spiderflow.entity.MetaData;
import org.spiderflow.entity.MetaDataMapping;
import org.spiderflow.util.PagingUtils;
import org.spiderflow.vo.MetaDataMappingReq;
......@@ -19,6 +21,12 @@ public class MetaDataMappingService extends ServiceImpl<MetaDataMappingDao, Meta
public PageVo<MetaDataMapping> getPageList(MetaDataMappingReq req) {
LambdaQueryWrapper<MetaDataMapping> wrapper = new LambdaQueryWrapper<>();
if (StringUtils.isNotBlank(req.getTableSqlName())) {
wrapper.like(MetaDataMapping::getTableSqlname, req.getTableSqlName());
}
if (StringUtils.isNotBlank(req.getFieldName())) {
wrapper.like(MetaDataMapping::getFieldName, req.getFieldName());
}
return PagingUtils.paging(req, metaDataMappingDao, wrapper, MetaDataMapping.class);
}
......
spider-flow-web/src/main/java/org/spiderflow/service/MetaDataService.java
package org.spiderflow.service;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.convert.Convert;
import cn.hutool.core.date.DateUtil;
import cn.hutool.poi.excel.ExcelReader;
import cn.hutool.poi.excel.ExcelUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import org.apache.commons.lang3.StringUtils;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import org.apache.commons.lang3.StringUtils;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import lombok.RequiredArgsConstructor;
import org.spiderflow.core.exception.BusinessException;
import org.spiderflow.dao.MetaDataDao;
import org.spiderflow.dao.MetaDataMappingDao;
import org.spiderflow.entity.MetaData;
import org.spiderflow.entity.MetaDataMapping;
import org.spiderflow.util.PagingUtils;
import org.spiderflow.vo.MetaDataReq;
import org.spiderflow.vo.MetaDataVo;
import org.spiderflow.vo.PageVo;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class MetaDataService extends ServiceImpl<MetaDataDao, MetaData> {
@Resource
private MetaDataDao metaDataDao;
@Resource
private MetaDataMappingDao metaDataMappingDao;
public PageVo<MetaDataVo> getPageList(MetaDataReq req) {
List<MetaDataVo> list = metaDataDao.getPage(req);
......@@ -36,12 +53,6 @@ public class MetaDataService extends ServiceImpl<MetaDataDao, MetaData> {
Long categoryId = req.getCategoryId();
// 获取模糊查询的条件
String metaName = req.getMetaName();
// 判断模糊查询的条件是否为空或空字符串
if (StringUtils.isEmpty(metaName)) {
// 如果为空或空字符串，设置req对象的metaName属性为null
req.setMetaName(null);
}
// 创建一个查询条件对象，设置categoryId等于参数值
LambdaQueryWrapper<MetaData> wrapper = new LambdaQueryWrapper<>();
wrapper.eq(MetaData::getCategoryId, categoryId);
// 判断模糊查询的条件是否为空或空字符串
......@@ -49,13 +60,76 @@ public class MetaDataService extends ServiceImpl<MetaDataDao, MetaData> {
// 如果不为空或空字符串，添加一个模糊查询的条件
wrapper.like(MetaData::getMetaName, metaName);
}
// 调用PagingUtils工具类的paging方法，传入分页请求对象，数据访问对象，查询条件对象和实体类类型，返回分页结果对象
return PagingUtils.paging(req, metaDataDao, wrapper, MetaData.class);
// LambdaQueryWrapper<MetaDataMapping> mappingWrapper = new LambdaQueryWrapper<>();
// List<MetaData> metaDatas = metaDataDao.selectList(wrapper.eq(MetaData::getCategoryId, categoryId));
// MetaDataReq metaDataReq = new MetaDataReq();
// ArrayList<Object> list = new ArrayList<>();
// for (MetaData metaData : metaDatas) {
// BeanUtil.copyProperties(metaData, metaDataReq);
// List<MetaDataMapping> metaDataMappings = metaDataMappingDao.selectList(mappingWrapper.eq(MetaDataMapping::getMetadataId, metaData.getId()));
// for (MetaDataMapping mapping : metaDataMappings) {
// Long queryDbId = mapping.getQueryDbId();
// metaDataReq.setQueryDbId(queryDbId);
// }
// list.add(metaDataReq);
// }
// return list;
}
public Object addExcleData(MultipartFile file, String dataLevel,Long categoryId,Long queryDbId) {
ExcelReader reader;
try {
reader = ExcelUtil.getReader(file.getInputStream());
} catch (IOException e) {
e.printStackTrace();
throw new BusinessException("文件解析失败");
}
List<List<Object>> readList = reader.read(1);
for (List<Object> objectList : readList) {
int index = 0;
MetaData metaData = new MetaData();
metaData.setEnglishName(Convert.toStr(CollUtil.get(objectList, 0)));
metaData.setDataType(Convert.toStr(CollUtil.get(objectList, 1)));
metaData.setNotNull(Convert.toStr(CollUtil.get(objectList, 2)));
metaData.setDataLength(Convert.toStr(CollUtil.get(objectList, 3)));
metaData.setFieldPrecision(Convert.toStr(CollUtil.get(objectList, 4)));
metaData.setMetaName(Convert.toStr(CollUtil.get(objectList, 5)));
String dataCode = Convert.toStr(CollUtil.get(objectList, 6)) + "_" + Convert.toStr(CollUtil.get(objectList, 0));
metaData.setDataCode(dataCode);
metaData.setCreateTime(DateUtil.date());
metaData.setDataLevel(dataLevel);
metaData.setCategoryId(categoryId);
metaData.setMutiMetaFlag(0);
List<MetaData> list = metaDataDao.selectList(Wrappers.lambdaQuery(MetaData.builder().dataCode(dataCode).build()));
if (list != null && list.size() > 0) {
for (MetaData one : list) {
metaDataDao.deleteById(one.getId());
}
}
metaDataDao.insert(metaData);
MetaDataMapping mapping = new MetaDataMapping();
String tableSqlname=Convert.toStr(CollUtil.get(objectList, 6));
mapping.setTableSqlname(tableSqlname);
mapping.setTableName(Convert.toStr(CollUtil.get(objectList, 7)));
mapping.setFieldName(metaData.getEnglishName());
mapping.setFieldType(metaData.getDataType());
mapping.setLength(metaData.getDataLength());
mapping.setPrecision(metaData.getFieldPrecision());
mapping.setIsPk(0);
mapping.setMetadataId(metaData.getId());
mapping.setCreateTime(DateUtil.date());
mapping.setQueryDbId(queryDbId);
List<MetaDataMapping> mappings= metaDataMappingDao.selectRepeat(tableSqlname);
if (mappings != null && mappings.size() > 0) {
for (MetaDataMapping one : mappings) {
metaDataMappingDao.deleteById(one.getId());
}
}
metaDataMappingDao.insert(mapping);
}
return "导入成功";
}
}
spider-flow-web/src/main/java/org/spiderflow/vo/MetaDataReq.java
......@@ -49,10 +49,12 @@ public class MetaDataReq extends PageReqVo {
/**
* 所处级别
*/
private String datLevel;
private String dataLevel;
/**
* 所属领域id
*/
private Long categoryId;
private Long queryDbId;
}
spider-flow-web/src/main/resources/application.yml
......@@ -71,3 +71,13 @@ logging:
selenium:
driver:
chrome: D:\Chrome Driver\chromedriver_win.exe
# mybatis plus 配置
mybatis-plus:
configuration:
jdbc-type-for-null: null
log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
global-config:
db-config:
select-strategy: not_empty
update-strategy: not_empty
spider-flow-web/src/main/resources/mapper/MetaDataMappingMapper.xml
......@@ -4,4 +4,16 @@
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="org.spiderflow.dao.MetaDataMappingDao">
<select id="selectByMetaIdList" resultType="java.util.List">
select * from meta_data_mapping where metadata_id=#{id}
</select>
<select id="selectRepeat" resultType="org.spiderflow.entity.MetaDataMapping">
SELECT
*
FROM
meta_data_mapping
WHERE
table_sqlname = #{tableSqlname}
AND field_name IN(SELECT field_name FROM meta_data_mapping GROUP BY field_name HAVING COUNT(field_name) > 1)
</select>
</mapper>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment