Skip to content

N
nnjcy-data-model
Commit 4d210969 by wqc
Options

增加保存连接库id到xml

parent cb61c745
Showing with 55 additions and 21 deletions
dataoperation-server/src/main/java/com/zq/dataoperation/controller/CommonQueryController.java
...@@ -104,7 +104,7 @@ public class CommonQueryController { ...@@ -104,7 +104,7 @@ public class CommonQueryController {
return ResultVo.success(commonQueryService.runSelect(body)); return ResultVo.success(commonQueryService.runSelect(body));
} }
@ApiOperation("自定义sql查询") @ApiOperation("关联表sql查询")
@PostMapping("/customSelect") @PostMapping("/customSelect")
public ResultVo customSelect(@RequestBody Map<String, Object> body) throws Exception { public ResultVo customSelect(@RequestBody Map<String, Object> body) throws Exception {
return ResultVo.success(commonQueryService.customSelect(body)); return ResultVo.success(commonQueryService.customSelect(body));
......
dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java
...@@ -9,6 +9,7 @@ import com.zq.dataoperation.dao.MetaDataDao; ...@@ -9,6 +9,7 @@ import com.zq.dataoperation.dao.MetaDataDao;
import com.zq.dataoperation.entity.DataCleanExpressCategory; import com.zq.dataoperation.entity.DataCleanExpressCategory;
import com.zq.dataoperation.entity.DataCleanRecord; import com.zq.dataoperation.entity.DataCleanRecord;
import com.zq.dataoperation.entity.DataCleanRule; import com.zq.dataoperation.entity.DataCleanRule;
import com.zq.dataoperation.service.CommonQueryService;
import com.zq.dataoperation.service.DataCleanRecordService; import com.zq.dataoperation.service.DataCleanRecordService;
import com.zq.dataoperation.service.DataCleanRuleService; import com.zq.dataoperation.service.DataCleanRuleService;
import com.zq.dataoperation.service.DataCleanService; import com.zq.dataoperation.service.DataCleanService;
...@@ -100,7 +101,7 @@ public class DataCleanController { ...@@ -100,7 +101,7 @@ public class DataCleanController {
return null; return null;
} }
@ApiOperation("新增清洗记录") @ApiOperation("生成报告")
@PostMapping("/addCleanList") @PostMapping("/addCleanList")
public ResultVo addCleanList(@RequestBody DataCleanRecord dataCleanRecord){ public ResultVo addCleanList(@RequestBody DataCleanRecord dataCleanRecord){
AssertUtils.notNull(dataCleanRecord.getQueryDbId(),"缺少连接库ID"); AssertUtils.notNull(dataCleanRecord.getQueryDbId(),"缺少连接库ID");
...@@ -114,10 +115,10 @@ public class DataCleanController { ...@@ -114,10 +115,10 @@ public class DataCleanController {
return ResultVo.success(dataCleanRecordService.getCleanList(req)); return ResultVo.success(dataCleanRecordService.getCleanList(req));
} }
@ApiOperation("脏数据") @ApiOperation("查看干净/脏数据")
@PostMapping("/getDirtyDetails") @PostMapping("/getDirtyDetails")
public ResultVo getDirtyDetails(@RequestBody DataCleanRecordReq req){ public ResultVo getDirtyDetails(@RequestBody Map<String, Object> body){
return ResultVo.success(dataCleanRecordService.getDirtyDetails(req)); return ResultVo.success(dataCleanRecordService.getDirtyDetails(body));
} }
} }
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRecord.java
...@@ -92,6 +92,12 @@ public class DataCleanRecord { ...@@ -92,6 +92,12 @@ public class DataCleanRecord {
private Long dataDirtyCount; private Long dataDirtyCount;
/** /**
* 源数据表名
*/
@ApiModelProperty("源数据表名")
private String originalTable;
/**
* 创建时间 * 创建时间
*/ */
@ApiModelProperty("创建时间") @ApiModelProperty("创建时间")
......
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
...@@ -21,6 +21,7 @@ import com.zq.dataoperation.holder.DatabaseHolder; ...@@ -21,6 +21,7 @@ import com.zq.dataoperation.holder.DatabaseHolder;
import com.zq.dataoperation.utils.IdentityUtils; import com.zq.dataoperation.utils.IdentityUtils;
import com.zq.dataoperation.utils.SqlUtils; import com.zq.dataoperation.utils.SqlUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.checkerframework.checker.units.qual.A;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.BatchPreparedStatementSetter;
...@@ -66,7 +67,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -66,7 +67,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
} }
@Transactional @Transactional
public void updateDatasource(QueryDb queryDb){ public void updateDatasource(QueryDb queryDb) {
queryDb.setUpdateTime(DateUtil.date()); queryDb.setUpdateTime(DateUtil.date());
AssertUtils.isTrue(queryDbService.updateById(queryDb), "操作失败！"); AssertUtils.isTrue(queryDbService.updateById(queryDb), "操作失败！");
DatabaseHolder.remove(queryDb.getId()); DatabaseHolder.remove(queryDb.getId());
...@@ -74,7 +75,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -74,7 +75,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
} }
@Transactional @Transactional
public void deleteDatasource(Integer id){ public void deleteDatasource(Integer id) {
AssertUtils.isTrue(queryDbService.removeById(id), "操作失败！"); AssertUtils.isTrue(queryDbService.removeById(id), "操作失败！");
DatabaseHolder.remove(id); DatabaseHolder.remove(id);
} }
...@@ -129,10 +130,10 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -129,10 +130,10 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
String pkMetaTwo = jsonObject.get("pkMetaTwo").toString(); String pkMetaTwo = jsonObject.get("pkMetaTwo").toString();
String querySql = jsonObject.get("querySql").toString(); String querySql = jsonObject.get("querySql").toString();
if (StringUtils.isBlank(tableTwo)||StringUtils.isBlank(pkMetaTwo)){ if (StringUtils.isBlank(tableTwo)||StringUtils.isBlank(pkMetaTwo)){
String sql = "SELECT * FROM " + tableOne +" "+querySql; String sql = "SELECT * FROM " + tableOne +" WHERE "+querySql;
return commonQuery(sql,queryDbId); return commonQuery(sql,queryDbId);
}else{ }else{
String leftSql="SELECT * FROM "+tableOne +" t LEFT JOIN "+tableTwo+" h ON t."+pkMetaOne+" = h."+pkMetaTwo+" "+querySql; String leftSql="SELECT * FROM "+tableOne +" t LEFT JOIN "+tableTwo+" h ON t."+pkMetaOne+" = h."+pkMetaTwo+" WHERE "+querySql;
return commonQuery(leftSql,queryDbId); return commonQuery(leftSql,queryDbId);
} }
...@@ -172,7 +173,14 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -172,7 +173,14 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
public List<Map<String, Object>> commonQuery(String sql, Integer queryId) { public List<Map<String, Object>> commonQuery(String sql, Integer queryId) {
JdbcTemplate jdbcTemplate = DatabaseHolder.getJdbcTemplate(queryId); JdbcTemplate jdbcTemplate = DatabaseHolder.getJdbcTemplate(queryId);
return jdbcTemplate.queryForList(sql); List<Map<String, Object>> mapList =null;
try{
mapList = jdbcTemplate.queryForList(sql);
}catch (Exception E){
log.warn("抱歉！不支持此查询");
}
AssertUtils.notNull(mapList,"抱歉！无查询结果");
return mapList;
} }
public Object getTables(Integer id) throws Exception { public Object getTables(Integer id) throws Exception {
...@@ -255,11 +263,11 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -255,11 +263,11 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
List<Map<String, Object>> dataList = null; List<Map<String, Object>> dataList = null;
List<Map<String, Object>> dirtyList = null; List<Map<String, Object>> dirtyList = null;
if (matches) { if (matches) {
String dataSql="SELECT * FROM " + tableName + " WHERE " + metaData + " REGEXP '" + idCardRegex + "'"+" OR LENGTH("+metaData+")=18"; String dataSql = "SELECT * FROM " + tableName + " WHERE " + metaData + " REGEXP '" + idCardRegex + "'" + " OR LENGTH(" + metaData + ")=18";
dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dataSql); dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dataSql);
map.put("dataList", dataList); map.put("dataList", dataList);
} else { } else {
String dirtysql = sql + " WHERE " + metaData + " NOT REGEXP '" + idCardRegex + "'"+" AND LENGTH("+metaData+")<18"; String dirtysql = sql + " WHERE " + metaData + " NOT REGEXP '" + idCardRegex + "'" + " AND LENGTH(" + metaData + ")<18";
dirtyList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dirtysql); dirtyList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dirtysql);
map.put("dirtyList", dirtyList); map.put("dirtyList", dirtyList);
} }
...@@ -419,7 +427,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -419,7 +427,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
int index = 1; int index = 1;
for (String key : finalSet) { for (String key : finalSet) {
try { try {
String keyStr =String.valueOf(map.get(key)); String keyStr = String.valueOf(map.get(key));
preparedStatement.setObject(index++, keyStr); preparedStatement.setObject(index++, keyStr);
} catch (NullPointerException e) { } catch (NullPointerException e) {
e.printStackTrace(); e.printStackTrace();
......
dataoperation-server/src/main/java/com/zq/dataoperation/service/DataCleanRecordService.java
package com.zq.dataoperation.service; package com.zq.dataoperation.service;
import cn.hutool.core.date.DateUtil; import cn.hutool.core.date.DateUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.common.utils.AssertUtils; import com.zq.common.utils.AssertUtils;
...@@ -22,6 +24,7 @@ import org.springframework.stereotype.Service; ...@@ -22,6 +24,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Map;
@Service @Service
public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, DataCleanRecord> { public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, DataCleanRecord> {
...@@ -34,6 +37,8 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data ...@@ -34,6 +37,8 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
private DataCollectTaskLogMapper dataCollectTaskLogMapper; private DataCollectTaskLogMapper dataCollectTaskLogMapper;
@Resource @Resource
private DataCollectMapper dataCollectMapper; private DataCollectMapper dataCollectMapper;
@Resource
private CommonQueryService commonQueryService;
public Object runAndSave(DataCleanRecord dataCleanRecord) { public Object runAndSave(DataCleanRecord dataCleanRecord) {
...@@ -62,6 +67,7 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data ...@@ -62,6 +67,7 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
// BigDecimal dirtyCount = BigDecimal.valueOf(collectCount).subtract(BigDecimal.valueOf(cleanCount));//脏数据 // BigDecimal dirtyCount = BigDecimal.valueOf(collectCount).subtract(BigDecimal.valueOf(cleanCount));//脏数据
DataCleanRecord build = DataCleanRecord.builder() DataCleanRecord build = DataCleanRecord.builder()
.originalTable(dataCleanRecord.getOriginalTable())
.queryDbId(dataCleanRecord.getQueryDbId()) .queryDbId(dataCleanRecord.getQueryDbId())
.dataCollectTaskLogId(collectSettingId) .dataCollectTaskLogId(collectSettingId)
.tableName(dataCleanRecord.getTableName()) .tableName(dataCleanRecord.getTableName())
...@@ -91,7 +97,11 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data ...@@ -91,7 +97,11 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
return PagingUtils.paging(req, dataCleanRecordDao, wrapper.orderByDesc(), DataCleanRecord.class); return PagingUtils.paging(req, dataCleanRecordDao, wrapper.orderByDesc(), DataCleanRecord.class);
} }
public Object getDirtyDetails(DataCleanRecordReq req) { public Object getDirtyDetails(Map<String, Object> body) {
return null; JSONObject jsonObject = JSONUtil.parseObj(body);
Integer queryDbId = Integer.valueOf(jsonObject.get("queryDbId").toString());
String tableName = jsonObject.get("tableName").toString();
String sql="SELECT * FROM " + tableName;
return commonQueryService.commonQuery(sql,queryDbId);
} }
} }
dataoperation-server/src/main/java/com/zq/dataoperation/vo/DataCleanRecordReq.java
...@@ -81,6 +81,12 @@ public class DataCleanRecordReq extends PageReqVo { ...@@ -81,6 +81,12 @@ public class DataCleanRecordReq extends PageReqVo {
private Long dataDirtyCount; private Long dataDirtyCount;
/** /**
* 源数据表名
*/
@ApiModelProperty("源数据表名")
private String originalTable;
/**
* 创建时间 * 创建时间
*/ */
@ApiModelProperty("创建时间") @ApiModelProperty("创建时间")
......
spider-flow-core/src/main/java/com/zq/spiderflow/core/mapper/SpiderFlowMapper.java
...@@ -34,11 +34,11 @@ public interface SpiderFlowMapper extends BaseMapper<SpiderFlow>{ ...@@ -34,11 +34,11 @@ public interface SpiderFlowMapper extends BaseMapper<SpiderFlow>{
}) })
IPage<SpiderFlow> selectSpiderPage(Page<SpiderFlow> page,@Param("name") String name); IPage<SpiderFlow> selectSpiderPage(Page<SpiderFlow> page,@Param("name") String name);
@Insert("insert into sp_flow(id,name,xml,flow_type,enabled) values(#{id},#{name},#{xml},#{flowType},'0')") @Insert("insert into sp_flow(id,name,xml,query_db_id,flow_type,enabled) values(#{id},#{name},#{xml},#{queryDbId},#{flowType},'0')")
int insertSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml, @Param("flowType") String flowType); int insertSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml,@Param("queryDbId") String queryDbId,@Param("flowType") String flowType);
@Update("update sp_flow set name = #{name},xml = #{xml} where id = #{id}") @Update("update sp_flow set name = #{name},xml = #{xml},query_db_id=#{queryDbId} where id = #{id}")
int updateSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml); int updateSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml,@Param("queryDbId") String queryDbId);
@Update("update sp_flow set execute_count = 0 where id = #{id}") @Update("update sp_flow set execute_count = 0 where id = #{id}")
int resetExecuteCount(@Param("id") String id); int resetExecuteCount(@Param("id") String id);
......
spider-flow-core/src/main/java/com/zq/spiderflow/core/model/SpiderFlow.java
...@@ -58,5 +58,8 @@ public class SpiderFlow { ...@@ -58,5 +58,8 @@ public class SpiderFlow {
@TableField(exist = false) @TableField(exist = false)
private Integer running; private Integer running;
//数据库连接ID
private String queryDbId;
} }
spider-flow-core/src/main/java/com/zq/spiderflow/core/service/SpiderFlowService.java
...@@ -120,7 +120,7 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow> ...@@ -120,7 +120,7 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
spiderFlow.setNextExecuteTime(trigger.getStartTime()); spiderFlow.setNextExecuteTime(trigger.getStartTime());
} }
if(StringUtils.isNotEmpty(spiderFlow.getId())){ //update 任务 if(StringUtils.isNotEmpty(spiderFlow.getId())){ //update 任务
sfMapper.updateSpiderFlow(spiderFlow.getId(), spiderFlow.getName(), spiderFlow.getXml()); sfMapper.updateSpiderFlow(spiderFlow.getId(), spiderFlow.getName(), spiderFlow.getXml(),spiderFlow.getQueryDbId());
spiderJobManager.remove(spiderFlow.getId()); spiderJobManager.remove(spiderFlow.getId());
spiderFlow = getById(spiderFlow.getId()); spiderFlow = getById(spiderFlow.getId());
if("1".equals(spiderFlow.getEnabled()) && StringUtils.isNotEmpty(spiderFlow.getCron())){ if("1".equals(spiderFlow.getEnabled()) && StringUtils.isNotEmpty(spiderFlow.getCron())){
...@@ -128,7 +128,7 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow> ...@@ -128,7 +128,7 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
} }
}else{//insert 任务 }else{//insert 任务
String id = UUID.randomUUID().toString().replace("-", ""); String id = UUID.randomUUID().toString().replace("-", "");
sfMapper.insertSpiderFlow(id, spiderFlow.getName(), spiderFlow.getXml(),spiderFlow.getFlowType()); sfMapper.insertSpiderFlow(id, spiderFlow.getName(), spiderFlow.getXml(),spiderFlow.getQueryDbId(),spiderFlow.getFlowType());
spiderFlow.setId(id); spiderFlow.setId(id);
} }
File file = new File(workspace,spiderFlow.getId() + File.separator + "xmls" + File.separator + System.currentTimeMillis() + ".xml"); File file = new File(workspace,spiderFlow.getId() + File.separator + "xmls" + File.separator + System.currentTimeMillis() + ".xml");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment