Skip to content

N
nnjcy-data-model
Commit 4d210969 by wqc
Options

增加保存连接库id到xml

parent cb61c745
Showing with 55 additions and 21 deletions
dataoperation-server/src/main/java/com/zq/dataoperation/controller/CommonQueryController.java
......@@ -104,7 +104,7 @@ public class CommonQueryController {
return ResultVo.success(commonQueryService.runSelect(body));
}
@ApiOperation("自定义sql查询")
@ApiOperation("关联表sql查询")
@PostMapping("/customSelect")
public ResultVo customSelect(@RequestBody Map<String, Object> body) throws Exception {
return ResultVo.success(commonQueryService.customSelect(body));
......
dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java
......@@ -9,6 +9,7 @@ import com.zq.dataoperation.dao.MetaDataDao;
import com.zq.dataoperation.entity.DataCleanExpressCategory;
import com.zq.dataoperation.entity.DataCleanRecord;
import com.zq.dataoperation.entity.DataCleanRule;
import com.zq.dataoperation.service.CommonQueryService;
import com.zq.dataoperation.service.DataCleanRecordService;
import com.zq.dataoperation.service.DataCleanRuleService;
import com.zq.dataoperation.service.DataCleanService;
......@@ -100,7 +101,7 @@ public class DataCleanController {
return null;
}
@ApiOperation("新增清洗记录")
@ApiOperation("生成报告")
@PostMapping("/addCleanList")
public ResultVo addCleanList(@RequestBody DataCleanRecord dataCleanRecord){
AssertUtils.notNull(dataCleanRecord.getQueryDbId(),"缺少连接库ID");
......@@ -114,10 +115,10 @@ public class DataCleanController {
return ResultVo.success(dataCleanRecordService.getCleanList(req));
}
@ApiOperation("脏数据")
@ApiOperation("查看干净/脏数据")
@PostMapping("/getDirtyDetails")
public ResultVo getDirtyDetails(@RequestBody DataCleanRecordReq req){
return ResultVo.success(dataCleanRecordService.getDirtyDetails(req));
public ResultVo getDirtyDetails(@RequestBody Map<String, Object> body){
return ResultVo.success(dataCleanRecordService.getDirtyDetails(body));
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRecord.java
......@@ -92,6 +92,12 @@ public class DataCleanRecord {
private Long dataDirtyCount;
/**
* 源数据表名
*/
@ApiModelProperty("源数据表名")
private String originalTable;
/**
* 创建时间
*/
@ApiModelProperty("创建时间")
......
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
......@@ -21,6 +21,7 @@ import com.zq.dataoperation.holder.DatabaseHolder;
import com.zq.dataoperation.utils.IdentityUtils;
import com.zq.dataoperation.utils.SqlUtils;
import org.apache.commons.lang3.StringUtils;
import org.checkerframework.checker.units.qual.A;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
......@@ -66,7 +67,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
@Transactional
public void updateDatasource(QueryDb queryDb){
public void updateDatasource(QueryDb queryDb) {
queryDb.setUpdateTime(DateUtil.date());
AssertUtils.isTrue(queryDbService.updateById(queryDb), "操作失败！");
DatabaseHolder.remove(queryDb.getId());
......@@ -74,7 +75,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
@Transactional
public void deleteDatasource(Integer id){
public void deleteDatasource(Integer id) {
AssertUtils.isTrue(queryDbService.removeById(id), "操作失败！");
DatabaseHolder.remove(id);
}
......@@ -129,10 +130,10 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
String pkMetaTwo = jsonObject.get("pkMetaTwo").toString();
String querySql = jsonObject.get("querySql").toString();
if (StringUtils.isBlank(tableTwo)||StringUtils.isBlank(pkMetaTwo)){
String sql = "SELECT * FROM " + tableOne +" "+querySql;
String sql = "SELECT * FROM " + tableOne +" WHERE "+querySql;
return commonQuery(sql,queryDbId);
}else{
String leftSql="SELECT * FROM "+tableOne +" t LEFT JOIN "+tableTwo+" h ON t."+pkMetaOne+" = h."+pkMetaTwo+" "+querySql;
String leftSql="SELECT * FROM "+tableOne +" t LEFT JOIN "+tableTwo+" h ON t."+pkMetaOne+" = h."+pkMetaTwo+" WHERE "+querySql;
return commonQuery(leftSql,queryDbId);
}
......@@ -172,7 +173,14 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
public List<Map<String, Object>> commonQuery(String sql, Integer queryId) {
JdbcTemplate jdbcTemplate = DatabaseHolder.getJdbcTemplate(queryId);
return jdbcTemplate.queryForList(sql);
List<Map<String, Object>> mapList =null;
try{
mapList = jdbcTemplate.queryForList(sql);
}catch (Exception E){
log.warn("抱歉！不支持此查询");
}
AssertUtils.notNull(mapList,"抱歉！无查询结果");
return mapList;
}
public Object getTables(Integer id) throws Exception {
......@@ -255,11 +263,11 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
List<Map<String, Object>> dataList = null;
List<Map<String, Object>> dirtyList = null;
if (matches) {
String dataSql="SELECT * FROM " + tableName + " WHERE " + metaData + " REGEXP '" + idCardRegex + "'"+" OR LENGTH("+metaData+")=18";
String dataSql = "SELECT * FROM " + tableName + " WHERE " + metaData + " REGEXP '" + idCardRegex + "'" + " OR LENGTH(" + metaData + ")=18";
dataList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dataSql);
map.put("dataList", dataList);
} else {
String dirtysql = sql + " WHERE " + metaData + " NOT REGEXP '" + idCardRegex + "'"+" AND LENGTH("+metaData+")<18";
String dirtysql = sql + " WHERE " + metaData + " NOT REGEXP '" + idCardRegex + "'" + " AND LENGTH(" + metaData + ")<18";
dirtyList = DatabaseHolder.getJdbcTemplate(datasourceId).queryForList(dirtysql);
map.put("dirtyList", dirtyList);
}
......@@ -419,7 +427,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
int index = 1;
for (String key : finalSet) {
try {
String keyStr =String.valueOf(map.get(key));
String keyStr = String.valueOf(map.get(key));
preparedStatement.setObject(index++, keyStr);
} catch (NullPointerException e) {
e.printStackTrace();
......
dataoperation-server/src/main/java/com/zq/dataoperation/service/DataCleanRecordService.java
package com.zq.dataoperation.service;
import cn.hutool.core.date.DateUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.common.utils.AssertUtils;
......@@ -22,6 +24,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
import java.util.Map;
@Service
public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, DataCleanRecord> {
......@@ -34,6 +37,8 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
private DataCollectTaskLogMapper dataCollectTaskLogMapper;
@Resource
private DataCollectMapper dataCollectMapper;
@Resource
private CommonQueryService commonQueryService;
public Object runAndSave(DataCleanRecord dataCleanRecord) {
......@@ -62,6 +67,7 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
// BigDecimal dirtyCount = BigDecimal.valueOf(collectCount).subtract(BigDecimal.valueOf(cleanCount));//脏数据
DataCleanRecord build = DataCleanRecord.builder()
.originalTable(dataCleanRecord.getOriginalTable())
.queryDbId(dataCleanRecord.getQueryDbId())
.dataCollectTaskLogId(collectSettingId)
.tableName(dataCleanRecord.getTableName())
......@@ -91,7 +97,11 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
return PagingUtils.paging(req, dataCleanRecordDao, wrapper.orderByDesc(), DataCleanRecord.class);
}
public Object getDirtyDetails(DataCleanRecordReq req) {
return null;
public Object getDirtyDetails(Map<String, Object> body) {
JSONObject jsonObject = JSONUtil.parseObj(body);
Integer queryDbId = Integer.valueOf(jsonObject.get("queryDbId").toString());
String tableName = jsonObject.get("tableName").toString();
String sql="SELECT * FROM " + tableName;
return commonQueryService.commonQuery(sql,queryDbId);
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/vo/DataCleanRecordReq.java
......@@ -81,6 +81,12 @@ public class DataCleanRecordReq extends PageReqVo {
private Long dataDirtyCount;
/**
* 源数据表名
*/
@ApiModelProperty("源数据表名")
private String originalTable;
/**
* 创建时间
*/
@ApiModelProperty("创建时间")
......
spider-flow-core/src/main/java/com/zq/spiderflow/core/mapper/SpiderFlowMapper.java
......@@ -34,11 +34,11 @@ public interface SpiderFlowMapper extends BaseMapper<SpiderFlow>{
})
IPage<SpiderFlow> selectSpiderPage(Page<SpiderFlow> page,@Param("name") String name);
@Insert("insert into sp_flow(id,name,xml,flow_type,enabled) values(#{id},#{name},#{xml},#{flowType},'0')")
int insertSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml, @Param("flowType") String flowType);
@Insert("insert into sp_flow(id,name,xml,query_db_id,flow_type,enabled) values(#{id},#{name},#{xml},#{queryDbId},#{flowType},'0')")
int insertSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml,@Param("queryDbId") String queryDbId,@Param("flowType") String flowType);
@Update("update sp_flow set name = #{name},xml = #{xml} where id = #{id}")
int updateSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml);
@Update("update sp_flow set name = #{name},xml = #{xml},query_db_id=#{queryDbId} where id = #{id}")
int updateSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml,@Param("queryDbId") String queryDbId);
@Update("update sp_flow set execute_count = 0 where id = #{id}")
int resetExecuteCount(@Param("id") String id);
......
spider-flow-core/src/main/java/com/zq/spiderflow/core/model/SpiderFlow.java
......@@ -58,5 +58,8 @@ public class SpiderFlow {
@TableField(exist = false)
private Integer running;
//数据库连接ID
private String queryDbId;
}
spider-flow-core/src/main/java/com/zq/spiderflow/core/service/SpiderFlowService.java
......@@ -120,7 +120,7 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
spiderFlow.setNextExecuteTime(trigger.getStartTime());
}
if(StringUtils.isNotEmpty(spiderFlow.getId())){ //update 任务
sfMapper.updateSpiderFlow(spiderFlow.getId(), spiderFlow.getName(), spiderFlow.getXml());
sfMapper.updateSpiderFlow(spiderFlow.getId(), spiderFlow.getName(), spiderFlow.getXml(),spiderFlow.getQueryDbId());
spiderJobManager.remove(spiderFlow.getId());
spiderFlow = getById(spiderFlow.getId());
if("1".equals(spiderFlow.getEnabled()) && StringUtils.isNotEmpty(spiderFlow.getCron())){
......@@ -128,7 +128,7 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
}
}else{//insert 任务
String id = UUID.randomUUID().toString().replace("-", "");
sfMapper.insertSpiderFlow(id, spiderFlow.getName(), spiderFlow.getXml(),spiderFlow.getFlowType());
sfMapper.insertSpiderFlow(id, spiderFlow.getName(), spiderFlow.getXml(),spiderFlow.getQueryDbId(),spiderFlow.getFlowType());
spiderFlow.setId(id);
}
File file = new File(workspace,spiderFlow.getId() + File.separator + "xmls" + File.separator + System.currentTimeMillis() + ".xml");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment