Skip to content

N
nnjcy-data-model
Commit 7d2fadcb by wqc
Options

清洗记录调整

parent ec165856
Showing with 212 additions and 111 deletions
dataoperation-server/pom.xml
...@@ -118,6 +118,12 @@ ...@@ -118,6 +118,12 @@
<version>1.0.0</version> <version>1.0.0</version>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<dependency>
<groupId>com.zq</groupId>
<artifactId>data-collect-server</artifactId>
<version>1.0.0</version>
<scope>compile</scope>
</dependency>
</dependencies> </dependencies>
<build> <build>
......
dataoperation-server/src/main/java/com/zq/dataoperation/config/MybatisPlusConfig.java 0 → 100644
package com.zq.dataoperation.config;
import com.baomidou.mybatisplus.annotation.DbType;
import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class MybatisPlusConfig {
@Bean
public MybatisPlusInterceptor mybatisPlusInterceptor(){
MybatisPlusInterceptor interceptor=new MybatisPlusInterceptor();
interceptor.addInnerInterceptor(new PaginationInnerInterceptor(DbType.MYSQL));
return interceptor;
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java
...@@ -3,6 +3,7 @@ package com.zq.dataoperation.controller; ...@@ -3,6 +3,7 @@ package com.zq.dataoperation.controller;
import cn.hutool.core.date.DateUtil; import cn.hutool.core.date.DateUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.zq.common.utils.AssertUtils; import com.zq.common.utils.AssertUtils;
import com.zq.common.vo.PageVo;
import com.zq.common.vo.ResultVo; import com.zq.common.vo.ResultVo;
import com.zq.dataoperation.dao.MetaDataDao; import com.zq.dataoperation.dao.MetaDataDao;
import com.zq.dataoperation.entity.DataCleanExpressCategory; import com.zq.dataoperation.entity.DataCleanExpressCategory;
...@@ -12,6 +13,7 @@ import com.zq.dataoperation.service.DataCleanRecordService; ...@@ -12,6 +13,7 @@ import com.zq.dataoperation.service.DataCleanRecordService;
import com.zq.dataoperation.service.DataCleanRuleService; import com.zq.dataoperation.service.DataCleanRuleService;
import com.zq.dataoperation.service.DataCleanService; import com.zq.dataoperation.service.DataCleanService;
import com.zq.dataoperation.entity.MetaData; import com.zq.dataoperation.entity.MetaData;
import com.zq.dataoperation.vo.DataCleanRecordReq;
import io.swagger.annotations.Api; import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
...@@ -102,9 +104,20 @@ public class DataCleanController { ...@@ -102,9 +104,20 @@ public class DataCleanController {
@PostMapping("/addCleanList") @PostMapping("/addCleanList")
public ResultVo addCleanList(@RequestBody DataCleanRecord dataCleanRecord){ public ResultVo addCleanList(@RequestBody DataCleanRecord dataCleanRecord){
AssertUtils.notNull(dataCleanRecord.getQueryDbId(),"缺少连接库ID"); AssertUtils.notNull(dataCleanRecord.getQueryDbId(),"缺少连接库ID");
AssertUtils.notNull(dataCleanRecord.getDataCollectTaskLogId(),"缺少汇聚任务ID");
return ResultVo.success(dataCleanRecordService.runAndSave(dataCleanRecord)); return ResultVo.success(dataCleanRecordService.runAndSave(dataCleanRecord));
} }
@ApiOperation("清洗记录列表")
@PostMapping("/getCleanList")
public ResultVo<PageVo<DataCleanRecord>> getCleanList(@RequestBody DataCleanRecordReq req){
return ResultVo.success(dataCleanRecordService.getCleanList(req));
}
@ApiOperation("脏数据")
@PostMapping("/getDirtyDetails")
public ResultVo getDirtyDetails(@RequestBody DataCleanRecordReq req){
return ResultVo.success(dataCleanRecordService.getDirtyDetails(req));
}
} }
dataoperation-server/src/main/java/com/zq/dataoperation/dao/DataCleanRecordDao.java
...@@ -2,7 +2,9 @@ package com.zq.dataoperation.dao; ...@@ -2,7 +2,9 @@ package com.zq.dataoperation.dao;
import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zq.dataoperation.entity.DataCleanRecord; import com.zq.dataoperation.entity.DataCleanRecord;
import org.springframework.stereotype.Repository;
@Repository
public interface DataCleanRecordDao extends BaseMapper<DataCleanRecord> { public interface DataCleanRecordDao extends BaseMapper<DataCleanRecord> {
} }
dataoperation-server/src/main/java/com/zq/dataoperation/dao/DataCollectTaskLogMapper.java deleted 100644 → 0
package com.zq.dataoperation.dao;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zq.dataoperation.entity.DataCollectTaskLog;
public interface DataCollectTaskLogMapper extends BaseMapper<DataCollectTaskLog> {
}
\ No newline at end of file
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRecord.java
package com.zq.dataoperation.entity; package com.zq.dataoperation.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.annotation.TableName;
import com.zq.common.vo.PageReqVo;
import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
...@@ -22,9 +25,10 @@ import java.util.Date; ...@@ -22,9 +25,10 @@ import java.util.Date;
@TableName(value = "data_clean_record") @TableName(value = "data_clean_record")
public class DataCleanRecord { public class DataCleanRecord {
/** /**
* id * id
*/ */
@TableId(type = IdType.AUTO)
private Integer id; private Integer id;
/** /**
...@@ -48,14 +52,14 @@ public class DataCleanRecord { ...@@ -48,14 +52,14 @@ public class DataCleanRecord {
/** /**
* 清洗开始时间 * 清洗开始时间
*/ */
@ApiModelProperty("清洗开始时间") @ApiModelProperty("汇聚完成时间")
private Date startTime; private Date collectTime;
/** /**
* 结束时间 * 结束时间
*/ */
@ApiModelProperty("结束时间") @ApiModelProperty("清洗完成时间")
private Date endTime; private String cleanEndTime;
/** /**
* 花费时间 * 花费时间
...@@ -70,22 +74,22 @@ public class DataCleanRecord { ...@@ -70,22 +74,22 @@ public class DataCleanRecord {
private Integer status; private Integer status;
/** /**
* 花费时间 * 汇聚数据量
*/ */
@ApiModelProperty("花费时间") @ApiModelProperty("汇聚数据量")
private String dataCollectCount; private Long dataCollectCount;
/** /**
* 花费时间 * 清洗数据量
*/ */
@ApiModelProperty("花费时间") @ApiModelProperty("清洗数据量")
private String dataCleanCount; private Long dataCleanCount;
/** /**
* 花费时间 * 脏数据量
*/ */
@ApiModelProperty("花费时间") @ApiModelProperty("脏数据量")
private String dataDirtyCount; private Long dataDirtyCount;
/** /**
* 创建时间 * 创建时间
......
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRule.java
...@@ -2,6 +2,8 @@ package com.zq.dataoperation.entity; ...@@ -2,6 +2,8 @@ package com.zq.dataoperation.entity;
import java.util.Date; import java.util.Date;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.annotation.TableName;
import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
...@@ -27,6 +29,7 @@ public class DataCleanRule{ ...@@ -27,6 +29,7 @@ public class DataCleanRule{
/** /**
* id * id
*/ */
@TableId(type = IdType.AUTO)
private Integer id; private Integer id;
/** /**
......
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCollectTaskLog.java deleted 100644 → 0
package com.zq.dataoperation.entity;
import com.baomidou.mybatisplus.annotation.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Date;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@TableName(value = "data_collect_task_log")
public class DataCollectTaskLog {
/**
* 数据汇聚表id
*/
private Integer dataCollectSettingId;
/**
* 开始时间
*/
private Date startTime;
/**
* 结束时间
*/
private Date endTime;
/**
* 花费多少秒
*/
private Integer duration;
/**
* 状态
* 1运行中 2正常结束 3异常终止
*/
private Integer status;
/**
* 错误信息
* 异常终止时记录异常信息
*/
private String error;
/**
* 使用序列
* id
*/
@TableId(type = IdType.AUTO)
private Integer id;
/**
* createTime
*/
@TableField(fill = FieldFill.INSERT)
private Date createTime;
/**
* updateTime
*/
@TableField(fill = FieldFill.INSERT_UPDATE)
private Date updateTime;
}
dataoperation-server/src/main/java/com/zq/dataoperation/runner/DatabaseConnectRunner.java
...@@ -5,7 +5,6 @@ import com.alibaba.druid.pool.DruidPooledConnection; ...@@ -5,7 +5,6 @@ import com.alibaba.druid.pool.DruidPooledConnection;
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder; import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
import com.zq.common.exception.BusinessException; import com.zq.common.exception.BusinessException;
import com.zq.dataoperation.entity.QueryDb; import com.zq.dataoperation.entity.QueryDb;
import com.zq.dataoperation.utils.ConnectionUtil;
import com.zq.dataoperation.service.CommonQueryService; import com.zq.dataoperation.service.CommonQueryService;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationArguments;
......
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
...@@ -357,8 +357,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo ...@@ -357,8 +357,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
} }
}); });
List<Map<String, Object>> newCount = fromDbJdbcTemplate.queryForList("SELECT COUNT(1) FROM " + newTableName); List<Map<String, Object>> newCount = fromDbJdbcTemplate.queryForList("SELECT COUNT(1) AS newCount FROM " + newTableName);
List<Map<String, Object>> oldCount = fromDbJdbcTemplate.queryForList("SELECT COUNT(1) FROM " + oldTableName); List<Map<String, Object>> oldCount = fromDbJdbcTemplate.queryForList("SELECT COUNT(1) AS oldCount FROM " + oldTableName);
List<Map<String, Object>> newTable = fromDbJdbcTemplate.queryForList("SELECT * FROM " + newTableName); List<Map<String, Object>> newTable = fromDbJdbcTemplate.queryForList("SELECT * FROM " + newTableName);
long times = System.currentTimeMillis() - startTime; long times = System.currentTimeMillis() - startTime;
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
......
dataoperation-server/src/main/java/com/zq/dataoperation/service/DataCleanRecordService.java
package com.zq.dataoperation.service; package com.zq.dataoperation.service;
import cn.hutool.core.date.DateUtil; import cn.hutool.core.date.DateUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.common.utils.AssertUtils; import com.zq.common.utils.AssertUtils;
import com.zq.common.utils.PagingUtils;
import com.zq.common.vo.PageVo;
import com.zq.common.vo.ResultVo; import com.zq.common.vo.ResultVo;
import com.zq.datacollect.entity.DataCollectSetting;
import com.zq.datacollect.entity.DataCollectTaskLog;
import com.zq.datacollect.mapper.DataCollectMapper;
import com.zq.datacollect.mapper.DataCollectTaskLogMapper;
import com.zq.dataoperation.dao.DataCleanRecordDao; import com.zq.dataoperation.dao.DataCleanRecordDao;
import com.zq.dataoperation.dao.DataCollectTaskLogMapper;
import com.zq.dataoperation.dao.QueryDbDao; import com.zq.dataoperation.dao.QueryDbDao;
import com.zq.dataoperation.entity.DataCleanRecord; import com.zq.dataoperation.entity.DataCleanRecord;
import com.zq.dataoperation.entity.DataCollectTaskLog;
import com.zq.dataoperation.entity.QueryDb; import com.zq.dataoperation.entity.QueryDb;
import lombok.RequiredArgsConstructor; import com.zq.dataoperation.vo.DataCleanRecordReq;
import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.math.BigDecimal;
import java.util.Date;
import java.util.List;
@Slf4j
@RequiredArgsConstructor
@Service @Service
public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, DataCleanRecord> { public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, DataCleanRecord> {
...@@ -27,21 +33,44 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data ...@@ -27,21 +33,44 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
private QueryDbDao queryDbDao; private QueryDbDao queryDbDao;
@Resource @Resource
private DataCollectTaskLogMapper dataCollectTaskLogMapper; private DataCollectTaskLogMapper dataCollectTaskLogMapper;
@Resource
private DataCollectMapper dataCollectMapper;
public Object runAndSave(DataCleanRecord dataCleanRecord) { public Object runAndSave(DataCleanRecord dataCleanRecord) {
QueryDb queryDb = queryDbDao.selectById(dataCleanRecord.getQueryDbId()); QueryDb queryDb = queryDbDao.selectById(dataCleanRecord.getQueryDbId());
AssertUtils.notNull(queryDb, "无连接表ID"); AssertUtils.notNull(queryDb, "无连接表ID");
DataCollectTaskLog dataCollectTaskLog = dataCollectTaskLogMapper.selectById(dataCleanRecord.getDataCollectTaskLogId()); LambdaQueryWrapper<DataCollectSetting> wrapper = new LambdaQueryWrapper<>();
AssertUtils.notNull(dataCollectTaskLog, "无连接表ID"); wrapper.eq(DataCollectSetting::getToDbId, queryDb.getId()).eq(DataCollectSetting::getToDbId, 1);//数据库类型
List<DataCollectSetting> dataCollectSettings = dataCollectMapper.selectList(wrapper);
Integer collectSettingId = null;
for (DataCollectSetting collectSetting : dataCollectSettings) {
collectSettingId = collectSetting.getId();
}
LambdaQueryWrapper<DataCollectTaskLog> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(DataCollectTaskLog::getDataCollectSettingId,collectSettingId);
List<DataCollectTaskLog> taskLogs = dataCollectTaskLogMapper.selectList(queryWrapper);
Date endTime =null;
Integer status=null;
for (DataCollectTaskLog taskLog:taskLogs) {
endTime = taskLog.getEndTime();
status = taskLog.getStatus();
}
Long collectCount = dataCleanRecord.getDataCollectCount();
Long cleanCount = dataCleanRecord.getDataCleanCount();
// BigDecimal dirtyCount = BigDecimal.valueOf(collectCount).subtract(BigDecimal.valueOf(cleanCount));//脏数据
DataCleanRecord build = DataCleanRecord.builder() DataCleanRecord build = DataCleanRecord.builder()
.queryDbId(dataCleanRecord.getQueryDbId()) .queryDbId(dataCleanRecord.getQueryDbId())
.dataCollectTaskLogId(dataCollectTaskLog.getDataCollectSettingId()) .dataCollectTaskLogId(collectSettingId)
.tableName(dataCleanRecord.getTableName()) .tableName(dataCleanRecord.getTableName())
.endTime(DateUtil.date()) .cleanEndTime(dataCleanRecord.getCleanEndTime())
.status(dataCollectTaskLog.getStatus()) .collectTime(endTime)
.dataCollectCount(dataCleanRecord.getDataCollectCount()) .status(status)
.dataCleanCount(dataCleanRecord.getDataCleanCount()) .dataCollectCount(collectCount)
.dataCleanCount(cleanCount)
.dataDirtyCount(dataCleanRecord.getDataDirtyCount()) .dataDirtyCount(dataCleanRecord.getDataDirtyCount())
.createTime(DateUtil.date()) .createTime(DateUtil.date())
.build(); .build();
...@@ -52,6 +81,18 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data ...@@ -52,6 +81,18 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
} else { } else {
i = dataCleanRecordDao.updateById(build); i = dataCleanRecordDao.updateById(build);
} }
return i>0 ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败"); return i > 0 ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败");
}
public PageVo<DataCleanRecord> getCleanList(DataCleanRecordReq req) {
LambdaQueryWrapper<DataCleanRecord> wrapper = new LambdaQueryWrapper<>();
if (StringUtils.isNotBlank(req.getTableName())) {
wrapper.like(DataCleanRecord::getTableName, req.getTableName());
}
return PagingUtils.paging(req, dataCleanRecordDao, wrapper.orderByDesc(), DataCleanRecord.class);
}
public Object getDirtyDetails(DataCleanRecordReq req) {
return null;
} }
} }
dataoperation-server/src/main/java/com/zq/dataoperation/vo/DataCleanRecordReq.java 0 → 100644
package com.zq.dataoperation.vo;
import com.zq.common.vo.PageReqVo;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.Date;
/**
* (data_clean_record)实体类
*
* @author zq
* @since 2023-07-04 16:12:53
*/
@Data
public class DataCleanRecordReq extends PageReqVo {
/**
* id
*/
private Integer id;
/**
* 连接id
*/
@ApiModelProperty("连接id")
private Long queryDbId;
/**
* 表名
*/
@ApiModelProperty("表名")
private String tableName;
/**
* 任务汇聚表ID
*/
@ApiModelProperty("任务汇聚表ID")
private Integer dataCollectTaskLogId;
/**
* 清洗开始时间
*/
@ApiModelProperty("汇聚完成时间")
private Date collectTime;
/**
* 结束时间
*/
@ApiModelProperty("清洗耗时时间")
private String cleanEndTime;
/**
* 花费时间
*/
@ApiModelProperty("花费时间")
private String duration;
/**
* 1运行中 2正常结束 3异常终止
*/
@ApiModelProperty("状态")
private Integer status;
/**
* 汇聚数据量
*/
@ApiModelProperty("汇聚数据量")
private Long dataCollectCount;
/**
* 清洗数据量
*/
@ApiModelProperty("清洗数据量")
private Long dataCleanCount;
/**
* 脏数据量
*/
@ApiModelProperty("脏数据量")
private Long dataDirtyCount;
/**
* 创建时间
*/
@ApiModelProperty("创建时间")
private Date createTime;
/**
* 更新时间
*/
@ApiModelProperty("更新时间")
private Date updateTime;
}
spider-flow-web/src/main/java/com/zq/spiderflow/entity/MetaDataMapping.java
...@@ -12,8 +12,6 @@ import java.util.Date; ...@@ -12,8 +12,6 @@ import java.util.Date;
@Builder @Builder
@AllArgsConstructor @AllArgsConstructor
@NoArgsConstructor @NoArgsConstructor
//@KeySequence(value = "META_DATA_MAPPING_SEQ")
//@TableName(value = "META_DATA_MAPPING")
@TableName(value = "meta_data_mapping") @TableName(value = "meta_data_mapping")
public class MetaDataMapping { public class MetaDataMapping {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment