Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
N
nnjcy-data-model
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
梁家彪
nnjcy-data-model
Commits
7d2fadcb
Commit
7d2fadcb
authored
Oct 27, 2023
by
wqc
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
清洗记录调整
parent
ec165856
Hide whitespace changes
Inline
Side-by-side
Showing
13 changed files
with
212 additions
and
111 deletions
+212
-111
dataoperation-server/pom.xml
+6
-0
dataoperation-server/src/main/java/com/zq/dataoperation/config/MybatisPlusConfig.java
+17
-0
dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java
+14
-1
dataoperation-server/src/main/java/com/zq/dataoperation/dao/DataCleanRecordDao.java
+2
-0
dataoperation-server/src/main/java/com/zq/dataoperation/dao/DataCollectTaskLogMapper.java
+0
-8
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRecord.java
+18
-14
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRule.java
+3
-0
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCollectTaskLog.java
+0
-69
dataoperation-server/src/main/java/com/zq/dataoperation/runner/DatabaseConnectRunner.java
+0
-1
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
+2
-2
dataoperation-server/src/main/java/com/zq/dataoperation/service/DataCleanRecordService.java
+55
-14
dataoperation-server/src/main/java/com/zq/dataoperation/vo/DataCleanRecordReq.java
+95
-0
spider-flow-web/src/main/java/com/zq/spiderflow/entity/MetaDataMapping.java
+0
-2
No files found.
dataoperation-server/pom.xml
View file @
7d2fadcb
...
...
@@ -118,6 +118,12 @@
<version>
1.0.0
</version>
<scope>
compile
</scope>
</dependency>
<dependency>
<groupId>
com.zq
</groupId>
<artifactId>
data-collect-server
</artifactId>
<version>
1.0.0
</version>
<scope>
compile
</scope>
</dependency>
</dependencies>
<build>
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/config/MybatisPlusConfig.java
0 → 100644
View file @
7d2fadcb
package
com
.
zq
.
dataoperation
.
config
;
import
com.baomidou.mybatisplus.annotation.DbType
;
import
com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor
;
import
com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor
;
import
org.springframework.context.annotation.Bean
;
import
org.springframework.context.annotation.Configuration
;
@Configuration
public
class
MybatisPlusConfig
{
@Bean
public
MybatisPlusInterceptor
mybatisPlusInterceptor
(){
MybatisPlusInterceptor
interceptor
=
new
MybatisPlusInterceptor
();
interceptor
.
addInnerInterceptor
(
new
PaginationInnerInterceptor
(
DbType
.
MYSQL
));
return
interceptor
;
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java
View file @
7d2fadcb
...
...
@@ -3,6 +3,7 @@ package com.zq.dataoperation.controller;
import
cn.hutool.core.date.DateUtil
;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.zq.common.utils.AssertUtils
;
import
com.zq.common.vo.PageVo
;
import
com.zq.common.vo.ResultVo
;
import
com.zq.dataoperation.dao.MetaDataDao
;
import
com.zq.dataoperation.entity.DataCleanExpressCategory
;
...
...
@@ -12,6 +13,7 @@ import com.zq.dataoperation.service.DataCleanRecordService;
import
com.zq.dataoperation.service.DataCleanRuleService
;
import
com.zq.dataoperation.service.DataCleanService
;
import
com.zq.dataoperation.entity.MetaData
;
import
com.zq.dataoperation.vo.DataCleanRecordReq
;
import
io.swagger.annotations.Api
;
import
io.swagger.annotations.ApiOperation
;
import
lombok.RequiredArgsConstructor
;
...
...
@@ -102,9 +104,20 @@ public class DataCleanController {
@PostMapping
(
"/addCleanList"
)
public
ResultVo
addCleanList
(
@RequestBody
DataCleanRecord
dataCleanRecord
){
AssertUtils
.
notNull
(
dataCleanRecord
.
getQueryDbId
(),
"缺少连接库ID"
);
AssertUtils
.
notNull
(
dataCleanRecord
.
getDataCollectTaskLogId
(),
"缺少汇聚任务ID"
);
return
ResultVo
.
success
(
dataCleanRecordService
.
runAndSave
(
dataCleanRecord
));
}
@ApiOperation
(
"清洗记录列表"
)
@PostMapping
(
"/getCleanList"
)
public
ResultVo
<
PageVo
<
DataCleanRecord
>>
getCleanList
(
@RequestBody
DataCleanRecordReq
req
){
return
ResultVo
.
success
(
dataCleanRecordService
.
getCleanList
(
req
));
}
@ApiOperation
(
"脏数据"
)
@PostMapping
(
"/getDirtyDetails"
)
public
ResultVo
getDirtyDetails
(
@RequestBody
DataCleanRecordReq
req
){
return
ResultVo
.
success
(
dataCleanRecordService
.
getDirtyDetails
(
req
));
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/dao/DataCleanRecordDao.java
View file @
7d2fadcb
...
...
@@ -2,7 +2,9 @@ package com.zq.dataoperation.dao;
import
com.baomidou.mybatisplus.core.mapper.BaseMapper
;
import
com.zq.dataoperation.entity.DataCleanRecord
;
import
org.springframework.stereotype.Repository
;
@Repository
public
interface
DataCleanRecordDao
extends
BaseMapper
<
DataCleanRecord
>
{
}
dataoperation-server/src/main/java/com/zq/dataoperation/dao/DataCollectTaskLogMapper.java
deleted
100644 → 0
View file @
ec165856
package
com
.
zq
.
dataoperation
.
dao
;
import
com.baomidou.mybatisplus.core.mapper.BaseMapper
;
import
com.zq.dataoperation.entity.DataCollectTaskLog
;
public
interface
DataCollectTaskLogMapper
extends
BaseMapper
<
DataCollectTaskLog
>
{
}
\ No newline at end of file
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRecord.java
View file @
7d2fadcb
package
com
.
zq
.
dataoperation
.
entity
;
import
com.baomidou.mybatisplus.annotation.IdType
;
import
com.baomidou.mybatisplus.annotation.TableId
;
import
com.baomidou.mybatisplus.annotation.TableName
;
import
com.zq.common.vo.PageReqVo
;
import
io.swagger.annotations.ApiModelProperty
;
import
lombok.AllArgsConstructor
;
import
lombok.Builder
;
...
...
@@ -22,9 +25,10 @@ import java.util.Date;
@TableName
(
value
=
"data_clean_record"
)
public
class
DataCleanRecord
{
/**
/**
* id
*/
@TableId
(
type
=
IdType
.
AUTO
)
private
Integer
id
;
/**
...
...
@@ -48,14 +52,14 @@ public class DataCleanRecord {
/**
* 清洗开始时间
*/
@ApiModelProperty
(
"
清洗开始
时间"
)
private
Date
star
tTime
;
@ApiModelProperty
(
"
汇聚完成
时间"
)
private
Date
collec
tTime
;
/**
* 结束时间
*/
@ApiModelProperty
(
"
结束
时间"
)
private
Date
e
ndTime
;
@ApiModelProperty
(
"
清洗完成
时间"
)
private
String
cleanE
ndTime
;
/**
* 花费时间
...
...
@@ -70,22 +74,22 @@ public class DataCleanRecord {
private
Integer
status
;
/**
*
花费时间
*
汇聚数据量
*/
@ApiModelProperty
(
"
花费时间
"
)
private
String
dataCollectCount
;
@ApiModelProperty
(
"
汇聚数据量
"
)
private
Long
dataCollectCount
;
/**
*
花费时间
*
清洗数据量
*/
@ApiModelProperty
(
"
花费时间
"
)
private
String
dataCleanCount
;
@ApiModelProperty
(
"
清洗数据量
"
)
private
Long
dataCleanCount
;
/**
*
花费时间
*
脏数据量
*/
@ApiModelProperty
(
"
花费时间
"
)
private
String
dataDirtyCount
;
@ApiModelProperty
(
"
脏数据量
"
)
private
Long
dataDirtyCount
;
/**
* 创建时间
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRule.java
View file @
7d2fadcb
...
...
@@ -2,6 +2,8 @@ package com.zq.dataoperation.entity;
import
java.util.Date
;
import
com.baomidou.mybatisplus.annotation.IdType
;
import
com.baomidou.mybatisplus.annotation.TableId
;
import
com.baomidou.mybatisplus.annotation.TableName
;
import
io.swagger.annotations.ApiModelProperty
;
import
lombok.AllArgsConstructor
;
...
...
@@ -27,6 +29,7 @@ public class DataCleanRule{
/**
* id
*/
@TableId
(
type
=
IdType
.
AUTO
)
private
Integer
id
;
/**
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCollectTaskLog.java
deleted
100644 → 0
View file @
ec165856
package
com
.
zq
.
dataoperation
.
entity
;
import
com.baomidou.mybatisplus.annotation.*
;
import
lombok.AllArgsConstructor
;
import
lombok.Builder
;
import
lombok.Data
;
import
lombok.NoArgsConstructor
;
import
java.util.Date
;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@TableName
(
value
=
"data_collect_task_log"
)
public
class
DataCollectTaskLog
{
/**
* 数据汇聚表id
*/
private
Integer
dataCollectSettingId
;
/**
* 开始时间
*/
private
Date
startTime
;
/**
* 结束时间
*/
private
Date
endTime
;
/**
* 花费多少秒
*/
private
Integer
duration
;
/**
* 状态
* 1运行中 2正常结束 3异常终止
*/
private
Integer
status
;
/**
* 错误信息
* 异常终止时记录异常信息
*/
private
String
error
;
/**
* 使用序列
* id
*/
@TableId
(
type
=
IdType
.
AUTO
)
private
Integer
id
;
/**
* createTime
*/
@TableField
(
fill
=
FieldFill
.
INSERT
)
private
Date
createTime
;
/**
* updateTime
*/
@TableField
(
fill
=
FieldFill
.
INSERT_UPDATE
)
private
Date
updateTime
;
}
dataoperation-server/src/main/java/com/zq/dataoperation/runner/DatabaseConnectRunner.java
View file @
7d2fadcb
...
...
@@ -5,7 +5,6 @@ import com.alibaba.druid.pool.DruidPooledConnection;
import
com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder
;
import
com.zq.common.exception.BusinessException
;
import
com.zq.dataoperation.entity.QueryDb
;
import
com.zq.dataoperation.utils.ConnectionUtil
;
import
com.zq.dataoperation.service.CommonQueryService
;
import
lombok.extern.slf4j.Slf4j
;
import
org.springframework.boot.ApplicationArguments
;
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
View file @
7d2fadcb
...
...
@@ -357,8 +357,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
});
List
<
Map
<
String
,
Object
>>
newCount
=
fromDbJdbcTemplate
.
queryForList
(
"SELECT COUNT(1) FROM "
+
newTableName
);
List
<
Map
<
String
,
Object
>>
oldCount
=
fromDbJdbcTemplate
.
queryForList
(
"SELECT COUNT(1) FROM "
+
oldTableName
);
List
<
Map
<
String
,
Object
>>
newCount
=
fromDbJdbcTemplate
.
queryForList
(
"SELECT COUNT(1)
AS newCount
FROM "
+
newTableName
);
List
<
Map
<
String
,
Object
>>
oldCount
=
fromDbJdbcTemplate
.
queryForList
(
"SELECT COUNT(1)
AS oldCount
FROM "
+
oldTableName
);
List
<
Map
<
String
,
Object
>>
newTable
=
fromDbJdbcTemplate
.
queryForList
(
"SELECT * FROM "
+
newTableName
);
long
times
=
System
.
currentTimeMillis
()
-
startTime
;
Map
<
String
,
Object
>
map
=
new
HashMap
<>();
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/service/DataCleanRecordService.java
View file @
7d2fadcb
package
com
.
zq
.
dataoperation
.
service
;
import
cn.hutool.core.date.DateUtil
;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.baomidou.mybatisplus.extension.service.impl.ServiceImpl
;
import
com.zq.common.utils.AssertUtils
;
import
com.zq.common.utils.PagingUtils
;
import
com.zq.common.vo.PageVo
;
import
com.zq.common.vo.ResultVo
;
import
com.zq.datacollect.entity.DataCollectSetting
;
import
com.zq.datacollect.entity.DataCollectTaskLog
;
import
com.zq.datacollect.mapper.DataCollectMapper
;
import
com.zq.datacollect.mapper.DataCollectTaskLogMapper
;
import
com.zq.dataoperation.dao.DataCleanRecordDao
;
import
com.zq.dataoperation.dao.DataCollectTaskLogMapper
;
import
com.zq.dataoperation.dao.QueryDbDao
;
import
com.zq.dataoperation.entity.DataCleanRecord
;
import
com.zq.dataoperation.entity.DataCollectTaskLog
;
import
com.zq.dataoperation.entity.QueryDb
;
import
lombok.RequiredArgsConstructor
;
import
lombok.extern.slf4j.Slf4j
;
import
com.zq.dataoperation.vo.DataCleanRecordReq
;
import
org.apache.commons.lang3.StringUtils
;
import
org.springframework.stereotype.Service
;
import
javax.annotation.Resource
;
import
java.math.BigDecimal
;
import
java.util.Date
;
import
java.util.List
;
@Slf4j
@RequiredArgsConstructor
@Service
public
class
DataCleanRecordService
extends
ServiceImpl
<
DataCleanRecordDao
,
DataCleanRecord
>
{
...
...
@@ -27,21 +33,44 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
private
QueryDbDao
queryDbDao
;
@Resource
private
DataCollectTaskLogMapper
dataCollectTaskLogMapper
;
@Resource
private
DataCollectMapper
dataCollectMapper
;
public
Object
runAndSave
(
DataCleanRecord
dataCleanRecord
)
{
QueryDb
queryDb
=
queryDbDao
.
selectById
(
dataCleanRecord
.
getQueryDbId
());
AssertUtils
.
notNull
(
queryDb
,
"无连接表ID"
);
DataCollectTaskLog
dataCollectTaskLog
=
dataCollectTaskLogMapper
.
selectById
(
dataCleanRecord
.
getDataCollectTaskLogId
());
AssertUtils
.
notNull
(
dataCollectTaskLog
,
"无连接表ID"
);
LambdaQueryWrapper
<
DataCollectSetting
>
wrapper
=
new
LambdaQueryWrapper
<>();
wrapper
.
eq
(
DataCollectSetting:
:
getToDbId
,
queryDb
.
getId
()).
eq
(
DataCollectSetting:
:
getToDbId
,
1
);
//数据库类型
List
<
DataCollectSetting
>
dataCollectSettings
=
dataCollectMapper
.
selectList
(
wrapper
);
Integer
collectSettingId
=
null
;
for
(
DataCollectSetting
collectSetting
:
dataCollectSettings
)
{
collectSettingId
=
collectSetting
.
getId
();
}
LambdaQueryWrapper
<
DataCollectTaskLog
>
queryWrapper
=
new
LambdaQueryWrapper
<>();
queryWrapper
.
eq
(
DataCollectTaskLog:
:
getDataCollectSettingId
,
collectSettingId
);
List
<
DataCollectTaskLog
>
taskLogs
=
dataCollectTaskLogMapper
.
selectList
(
queryWrapper
);
Date
endTime
=
null
;
Integer
status
=
null
;
for
(
DataCollectTaskLog
taskLog:
taskLogs
)
{
endTime
=
taskLog
.
getEndTime
();
status
=
taskLog
.
getStatus
();
}
Long
collectCount
=
dataCleanRecord
.
getDataCollectCount
();
Long
cleanCount
=
dataCleanRecord
.
getDataCleanCount
();
// BigDecimal dirtyCount = BigDecimal.valueOf(collectCount).subtract(BigDecimal.valueOf(cleanCount));//脏数据
DataCleanRecord
build
=
DataCleanRecord
.
builder
()
.
queryDbId
(
dataCleanRecord
.
getQueryDbId
())
.
dataCollectTaskLogId
(
dataCollectTaskLog
.
getDataCollectSettingId
()
)
.
dataCollectTaskLogId
(
collectSettingId
)
.
tableName
(
dataCleanRecord
.
getTableName
())
.
endTime
(
DateUtil
.
date
())
.
status
(
dataCollectTaskLog
.
getStatus
())
.
dataCollectCount
(
dataCleanRecord
.
getDataCollectCount
())
.
dataCleanCount
(
dataCleanRecord
.
getDataCleanCount
())
.
cleanEndTime
(
dataCleanRecord
.
getCleanEndTime
())
.
collectTime
(
endTime
)
.
status
(
status
)
.
dataCollectCount
(
collectCount
)
.
dataCleanCount
(
cleanCount
)
.
dataDirtyCount
(
dataCleanRecord
.
getDataDirtyCount
())
.
createTime
(
DateUtil
.
date
())
.
build
();
...
...
@@ -52,6 +81,18 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
}
else
{
i
=
dataCleanRecordDao
.
updateById
(
build
);
}
return
i
>
0
?
ResultVo
.
success
(
"操作成功!"
)
:
ResultVo
.
fail
(
"操作失败"
);
return
i
>
0
?
ResultVo
.
success
(
"操作成功!"
)
:
ResultVo
.
fail
(
"操作失败"
);
}
public
PageVo
<
DataCleanRecord
>
getCleanList
(
DataCleanRecordReq
req
)
{
LambdaQueryWrapper
<
DataCleanRecord
>
wrapper
=
new
LambdaQueryWrapper
<>();
if
(
StringUtils
.
isNotBlank
(
req
.
getTableName
()))
{
wrapper
.
like
(
DataCleanRecord:
:
getTableName
,
req
.
getTableName
());
}
return
PagingUtils
.
paging
(
req
,
dataCleanRecordDao
,
wrapper
.
orderByDesc
(),
DataCleanRecord
.
class
);
}
public
Object
getDirtyDetails
(
DataCleanRecordReq
req
)
{
return
null
;
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/vo/DataCleanRecordReq.java
0 → 100644
View file @
7d2fadcb
package
com
.
zq
.
dataoperation
.
vo
;
import
com.zq.common.vo.PageReqVo
;
import
io.swagger.annotations.ApiModelProperty
;
import
lombok.Data
;
import
java.util.Date
;
/**
* (data_clean_record)实体类
*
* @author zq
* @since 2023-07-04 16:12:53
*/
@Data
public
class
DataCleanRecordReq
extends
PageReqVo
{
/**
* id
*/
private
Integer
id
;
/**
* 连接id
*/
@ApiModelProperty
(
"连接id"
)
private
Long
queryDbId
;
/**
* 表名
*/
@ApiModelProperty
(
"表名"
)
private
String
tableName
;
/**
* 任务汇聚表ID
*/
@ApiModelProperty
(
"任务汇聚表ID"
)
private
Integer
dataCollectTaskLogId
;
/**
* 清洗开始时间
*/
@ApiModelProperty
(
"汇聚完成时间"
)
private
Date
collectTime
;
/**
* 结束时间
*/
@ApiModelProperty
(
"清洗耗时时间"
)
private
String
cleanEndTime
;
/**
* 花费时间
*/
@ApiModelProperty
(
"花费时间"
)
private
String
duration
;
/**
* 1运行中 2正常结束 3异常终止
*/
@ApiModelProperty
(
"状态"
)
private
Integer
status
;
/**
* 汇聚数据量
*/
@ApiModelProperty
(
"汇聚数据量"
)
private
Long
dataCollectCount
;
/**
* 清洗数据量
*/
@ApiModelProperty
(
"清洗数据量"
)
private
Long
dataCleanCount
;
/**
* 脏数据量
*/
@ApiModelProperty
(
"脏数据量"
)
private
Long
dataDirtyCount
;
/**
* 创建时间
*/
@ApiModelProperty
(
"创建时间"
)
private
Date
createTime
;
/**
* 更新时间
*/
@ApiModelProperty
(
"更新时间"
)
private
Date
updateTime
;
}
spider-flow-web/src/main/java/com/zq/spiderflow/entity/MetaDataMapping.java
View file @
7d2fadcb
...
...
@@ -12,8 +12,6 @@ import java.util.Date;
@Builder
@AllArgsConstructor
@NoArgsConstructor
//@KeySequence(value = "META_DATA_MAPPING_SEQ")
//@TableName(value = "META_DATA_MAPPING")
@TableName
(
value
=
"meta_data_mapping"
)
public
class
MetaDataMapping
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment