Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
N
nnjcy-data-model
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
梁家彪
nnjcy-data-model
Commits
4d210969
Commit
4d210969
authored
Nov 27, 2023
by
wqc
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
增加保存连接库id到xml
parent
cb61c745
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
55 additions
and
21 deletions
+55
-21
dataoperation-server/src/main/java/com/zq/dataoperation/controller/CommonQueryController.java
+1
-1
dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java
+5
-4
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRecord.java
+6
-0
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
+16
-8
dataoperation-server/src/main/java/com/zq/dataoperation/service/DataCleanRecordService.java
+12
-2
dataoperation-server/src/main/java/com/zq/dataoperation/vo/DataCleanRecordReq.java
+6
-0
spider-flow-core/src/main/java/com/zq/spiderflow/core/mapper/SpiderFlowMapper.java
+4
-4
spider-flow-core/src/main/java/com/zq/spiderflow/core/model/SpiderFlow.java
+3
-0
spider-flow-core/src/main/java/com/zq/spiderflow/core/service/SpiderFlowService.java
+2
-2
No files found.
dataoperation-server/src/main/java/com/zq/dataoperation/controller/CommonQueryController.java
View file @
4d210969
...
...
@@ -104,7 +104,7 @@ public class CommonQueryController {
return
ResultVo
.
success
(
commonQueryService
.
runSelect
(
body
));
}
@ApiOperation
(
"
自定义
sql查询"
)
@ApiOperation
(
"
关联表
sql查询"
)
@PostMapping
(
"/customSelect"
)
public
ResultVo
customSelect
(
@RequestBody
Map
<
String
,
Object
>
body
)
throws
Exception
{
return
ResultVo
.
success
(
commonQueryService
.
customSelect
(
body
));
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java
View file @
4d210969
...
...
@@ -9,6 +9,7 @@ import com.zq.dataoperation.dao.MetaDataDao;
import
com.zq.dataoperation.entity.DataCleanExpressCategory
;
import
com.zq.dataoperation.entity.DataCleanRecord
;
import
com.zq.dataoperation.entity.DataCleanRule
;
import
com.zq.dataoperation.service.CommonQueryService
;
import
com.zq.dataoperation.service.DataCleanRecordService
;
import
com.zq.dataoperation.service.DataCleanRuleService
;
import
com.zq.dataoperation.service.DataCleanService
;
...
...
@@ -100,7 +101,7 @@ public class DataCleanController {
return
null
;
}
@ApiOperation
(
"
新增清洗记录
"
)
@ApiOperation
(
"
生成报告
"
)
@PostMapping
(
"/addCleanList"
)
public
ResultVo
addCleanList
(
@RequestBody
DataCleanRecord
dataCleanRecord
){
AssertUtils
.
notNull
(
dataCleanRecord
.
getQueryDbId
(),
"缺少连接库ID"
);
...
...
@@ -114,10 +115,10 @@ public class DataCleanController {
return
ResultVo
.
success
(
dataCleanRecordService
.
getCleanList
(
req
));
}
@ApiOperation
(
"脏数据"
)
@ApiOperation
(
"
查看干净/
脏数据"
)
@PostMapping
(
"/getDirtyDetails"
)
public
ResultVo
getDirtyDetails
(
@RequestBody
DataCleanRecordReq
req
){
return
ResultVo
.
success
(
dataCleanRecordService
.
getDirtyDetails
(
req
));
public
ResultVo
getDirtyDetails
(
@RequestBody
Map
<
String
,
Object
>
body
){
return
ResultVo
.
success
(
dataCleanRecordService
.
getDirtyDetails
(
body
));
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/entity/DataCleanRecord.java
View file @
4d210969
...
...
@@ -92,6 +92,12 @@ public class DataCleanRecord {
private
Long
dataDirtyCount
;
/**
* 源数据表名
*/
@ApiModelProperty
(
"源数据表名"
)
private
String
originalTable
;
/**
* 创建时间
*/
@ApiModelProperty
(
"创建时间"
)
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
View file @
4d210969
...
...
@@ -21,6 +21,7 @@ import com.zq.dataoperation.holder.DatabaseHolder;
import
com.zq.dataoperation.utils.IdentityUtils
;
import
com.zq.dataoperation.utils.SqlUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.checkerframework.checker.units.qual.A
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.jdbc.core.BatchPreparedStatementSetter
;
...
...
@@ -66,7 +67,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
@Transactional
public
void
updateDatasource
(
QueryDb
queryDb
){
public
void
updateDatasource
(
QueryDb
queryDb
)
{
queryDb
.
setUpdateTime
(
DateUtil
.
date
());
AssertUtils
.
isTrue
(
queryDbService
.
updateById
(
queryDb
),
"操作失败!"
);
DatabaseHolder
.
remove
(
queryDb
.
getId
());
...
...
@@ -74,7 +75,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
@Transactional
public
void
deleteDatasource
(
Integer
id
){
public
void
deleteDatasource
(
Integer
id
)
{
AssertUtils
.
isTrue
(
queryDbService
.
removeById
(
id
),
"操作失败!"
);
DatabaseHolder
.
remove
(
id
);
}
...
...
@@ -129,10 +130,10 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
String
pkMetaTwo
=
jsonObject
.
get
(
"pkMetaTwo"
).
toString
();
String
querySql
=
jsonObject
.
get
(
"querySql"
).
toString
();
if
(
StringUtils
.
isBlank
(
tableTwo
)||
StringUtils
.
isBlank
(
pkMetaTwo
)){
String
sql
=
"SELECT * FROM "
+
tableOne
+
" "
+
querySql
;
String
sql
=
"SELECT * FROM "
+
tableOne
+
"
WHERE
"
+
querySql
;
return
commonQuery
(
sql
,
queryDbId
);
}
else
{
String
leftSql
=
"SELECT * FROM "
+
tableOne
+
" t LEFT JOIN "
+
tableTwo
+
" h ON t."
+
pkMetaOne
+
" = h."
+
pkMetaTwo
+
" "
+
querySql
;
String
leftSql
=
"SELECT * FROM "
+
tableOne
+
" t LEFT JOIN "
+
tableTwo
+
" h ON t."
+
pkMetaOne
+
" = h."
+
pkMetaTwo
+
"
WHERE
"
+
querySql
;
return
commonQuery
(
leftSql
,
queryDbId
);
}
...
...
@@ -172,7 +173,14 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
public
List
<
Map
<
String
,
Object
>>
commonQuery
(
String
sql
,
Integer
queryId
)
{
JdbcTemplate
jdbcTemplate
=
DatabaseHolder
.
getJdbcTemplate
(
queryId
);
return
jdbcTemplate
.
queryForList
(
sql
);
List
<
Map
<
String
,
Object
>>
mapList
=
null
;
try
{
mapList
=
jdbcTemplate
.
queryForList
(
sql
);
}
catch
(
Exception
E
){
log
.
warn
(
"抱歉!不支持此查询"
);
}
AssertUtils
.
notNull
(
mapList
,
"抱歉!无查询结果"
);
return
mapList
;
}
public
Object
getTables
(
Integer
id
)
throws
Exception
{
...
...
@@ -255,11 +263,11 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
List
<
Map
<
String
,
Object
>>
dataList
=
null
;
List
<
Map
<
String
,
Object
>>
dirtyList
=
null
;
if
(
matches
)
{
String
dataSql
=
"SELECT * FROM "
+
tableName
+
" WHERE "
+
metaData
+
" REGEXP '"
+
idCardRegex
+
"'"
+
" OR LENGTH("
+
metaData
+
")=18"
;
String
dataSql
=
"SELECT * FROM "
+
tableName
+
" WHERE "
+
metaData
+
" REGEXP '"
+
idCardRegex
+
"'"
+
" OR LENGTH("
+
metaData
+
")=18"
;
dataList
=
DatabaseHolder
.
getJdbcTemplate
(
datasourceId
).
queryForList
(
dataSql
);
map
.
put
(
"dataList"
,
dataList
);
}
else
{
String
dirtysql
=
sql
+
" WHERE "
+
metaData
+
" NOT REGEXP '"
+
idCardRegex
+
"'"
+
" AND LENGTH("
+
metaData
+
")<18"
;
String
dirtysql
=
sql
+
" WHERE "
+
metaData
+
" NOT REGEXP '"
+
idCardRegex
+
"'"
+
" AND LENGTH("
+
metaData
+
")<18"
;
dirtyList
=
DatabaseHolder
.
getJdbcTemplate
(
datasourceId
).
queryForList
(
dirtysql
);
map
.
put
(
"dirtyList"
,
dirtyList
);
}
...
...
@@ -419,7 +427,7 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
int
index
=
1
;
for
(
String
key
:
finalSet
)
{
try
{
String
keyStr
=
String
.
valueOf
(
map
.
get
(
key
));
String
keyStr
=
String
.
valueOf
(
map
.
get
(
key
));
preparedStatement
.
setObject
(
index
++,
keyStr
);
}
catch
(
NullPointerException
e
)
{
e
.
printStackTrace
();
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/service/DataCleanRecordService.java
View file @
4d210969
package
com
.
zq
.
dataoperation
.
service
;
import
cn.hutool.core.date.DateUtil
;
import
cn.hutool.json.JSONObject
;
import
cn.hutool.json.JSONUtil
;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.baomidou.mybatisplus.extension.service.impl.ServiceImpl
;
import
com.zq.common.utils.AssertUtils
;
...
...
@@ -22,6 +24,7 @@ import org.springframework.stereotype.Service;
import
javax.annotation.Resource
;
import
java.util.Date
;
import
java.util.List
;
import
java.util.Map
;
@Service
public
class
DataCleanRecordService
extends
ServiceImpl
<
DataCleanRecordDao
,
DataCleanRecord
>
{
...
...
@@ -34,6 +37,8 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
private
DataCollectTaskLogMapper
dataCollectTaskLogMapper
;
@Resource
private
DataCollectMapper
dataCollectMapper
;
@Resource
private
CommonQueryService
commonQueryService
;
public
Object
runAndSave
(
DataCleanRecord
dataCleanRecord
)
{
...
...
@@ -62,6 +67,7 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
// BigDecimal dirtyCount = BigDecimal.valueOf(collectCount).subtract(BigDecimal.valueOf(cleanCount));//脏数据
DataCleanRecord
build
=
DataCleanRecord
.
builder
()
.
originalTable
(
dataCleanRecord
.
getOriginalTable
())
.
queryDbId
(
dataCleanRecord
.
getQueryDbId
())
.
dataCollectTaskLogId
(
collectSettingId
)
.
tableName
(
dataCleanRecord
.
getTableName
())
...
...
@@ -91,7 +97,11 @@ public class DataCleanRecordService extends ServiceImpl<DataCleanRecordDao, Data
return
PagingUtils
.
paging
(
req
,
dataCleanRecordDao
,
wrapper
.
orderByDesc
(),
DataCleanRecord
.
class
);
}
public
Object
getDirtyDetails
(
DataCleanRecordReq
req
)
{
return
null
;
public
Object
getDirtyDetails
(
Map
<
String
,
Object
>
body
)
{
JSONObject
jsonObject
=
JSONUtil
.
parseObj
(
body
);
Integer
queryDbId
=
Integer
.
valueOf
(
jsonObject
.
get
(
"queryDbId"
).
toString
());
String
tableName
=
jsonObject
.
get
(
"tableName"
).
toString
();
String
sql
=
"SELECT * FROM "
+
tableName
;
return
commonQueryService
.
commonQuery
(
sql
,
queryDbId
);
}
}
dataoperation-server/src/main/java/com/zq/dataoperation/vo/DataCleanRecordReq.java
View file @
4d210969
...
...
@@ -81,6 +81,12 @@ public class DataCleanRecordReq extends PageReqVo {
private
Long
dataDirtyCount
;
/**
* 源数据表名
*/
@ApiModelProperty
(
"源数据表名"
)
private
String
originalTable
;
/**
* 创建时间
*/
@ApiModelProperty
(
"创建时间"
)
...
...
spider-flow-core/src/main/java/com/zq/spiderflow/core/mapper/SpiderFlowMapper.java
View file @
4d210969
...
...
@@ -34,11 +34,11 @@ public interface SpiderFlowMapper extends BaseMapper<SpiderFlow>{
})
IPage
<
SpiderFlow
>
selectSpiderPage
(
Page
<
SpiderFlow
>
page
,
@Param
(
"name"
)
String
name
);
@Insert
(
"insert into sp_flow(id,name,xml,
flow_type,enabled) values(#{id},#{name},#{xml
},#{flowType},'0')"
)
int
insertSpiderFlow
(
@Param
(
"id"
)
String
id
,
@Param
(
"name"
)
String
name
,
@Param
(
"xml"
)
String
xml
,
@Param
(
"flowType"
)
String
flowType
);
@Insert
(
"insert into sp_flow(id,name,xml,
query_db_id,flow_type,enabled) values(#{id},#{name},#{xml},#{queryDbId
},#{flowType},'0')"
)
int
insertSpiderFlow
(
@Param
(
"id"
)
String
id
,
@Param
(
"name"
)
String
name
,
@Param
(
"xml"
)
String
xml
,
@Param
(
"queryDbId"
)
String
queryDbId
,
@Param
(
"flowType"
)
String
flowType
);
@Update
(
"update sp_flow set name = #{name},xml = #{xml} where id = #{id}"
)
int
updateSpiderFlow
(
@Param
(
"id"
)
String
id
,
@Param
(
"name"
)
String
name
,
@Param
(
"xml"
)
String
xml
);
@Update
(
"update sp_flow set name = #{name},xml = #{xml}
,query_db_id=#{queryDbId}
where id = #{id}"
)
int
updateSpiderFlow
(
@Param
(
"id"
)
String
id
,
@Param
(
"name"
)
String
name
,
@Param
(
"xml"
)
String
xml
,
@Param
(
"queryDbId"
)
String
queryDbId
);
@Update
(
"update sp_flow set execute_count = 0 where id = #{id}"
)
int
resetExecuteCount
(
@Param
(
"id"
)
String
id
);
...
...
spider-flow-core/src/main/java/com/zq/spiderflow/core/model/SpiderFlow.java
View file @
4d210969
...
...
@@ -58,5 +58,8 @@ public class SpiderFlow {
@TableField
(
exist
=
false
)
private
Integer
running
;
//数据库连接ID
private
String
queryDbId
;
}
spider-flow-core/src/main/java/com/zq/spiderflow/core/service/SpiderFlowService.java
View file @
4d210969
...
...
@@ -120,7 +120,7 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
spiderFlow
.
setNextExecuteTime
(
trigger
.
getStartTime
());
}
if
(
StringUtils
.
isNotEmpty
(
spiderFlow
.
getId
())){
//update 任务
sfMapper
.
updateSpiderFlow
(
spiderFlow
.
getId
(),
spiderFlow
.
getName
(),
spiderFlow
.
getXml
());
sfMapper
.
updateSpiderFlow
(
spiderFlow
.
getId
(),
spiderFlow
.
getName
(),
spiderFlow
.
getXml
()
,
spiderFlow
.
getQueryDbId
()
);
spiderJobManager
.
remove
(
spiderFlow
.
getId
());
spiderFlow
=
getById
(
spiderFlow
.
getId
());
if
(
"1"
.
equals
(
spiderFlow
.
getEnabled
())
&&
StringUtils
.
isNotEmpty
(
spiderFlow
.
getCron
())){
...
...
@@ -128,7 +128,7 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
}
}
else
{
//insert 任务
String
id
=
UUID
.
randomUUID
().
toString
().
replace
(
"-"
,
""
);
sfMapper
.
insertSpiderFlow
(
id
,
spiderFlow
.
getName
(),
spiderFlow
.
getXml
(),
spiderFlow
.
getFlowType
());
sfMapper
.
insertSpiderFlow
(
id
,
spiderFlow
.
getName
(),
spiderFlow
.
getXml
(),
spiderFlow
.
get
QueryDbId
(),
spiderFlow
.
get
FlowType
());
spiderFlow
.
setId
(
id
);
}
File
file
=
new
File
(
workspace
,
spiderFlow
.
getId
()
+
File
.
separator
+
"xmls"
+
File
.
separator
+
System
.
currentTimeMillis
()
+
".xml"
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment