Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
N
nnjcy-data-model
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
梁家彪
nnjcy-data-model
Commits
f5c2f250
Commit
f5c2f250
authored
Sep 18, 2023
by
wqc
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
调整代码
parent
c1c9ff0f
Hide whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
559 additions
and
11 deletions
+559
-11
dataoperation-server/pom.xml
+0
-6
dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java
+2
-3
dataoperation-server/src/main/java/com/zq/dataoperation/dao/MetaDataDao.java
+14
-0
dataoperation-server/src/main/java/com/zq/dataoperation/dao/MetaDataMappingDao.java
+22
-0
dataoperation-server/src/main/java/com/zq/dataoperation/entity/MetaData.java
+93
-0
dataoperation-server/src/main/java/com/zq/dataoperation/entity/MetaDataMapping.java
+84
-0
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
+2
-2
dataoperation-server/src/main/java/com/zq/dataoperation/vo/MetaDataMappingReq.java
+51
-0
dataoperation-server/src/main/java/com/zq/dataoperation/vo/MetaDataReq.java
+63
-0
dataoperation-server/src/main/java/com/zq/dataoperation/vo/MetaDataVo.java
+83
-0
spider-flow-core/src/main/java/com/zq/spiderflow/core/executor/shape/IdentityExecutor.java
+58
-0
spider-flow-core/src/main/java/com/zq/spiderflow/core/utils/IdentityUtils.java
+87
-0
No files found.
dataoperation-server/pom.xml
View file @
f5c2f250
...
...
@@ -113,12 +113,6 @@
</dependency>
<dependency>
<groupId>
com.zq
</groupId>
<artifactId>
spider-flow-web
</artifactId>
<version>
1.0.0
</version>
<scope>
compile
</scope>
</dependency>
<dependency>
<groupId>
com.zq
</groupId>
<artifactId>
sys-server
</artifactId>
<version>
1.0.0
</version>
<scope>
compile
</scope>
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java
View file @
f5c2f250
...
...
@@ -3,14 +3,13 @@ package com.zq.dataoperation.controller;
import
cn.hutool.core.date.DateUtil
;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.zq.common.utils.AssertUtils
;
import
com.zq.common.vo.PageVo
;
import
com.zq.common.vo.ResultVo
;
import
com.zq.dataoperation.dao.MetaDataDao
;
import
com.zq.dataoperation.entity.DataCleanExpressCategory
;
import
com.zq.dataoperation.entity.DataCleanRule
;
import
com.zq.dataoperation.service.DataCleanRuleService
;
import
com.zq.dataoperation.service.DataCleanService
;
import
com.zq.spiderflow.dao.MetaDataDao
;
import
com.zq.spiderflow.entity.MetaData
;
import
com.zq.dataoperation.entity.MetaData
;
import
io.swagger.annotations.Api
;
import
io.swagger.annotations.ApiOperation
;
import
lombok.RequiredArgsConstructor
;
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/dao/MetaDataDao.java
0 → 100644
View file @
f5c2f250
package
com
.
zq
.
dataoperation
.
dao
;
import
com.baomidou.mybatisplus.core.mapper.BaseMapper
;
import
com.zq.dataoperation.entity.MetaData
;
import
com.zq.dataoperation.vo.MetaDataReq
;
import
com.zq.dataoperation.vo.MetaDataVo
;
import
org.apache.ibatis.annotations.Param
;
import
java.util.List
;
public
interface
MetaDataDao
extends
BaseMapper
<
MetaData
>
{
List
<
MetaDataVo
>
getPage
(
@Param
(
"req"
)
MetaDataReq
req
);
}
dataoperation-server/src/main/java/com/zq/dataoperation/dao/MetaDataMappingDao.java
0 → 100644
View file @
f5c2f250
package
com
.
zq
.
dataoperation
.
dao
;
import
com.baomidou.mybatisplus.core.mapper.BaseMapper
;
import
com.zq.dataoperation.entity.MetaDataMapping
;
import
com.zq.dataoperation.vo.MetaDataReq
;
import
org.springframework.stereotype.Repository
;
import
java.util.List
;
/**
* @author Administrator
*/
@Repository
public
interface
MetaDataMappingDao
extends
BaseMapper
<
MetaDataMapping
>
{
List
<
MetaDataMapping
>
selectRepeat
(
String
tableSqlname
);
List
<
MetaDataMapping
>
getSqlName
();
List
<
MetaDataMapping
>
selectByCateId
(
Long
id
);
List
<
MetaDataReq
>
getMetaByTable
(
String
tableSqlName
);
}
dataoperation-server/src/main/java/com/zq/dataoperation/entity/MetaData.java
0 → 100644
View file @
f5c2f250
package
com
.
zq
.
dataoperation
.
entity
;
import
com.baomidou.mybatisplus.annotation.*
;
import
lombok.AllArgsConstructor
;
import
lombok.Builder
;
import
lombok.Data
;
import
lombok.NoArgsConstructor
;
import
java.util.Date
;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
//@KeySequence(value = "METADATA_SEQ")
//@TableName(value = "METADATA")
@TableName
(
value
=
"meta_data"
)
public
class
MetaData
{
/**
* 使用序列
* id
*/
@TableId
(
type
=
IdType
.
AUTO
)
private
Long
id
;
/**
* 数据元名称
*/
private
String
metaName
;
/**
* 数据元类型
*/
private
String
dataType
;
/**
* 数据元长度
*/
private
String
dataLength
;
/**
* 精度
*/
private
String
fieldPrecision
;
/**
* 元数据代码
*/
private
String
dataCode
;
/**
* 数据元英文名称
*/
private
String
englishName
;
/**
* 所处级别
*/
private
String
dataLevel
;
/**
* 是否非空
*/
private
String
notNull
;
/**
* 所属领域id
*/
private
Long
categoryId
;
/**
* 是否复合元数据0否1是
*/
private
Integer
mutiMetaFlag
;
/**
* 复合元数据表达式
*/
private
String
mutiMetaExpress
;
/**
* createTime
*/
@TableField
(
fill
=
FieldFill
.
INSERT
)
private
Date
createTime
;
/**
* updateTime
*/
@TableField
(
fill
=
FieldFill
.
INSERT_UPDATE
)
private
Date
updateTime
;
}
dataoperation-server/src/main/java/com/zq/dataoperation/entity/MetaDataMapping.java
0 → 100644
View file @
f5c2f250
package
com
.
zq
.
dataoperation
.
entity
;
import
com.baomidou.mybatisplus.annotation.*
;
import
lombok.AllArgsConstructor
;
import
lombok.Builder
;
import
lombok.Data
;
import
lombok.NoArgsConstructor
;
import
java.util.Date
;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
//@KeySequence(value = "META_DATA_MAPPING_SEQ")
//@TableName(value = "META_DATA_MAPPING")
@TableName
(
value
=
"meta_data_mapping"
)
public
class
MetaDataMapping
{
/**
* 使用序列
* id
*/
@TableId
(
type
=
IdType
.
AUTO
)
private
Long
id
;
/**
* 表名
*/
private
String
tableName
;
/**
* 数据库表名
*/
private
String
tableSqlname
;
/**
* 列名
*/
private
String
fieldName
;
/**
* 列类型
*/
private
String
fieldType
;
/**
* 列长度
*/
private
String
length
;
/**
* 列精度
*/
@TableField
(
"`precision`"
)
private
String
precision
;
/**
* 是否主键
*/
private
Integer
isPk
;
/**
* 数据元id
*/
private
Long
metadataId
;
/**
* 查询数据源id
*/
private
Long
queryDbId
;
/**
* createTime
*/
@TableField
(
fill
=
FieldFill
.
INSERT
)
private
Date
createTime
;
/**
* updateTime
*/
@TableField
(
fill
=
FieldFill
.
INSERT_UPDATE
)
private
Date
updateTime
;
}
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
View file @
f5c2f250
...
...
@@ -9,13 +9,13 @@ import com.zq.common.exception.BusinessException;
import
com.zq.common.vo.PageVo
;
import
com.zq.common.vo.ResultVo
;
import
com.zq.dataoperation.dao.CommonQuerySettingDao
;
import
com.zq.dataoperation.dao.MetaDataMappingDao
;
import
com.zq.dataoperation.dao.QueryDbDao
;
import
com.zq.dataoperation.entity.CommonQuerySetting
;
import
com.zq.dataoperation.entity.MetaDataMapping
;
import
com.zq.dataoperation.entity.QueryDb
;
import
com.zq.dataoperation.utils.ConnectionUtil
;
import
com.zq.dataoperation.utils.SqlUtils
;
import
com.zq.spiderflow.dao.MetaDataMappingDao
;
import
com.zq.spiderflow.entity.MetaDataMapping
;
import
org.springframework.stereotype.Service
;
import
org.springframework.web.bind.annotation.RequestBody
;
...
...
dataoperation-server/src/main/java/com/zq/dataoperation/vo/MetaDataMappingReq.java
0 → 100644
View file @
f5c2f250
package
com
.
zq
.
dataoperation
.
vo
;
import
com.zq.common.vo.PageReqVo
;
import
lombok.Data
;
@Data
public
class
MetaDataMappingReq
extends
PageReqVo
{
private
Long
id
;
/**
* 表名
*/
private
String
tableName
;
/**
* 数据库表名
*/
private
String
tableSqlname
;
/**
* 列名
*/
private
String
fieldName
;
/**
* 列类型
*/
private
String
fieldType
;
/**
* 列长度
*/
private
Integer
length
;
/**
* 列精度
*/
private
Integer
precision
;
/**
* 是否主键
*/
private
Integer
isPk
;
/**
* 数据元id
*/
private
Long
metadataId
;
}
dataoperation-server/src/main/java/com/zq/dataoperation/vo/MetaDataReq.java
0 → 100644
View file @
f5c2f250
package
com
.
zq
.
dataoperation
.
vo
;
import
com.baomidou.mybatisplus.annotation.IdType
;
import
com.baomidou.mybatisplus.annotation.TableId
;
import
com.zq.common.vo.PageReqVo
;
import
lombok.AllArgsConstructor
;
import
lombok.Builder
;
import
lombok.Data
;
import
lombok.NoArgsConstructor
;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public
class
MetaDataReq
extends
PageReqVo
{
/**
* 使用序列
* id
*/
@TableId
(
type
=
IdType
.
INPUT
)
private
Long
id
;
/**
* 数据元名称
*/
private
String
metaName
;
/**
* 数据元类型
*/
private
String
dataType
;
/**
* 精度
*/
private
String
fieldPrecision
;
/**
* 元数据代码
*/
private
String
dataCode
;
/**
* 数据元英文名称
*/
private
String
englishName
;
/**
* 所处级别
*/
private
String
dataLevel
;
/**
* 所属领域id
*/
private
Long
categoryId
;
private
Long
queryDbId
;
private
String
tableSqlName
;
}
dataoperation-server/src/main/java/com/zq/dataoperation/vo/MetaDataVo.java
0 → 100644
View file @
f5c2f250
package
com
.
zq
.
dataoperation
.
vo
;
import
com.baomidou.mybatisplus.annotation.FieldFill
;
import
com.baomidou.mybatisplus.annotation.TableField
;
import
lombok.Builder
;
import
lombok.Data
;
import
java.util.Date
;
@Data
@Builder
public
class
MetaDataVo
{
private
Long
id
;
/**
* 数据元名称
*/
private
String
metaName
;
/**
* 数据元类型
*/
private
String
dataType
;
/**
* 数据元长度
*/
private
String
dataLength
;
/**
* 精度
*/
private
String
fieldPrecision
;
/**
* 元数据代码
*/
private
String
dataCode
;
/**
* 数据元英文名称
*/
private
String
englishName
;
/**
* 所处级别
*/
private
String
dataLevel
;
/**
* 是否非空
*/
private
String
notNull
;
/**
* 所属领域id
*/
private
Long
categoryId
;
/**
* 是否复合元数据
*/
private
Integer
mutiMetaFlag
;
/**
* 复合元数据表达式
*/
private
String
mutiMetaExpress
;
/**
* createTime
*/
@TableField
(
fill
=
FieldFill
.
INSERT
)
private
Date
createTime
;
/**
* updateTime
*/
@TableField
(
fill
=
FieldFill
.
INSERT_UPDATE
)
private
Date
updateTime
;
private
String
categoryName
;
}
spider-flow-core/src/main/java/com/zq/spiderflow/core/executor/shape/IdentityExecutor.java
0 → 100644
View file @
f5c2f250
package
com
.
zq
.
spiderflow
.
core
.
executor
.
shape
;
import
com.zq.spiderflow.context.SpiderContext
;
import
com.zq.spiderflow.core.utils.IdentityUtils
;
import
com.zq.spiderflow.executor.ShapeExecutor
;
import
com.zq.spiderflow.model.Shape
;
import
com.zq.spiderflow.model.SpiderNode
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.exception.ExceptionUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.stereotype.Component
;
import
java.util.Map
;
@Component
public
class
IdentityExecutor
implements
ShapeExecutor
{
public
static
final
String
META_DATA
=
"metaData"
;
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
ExecuteSQLExecutor
.
class
);
@Override
public
Shape
shape
()
{
Shape
shape
=
new
Shape
();
//web界面上显示的图标
shape
.
setImage
(
""
);
//拖放至容器里时默认的节点名称
shape
.
setLabel
(
"身份证清洗"
);
//模板文件名
shape
.
setName
(
"IdentityExecutor"
);
//鼠标移动至图标上显示的名称
shape
.
setTitle
(
"身份证清洗"
);
return
shape
;
}
@Override
public
String
supportShape
()
{
return
"IdentityExecutor"
;
}
@Override
public
void
execute
(
SpiderNode
node
,
SpiderContext
context
,
Map
<
String
,
Object
>
variables
)
{
String
metaData
=
node
.
getStringJsonValue
(
META_DATA
);
if
(
StringUtils
.
isNotBlank
(
metaData
))
{
logger
.
warn
(
"字段不能为空!"
);
}
if
(
IdentityUtils
.
isLegalIdCard
(
metaData
)!=
true
){
logger
.
warn
(
"字段不符合身份证!"
);
}
else
if
(
IdentityUtils
.
is15BitLegalIdCard
(
metaData
)!=
true
){
logger
.
warn
(
"身份证不满15位!"
);
}
else
if
(
IdentityUtils
.
is18BitLegalIdCard
(
metaData
)!=
true
){
logger
.
warn
(
"身份证不满18位!"
);
}
else
{
// ExceptionUtils.wrapAndThrow(e);
}
}
}
spider-flow-core/src/main/java/com/zq/spiderflow/core/utils/IdentityUtils.java
0 → 100644
View file @
f5c2f250
package
com
.
zq
.
spiderflow
.
core
.
utils
;
import
java.util.stream.IntStream
;
/**
* 身份证号码验证
* 1、号码的结构
* 公民身份号码是特征组合码,由十七位数字本体码和一位校验码组成。从左至右依次为:六位数字地址码,
* 八位数字出生日期码,三位数字顺序码和一位数字校验码。
* 2、地址码(前六位数)
* 表示编码对象常住户口所在县(市、旗、区)的行政区划代码,按GB/T2260的规定执行。
* 3、出生日期码(第七位至十四位)
* 表示编码对象出生的年、月、日,按GB/T7408的规定执行,年、月、日代码之间不用分隔符。
* 4、顺序码(第十五位至十七位)
* 表示在同一地址码所标识的区域范围内,对同年、同月、同日出生的人编定的顺序号,
* 顺序码的奇数分配给男性,偶数分配给女性。
* 5、校验码(第十八位数)
* (1)十七位数字本体码加权求和公式 S = Sum(Ai Wi), i = 0, , 16 ,先对前17位数字的权求和 ;
* Ai:表示第i位置上的身份证号码数字值; Wi:表示第i位置上的加权因子 Wi: 7 9 10 5 8 4 2 1 6 3 7 9 10 5 8 4 2
* (2)计算模 Y = mod(S, 11)
* (3)通过模( 0 1 2 3 4 5 6 7 8 9 10)得到对应的校验码 Y:1 0 X 9 8 7 6 5 4 3 2
*
*/
public
class
IdentityUtils
{
/**
* 身份证校验码
*/
private
static
final
int
[]
COEFFICIENT_ARRAY
=
{
7
,
9
,
10
,
5
,
8
,
4
,
2
,
1
,
6
,
3
,
7
,
9
,
10
,
5
,
8
,
4
,
2
};
/**
* 身份证号的尾数规则
*/
private
static
final
String
[]
IDENTITY_MANTISSA
=
{
"1"
,
"0"
,
"X"
,
"9"
,
"8"
,
"7"
,
"6"
,
"5"
,
"4"
,
"3"
,
"2"
};
private
static
final
String
IDENTITY_PATTERN_15
=
"^[1-9]\\d{7}((0\\d)|(1[0-2]))(([0-2]\\d)|3[0-1])\\d{3}$"
;
// 修改前未做X位的校验:"^[1-9]\\d{5}[1-9]\\d{3}((0\\d)|(1[0-2]))(([0-2]\\d)|3[0-1])\\d{4}$";
// 修改后:"^[1-9]\\d{5}[1-9]\\d{3}((0\\d)|(1[0-2]))(([0-2]\\d)|3[0-1])\\d{3}[0-9Xx]$";
private
static
final
String
IDENTITY_PATTERN_18
=
"^[1-9]\\d{5}[1-9]\\d{3}((0\\d)|(1[0-2]))(([0-2]\\d)|3[0-1])\\d{3}[0-9Xx]$"
;
/**
* 身份证号校验
*/
public
static
boolean
isLegalIdCard
(
String
identity
)
{
if
(
identity
==
null
)
{
return
false
;
}
if
(
identity
.
length
()
==
15
)
{
return
is15BitLegalIdCard
(
identity
);
}
if
(
identity
.
length
()
==
18
)
{
return
is18BitLegalIdCard
(
identity
);
}
return
false
;
}
public
static
boolean
is15BitLegalIdCard
(
String
identity
)
{
return
identity
.
matches
(
IDENTITY_PATTERN_15
);
}
public
static
boolean
is18BitLegalIdCard
(
String
identity
)
{
if
(
identity
.
length
()
!=
18
)
{
return
false
;
}
if
(!
identity
.
matches
(
IDENTITY_PATTERN_18
))
{
return
false
;
}
// 将字符串对象中的字符转换为一个字符数组
char
[]
chars
=
identity
.
toCharArray
();
long
sum
=
IntStream
.
range
(
0
,
17
).
map
(
index
->
{
char
ch
=
chars
[
index
];
// 通俗理解:digit()是个边界值判断,不过边界返回字符数字本身数值,超过边界即返回 -1.
int
digit
=
Character
.
digit
(
ch
,
10
);
int
coefficient
=
COEFFICIENT_ARRAY
[
index
];
return
digit
*
coefficient
;
}).
summaryStatistics
().
getSum
();
// 计算出的尾数索引
int
mantissaIndex
=
(
int
)
(
sum
%
11
);
String
mantissa
=
IDENTITY_MANTISSA
[
mantissaIndex
];
String
lastChar
=
identity
.
substring
(
17
);
return
lastChar
.
equalsIgnoreCase
(
mantissa
);
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment