调整代码

dataoperation-server/pom.xml

@@ -113,12 +113,6 @@
         </dependency>
         <dependency>
             <groupId>com.zq</groupId>
-            <artifactId>spider-flow-web</artifactId>
-            <version>1.0.0</version>
-            <scope>compile</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.zq</groupId>
             <artifactId>sys-server</artifactId>
             <version>1.0.0</version>
             <scope>compile</scope>

dataoperation-server/src/main/java/com/zq/dataoperation/controller/DataCleanController.java

@@ -3,14 +3,13 @@ package com.zq.dataoperation.controller;
 import cn.hutool.core.date.DateUtil;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.zq.common.utils.AssertUtils;
-import com.zq.common.vo.PageVo;
 import com.zq.common.vo.ResultVo;
+import com.zq.dataoperation.dao.MetaDataDao;
 import com.zq.dataoperation.entity.DataCleanExpressCategory;
 import com.zq.dataoperation.entity.DataCleanRule;
 import com.zq.dataoperation.service.DataCleanRuleService;
 import com.zq.dataoperation.service.DataCleanService;
-import com.zq.spiderflow.dao.MetaDataDao;
-import com.zq.spiderflow.entity.MetaData;
+import com.zq.dataoperation.entity.MetaData;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import lombok.RequiredArgsConstructor;

dataoperation-server/src/main/java/com/zq/dataoperation/dao/MetaDataDao.java

+package com.zq.dataoperation.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.zq.dataoperation.entity.MetaData;
+import com.zq.dataoperation.vo.MetaDataReq;
+import com.zq.dataoperation.vo.MetaDataVo;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+public interface MetaDataDao extends BaseMapper<MetaData> {
+
+    List<MetaDataVo> getPage(@Param("req") MetaDataReq req);
+}

dataoperation-server/src/main/java/com/zq/dataoperation/dao/MetaDataMappingDao.java

+package com.zq.dataoperation.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.zq.dataoperation.entity.MetaDataMapping;
+import com.zq.dataoperation.vo.MetaDataReq;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+/**
+ * @author Administrator
+ */
+@Repository
+public interface MetaDataMappingDao extends BaseMapper<MetaDataMapping> {
+    List<MetaDataMapping> selectRepeat(String tableSqlname);
+
+    List<MetaDataMapping> getSqlName();
+
+    List<MetaDataMapping> selectByCateId(Long id);
+
+    List<MetaDataReq> getMetaByTable(String tableSqlName);
+}

dataoperation-server/src/main/java/com/zq/dataoperation/entity/MetaData.java

+package com.zq.dataoperation.entity;
+
+import com.baomidou.mybatisplus.annotation.*;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Date;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+//@KeySequence(value = "METADATA_SEQ")
+//@TableName(value = "METADATA")
+@TableName(value = "meta_data")
+public class MetaData {
+
+    /**
+     * 使用序列
+     * id
+     */
+    @TableId(type = IdType.AUTO)
+    private Long id;
+
+    /**
+     *  数据元名称
+     */
+    private String metaName;
+
+    /**
+     *  数据元类型
+     */
+    private String dataType;
+
+    /**
+     *  数据元长度
+     */
+    private String dataLength;
+
+    /**
+     *  精度
+     */
+    private String fieldPrecision;
+
+    /**
+     *  元数据代码
+     */
+    private String dataCode;
+
+    /**
+     *  数据元英文名称
+     */
+    private String englishName;
+
+    /**
+     *  所处级别
+     */
+    private String dataLevel;
+
+    /**
+     *  是否非空
+     */
+    private String notNull;
+
+    /**
+     *  所属领域id
+     */
+    private Long categoryId;
+
+    /**
+     *  是否复合元数据0否1是
+     */
+    private Integer mutiMetaFlag;
+
+    /**
+     *  复合元数据表达式
+     */
+    private String mutiMetaExpress;
+
+    /**
+     *  createTime
+     */
+    @TableField(fill = FieldFill.INSERT)
+    private Date createTime;
+
+    /**
+     *  updateTime
+     */
+    @TableField(fill = FieldFill.INSERT_UPDATE)
+    private Date updateTime;
+}

dataoperation-server/src/main/java/com/zq/dataoperation/entity/MetaDataMapping.java

+package com.zq.dataoperation.entity;
+
+import com.baomidou.mybatisplus.annotation.*;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Date;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+//@KeySequence(value = "META_DATA_MAPPING_SEQ")
+//@TableName(value = "META_DATA_MAPPING")
+@TableName(value = "meta_data_mapping")
+public class MetaDataMapping {
+
+    /**
+     * 使用序列
+     * id
+     */
+    @TableId(type = IdType.AUTO)
+    private Long id;
+
+    /**
+     *  表名
+     */
+    private String tableName;
+
+    /**
+     *  数据库表名
+     */
+    private String tableSqlname;
+
+    /**
+     *  列名
+     */
+    private String fieldName;
+
+    /**
+     *  列类型
+     */
+    private String fieldType;
+
+    /**
+     *  列长度
+     */
+    private String length;
+
+    /**
+     *  列精度
+     */
+    @TableField("`precision`")
+    private String precision;
+
+    /**
+     *  是否主键
+     */
+    private Integer isPk;
+
+    /**
+     *  数据元id
+     */
+    private Long metadataId;
+
+    /**
+     *  查询数据源id
+     */
+    private Long queryDbId;
+
+    /**
+     *  createTime
+     */
+    @TableField(fill = FieldFill.INSERT)
+    private Date createTime;
+
+    /**
+     *  updateTime
+     */
+    @TableField(fill = FieldFill.INSERT_UPDATE)
+    private Date updateTime;
+}

dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java

@@ -9,13 +9,13 @@ import com.zq.common.exception.BusinessException;
 import com.zq.common.vo.PageVo;
 import com.zq.common.vo.ResultVo;
 import com.zq.dataoperation.dao.CommonQuerySettingDao;
+import com.zq.dataoperation.dao.MetaDataMappingDao;
 import com.zq.dataoperation.dao.QueryDbDao;
 import com.zq.dataoperation.entity.CommonQuerySetting;
+import com.zq.dataoperation.entity.MetaDataMapping;
 import com.zq.dataoperation.entity.QueryDb;
 import com.zq.dataoperation.utils.ConnectionUtil;
 import com.zq.dataoperation.utils.SqlUtils;
-import com.zq.spiderflow.dao.MetaDataMappingDao;
-import com.zq.spiderflow.entity.MetaDataMapping;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;
 

dataoperation-server/src/main/java/com/zq/dataoperation/vo/MetaDataMappingReq.java

+package com.zq.dataoperation.vo;
+
+
+import com.zq.common.vo.PageReqVo;
+import lombok.Data;
+
+@Data
+public class MetaDataMappingReq extends PageReqVo {
+
+    private Long id;
+
+    /**
+     *  表名
+     */
+    private String tableName;
+
+    /**
+     *  数据库表名
+     */
+    private String tableSqlname;
+
+    /**
+     *  列名
+     */
+    private String fieldName;
+
+    /**
+     *  列类型
+     */
+    private String fieldType;
+
+    /**
+     *  列长度
+     */
+    private Integer length;
+
+    /**
+     *  列精度
+     */
+    private Integer precision;
+
+    /**
+     *  是否主键
+     */
+    private Integer isPk;
+
+    /**
+     *  数据元id
+     */
+    private Long metadataId;
+}

dataoperation-server/src/main/java/com/zq/dataoperation/vo/MetaDataReq.java

+package com.zq.dataoperation.vo;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.zq.common.vo.PageReqVo;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MetaDataReq extends PageReqVo {
+
+    /**
+     * 使用序列
+     * id
+     */
+    @TableId(type = IdType.INPUT)
+    private Long id;
+
+    /**
+     *  数据元名称
+     */
+    private String metaName;
+
+    /**
+     *  数据元类型
+     */
+    private String dataType;
+
+
+    /**
+     *  精度
+     */
+    private String fieldPrecision;
+
+    /**
+     *  元数据代码
+     */
+    private String dataCode;
+
+    /**
+     *  数据元英文名称
+     */
+    private String englishName;
+
+    /**
+     *  所处级别
+     */
+    private String dataLevel;
+
+    /**
+     *  所属领域id
+     */
+    private Long categoryId;
+
+    private Long queryDbId;
+
+    private String tableSqlName;
+}

dataoperation-server/src/main/java/com/zq/dataoperation/vo/MetaDataVo.java

+package com.zq.dataoperation.vo;
+
+import com.baomidou.mybatisplus.annotation.FieldFill;
+import com.baomidou.mybatisplus.annotation.TableField;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.Date;
+
+@Data
+@Builder
+public class MetaDataVo {
+    private Long id;
+
+    /**
+     *  数据元名称
+     */
+    private String metaName;
+
+    /**
+     *  数据元类型
+     */
+    private String dataType;
+
+    /**
+     *  数据元长度
+     */
+    private String dataLength;
+
+    /**
+     *  精度
+     */
+    private String fieldPrecision;
+
+    /**
+     *  元数据代码
+     */
+    private String dataCode;
+
+    /**
+     *  数据元英文名称
+     */
+    private String englishName;
+
+    /**
+     *  所处级别
+     */
+    private String dataLevel;
+
+    /**
+     *  是否非空
+     */
+    private String notNull;
+
+    /**
+     *  所属领域id
+     */
+    private Long categoryId;
+
+    /**
+     *  是否复合元数据
+     */
+    private Integer mutiMetaFlag;
+
+    /**
+     *  复合元数据表达式
+     */
+    private String mutiMetaExpress;
+
+    /**
+     *  createTime
+     */
+    @TableField(fill = FieldFill.INSERT)
+    private Date createTime;
+
+    /**
+     *  updateTime
+     */
+    @TableField(fill = FieldFill.INSERT_UPDATE)
+    private Date updateTime;
+
+    private String categoryName;
+}

spider-flow-core/src/main/java/com/zq/spiderflow/core/executor/shape/IdentityExecutor.java

+package com.zq.spiderflow.core.executor.shape;
+
+import com.zq.spiderflow.context.SpiderContext;
+import com.zq.spiderflow.core.utils.IdentityUtils;
+import com.zq.spiderflow.executor.ShapeExecutor;
+import com.zq.spiderflow.model.Shape;
+import com.zq.spiderflow.model.SpiderNode;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import java.util.Map;
+
+@Component
+public class IdentityExecutor implements ShapeExecutor {
+
+    public static final String META_DATA = "metaData";
+    private static final Logger logger = LoggerFactory.getLogger(ExecuteSQLExecutor.class);
+
+    @Override
+    public Shape shape() {
+        Shape shape = new Shape();
+        //web界面上显示的图标
+        shape.setImage("");
+        //拖放至容器里时默认的节点名称
+        shape.setLabel("身份证清洗");
+        //模板文件名
+        shape.setName("IdentityExecutor");
+        //鼠标移动至图标上显示的名称
+        shape.setTitle("身份证清洗");
+        return shape;
+    }
+
+    @Override
+    public String supportShape() {
+        return "IdentityExecutor";
+    }
+
+    @Override
+    public void execute(SpiderNode node, SpiderContext context, Map<String, Object> variables) {
+        String metaData = node.getStringJsonValue(META_DATA);
+        if (StringUtils.isNotBlank(metaData)) {
+            logger.warn("字段不能为空！");
+        }
+        if(IdentityUtils.isLegalIdCard(metaData)!=true){
+            logger.warn("字段不符合身份证！");
+        }else if(IdentityUtils.is15BitLegalIdCard(metaData)!=true){
+            logger.warn("身份证不满15位！");
+        }else if(IdentityUtils.is18BitLegalIdCard(metaData)!=true){
+            logger.warn("身份证不满18位！");
+        }else {
+
+//            ExceptionUtils.wrapAndThrow(e);
+        }
+    }
+}

spider-flow-core/src/main/java/com/zq/spiderflow/core/utils/IdentityUtils.java

+package com.zq.spiderflow.core.utils;
+
+import java.util.stream.IntStream;
+/**
+ * 身份证号码验证
+ * 1、号码的结构
+ * 公民身份号码是特征组合码，由十七位数字本体码和一位校验码组成。从左至右依次为：六位数字地址码，
+ * 八位数字出生日期码，三位数字顺序码和一位数字校验码。
+ * 2、地址码(前六位数）
+ * 表示编码对象常住户口所在县(市、旗、区)的行政区划代码，按GB/T2260的规定执行。
+ * 3、出生日期码（第七位至十四位）
+ * 表示编码对象出生的年、月、日，按GB/T7408的规定执行，年、月、日代码之间不用分隔符。
+ * 4、顺序码（第十五位至十七位）
+ * 表示在同一地址码所标识的区域范围内，对同年、同月、同日出生的人编定的顺序号，
+ * 顺序码的奇数分配给男性，偶数分配给女性。
+ * 5、校验码（第十八位数）
+ * （1）十七位数字本体码加权求和公式 S = Sum(Ai Wi), i = 0, , 16 ，先对前17位数字的权求和 ;
+ * Ai:表示第i位置上的身份证号码数字值; Wi:表示第i位置上的加权因子 Wi: 7 9 10 5 8 4 2 1 6 3 7 9 10 5 8 4 2
+ * （2）计算模 Y = mod(S, 11)
+ * （3）通过模( 0 1 2 3 4 5 6 7 8 9 10)得到对应的校验码 Y:1 0 X 9 8 7 6 5 4 3 2
+ *
+ */
+
+public class IdentityUtils {
+    /**
+     * 身份证校验码
+     */
+    private static final int[] COEFFICIENT_ARRAY = {7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2};
+
+    /**
+     * 身份证号的尾数规则
+     */
+    private static final String[] IDENTITY_MANTISSA = {"1", "0", "X", "9", "8", "7", "6", "5", "4", "3", "2"};
+
+    private static final String IDENTITY_PATTERN_15 = "^[1-9]\\d{7}((0\\d)|(1[0-2]))(([0-2]\\d)|3[0-1])\\d{3}$";
+
+    // 修改前未做X位的校验："^[1-9]\\d{5}[1-9]\\d{3}((0\\d)|(1[0-2]))(([0-2]\\d)|3[0-1])\\d{4}$";
+    // 修改后："^[1-9]\\d{5}[1-9]\\d{3}((0\\d)|(1[0-2]))(([0-2]\\d)|3[0-1])\\d{3}[0-9Xx]$";
+    private static final String IDENTITY_PATTERN_18 = "^[1-9]\\d{5}[1-9]\\d{3}((0\\d)|(1[0-2]))(([0-2]\\d)|3[0-1])\\d{3}[0-9Xx]$";
+
+
+    /**
+     * 身份证号校验
+     */
+    public static boolean isLegalIdCard(String identity) {
+        if (identity == null) {
+            return false;
+        }
+        if (identity.length() == 15) {
+            return is15BitLegalIdCard(identity);
+        }
+        if (identity.length() == 18) {
+            return is18BitLegalIdCard(identity);
+        }
+        return false;
+    }
+
+    public static boolean is15BitLegalIdCard(String identity) {
+        return identity.matches(IDENTITY_PATTERN_15);
+    }
+
+    public static boolean is18BitLegalIdCard(String identity) {
+        if (identity.length() != 18) {
+            return false;
+        }
+        if (!identity.matches(IDENTITY_PATTERN_18)) {
+            return false;
+        }
+        // 将字符串对象中的字符转换为一个字符数组
+        char[] chars = identity.toCharArray();
+        long sum = IntStream.range(0, 17).map(index -> {
+            char ch = chars[index];
+            // 通俗理解：digit()是个边界值判断，不过边界返回字符数字本身数值，超过边界即返回 -1.
+            int digit = Character.digit(ch, 10);
+            int coefficient = COEFFICIENT_ARRAY[index];
+            return digit * coefficient;
+        }).summaryStatistics().getSum();
+
+        // 计算出的尾数索引
+        int mantissaIndex = (int) (sum % 11);
+        String mantissa = IDENTITY_MANTISSA[mantissaIndex];
+
+        String lastChar = identity.substring(17);
+        return lastChar.equalsIgnoreCase(mantissa);
+    }
+
+}