完善数据库、文件、接口汇聚

data-collect-server/pom.xml

@@ -80,6 +80,12 @@
             <artifactId>common-client</artifactId>
             <artifactId>common-client</artifactId>
             <version>1.0.0</version>
             <version>1.0.0</version>
         </dependency>
         </dependency>
+
+        <dependency>
+            <groupId>com.alibaba.fastjson2</groupId>
+            <artifactId>fastjson2</artifactId>
+            <version>2.0.41</version>
+        </dependency>
     </dependencies>
     </dependencies>
 
 
     <build>
     <build>

data-collect-server/src/main/java/com/zq/datacollect/controller/DataCollecSiteController.java

+package com.zq.datacollect.controller;
+
+import com.zq.common.utils.AssertUtils;
+import com.zq.common.vo.ResultVo;
+import com.zq.datacollect.service.DataCollecSiteService;
+import com.zq.datacollect.vo.DataCollectSiteAddReq;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.web.bind.annotation.*;
+
+import javax.annotation.Resource;
+
+@RestController
+@Api(tags = "网站汇聚")
+@RequestMapping(value = "/data/collect/site")
+public class DataCollecSiteController {
+
+    @Resource
+    private DataCollecSiteService dataCollecSiteService;
+
+    @ApiOperation("根据ID获取网站汇聚")
+    @GetMapping(value = "/get/{dataCollectSettingId}")
+    public ResultVo get(@PathVariable Integer dataCollectSettingId) {
+        return ResultVo.success(dataCollecSiteService.get(dataCollectSettingId));
+    }
+
+    @ApiOperation("新增网站汇聚")
+    @PostMapping(value = "/add")
+    public ResultVo add(@RequestBody DataCollectSiteAddReq dataCollectSiteAddReq) {
+        dataCollecSiteService.add(dataCollectSiteAddReq);
+        return ResultVo.success("添加成功");
+    }
+
+    @ApiOperation("修改网站汇聚")
+    @PutMapping(value = "/update")
+    public ResultVo update(@RequestBody DataCollectSiteAddReq dataCollectSiteAddReq) {
+        AssertUtils.notNull(dataCollectSiteAddReq.getDataCollectSetting().getId(), "修改失败，id不能为空");
+        dataCollecSiteService.update(dataCollectSiteAddReq);
+        return ResultVo.success("修改成功");
+    }
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/controller/DataCollectApiController.java

+package com.zq.datacollect.controller;
+
+import com.zq.common.utils.AssertUtils;
+import com.zq.common.vo.ResultVo;
+import com.zq.datacollect.service.DataCollectApiService;
+import com.zq.datacollect.vo.DataCollectApiAddReq;
+import com.zq.datacollect.vo.DataCollectApiExecuteReq;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+
+import javax.annotation.Resource;
+
+@RestController
+@Api(tags = "接口汇聚")
+@RequestMapping(value = "/data/collect/api")
+public class DataCollectApiController {
+
+    @Resource
+    private DataCollectApiService dataCollectApiService;
+
+    @ApiOperation("根据ID获取接口汇聚")
+    @GetMapping(value = "/get/{dataCollectSettingId}")
+    public ResultVo get(@PathVariable Integer dataCollectSettingId) {
+        return ResultVo.success(dataCollectApiService.get(dataCollectSettingId));
+    }
+
+    @ApiOperation("新增接口汇聚")
+    @PostMapping(value = "/add")
+    public ResultVo add(@RequestBody DataCollectApiAddReq dataCollectApiAddReq) {
+        dataCollectApiService.add(dataCollectApiAddReq);
+        return ResultVo.success("添加成功");
+    }
+
+    @ApiOperation("修改接口汇聚")
+    @PutMapping(value = "/update")
+    public ResultVo update(@RequestBody DataCollectApiAddReq dataCollectApiAddReq) {
+        AssertUtils.notNull(dataCollectApiAddReq.getDataCollectSetting().getId(), "修改失败，id不能为空");
+        dataCollectApiService.update(dataCollectApiAddReq);
+        return ResultVo.success("修改成功");
+    }
+
+    @ApiOperation("执行接口汇聚")
+    @PostMapping(value = "/execute")
+    public ResultVo executeDb(@RequestParam("dataCollectSettingId") Integer dataCollectSettingId, @RequestBody DataCollectApiExecuteReq dataCollectApiExecuteReq) {
+        dataCollectApiService.execute(dataCollectSettingId, dataCollectApiExecuteReq);
+        return ResultVo.success("汇聚执行成功");
+    }
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/controller/DataCollectDbController.java

+package com.zq.datacollect.controller;
+
+import com.zq.common.utils.AssertUtils;
+import com.zq.common.vo.ResultVo;
+import com.zq.datacollect.service.DataCollectDbService;
+import com.zq.datacollect.vo.DataCollectDbAddReq;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.web.bind.annotation.*;
+
+import javax.annotation.Resource;
+
+@RestController
+@Api(tags = "数据库汇聚")
+@RequestMapping(value = "/data/collect/db")
+public class DataCollectDbController {
+
+    @Resource
+    private DataCollectDbService dataCollectDbService;
+
+    @ApiOperation("根据ID获取数据库汇聚")
+    @GetMapping(value = "/get/{dataCollectSettingId}")
+    public ResultVo get(@PathVariable Integer dataCollectSettingId) {
+        return ResultVo.success(dataCollectDbService.get(dataCollectSettingId));
+    }
+
+    @ApiOperation("新增数据库汇聚")
+    @PostMapping(value = "/add")
+    public ResultVo add(@RequestBody DataCollectDbAddReq dataCollectDbAddReq) {
+        dataCollectDbService.add(dataCollectDbAddReq);
+        return ResultVo.success("添加成功");
+    }
+
+    @ApiOperation("修改数据库汇聚")
+    @PutMapping(value = "/update")
+    public ResultVo update(@RequestBody DataCollectDbAddReq dataCollectDbAddReq) {
+        AssertUtils.notNull(dataCollectDbAddReq.getDataCollectSetting().getId(), "修改失败，id不能为空");
+        dataCollectDbService.update(dataCollectDbAddReq);
+        return ResultVo.success("修改成功");
+    }
+
+    @ApiOperation("执行数据库汇聚")
+    @PostMapping(value = "/execute")
+    public ResultVo executeDb(@RequestParam("dataCollectSettingId") Integer dataCollectSettingId) {
+        dataCollectDbService.execute(dataCollectSettingId);
+        return ResultVo.success("汇聚执行成功");
+    }
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/controller/DataCollectFileController.java

+package com.zq.datacollect.controller;
+
+import com.zq.common.utils.AssertUtils;
+import com.zq.common.vo.ResultVo;
+import com.zq.datacollect.service.DataCollectFileService;
+import com.zq.datacollect.vo.DataCollectFileAddReq;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+
+import javax.annotation.Resource;
+
+@RestController
+@Api(tags = "文件汇聚")
+@RequestMapping(value = "/data/collect/file")
+public class DataCollectFileController {
+
+    @Resource
+    private DataCollectFileService dataCollectFileService;
+
+    @ApiOperation("根据ID获取文件汇聚")
+    @GetMapping(value = "/get/{dataCollectSettingId}")
+    public ResultVo get(@PathVariable Integer dataCollectSettingId) {
+        return ResultVo.success(dataCollectFileService.get(dataCollectSettingId));
+    }
+
+    @ApiOperation("新增文件汇聚")
+    @PostMapping(value = "/add")
+    public ResultVo add(@RequestBody DataCollectFileAddReq dataCollectFileAddReq) {
+        dataCollectFileService.add(dataCollectFileAddReq);
+        return ResultVo.success("添加成功");
+    }
+
+    @ApiOperation("修改网站汇聚")
+    @PutMapping(value = "/update")
+    public ResultVo update(@RequestBody DataCollectFileAddReq dataCollectFileAddReq) {
+        AssertUtils.notNull(dataCollectFileAddReq.getDataCollectSetting().getId(), "修改失败，id不能为空");
+        dataCollectFileService.update(dataCollectFileAddReq);
+        return ResultVo.success("修改成功");
+    }
+
+    @ApiOperation("解析Excel文件")
+    @PostMapping(value = "/analysis/excel")
+    public ResultVo analysisExcel(MultipartFile file) {
+        return ResultVo.success(dataCollectFileService.analysisExcel(file));
+    }
+
+    @ApiOperation("执行文件汇聚")
+    @PostMapping(value = "/execute")
+    public ResultVo executeDb(@RequestParam("dataCollectSettingId") Integer dataCollectSettingId, MultipartFile file) {
+        dataCollectFileService.execute(dataCollectSettingId, file);
+        return ResultVo.success("汇聚执行成功");
+    }
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/entity/DataCollectSetting.java

@@ -50,6 +50,11 @@ public class DataCollectSetting {
     private String toTable;
     private String toTable;
 
 
     /**
     /**
+     * 表不存在时创建表（0否 1是）
+     */
+    private Integer createTable = 0;
+
+    /**
      * 汇聚任务的名称
      * 汇聚任务的名称
      */
      */
     private String taskName;
     private String taskName;

data-collect-server/src/main/java/com/zq/datacollect/entity/DataCollectSettingApi.java

@@ -49,7 +49,7 @@ public class DataCollectSettingApi {
     /**
     /**
      * 请求头参数
      * 请求头参数
      */
      */
-    private String heads;
+    private String headers;
 
 
     /**
     /**
      * 请求体格式
      * 请求体格式

data-collect-server/src/main/java/com/zq/datacollect/entity/DataCollectSettingApiPara.java

@@ -24,7 +24,7 @@ public class DataCollectSettingApiPara {
     /**
     /**
      * id
      * id
      */
      */
-    private Integer dataCollectSettingApiId;
+    private Integer dataCollectSettingId;
 
 
     /**
     /**
      * 参数名称
      * 参数名称

data-collect-server/src/main/java/com/zq/datacollect/mapper/DataCollectTestFromMapper.java

+package com.zq.datacollect.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.zq.datacollect.entity.DataCollectTestFrom;
+
+public interface DataCollectTestFromMapper extends BaseMapper<DataCollectTestFrom> {
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/service/DataCollecSiteService.java

+package com.zq.datacollect.service;
+
+import com.zq.datacollect.entity.DataCollectSettingSite;
+import com.zq.datacollect.vo.DataCollectSettingAddReq;
+import com.zq.datacollect.vo.DataCollectSiteAddReq;
+
+public interface DataCollecSiteService {
+
+    DataCollectSettingSite get(Integer dataCollectSettingId);
+
+    void add(DataCollectSiteAddReq dataCollectSiteAddReq);
+
+    void update(DataCollectSiteAddReq dataCollectSiteAddReq);
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/service/DataCollectApiService.java

+package com.zq.datacollect.service;
+
+import com.zq.datacollect.entity.DataCollectSettingApi;
+import com.zq.datacollect.vo.DataCollectApiAddReq;
+import com.zq.datacollect.vo.DataCollectApiExecuteReq;
+
+public interface DataCollectApiService {
+
+    DataCollectSettingApi get(Integer dataCollectSettingId);
+
+    void add(DataCollectApiAddReq dataCollectApiAddReq);
+
+    void update(DataCollectApiAddReq dataCollectApiAddReq);
+
+    void execute(Integer dataCollectSettingId, DataCollectApiExecuteReq dataCollectApiExecuteReq);
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/service/DataCollectDbService.java

+package com.zq.datacollect.service;
+
+import com.zq.datacollect.entity.DataCollectSettingDb;
+import com.zq.datacollect.vo.DataCollectDbAddReq;
+
+public interface DataCollectDbService {
+
+    DataCollectSettingDb get(Integer dataCollectSettingId);
+
+    void add(DataCollectDbAddReq dataCollectDbAddReq);
+
+    void update(DataCollectDbAddReq dataCollectDbAddReq);
+
+    void execute(Integer dataCollectSettingId);
+}

data-collect-server/src/main/java/com/zq/datacollect/service/DataCollectFileService.java

+package com.zq.datacollect.service;
+
+import com.zq.datacollect.entity.DataCollectSettingFile;
+import com.zq.datacollect.vo.DataCollectFileAddReq;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.util.List;
+import java.util.Map;
+
+public interface DataCollectFileService {
+
+    DataCollectSettingFile get(Integer dataCollectSettingId);
+
+    void add(DataCollectFileAddReq dataCollectFileAddReq);
+
+    void update(DataCollectFileAddReq dataCollectFileAddReq);
+
+    List<Map<String, Object>> analysisExcel(MultipartFile multipartFile);
+
+    void execute(Integer dataCollectSettingId, MultipartFile file);
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollecSiteServiceImpl.java

+package com.zq.datacollect.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.zq.common.utils.AssertUtils;
+import com.zq.datacollect.entity.DataCollectSetting;
+import com.zq.datacollect.entity.DataCollectSettingSite;
+import com.zq.datacollect.mapper.DataCollectMapper;
+import com.zq.datacollect.mapper.DataCollectSettingSiteMapper;
+import com.zq.datacollect.service.DataCollecSiteService;
+import com.zq.datacollect.vo.DataCollectSiteAddReq;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import javax.annotation.Resource;
+
+@Service
+public class DataCollecSiteServiceImpl extends ServiceImpl<DataCollectSettingSiteMapper, DataCollectSettingSite> implements DataCollecSiteService {
+
+    @Resource
+    private DataCollectMapper dataCollectMapper;
+
+    @Override
+    public DataCollectSettingSite get(Integer dataCollectSettingId) {
+        return lambdaQuery().eq(DataCollectSettingSite::getDataCollectSettingId, dataCollectSettingId).one();
+    }
+
+    @Override
+    @Transactional
+    public void add(DataCollectSiteAddReq dataCollectSiteAddReq) {
+        AssertUtils.isTrue(dataCollectMapper.insert(dataCollectSiteAddReq.getDataCollectSetting())==1, "添加失败");
+        DataCollectSettingSite settingSite = dataCollectSiteAddReq.getDataCollectSettingSite();
+        settingSite.setDataCollectSettingId(dataCollectSiteAddReq.getDataCollectSetting().getId());
+        AssertUtils.isTrue(save(settingSite), "添加失败");
+    }
+
+    @Override
+    @Transactional
+    public void update(DataCollectSiteAddReq dataCollectSiteAddReq) {
+        DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectSiteAddReq.getDataCollectSetting().getId());
+        AssertUtils.isTrue(dataCollectSetting.getDataType()==dataCollectSiteAddReq.getDataCollectSetting().getDataType(), "修改失败，数据来源不允许修改");
+        AssertUtils.isTrue(updateById(dataCollectSiteAddReq.getDataCollectSettingSite()), "修改失败");
+        AssertUtils.isTrue(dataCollectMapper.updateById(dataCollectSiteAddReq.getDataCollectSetting())==1, "修改失败");
+    }
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectApiServiceImpl.java

+package com.zq.datacollect.service.impl;
+
+import cn.hutool.core.bean.BeanUtil;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.http.HttpRequest;
+import cn.hutool.http.HttpUtil;
+import com.alibaba.fastjson2.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.zq.common.exception.BusinessException;
+import com.zq.common.utils.AssertUtils;
+import com.zq.datacollect.entity.DataCollectSetting;
+import com.zq.datacollect.entity.DataCollectSettingApi;
+import com.zq.datacollect.entity.DataCollectSettingApiPara;
+import com.zq.datacollect.holder.DatabaseHolder;
+import com.zq.datacollect.mapper.DataCollectMapper;
+import com.zq.datacollect.mapper.DataCollectSettingApiMapper;
+import com.zq.datacollect.mapper.DataCollectSettingApiParaMapper;
+import com.zq.datacollect.service.DataCollectApiService;
+import com.zq.datacollect.util.SqlUtil;
+import com.zq.datacollect.vo.DataCollectApiAddReq;
+import com.zq.datacollect.vo.DataCollectApiExecuteReq;
+import com.zq.datacollect.vo.DataCollectSettingApiResp;
+import org.springframework.jdbc.core.BatchPreparedStatementSetter;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import javax.annotation.Resource;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Service
+public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApiMapper, DataCollectSettingApi> implements DataCollectApiService {
+
+    @Resource
+    private DataCollectMapper dataCollectMapper;
+
+    @Resource
+    private DataCollectSettingApiParaMapper dataCollectSettingApiParaMapper;
+
+    @Override
+    public DataCollectSettingApiResp get(Integer dataCollectSettingId) {
+        DataCollectSettingApi settingApi = lambdaQuery().eq(DataCollectSettingApi::getDataCollectSettingId, dataCollectSettingId).one();
+        List<DataCollectSettingApiPara> dataCollectSettingApiParas = dataCollectSettingApiParaMapper.selectList(new LambdaQueryWrapper<DataCollectSettingApiPara>().eq(DataCollectSettingApiPara::getDataCollectSettingId, settingApi.getDataCollectSettingId()));
+        DataCollectSettingApiResp dataCollectSettingApiResp = BeanUtil.copyProperties(settingApi, DataCollectSettingApiResp.class);
+        dataCollectSettingApiResp.setDataCollectSettingApiParas(dataCollectSettingApiParas);
+        return dataCollectSettingApiResp;
+    }
+
+    @Override
+    @Transactional
+    public void add(DataCollectApiAddReq dataCollectApiAddReq) {
+        AssertUtils.isTrue(dataCollectMapper.insert(dataCollectApiAddReq.getDataCollectSetting())==1, "添加失败");
+        DataCollectSettingApi settingApi = dataCollectApiAddReq.getDataCollectSettingApi();
+        settingApi.setDataCollectSettingId(dataCollectApiAddReq.getDataCollectSetting().getId());
+        AssertUtils.isTrue(save(settingApi), "添加失败");
+        for (DataCollectSettingApiPara settingApiPara : dataCollectApiAddReq.getDataCollectSettingApiParas()) {
+            settingApiPara.setDataCollectSettingId(settingApi.getDataCollectSettingId());
+            AssertUtils.isTrue(dataCollectSettingApiParaMapper.insert(settingApiPara)==1, "添加失败");
+        }
+    }
+
+    @Override
+    @Transactional
+    public void update(DataCollectApiAddReq dataCollectApiAddReq) {
+        DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectApiAddReq.getDataCollectSetting().getId());
+        AssertUtils.isTrue(dataCollectSetting.getDataType()==dataCollectApiAddReq.getDataCollectSetting().getDataType(), "修改失败，数据来源不允许修改");
+        AssertUtils.isTrue(updateById(dataCollectApiAddReq.getDataCollectSettingApi()), "修改失败");
+        for (DataCollectSettingApiPara dataCollectSettingApiPara : dataCollectApiAddReq.getDataCollectSettingApiParas()) {
+            if(dataCollectSettingApiPara.getId() == null){
+                AssertUtils.isTrue(dataCollectSettingApiParaMapper.insert(dataCollectSettingApiPara)==1, "修改失败");
+            }else {
+                AssertUtils.isTrue(dataCollectSettingApiParaMapper.updateById(dataCollectSettingApiPara)==1, "修改失败");
+            }
+        }
+        AssertUtils.isTrue(dataCollectMapper.updateById(dataCollectApiAddReq.getDataCollectSetting())==1, "修改失败");
+    }
+
+    @Override
+    @Transactional
+    public void execute(Integer dataCollectSettingId, DataCollectApiExecuteReq dataCollectApiExecuteReq) {
+        DataCollectSettingApi settingApi = lambdaQuery().eq(DataCollectSettingApi::getDataCollectSettingId, dataCollectSettingId).one();
+        List<DataCollectSettingApiPara> apiParas;
+        if(dataCollectApiExecuteReq.getParas() != null && dataCollectApiExecuteReq.getParas().size() > 0){
+            apiParas = dataCollectApiExecuteReq.getParas();
+        }else {
+            apiParas = dataCollectSettingApiParaMapper.selectList(new LambdaQueryWrapper<DataCollectSettingApiPara>().eq(DataCollectSettingApiPara::getDataCollectSettingId, dataCollectSettingId));
+        }
+        String url = settingApi.getApiUrl() + (apiParas.size()==0?"": urlParams(apiParas));
+
+        HttpRequest request;
+        if(settingApi.getRequestType() == 1){
+            request = HttpUtil.createGet(url);
+        }else if(settingApi.getRequestType() == 2){
+            request = HttpUtil.createPost(url);
+        }else {
+            throw new BusinessException("不支持的请求类型");
+        }
+
+        if(StrUtil.isNotBlank(dataCollectApiExecuteReq.getHeaders())){
+            for (Map.Entry<String, Object> entry : JSONObject.parseObject(dataCollectApiExecuteReq.getHeaders()).entrySet()) {
+                request.header(entry.getKey(), entry.getValue().toString());
+            }
+        }
+        else if(StrUtil.isNotBlank(settingApi.getHeaders())){
+            for (Map.Entry<String, Object> entry : JSONObject.parseObject(settingApi.getHeaders()).entrySet()) {
+                request.header(entry.getKey(), entry.getValue().toString());
+            }
+        }
+
+        if(StrUtil.isNotBlank(dataCollectApiExecuteReq.getBody())){
+            request.body(dataCollectApiExecuteReq.getBody());
+        }else if(StrUtil.isNotBlank(settingApi.getBody())){
+            request.body(settingApi.getBody());
+        }
+
+        String body = request.execute().body();
+        log.error(body);
+        LinkedList<Map<String, Object>> list = JSONObject.parseObject(body).getJSONObject("data").getJSONArray("rows").to(LinkedList.class);
+
+        DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectSettingId);
+        JdbcTemplate toDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(dataCollectSetting.getToDbId());
+        Set<String> set = list.get(0).keySet();
+        set.remove("id");
+        if(dataCollectSetting.getCreateTable()==1 && toDbJdbcTemplate.queryForList("SHOW TABLES LIKE '"+ dataCollectSetting.getToTable() + "'").size()==0){
+            toDbJdbcTemplate.execute(SqlUtil.joinCreateTableSql(dataCollectSetting.getToTable(), set));
+        }
+        toDbJdbcTemplate.batchUpdate(SqlUtil.jointInsertSql(dataCollectSetting.getToTable(), set), new BatchPreparedStatementSetter() {
+            @Override
+            public void setValues(PreparedStatement preparedStatement, int i) throws SQLException {
+                Map<String, Object> map = list.get(i);
+                int index = 1;
+                for (String key : set) {
+                    preparedStatement.setObject(index++, map.get(key));
+                }
+            }
+
+            @Override
+            public int getBatchSize() {
+                return list.size();
+            }
+        });
+    }
+
+    public String urlParams(List<DataCollectSettingApiPara> settingApiParas){
+        StringBuilder url = new StringBuilder();
+        for (int i = 0; i < settingApiParas.size(); i++) {
+            DataCollectSettingApiPara settingApiPara = settingApiParas.get(i);
+            if(i==0){
+                url.append("?");
+            }else {
+                url.append("&");
+            }
+            url.append(settingApiPara.getKeyName()).append("=").append(settingApiPara.getKeyValue());
+        }
+        return url.toString();
+    }
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectDbServiceImpl.java

+package com.zq.datacollect.service.impl;
+
+import cn.hutool.core.util.StrUtil;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.zq.common.utils.AssertUtils;
+import com.zq.datacollect.entity.DataCollectSetting;
+import com.zq.datacollect.entity.DataCollectSettingDb;
+import com.zq.datacollect.holder.DatabaseHolder;
+import com.zq.datacollect.mapper.DataCollectMapper;
+import com.zq.datacollect.mapper.DataCollectSettingDbMapper;
+import com.zq.datacollect.service.DataCollectDbService;
+import com.zq.datacollect.util.SqlUtil;
+import com.zq.datacollect.vo.DataCollectDbAddReq;
+import org.springframework.jdbc.core.BatchPreparedStatementSetter;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import javax.annotation.Resource;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Service
+public class DataCollectDbServiceImpl extends ServiceImpl<DataCollectSettingDbMapper, DataCollectSettingDb> implements DataCollectDbService {
+
+    @Resource
+    private DataCollectMapper dataCollectMapper;
+
+    @Override
+    public DataCollectSettingDb get(Integer dataCollectSettingId) {
+        return lambdaQuery().eq(DataCollectSettingDb::getDataCollectSettingId, dataCollectSettingId).one();
+    }
+
+    @Override
+    @Transactional
+    public void add(DataCollectDbAddReq dataCollectDbAddReq) {
+        AssertUtils.isTrue(dataCollectMapper.insert(dataCollectDbAddReq.getDataCollectSetting())==1, "添加失败");
+        DataCollectSettingDb settingDb = dataCollectDbAddReq.getDataCollectSettingDb();
+        settingDb.setDataCollectSettingId(dataCollectDbAddReq.getDataCollectSetting().getId());
+        AssertUtils.isTrue(save(settingDb), "添加失败");
+    }
+
+    @Override
+    @Transactional
+    public void update(DataCollectDbAddReq dataCollectDbAddReq) {
+        DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectDbAddReq.getDataCollectSetting().getId());
+        AssertUtils.isTrue(dataCollectSetting.getDataType()==dataCollectDbAddReq.getDataCollectSetting().getDataType(), "修改失败，数据来源不允许修改");
+        AssertUtils.isTrue(updateById(dataCollectDbAddReq.getDataCollectSettingDb()), "修改失败");
+        AssertUtils.isTrue(dataCollectMapper.updateById(dataCollectDbAddReq.getDataCollectSetting())==1, "修改失败");
+    }
+
+    @Override
+    @Transactional
+    public void execute(Integer dataCollectSettingId) {
+        DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectSettingId);
+        DataCollectSettingDb settingDb = getOne(new LambdaQueryWrapper<DataCollectSettingDb>().eq(DataCollectSettingDb::getDataCollectSettingId, dataCollectSettingId));
+        JdbcTemplate toDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(dataCollectSetting.getToDbId());
+        JdbcTemplate fromDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(settingDb.getFromDbId());
+        List<Map<String, Object>> list = fromDbJdbcTemplate.queryForList("SELECT * FROM " + settingDb.getFromTable());
+        Set<String> set = list.get(0).keySet();
+        set.remove("id");
+        if(dataCollectSetting.getCreateTable()==1 && toDbJdbcTemplate.queryForList("SHOW TABLES LIKE '"+ dataCollectSetting.getToTable() + "'").size()==0){
+            toDbJdbcTemplate.execute(SqlUtil.joinCreateTableSql(dataCollectSetting.getToTable(), set));
+        }
+        toDbJdbcTemplate.batchUpdate(SqlUtil.jointInsertSql(dataCollectSetting.getToTable(), set), new BatchPreparedStatementSetter() {
+            @Override
+            public void setValues(PreparedStatement preparedStatement, int i) throws SQLException {
+                Map<String, Object> map = list.get(i);
+                int index = 1;
+                for (String key : set) {
+                    preparedStatement.setObject(index++, map.get(key));
+                }
+            }
+
+            @Override
+            public int getBatchSize() {
+                return list.size();
+            }
+        });
+    }
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectFileServiceImpl.java

+package com.zq.datacollect.service.impl;
+
+import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.util.StrUtil;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.zq.common.exception.BusinessException;
+import com.zq.common.utils.AssertUtils;
+import com.zq.common.utils.EasyExcelUtil;
+import com.zq.datacollect.entity.DataCollectSetting;
+import com.zq.datacollect.entity.DataCollectSettingFile;
+import com.zq.datacollect.holder.DatabaseHolder;
+import com.zq.datacollect.mapper.DataCollectMapper;
+import com.zq.datacollect.mapper.DataCollectSettingFileMapper;
+import com.zq.datacollect.service.DataCollectFileService;
+import com.zq.datacollect.util.SqlUtil;
+import com.zq.datacollect.vo.DataCollectFileAddReq;
+import org.springframework.jdbc.core.BatchPreparedStatementSetter;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.multipart.MultipartFile;
+
+import javax.annotation.Resource;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Service
+public class DataCollectFileServiceImpl extends ServiceImpl<DataCollectSettingFileMapper, DataCollectSettingFile> implements DataCollectFileService {
+
+    @Resource
+    private DataCollectMapper dataCollectMapper;
+
+    @Override
+    public DataCollectSettingFile get(Integer dataCollectSettingId) {
+        return lambdaQuery().eq(DataCollectSettingFile::getDataCollectSettingId, dataCollectSettingId).one();
+    }
+
+    @Override
+    @Transactional
+    public void add(DataCollectFileAddReq dataCollectFileAddReq) {
+        AssertUtils.isTrue(dataCollectMapper.insert(dataCollectFileAddReq.getDataCollectSetting())==1, "添加失败");
+        DataCollectSettingFile settingFile = dataCollectFileAddReq.getDataCollectSettingFile();
+        settingFile.setDataCollectSettingId(dataCollectFileAddReq.getDataCollectSetting().getId());
+        AssertUtils.isTrue(save(settingFile), "添加失败");
+    }
+
+    @Override
+    @Transactional
+    public void update(DataCollectFileAddReq dataCollectFileAddReq) {
+        DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectFileAddReq.getDataCollectSetting().getId());
+        AssertUtils.isTrue(dataCollectSetting.getDataType()==dataCollectFileAddReq.getDataCollectSetting().getDataType(), "修改失败，数据来源不允许修改");
+        AssertUtils.isTrue(updateById(dataCollectFileAddReq.getDataCollectSettingFile()), "修改失败");
+        AssertUtils.isTrue(dataCollectMapper.updateById(dataCollectFileAddReq.getDataCollectSetting())==1, "修改失败");
+    }
+
+    @Override
+    public List<Map<String, Object>> analysisExcel(MultipartFile file) {
+        String suffix = FileUtil.extName(file.getOriginalFilename());
+        AssertUtils.isTrue(suffix.equals("xls") || suffix.equals("xlsx") || suffix.equals("et"), "不支持的文件类型");
+        try {
+            return EasyExcelUtil.read(file.getInputStream());
+        } catch (Exception e) {
+            throw new BusinessException("文件读取失败，请检查后重试");
+        }
+    }
+
+    @Override
+    @Transactional
+    public void execute(Integer dataCollectSettingId, MultipartFile file) {
+        List<Map<String, Object>> list = analysisExcel(file);
+        DataCollectSetting dataCollectSetting = dataCollectMapper.selectById(dataCollectSettingId);
+        JdbcTemplate toDbJdbcTemplate = DatabaseHolder.getJdbcTemplate(dataCollectSetting.getToDbId());
+        Set<String> set = list.get(0).keySet();
+        set.remove("id");
+        if(dataCollectSetting.getCreateTable()==1 && toDbJdbcTemplate.queryForList("SHOW TABLES LIKE '"+ dataCollectSetting.getToTable() + "'").size()==0){
+            toDbJdbcTemplate.execute(SqlUtil.joinCreateTableSql(dataCollectSetting.getToTable(), set));
+        }
+        toDbJdbcTemplate.batchUpdate(SqlUtil.jointInsertSql(dataCollectSetting.getToTable(), set), new BatchPreparedStatementSetter() {
+            @Override
+            public void setValues(PreparedStatement preparedStatement, int i) throws SQLException {
+                Map<String, Object> map = list.get(i);
+                int index = 1;
+                for (String key : set) {
+                    preparedStatement.setObject(index++, map.get(key));
+                }
+            }
+
+            @Override
+            public int getBatchSize() {
+                return list.size();
+            }
+        });
+    }
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectServiceImpl.java

@@ -63,11 +63,15 @@ public class DataCollectServiceImpl extends ServiceImpl<DataCollectMapper, DataC
 //    }
 //    }
 
 
     @Override
     @Override
+    @Transactional
     public void delete(Integer id){
     public void delete(Integer id){
         DataCollectSetting dataCollectSetting = getById(id);
         DataCollectSetting dataCollectSetting = getById(id);
         UpdateWrapper wrapper = new UpdateWrapper();
         UpdateWrapper wrapper = new UpdateWrapper();
         wrapper.eq("data_collect_setting_id", dataCollectSetting.getId());
         wrapper.eq("data_collect_setting_id", dataCollectSetting.getId());
         getDao(dataCollectSetting.getDataType()).delete(wrapper);
         getDao(dataCollectSetting.getDataType()).delete(wrapper);
+        if(dataCollectSetting.getDataType()==3){
+            dataCollectSettingApiParaMapper.delete(wrapper);
+        }
         AssertUtils.isTrue(removeById(id), "删除失败");
         AssertUtils.isTrue(removeById(id), "删除失败");
     }
     }
 
 
@@ -93,4 +97,4 @@ public class DataCollectServiceImpl extends ServiceImpl<DataCollectMapper, DataC
                 throw new BusinessException("不支持的类型");
                 throw new BusinessException("不支持的类型");
         }
         }
     }
     }
-}
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/util/DataCollectTaskUtil.java

+package com.zq.datacollect.util;
+
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.http.HttpRequest;
+import cn.hutool.http.HttpResponse;
+import cn.hutool.http.HttpUtil;
+import cn.hutool.json.JSONUtil;
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
+import com.zq.common.exception.BusinessException;
+import com.zq.datacollect.entity.*;
+import org.springframework.jdbc.core.BatchPreparedStatementSetter;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class DataCollectTaskUtil {
+
+    public static void database(JdbcTemplate thisJdbc, DataCollectSetting setting, DataCollectSettingDbOld settingDb) throws SQLException {
+        DruidDataSource druidDataSource = DruidDataSourceBuilder.create().build();
+        String url = "";
+        String className = "";
+        switch (settingDb.getDbType()) {
+            case 1:
+                className = "com.mysql.cj.jdbc.Driver";
+                url = "jdbc:mysql://" + settingDb.getDbIp() + ":" + settingDb.getDbPort() + "/" + settingDb.getDbName() + "?characterEncoding=utf8&useSSL=false&serverTimezone=GMT%2B8&autoReconnect=true";
+                break;
+            case 2:
+                className = "oracle.jdbc.driver.OracleDriver";
+                url = "jdbc:oracle:thin:@//" + settingDb.getDbIp() + ":" + settingDb.getDbPort() + "/" + settingDb.getDbName();
+                break;
+            case 3:
+                className = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
+                url = "jdbc:sqlserver://" + settingDb.getDbIp() + ":" + settingDb.getDbPort() + "/" + settingDb.getDbName();
+                break;
+            default:
+                throw new BusinessException("不支持的数据库");
+        }
+        druidDataSource.setDriverClassName(className);
+        druidDataSource.setUrl(url);
+        druidDataSource.setUsername(settingDb.getUsername());
+        druidDataSource.setPassword(settingDb.getPassword());
+        druidDataSource.init();
+        JdbcTemplate template = new JdbcTemplate(druidDataSource);
+        List<Map<String, Object>> list = template.queryForList("select * from " + setting.getToTable());
+        Set<String> set = list.get(0).keySet();
+        set.remove("id");
+        thisJdbc.batchUpdate(sql(setting.getToTable(), set), new BatchPreparedStatementSetter() {
+            @Override
+            public void setValues(PreparedStatement preparedStatement, int i) throws SQLException {
+                Map<String, Object> map = list.get(i);
+                int index = 1;
+                for (String key : set) {
+                    preparedStatement.setObject(index++, map.get(key));
+                }
+            }
+
+            @Override
+            public int getBatchSize() {
+                return list.size();
+            }
+        });
+    }
+
+    public static void file(JdbcTemplate thisJdbc, DataCollectSetting setting, DataCollectSettingFile settingFile, MultipartFile[] multipartFiles) throws IOException {
+        List<File> ocrFiles = new ArrayList<>();
+        List<File> toDbFiles = new ArrayList<>();
+        String fileFolder = settingFile.getFileFolder();
+        for (MultipartFile multipartFile : multipartFiles) {
+            File file = new File(fileFolder + File.separator + "." + multipartFile.getOriginalFilename());
+            multipartFile.transferTo(file);
+            if (settingFile.getIfOcr() == 1) {
+                ocrFiles.add(file);
+            }
+            if (settingFile.getIfToDatabase() == 1) {
+                toDbFiles.add(file);
+            }
+
+            for (File ocrFile : ocrFiles) {
+                //ocr
+            }
+
+            for (File toDbFile : toDbFiles) {
+                //入库
+            }
+        }
+    }
+
+    public static void api(JdbcTemplate thisJdbc, DataCollectSetting setting, DataCollectSettingApi settingApi, List<DataCollectSettingApiPara> settingApiParas){
+        HttpRequest request;
+        if(settingApi.getRequestType() == 1){
+            request = HttpUtil.createGet(settingApi.getApiUrl());
+        }else if(settingApi.getRequestType() == 2){
+            request = HttpUtil.createPost(settingApi.getApiUrl());
+        }else {
+            throw new BusinessException("不支持的请求类型");
+        }
+
+        if(StrUtil.isNotBlank(settingApi.getHeaders())){
+            for (Map.Entry<String, Object> entry : JSONUtil.parseObj(settingApi.getHeaders()).entrySet()) {
+                request.header(entry.getKey(), entry.getValue().toString());
+            }
+        }
+
+        if(StrUtil.isNotBlank(settingApi.getBody())){
+            request.body(settingApi.getBody());
+        }
+
+        for (DataCollectSettingApiPara settingApiPara : settingApiParas) {
+            request.form(settingApiPara.getKeyName(), settingApiPara.getKeyValue());
+        }
+
+        HttpResponse response = request.execute();
+        System.out.println(response.body());
+    }
+
+    public static String sql(String tableName, Set<String> cloums){
+        StringBuilder sql = new StringBuilder("INSERT INTO " + tableName + "(" + StrUtil.join(",", cloums) + ") VALUES (");
+        for (int i = 0; i < cloums.size(); i++) {
+            if(i==0){
+                sql.append("?");
+            }else {
+                sql.append(",?");
+            }
+        }
+        sql.append(")");
+        System.out.println(sql.toString());
+        return sql.toString();
+    }
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/util/SqlUtil.java

+package com.zq.datacollect.util;
+
+import java.util.Set;
+
+public class SqlUtil {
+
+    public static String jointInsertSql(String tableName, Set<String> cloums){
+        StringBuilder fileds = new StringBuilder();
+        for (String cloum : cloums) {
+            fileds.append(cloum.replaceAll("\\p{Punct}", "")).append(",");
+        }
+        fileds.deleteCharAt(fileds.length()-1);
+        StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(fileds).append(") VALUES (");
+        for (int i = 0; i < cloums.size(); i++) {
+            if(i==0){
+                sql.append("?");
+            }else {
+                sql.append(",?");
+            }
+        }
+        sql.append(")");
+        return sql.toString();
+    }
+
+    public static String joinCreateTableSql(String tableName, Set<String> cloums){
+        StringBuilder sql = new StringBuilder("CREATE TABLE ").append(tableName).append("(");
+        for (String cloum : cloums) {
+            sql.append(cloum.replaceAll("\\p{Punct}", "")).append(" VARCHAR(255),");
+        }
+        sql.deleteCharAt(sql.length()-1).append(")");
+        return sql.toString();
+    }
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/vo/DataCollectApiExecuteReq.java

+package com.zq.datacollect.vo;
+
+import com.zq.datacollect.entity.DataCollectSettingApiPara;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class DataCollectApiExecuteReq {
+
+    /**
+     * 请求头
+     */
+    private String headers;
+
+    /**
+     * 请求体
+     */
+    private String body;
+
+    /**
+     * 参数
+     */
+    private List<DataCollectSettingApiPara> paras;
+}
\ No newline at end of file

data-collect-server/src/main/java/com/zq/datacollect/vo/DataCollectReq.java

@@ -45,6 +45,12 @@ public class DataCollectReq extends PageReqVo {
     private String toTable;
     private String toTable;
 
 
     /**
     /**
+     * 表不存在时创建表（0否 1是）
+     */
+    private Integer createTable;
+
+
+    /**
      * 汇聚任务的名称
      * 汇聚任务的名称
      */
      */
     private String taskName;
     private String taskName;

data-collect-server/src/main/java/com/zq/datacollect/vo/DataCollectSettingApiResp.java

+package com.zq.datacollect.vo;
+
+import com.zq.datacollect.entity.DataCollectSettingApi;
+import com.zq.datacollect.entity.DataCollectSettingApiPara;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class DataCollectSettingApiResp extends DataCollectSettingApi {
+
+    private List<DataCollectSettingApiPara> dataCollectSettingApiParas;
+}
\ No newline at end of file