Skip to content

N
nnjcy-data-model
Commit 4190cf2d by 梁家彪
Options

commit all

parent 24f51ccd
Showing with 282 additions and 73 deletions
config-server/src/main/resources/properties/config-test.yml
......@@ -13,6 +13,9 @@ spiderflow.name: SPIDER-FLOW
model.port: 9877
model.name: MODEL-SERVER
data.port: 9688
data.name: DATAOPERATION-SERVER
datacollect.port: 9697
datacollect.name: DATA-COLLECT-SERVER
......
data-collect-server/pom.xml
......@@ -93,9 +93,9 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
<!-- <configuration>-->
<!-- <skip>true</skip>-->
<!-- </configuration>-->
</plugin>
</plugins>
......
data-collect-server/src/main/java/com/zq/datacollect/DataCollectAppliction.java → data-collect-server/src/main/java/com/zq/datacollect/DataCollectApplication.java
......@@ -6,9 +6,9 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
@MapperScan({"com.zq.datacollect.mapper"})
public class DataCollectAppliction {
public class DataCollectApplication {
public static void main(String[] args) {
SpringApplication.run(DataCollectAppliction.class, args);
SpringApplication.run(DataCollectApplication.class, args);
}
}
\ No newline at end of file
data-collect-server/src/main/java/com/zq/datacollect/holder/DatabaseHolder.java
package com.zq.datacollect.holder;
import com.alibaba.druid.pool.DruidDataSource;
import org.springframework.jdbc.core.JdbcTemplate;
import java.sql.Connection;
......@@ -24,7 +25,7 @@ public class DatabaseHolder {
public static Connection getConnection(Integer id){
try {
return jdbcTemplateMap.get(id).getDataSource().getConnection();
return ((DruidDataSource) jdbcTemplateMap.get(id).getDataSource()).getConnection().getConnection();
} catch (SQLException e) {
throw new RuntimeException(e);
}
......
dataoperation-server/pom.xml
......@@ -76,11 +76,11 @@
<artifactId>poi</artifactId>
<version>4.1.1</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.oracle</groupId>-->
<!-- <artifactId>ojdbc6</artifactId>-->
<!-- <version>11.2.0.3</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>com.oracle</groupId>-->
<!-- <artifactId>ojdbc6</artifactId>-->
<!-- <version>11.2.0.3</version>-->
<!-- </dependency>-->
<!-- https://mvnrepository.com/artifact/com.oracle.database.nls/orai18n -->
<!-- swagger start-->
<dependency>
......
dataoperation-server/src/main/java/com/zq/dataoperation/controller/CommonQueryController.java
......@@ -60,9 +60,8 @@ public class CommonQueryController {
AssertUtils.notNull(queryDb.getDbType(), "缺少数据库类型");
AssertUtils.hasText(queryDb.getUsername(), "缺少连接用户名");
AssertUtils.hasText(queryDb.getPassword(), "缺少连接密码");
queryDb.setUpdateTime(DateUtil.date());
boolean b = queryDbService.updateById(queryDb);
return b ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败");
commonQueryService.updateDatasource(queryDb);
return ResultVo.success("操作成功！");
}
/**
......@@ -73,10 +72,10 @@ public class CommonQueryController {
*/
@ApiOperation("根据ID删除数据源")
@DeleteMapping("/deleteDatasource/{id}")
public ResultVo deleteDatasource(@PathVariable Long id) {
public ResultVo deleteDatasource(@PathVariable Integer id) {
AssertUtils.notNull(id, "删除至少需要一个ID！");
boolean b = queryDbService.removeById(id);
return b ? ResultVo.success("操作成功！") : ResultVo.fail("操作失败！");
commonQueryService.deleteDatasource(id);
return ResultVo.success("操作成功！");
}
......
dataoperation-server/src/main/java/com/zq/dataoperation/holder/DatabaseHolder.java
package com.zq.dataoperation.holder;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
import com.zq.common.exception.BusinessException;
import com.zq.common.utils.AssertUtils;
import org.springframework.jdbc.core.JdbcTemplate;
import java.sql.Connection;
......@@ -9,14 +13,33 @@ import java.util.Map;
public class DatabaseHolder {
private static Map<Integer, JdbcTemplate> jdbcTemplateMap;
public static void set(Map<Integer, JdbcTemplate> map) {
jdbcTemplateMap = map;
private static Map<Integer, JdbcTemplate> jdbcTemplateMap = new HashMap<>();
public static void add(Integer dbType, String dbIp, Integer dbPort, String dbName, String username, String password, Integer id) {
DruidDataSource druidDataSource = new DruidDataSource();
String url = "";
String className = "";
switch (dbType) {
case 1:
className = "com.mysql.cj.jdbc.Driver";
url = "jdbc:mysql://" + dbIp + ":" + dbPort + "/" + dbName + "?characterEncoding=utf8&useSSL=false&serverTimezone=GMT%2B8&autoReconnect=true";
break;
case 2:
className = "oracle.jdbc.driver.OracleDriver";
url = "jdbc:oracle:thin:@//" + dbIp + ":" + dbPort + "/" + dbName;
break;
case 3:
className = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
url = "jdbc:sqlserver://" + dbIp + ":" + dbPort + "/" + dbName;
break;
default:
throw new BusinessException("不支持的数据库");
}
public static void add(Integer id, JdbcTemplate jdbcTemplate){
jdbcTemplateMap.put(id, jdbcTemplate);
druidDataSource.setDriverClassName(className);
druidDataSource.setUrl(url);
druidDataSource.setUsername(username);
druidDataSource.setPassword(password);
jdbcTemplateMap.put(id, new JdbcTemplate(druidDataSource));
}
public static JdbcTemplate getJdbcTemplate(Integer id){
......@@ -27,7 +50,13 @@ public class DatabaseHolder {
try {
return jdbcTemplateMap.get(id).getDataSource().getConnection();
} catch (SQLException e) {
throw new RuntimeException(e);
e.printStackTrace();
throw new BusinessException("获取数据库连接失败");
}
}
public static void remove(Integer id){
((DruidDataSource)jdbcTemplateMap.get(id).getDataSource()).close();
jdbcTemplateMap.remove(id);
}
}
\ No newline at end of file
dataoperation-server/src/main/java/com/zq/dataoperation/runner/DatabaseRunner.java
package com.zq.dataoperation.runner;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
import com.zq.common.exception.BusinessException;
import com.zq.dataoperation.entity.QueryDb;
import com.zq.dataoperation.holder.DatabaseHolder;
import com.zq.dataoperation.service.CommonQueryService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j
@Component
......@@ -25,39 +19,12 @@ public class DatabaseRunner implements ApplicationRunner {
@Resource
private CommonQueryService service;
private Map<Integer, JdbcTemplate> jdbcTemplateMap = new HashMap<>();
@Override
public void run(ApplicationArguments args) throws SQLException {
public void run(ApplicationArguments args) {
List<QueryDb> datasource = service.getDatasource();
for (QueryDb ds : datasource) {
DruidDataSource druidDataSource = DruidDataSourceBuilder.create().build();
String url = "";
String className = "";
switch (ds.getDbType()) {
case 1:
className = "com.mysql.cj.jdbc.Driver";
url = "jdbc:mysql://" + ds.getDbIp() + ":" + ds.getDbPort() + "/" + ds.getDbName() + "?characterEncoding=utf8&useSSL=false&serverTimezone=GMT%2B8&autoReconnect=true";
break;
case 2:
className = "oracle.jdbc.driver.OracleDriver";
url = "jdbc:oracle:thin:@//" + ds.getDbIp() + ":" + ds.getDbPort() + "/" + ds.getDbName();
break;
case 3:
className = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
url = "jdbc:sqlserver://" + ds.getDbIp() + ":" + ds.getDbPort() + "/" + ds.getDbName();
break;
default:
throw new BusinessException("不支持的数据库");
}
druidDataSource.setDriverClassName(className);
druidDataSource.setUrl(url);
druidDataSource.setUsername(ds.getUsername());
druidDataSource.setPassword(ds.getPassword());
druidDataSource.init();
jdbcTemplateMap.put(ds.getId(), new JdbcTemplate(druidDataSource));
DatabaseHolder.add(ds.getDbType(), ds.getDbIp(), ds.getDbPort(), ds.getDbName(), ds.getUsername(), ds.getPassword(), ds.getId());
}
DatabaseHolder.set(jdbcTemplateMap);
log.info("成功加载数据库：{}个", jdbcTemplateMap.size());
log.info("成功加载数据库：{}个", datasource.size());
}
}
\ No newline at end of file
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
package com.zq.dataoperation.service;
import cn.hutool.core.date.DateUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.alibaba.druid.pool.DruidPooledConnection;
......@@ -7,6 +8,7 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zq.common.context.ThreadContext;
import com.zq.common.exception.BusinessException;
import com.zq.common.utils.AssertUtils;
import com.zq.common.vo.PageVo;
import com.zq.common.vo.ResultVo;
import com.zq.dataoperation.dao.CommonQuerySettingDao;
......@@ -50,9 +52,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
public ResultVo addDatasource(QueryDb queryDb) {
if (queryDbDao.insert(queryDb) != 1) {
throw new BusinessException("添加失败");
}
AssertUtils.isTrue(queryDbDao.insert(queryDb) == 1, "添加失败，请检查后重试");
DatabaseHolder.add(queryDb.getDbType(), queryDb.getDbIp(), queryDb.getDbPort(), queryDb.getDbName(), queryDb.getUsername(), queryDb.getPassword(), queryDb.getId());
return ResultVo.success();
}
......@@ -61,9 +62,21 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
}
public void add(CommonQuerySetting commonQuerySetting) {
if (!save(commonQuerySetting)) {
throw new BusinessException("添加失败");
AssertUtils.isTrue(save(commonQuerySetting), "添加失败");
}
@Transactional
public void updateDatasource(QueryDb queryDb){
queryDb.setUpdateTime(DateUtil.date());
AssertUtils.isTrue(queryDbService.updateById(queryDb), "操作失败！");
DatabaseHolder.remove(queryDb.getId());
DatabaseHolder.add(queryDb.getDbType(), queryDb.getDbIp(), queryDb.getDbPort(), queryDb.getDbName(), queryDb.getUsername(), queryDb.getPassword(), queryDb.getId());
}
@Transactional
public void deleteDatasource(Integer id){
AssertUtils.isTrue(queryDbService.removeById(id), "操作失败！");
DatabaseHolder.remove(id);
}
public List<Map<String, Object>> run(Map<String, Object> body) {
......@@ -345,7 +358,8 @@ public class CommonQueryService extends ServiceImpl<CommonQuerySettingDao, Commo
// }
return list;
}
@Resource
private QueryDbService queryDbService;
@Transactional
public Object sqlOutput(Map<String, Object> body) throws Exception {
......
dataoperation-server/src/main/java/com/zq/dataoperation/service/DataCleanRecordService.java
......@@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.math.BigDecimal;
import java.util.Date;
import java.util.List;
......
flow-component/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nnjcy-data-model</artifactId>
<groupId>com.zq</groupId>
<version>1.0.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>flow-component</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>com.zq</groupId>
<artifactId>spider-flow-api</artifactId>
</dependency>
<dependency>
<groupId>com.zq</groupId>
<artifactId>common-client</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
flow-component/src/main/java/com/zq/spiderflow/readfile/executor/ReadFileExecutor.java 0 → 100644
package com.zq.spiderflow.readfile.executor;
import cn.hutool.core.io.FileUtil;
import com.alibaba.excel.EasyExcel;
import com.zq.common.utils.AssertUtils;
import com.zq.common.utils.EasyExcelUtil;
import com.zq.spiderflow.context.SpiderContext;
import com.zq.spiderflow.executor.ShapeExecutor;
import com.zq.spiderflow.model.Shape;
import com.zq.spiderflow.model.SpiderNode;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.File;
import java.util.Map;
@Component
public class ReadFileExecutor implements ShapeExecutor {
@Value("${spider.workspace}")
private String workspace;
public static final String FILE_NAME = "fileName";
@Override
public Shape shape() {
Shape shape = new Shape();
shape.setImage("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAAAAXNSR0IArs4c6QAAD+NJREFUeF7tnX2MXUUZxp/3tt1+YWhx6d7dxQ/aaNqKIhCiJVIJlX8IRSMCxWC0GmmwC3tvY/mIH7QRFdhk926RSJtIQSShjWikhETNQiKRxmhQEVojUgFt9yxsKBtx22679zVn9+4CZe+9556ZOefcc56b9K/OvPPO876/nTlz5swI+KMCVKCqAkJtqAAVqK4AAWF2UIEaChAQpgcVICDMASoQTgGOIOF0Y62MKEBAMhJodjOcAvEDsltndRzGXN/9I4cxJ1w3WCuMAvMX4/iiQZzYtxLjuErGw9hIe514ANmts9oP4mwVrAFwHoAPpV3oRPdPcBBl/FqAvYMF/Bkimmh/I3QuckDO6NXOEzk8AOBCALMq/yLsMpuqosDUCPIPFdww1C0DVAqIFJC2fl0jih0AllL8RCswAkXJ68T3sj71igwQf+Q4nsMfBOhMdGrQuSkFxgA85HXg61mGJBpAduus/CB+CcVa5l9TKTAOxe1eAVuz+lwSCSCVqdXjAFqaKj3orK/AKARdXrfszKIc7gFRlXw/fgJgfRYFTkmfR8uCS17tlqdT0p/A3XAOyMot2vL6IuwFcG5gr1gwcQoocBCK1UNFOZA45xw65ByQxXfoqXPn4WUApzrsB01Ho8D+Y0ex6vAtMhJNc/G34hyQju3aWj6CfxKQ+INtyYOB3HxcfmiDjFqyl2gzUQHijyALEq0EnWtEgZ1eN76WhZUtAtJIWrDslAL+W/deryA3pV0SApL2CLvr3ygUN3lFucddE/FbJiDxx6B5PVAMq6BrqCC7mrcTtT0nIGmNbHT9OiCKKweL8kx0TUbXEgGJTus0t7Q/Nx+rD22Q4bR1spkAGVHgzbQFIOr+CHAKJv/5nxpY+wnw9GAHVqdtY2OzALLTK8hXrUUzy4ZUpW0bLpYyHoag1bIUu7xuXJOm5d/mAESwx+uWyy0HM9Pm2kp6tQAP2R5JAPSkafmXgGQVk8lNpN8EcJdlCcag2JSW5V8CYjk7mslcZSPpvdZ3Wqdo+ZeANFNGO/C1Y7suKB/Bo8DEARo2fwfKgi81+xZ5AmIzJZrUVmXHtf9JwgrLXXhGFNcOFmW/ZbuRmSMgkUmd7Iba+/TcsuBRB2cGDBw7iiuadYs8AUl23kbqXVuvrhXBfU6WfwuyLtLOWGqMgFgSMi1m8v26HoofOfg8oSmXfwlIWjLbVj/85d8SboPgu7B9bpqiq9mWfwmIrcRKkZ2J5d/F6IVio+VujWkZXxjaJHss23VmjoA4k7a5DVdWth6xvfzrH/6ggquaZfmXgDR3Hjv1vnKO8m+zvPxLQJymWPMbb+vTpSLwv/WwfSrNwOwyvvyfTXIwySoRkCRHJyG+TSz/5vCwg5Wtnbn56EryCSkEJCFJmHQ38n26ETKxsdH26TQ9XjduTuoWeQKS9MxMkH/5kvqAbHKwRf4mryA9CerqtCsEJIlRSahPld2/PwVwtWUXVYFrknj4AwGxHOm0m3O1/AtgTBSrknb4AwFJe0Y76F9lZesx28u/STwgm4A4SKAsmPR3/6rgCdvLv/7hDzIfn03KCSkEJAvZ7KiPS/r1gpziSQcXI+08dhTFJGyRJyCOkicrZivLv3db39gI9Jz2Br69b4v4dyXG9iMgsUmfkob9+ycP4YfAxAEQdvMpAbt/7XZohphX7gcxu/6Ax/4kmqY0L/8SkESnXvM4V1n+9R/abV+1N1IWXBrX7l8C0jw5mHhPK8u//u7fpVadVQwLsDqOwx8IiNVI0pir5V8AsdyPSECY09YVcHasqWBPbh7WRbn7l4BYTw8a9BWoHP5wn3U1BPd43dJl3W4VgwQkKqWz1o6//HsQ34HgNstdVxVcMtQtA5btzmiOgEShckbbqKxs9Vk/+xfYP7uMS6L4GpGAZDR5o+p25T2Y/zWi3bN/I3qJSECiypQMt+No+TeS/VoEJMOJG2XXK8u/T1n7ZFewZ/Y4rnc9zSIgUWZJxtuqHP7wK0t7tkYqH1g5PTmegGQ8aaPufr6kmy3dauUDcrHrLxAJSNQZkvX2Jq9+uxOAD4rJj4BMq8fdvCaJlLi6lRMb/VutTDY2EhACkrjctuZQvqT+0q/J6SgEhIBYy8fEGcqX1N+Gst7AMQJCQGqnT2uftrcIlpUVsw0SrW7VHHD8eAtefG2jeHULByxAQCpC8YvCgBkTsFhHST9VBr4P4DwACwNWs1VsBIJny8Atph8wERACYispJ+y09ehCmTNxNu71lt4jmPin/jVtegK3Dm2W/4UxREAISJi8qVonX1L/eJ2LrBo1N/aEV5BQe6wICAExT7+KhXxJ/e8f/KN1kvdTfMMryo8bdYyAEJBGc2bG8ku26bJcGc8BmGfFoH0jR2U2lg92iX9yTeAfASEggZOlVsG2fr1dFN+yYsyVEcEWr1u2NmKegBCQRvKlatn2kj6kwBetGHNlRPGAV5SvNGKegBCQRvKlatl8SfcC+KQVY+6M/M4ryKcbMU9ACEgj+VILkH22rxmw4tg7jezzCvKRRuwSEALSSL4QkMbV4laTac0yvJs3X1KOIDPDQ0AICEBAqg4tBISAEJAaMy8CQkAICAEJ8nDGZ5AVQWSKsQxXscKKz+3uYZWbrMdnED6D1M+gbI8gzwNYWV+kqiVegtb9cu9+CD5g0MbzXkHOaqQ+34PwPUgj+eLyPUjd6Y+FUapuGyd3kIAQEAJSQwECQkAICAGpnwNZfEjv2KbnjI9jeU6wTE0PVBBsBNBaX+mqJepOfzjFqq4uT1Y0yLx3zav79HoIehP2cRMBMYgxATEQb6qqf1FMyzzcL8DnLJizbYKAGChKQAzEm6qa0AMVptwjIAYxJiAG4vlVE32gwmTfCIhBjAmIgXhNcKACFHhhqCAfrtVNPqTzId0Ag+pV2/v1B6q41Ylxe0Zf8wqyhICEE5QjSDjdJmrlS7obwJUGJpxXVcXIUFEWEZBwUhOQcLpNAfKnyhm5BlYcV1Uc8YqygICE05mAhNNtCpBm+Bz2mFeQmofO8RmEzyAGGFSvaiGxnPh1klECYqAyRxAD8QjItHh1l5JPlpmbFSuKpHkvFgEhIAZ/XyerEhBjCU0NcIploCCnWAbicQThCGKQPhxBjMWzY4AjiIGOHEEMxOMIwhHEIH0yMYI4P1BBBb8RYI5BIDiCGIjHEcRAPAsjSN3l0XxJjwKYa+AmATEQj4AYiEdAOMUySJ9MTLFMt5pwBAmfYTybd1o7SwfH5fv0IhGcU1YsF8EymB+ocD6AmhsB68SfgBAQbS0fgX8DavhEMgTk9Hv0lNxxPJjAb8YJCAGJF5D8Nl2JMh4DcGb4WDirSUDCS8splukUq61HF8oc+M8J7w8fB6c1CUh4eQmIMSAlvVeADeFj4LwmAQkvMQExAcR/7ph1HP8Nr38kNf/uFaTm3R98D1I1DgTEBJC2kl4mwJ5I0jx8I694Bal57QABISD10yvEKla+pHcB2FzfeKwlhr2CnF7LAwJCQOpnaAhA2kv6ewUuqG881hKjXkEWEpBQMeAUy2SKZWEbSKioNVgpyD4p7sWaWVQCQkAmzt4iIASkzt/dEFMsjiDTmgYZpZzvKTs5wjy0oaJIXN+kExAC0uB0d8biqd3uTkAICAGpoQABISAEhIAEyQE+gwRRqUoZTrEMxLNQNUjychWLq1iJXMVKykZCAkJACEg1BfgepGpu8EVhzC8KOYIEn0PW1YrvQaqI2cTvQeoGPaK/7pxicYrFKRanWMGHq0pJTrE4xeJerBrYEBACQkAISJCRNZ7NinwGCRKbyTJ1teJDOh/Sg6fTWyX5ojCMapN1OMXiFItTLE6xgvwF4RSLp7u/O084gnAE4QjCESS5I8goFH+s6Z7gQgC5IF2oUqYMxVOxt5HD+VCDs5OB572CnNWIDvyisKJWHG/SW+/U98yei5cAnNZI0Fg2pAKK10+M4YPDN0vgg/oISEyALOnTVTnBzwAsDRluVgunwIGy4tpXi7I3SHUCEgMglZPc/SkLR44gWWq/zOvI4ULvRvEPgaj5IyBxAFLSAQAX1wsO/9+pAk94BVlTrwUCEjEg7X36eRU8Ui8w/H/3CojiisGi/KJWSwQkYkDyJX0YwNXuw88WAiiwyyvIOgISQKmoVrHyJX0FwPsCuMQi7hX4t1eQmpcWcQSJfgQZN3wn4T5tstNC2SvILI4gAQIe4Qhi+uVdgN6wSEAFgmzCvA/A+oD2ZirGrSbTqgTYi5Xv12eh+KiB4KxqSwHB37xu+RhHkACCRjWCtJW0V4BiAJdYxLECCvQNFWQTAQkgdFSALNmmZ+fK+EsAl1jEsQLlHD7+6o3yVwISQOioAPFd4SgSICCOiwQZPXwXuIoV8SrWVNzzJfUv7rzMcR7Q/MwKPOYVZG0QcQhITIBMjCR9ul0E1wUJFMvYUUAVO4aKEvhOegISIyATkPTrGinjOgguBXCKnTSglZMUeBOKxzWHHUPd4u+DC/wjIDED8vZItfXomTLrrQ+CRDB3HFiRA+YEjmiDBUUwZ1zRmcPEdylOflG04TuuOYzJON7x0K3jGB3aLP8K2zECkiBAwgaR9dwpQEAIiLvsSoFlAkJAUpDG7rpAQAiIu+xKgWUCQkBSkMbuukBACIi77EqBZQJCQFKQxu66QEAIiLvsSoFlAkJAUpDG7rpAQAiIu+xKgWUCQkBSkMbuukBACIi77EqBZQJCQFKQxu66QEAIiLvsSoFlAkJAUpDG7rpAQAiIu+xKgWUCQkBSkMbuukBACIi77EqBZQJCQFKQxu66QEBsAgIM5OZj3aENMuwuZLQclQJn9GrneA67FbjAoE2ezfs28cYAPAfgMASjBqKyahIUUHQCOBtAzRPg67hKQJIQS/qQWAUISGJDQ8eSoAABSUIU6ENiFSAgiQ0NHUuCAgQkCVGgD4lVgIAkNjR0LAkKpAOQxXfoqXPn4RDw1tm4SVCXPjS9Aj4gqwaLst9lT8Slcd92x3ZdUB7FyxC0um6L9jOlwDOquHKoKAdc9to5ICu3aMvhRXjS8K2pSw1ouzkV2JWbjy7XuyucAwJVyW/D3VBsbM440OtEKqDY6hVli2vf3APiX2bTq2slh58DaHHdIdrPhgIq+Eyjl/aEUSYSQDjNChMa1qmhwMBpb+DSfVvE36Pn9BcJIH4P2vt0hcrENc0cRZyGNOXGFcMKfML1w/mUipEB4jc4cW+g4m4AK1IeRnbPjQL7VXBDFFOrWACpLPu2lkfRBWAjl37dZFEKrR6A4kGvgK0Q0Sj7F+kIMt0xVenYgffqKN5fVnRKDsuh/M4jysA3Q1sqGM4pXjh6DC8evkVG4vA5HkDi6CnbpAIhFCAgIURjlewoQECyE2v2NIQCBCSEaKySHQUISHZizZ6GUICAhBCNVbKjAAHJTqzZ0xAKEJAQorFKdhT4P+mlhYywZtelAAAAAElFTkSuQmCC");
shape.setLabel("Excel文件");
shape.setName("Excel");
shape.setTitle("Excel文件");
return shape;
}
@Override
public String supportShape() {
return "Excel";
}
@Override
public void execute(SpiderNode node, SpiderContext context, Map<String, Object> variables) throws Exception {
String fileName = node.getStringJsonValue(FILE_NAME);
String suffix = FileUtil.extName(fileName);
AssertUtils.isTrue(suffix.equals("xls") || suffix.equals("xlsx") || suffix.equals("et"), "不支持的文件类型");
variables.put(node.getNodeId() + "_" + node.getNodeName(), EasyExcelUtil.readAll(new File(workspace, context.getFlowId() + File.separator + "file" + File.separator + fileName)));
}
}
\ No newline at end of file
pom.xml
......@@ -81,6 +81,12 @@
<artifactId>spider-flow-selenium</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.zq</groupId>
<artifactId>flow-component</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
</dependencyManagement>
......@@ -107,8 +113,9 @@
<module>spider-flow-core</module>
<module>spider-flow-web</module>
<module>spider-flow-selenium</module>
<module>dataoperation-server</module>
<module>data-collect-server</module>
<module>dataoperation-server</module>
<module>flow-component</module>
</modules>
<profiles>
......
spider-flow-api/src/main/java/com/zq/spiderflow/context/SpiderContext.java
......@@ -86,6 +86,10 @@ public class SpiderContext extends HashMap<String, Object>{
}
public void writeResult(SpiderOutput output){
}
public SubThreadPoolExecutor getThreadPool() {
return threadPool;
}
......
spider-flow-api/src/main/java/com/zq/spiderflow/executor/ShapeExecutor.java
package com.zq.spiderflow.executor;
import java.sql.SQLException;
import java.util.Map;
import com.zq.spiderflow.context.SpiderContext;
......@@ -36,7 +35,7 @@ public interface ShapeExecutor {
* @param context 爬虫上下文
* @param variables 节点流程的全部变量的集合
*/
void execute(SpiderNode node, SpiderContext context, Map<String, Object> variables) throws SQLException;
void execute(SpiderNode node, SpiderContext context, Map<String, Object> variables) throws Exception;
default boolean allowExecuteNext(SpiderNode node, SpiderContext context, Map<String, Object> variables){
return true;
......
spider-flow-web/pom.xml
......@@ -99,6 +99,12 @@
<artifactId>springfox-swagger-ui</artifactId>
<version>${swagger.version}</version>
</dependency>
<dependency>
<groupId>com.zq</groupId>
<artifactId>flow-component</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
<build>
......
spider-flow-web/src/main/java/com/zq/spiderflow/controller/ModelController.java 0 → 100644
package com.zq.spiderflow.controller;
import cn.hutool.core.util.StrUtil;
import com.zq.common.vo.ResultVo;
import com.zq.spiderflow.service.ModelService;
import com.zq.spiderflow.util.AssertUtils;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
@RestController
@RequestMapping("/spiderflow/model")
public class ModelController {
@Resource
private ModelService modelService;
@PostMapping("/test/file")
public ResultVo uploadTestFile(@RequestParam("flowId") String flowId, MultipartFile file) throws Exception {
AssertUtils.isTrue(StrUtil.isNotBlank(flowId), "流程id不能为空");
return ResultVo.success(modelService.uploadTestFile(flowId, file));
}
}
\ No newline at end of file
spider-flow-web/src/main/java/com/zq/spiderflow/model/SpiderWebSocketContext.java
......@@ -35,6 +35,10 @@ public class SpiderWebSocketContext extends SpiderContext {
this.debug = debug;
}
public void writeResult(SpiderOutput output){
this.write(new WebSocketEvent<>("result", output));
}
@Override
public void addOutput(SpiderOutput output) {
this.write(new WebSocketEvent<>("output", output));
......
spider-flow-web/src/main/java/com/zq/spiderflow/service/ModelService.java 0 → 100644
package com.zq.spiderflow.service;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.List;
import java.util.Map;
public interface ModelService {
Map<String, Object> uploadTestFile(String flowId, MultipartFile file) throws Exception;
}
\ No newline at end of file
spider-flow-web/src/main/java/com/zq/spiderflow/service/impl/ModelServiceImpl.java 0 → 100644
package com.zq.spiderflow.service.impl;
import cn.hutool.core.io.FileUtil;
import com.zq.common.utils.EasyExcelUtil;
import com.zq.spiderflow.service.ModelService;
import com.zq.spiderflow.util.AssertUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.util.*;
@Service
public class ModelServiceImpl implements ModelService {
@Value("${spider.workspace}")
private String workspace;
@Override
public Map<String, Object> uploadTestFile(String flowId, MultipartFile multipartFile) throws Exception {
String suffix = FileUtil.extName(multipartFile.getOriginalFilename());
AssertUtils.isTrue(suffix.equals("xls") || suffix.equals("xlsx") || suffix.equals("et"), "不支持的文件类型");
File file = new File(workspace, flowId + File.separator + "file" + File.separator + multipartFile.getOriginalFilename());
if(file.exists()){
FileUtil.del(file);
}else {
file.mkdirs();
}
multipartFile.transferTo(file);
return EasyExcelUtil.preview100(file);
}
}
\ No newline at end of file
spider-flow-web/src/main/java/com/zq/spiderflow/websocket/WebSocketEditorServer.java
package com.zq.spiderflow.websocket;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zq.spiderflow.core.Spider;
import com.zq.spiderflow.core.utils.SpiderFlowUtils;
import com.zq.spiderflow.model.SpiderWebSocketContext;
import com.zq.spiderflow.model.WebSocketEvent;
import com.zq.spiderflow.util.AssertUtils;
import org.springframework.stereotype.Component;
import javax.websocket.OnClose;
......@@ -35,11 +37,15 @@ public class WebSocketEditorServer {
context = new SpiderWebSocketContext(session);
context.setDebug(isDebug);
context.setRunning(true);
context.setFlowId(event.getString("flowId"));
String flowId = event.getString("flowId");
if(StrUtil.isBlank(flowId)){
context.write(new WebSocketEvent<>("error", "flowId不能为空！"));
}
context.setFlowId(flowId);
// new Thread(() -> {
String xml = event.getString("message");
if (xml != null) {
spider.runWithTest(SpiderFlowUtils.loadXMLFromString(xml), context);
spider. runWithTest(SpiderFlowUtils.loadXMLFromString(xml), context);
context.write(new WebSocketEvent<>("finish", null));
} else {
context.write(new WebSocketEvent<>("error", "xml不正确！"));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment