Skip to content

N
nnjcy-data-model
Commit 69a9202d by wqc
Options

org包名换zq包名，服务依赖互用问题

parent b4e75904
Showing with 708 additions and 726 deletions
dataoperation-server/pom.xml
......@@ -117,6 +117,7 @@
<version>1.0.0</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
......
dataoperation-server/src/main/java/com/zq/dataoperation/DataoperationApplication.java
......@@ -7,9 +7,9 @@ import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.scheduling.annotation.EnableScheduling;
@EnableScheduling
@MapperScan("com.zq.dataoperation.dao")
@MapperScan({"com.zq.dataoperation.dao","com.zq.spiderflow.dao"})
@EnableDiscoveryClient
@SpringBootApplication(scanBasePackages = {"com.zq.dataoperation", "com.zq.common.config"})
@SpringBootApplication//(scanBasePackages = {"com.zq.dataoperation", "com.zq.common.config","com.zq.spiderflow"})
public class DataoperationApplication {
public static void main(String[] args) {
......
dataoperation-server/src/main/java/com/zq/dataoperation/service/CommonQueryService.java
......@@ -15,14 +15,13 @@ import com.zq.dataoperation.dao.QueryDbDao;
import com.zq.dataoperation.entity.CommonQuerySetting;
import com.zq.dataoperation.entity.QueryDb;
import com.zq.dataoperation.utils.ConnectionUtil;
import org.spiderflow.dao.MetaDataMappingDao;
import org.spiderflow.entity.MetaDataMapping;
import zq.spiderflow.dao.MetaDataMappingDao;
import zq.spiderflow.entity.MetaDataMapping;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Wrapper;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......
spider-flow-api/src/main/java/org/spiderflow/Grammerable.java → spider-flow-api/src/main/java/zq/spiderflow/Grammerable.java
package org.spiderflow;
package zq.spiderflow;
import java.util.List;
import org.spiderflow.model.Grammer;
import zq.spiderflow.model.Grammer;
public interface Grammerable {
List<Grammer> grammers();
}
spider-flow-api/src/main/java/org/spiderflow/annotation/Comment.java → spider-flow-api/src/main/java/zq/spiderflow/annotation/Comment.java
package org.spiderflow.annotation;
package zq.spiderflow.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
......
spider-flow-api/src/main/java/org/spiderflow/annotation/Example.java → spider-flow-api/src/main/java/zq/spiderflow/annotation/Example.java
package org.spiderflow.annotation;
package zq.spiderflow.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
......
spider-flow-api/src/main/java/org/spiderflow/annotation/Return.java → spider-flow-api/src/main/java/zq/spiderflow/annotation/Return.java
package org.spiderflow.annotation;
package zq.spiderflow.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
......
spider-flow-api/src/main/java/org/spiderflow/common/CURDController.java → spider-flow-api/src/main/java/zq/spiderflow/common/CURDController.java
package org.spiderflow.common;
package zq.spiderflow.common;
import org.spiderflow.model.JsonBean;
import zq.spiderflow.model.JsonBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
......@@ -12,28 +12,28 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
public abstract class CURDController<S extends ServiceImpl<M, T>,M extends BaseMapper<T>, T> {
@Autowired
private S service;
@RequestMapping("/list")
public IPage<T> list(@RequestParam(name = "page",defaultValue = "1")Integer page, @RequestParam(name = "limit",defaultValue = "1")Integer size){
return service.page(new Page<T>(page, size), new QueryWrapper<T>().orderByDesc("create_date"));
}
@RequestMapping("get")
public JsonBean<T> get(String id) {
return new JsonBean<T>(service.getById(id));
}
@RequestMapping("delete")
public JsonBean<Boolean> delete(String id){
return new JsonBean<Boolean>(service.removeById(id));
}
@RequestMapping("save")
public JsonBean<Boolean> save(T t){
return new JsonBean<Boolean>(service.saveOrUpdate(t));
}
}
\ No newline at end of file
}
spider-flow-api/src/main/java/org/spiderflow/concurrent/ChildPriorThreadSubmitStrategy.java → spider-flow-api/src/main/java/zq/spiderflow/concurrent/ChildPriorThreadSubmitStrategy.java
package org.spiderflow.concurrent;
package zq.spiderflow.concurrent;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.model.SpiderNode;
import java.util.Comparator;
import java.util.PriorityQueue;
......
spider-flow-api/src/main/java/org/spiderflow/concurrent/LinkedThreadSubmitStrategy.java → spider-flow-api/src/main/java/zq/spiderflow/concurrent/LinkedThreadSubmitStrategy.java
package org.spiderflow.concurrent;
package zq.spiderflow.concurrent;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.model.SpiderNode;
import java.util.Comparator;
import java.util.List;
......
spider-flow-api/src/main/java/org/spiderflow/concurrent/ParentPriorThreadSubmitStrategy.java → spider-flow-api/src/main/java/zq/spiderflow/concurrent/ParentPriorThreadSubmitStrategy.java
package org.spiderflow.concurrent;
package zq.spiderflow.concurrent;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.model.SpiderNode;
import java.util.Comparator;
import java.util.PriorityQueue;
......
spider-flow-api/src/main/java/org/spiderflow/concurrent/RandomThreadSubmitStrategy.java → spider-flow-api/src/main/java/zq/spiderflow/concurrent/RandomThreadSubmitStrategy.java
package org.spiderflow.concurrent;
package zq.spiderflow.concurrent;
import org.apache.commons.lang3.RandomUtils;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.model.SpiderNode;
import java.util.Comparator;
import java.util.List;
......
spider-flow-api/src/main/java/org/spiderflow/concurrent/SpiderFlowThreadPoolExecutor.java → spider-flow-api/src/main/java/zq/spiderflow/concurrent/SpiderFlowThreadPoolExecutor.java
package org.spiderflow.concurrent;
package zq.spiderflow.concurrent;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.model.SpiderNode;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
......@@ -94,7 +94,7 @@ public class SpiderFlowThreadPoolExecutor {
this.futures = new Future[threads];
this.submitStrategy = submitStrategy;
}
/**
* 等待所有线程执行完毕
*/
......@@ -108,7 +108,7 @@ public class SpiderFlowThreadPoolExecutor {
submitStrategy.notifyAll();
}
}
private int index(){
for (int i = 0; i < threads; i++) {
if(futures[i] == null || futures[i].isDone()){
......@@ -129,7 +129,7 @@ public class SpiderFlowThreadPoolExecutor {
}
} catch (Throwable t) {
//忽略异常
}
}
}
}
......
spider-flow-api/src/main/java/org/spiderflow/concurrent/SpiderFutureTask.java → spider-flow-api/src/main/java/zq/spiderflow/concurrent/SpiderFutureTask.java
package org.spiderflow.concurrent;
package zq.spiderflow.concurrent;
import java.util.concurrent.FutureTask;
import org.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor;
import zq.spiderflow.model.SpiderNode;
public class SpiderFutureTask<V> extends FutureTask {
......
spider-flow-api/src/main/java/org/spiderflow/concurrent/ThreadSubmitStrategy.java → spider-flow-api/src/main/java/zq/spiderflow/concurrent/ThreadSubmitStrategy.java
package org.spiderflow.concurrent;
package zq.spiderflow.concurrent;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.model.SpiderNode;
import java.util.Comparator;
......
spider-flow-api/src/main/java/org/spiderflow/context/SpiderContext.java → spider-flow-api/src/main/java/zq/spiderflow/context/SpiderContext.java
package org.spiderflow.context;
package zq.spiderflow.context;
import org.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor;
import org.spiderflow.model.SpiderNode;
import org.spiderflow.model.SpiderOutput;
import zq.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor;
import zq.spiderflow.model.SpiderNode;
import zq.spiderflow.model.SpiderOutput;
import java.util.*;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.locks.ReentrantLock;
/**
* 爬虫上下文
......@@ -15,14 +14,14 @@ import java.util.concurrent.locks.ReentrantLock;
*
*/
public class SpiderContext extends HashMap<String, Object>{
private String id = UUID.randomUUID().toString().replace("-", "");
/**
* 流程ID
*/
private String flowId;
private static final long serialVersionUID = 8379177178417619790L;
/**
......@@ -102,11 +101,11 @@ public class SpiderContext extends HashMap<String, Object>{
public void setRootNode(SpiderNode rootNode) {
this.rootNode = rootNode;
}
public String getId() {
return id;
}
public CookieContext getCookieContext() {
return cookieContext;
}
......
spider-flow-api/src/main/java/org/spiderflow/enums/FlowNoticeType.java → spider-flow-api/src/main/java/zq/spiderflow/enums/FlowNoticeType.java
package org.spiderflow.enums;
package zq.spiderflow.enums;
/**
* 流程通知类型
*
*
* @author BillDowney
* @date 2020年4月4日 上午1:32:53
*/
......
spider-flow-api/src/main/java/org/spiderflow/enums/FlowNoticeWay.java → spider-flow-api/src/main/java/zq/spiderflow/enums/FlowNoticeWay.java
package org.spiderflow.enums;
package zq.spiderflow.enums;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* 流程通知方式
*
*
* @author BillDowney
* @date 2020年4月3日 下午3:26:18
*/
......
spider-flow-api/src/main/java/org/spiderflow/executor/FunctionExecutor.java → spider-flow-api/src/main/java/zq/spiderflow/executor/FunctionExecutor.java
package org.spiderflow.executor;
package zq.spiderflow.executor;
public interface FunctionExecutor {
String getFunctionPrefix();
}
spider-flow-api/src/main/java/org/spiderflow/executor/FunctionExtension.java → spider-flow-api/src/main/java/zq/spiderflow/executor/FunctionExtension.java
package org.spiderflow.executor;
package zq.spiderflow.executor;
public interface FunctionExtension {
Class<?> support();
}
spider-flow-api/src/main/java/org/spiderflow/executor/PluginConfig.java → spider-flow-api/src/main/java/zq/spiderflow/executor/PluginConfig.java
package org.spiderflow.executor;
package zq.spiderflow.executor;
import org.spiderflow.model.Plugin;
import zq.spiderflow.model.Plugin;
public interface PluginConfig {
Plugin plugin();
}
\ No newline at end of file
}
spider-flow-api/src/main/java/org/spiderflow/executor/ShapeExecutor.java → spider-flow-api/src/main/java/zq/spiderflow/executor/ShapeExecutor.java
package org.spiderflow.executor;
package zq.spiderflow.executor;
import java.util.Map;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.model.Shape;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.model.Shape;
import zq.spiderflow.model.SpiderNode;
/**
* 执行器接口
* @author jmxd
*
*/
public interface ShapeExecutor {
String LOOP_VARIABLE_NAME = "loopVariableName";
String LOOP_COUNT = "loopCount";
String THREAD_COUNT = "threadCount";
default Shape shape(){
return null;
}
/**
* 节点形状
* @return 节点形状名称
*/
String supportShape();
/**
* 执行器具体的功能实现
* @param node 当前要执行的爬虫节点
......@@ -35,11 +36,11 @@ public interface ShapeExecutor {
* @param variables 节点流程的全部变量的集合
*/
void execute(SpiderNode node, SpiderContext context, Map<String, Object> variables);
default boolean allowExecuteNext(SpiderNode node, SpiderContext context, Map<String, Object> variables){
return true;
}
default boolean isThread(){
return false;
}
......
spider-flow-api/src/main/java/org/spiderflow/expression/DynamicMethod.java → spider-flow-api/src/main/java/zq/spiderflow/expression/DynamicMethod.java
package org.spiderflow.expression;
package zq.spiderflow.expression;
import java.util.List;
public interface DynamicMethod {
Object execute(String methodName, List<Object> parameters);
}
spider-flow-api/src/main/java/org/spiderflow/io/SpiderResponse.java → spider-flow-api/src/main/java/zq/spiderflow/io/SpiderResponse.java
package org.spiderflow.io;
package zq.spiderflow.io;
import java.io.InputStream;
import java.util.Map;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import com.alibaba.fastjson.JSON;
......
spider-flow-api/src/main/java/org/spiderflow/listener/SpiderListener.java → spider-flow-api/src/main/java/zq/spiderflow/listener/SpiderListener.java
package org.spiderflow.listener;
package zq.spiderflow.listener;
import org.spiderflow.context.SpiderContext;
import zq.spiderflow.context.SpiderContext;
public interface SpiderListener {
......@@ -8,10 +8,10 @@ public interface SpiderListener {
* 开始执行之前
*/
void beforeStart(SpiderContext context);
/**
* 执行完毕之后
*/
void afterEnd(SpiderContext context);
}
spider-flow-api/src/main/java/org/spiderflow/model/Grammer.java → spider-flow-api/src/main/java/zq/spiderflow/model/Grammer.java
package org.spiderflow.model;
package zq.spiderflow.model;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
......@@ -6,24 +6,24 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.annotation.Return;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.annotation.Return;
public class Grammer {
private String owner;
private String method;
private String comment;
private String example;
private String function;
private List<String> returns;
public String getOwner() {
return owner;
}
......@@ -39,7 +39,7 @@ public class Grammer {
public void setMethod(String method) {
this.method = method;
}
public String getFunction() {
return function;
}
......@@ -67,11 +67,11 @@ public class Grammer {
public List<String> getReturns() {
return returns;
}
public void setReturns(List<String> returns) {
this.returns = returns;
}
public static List<Grammer> findGrammers(Class<?> clazz,String function,String owner,boolean mustStatic){
Method[] methods = clazz.getDeclaredMethods();
List<Grammer> grammers = new ArrayList<>();
......
spider-flow-api/src/main/java/org/spiderflow/model/JsonBean.java → spider-flow-api/src/main/java/zq/spiderflow/model/JsonBean.java
package org.spiderflow.model;
package zq.spiderflow.model;
public class JsonBean<T> {
private Integer code = 1;
private String message = "执行成功";
private T data;
public JsonBean(Integer code, String message, T data) {
......
spider-flow-api/src/main/java/org/spiderflow/model/Plugin.java → spider-flow-api/src/main/java/zq/spiderflow/model/Plugin.java
package org.spiderflow.model;
package zq.spiderflow.model;
public class Plugin {
private String name;
private String url;
public Plugin(String name, String url) {
this.name = name;
this.url = url;
......@@ -26,5 +26,5 @@ public class Plugin {
public void setUrl(String url) {
this.url = url;
}
}
spider-flow-api/src/main/java/org/spiderflow/model/Shape.java → spider-flow-api/src/main/java/zq/spiderflow/model/Shape.java
package org.spiderflow.model;
package zq.spiderflow.model;
public class Shape {
private String name;
private String label;
private String title;
private String image;
private String desc;
......
spider-flow-api/src/main/java/org/spiderflow/model/SpiderLog.java → spider-flow-api/src/main/java/zq/spiderflow/model/SpiderLog.java
package org.spiderflow.model;
package zq.spiderflow.model;
import java.util.ArrayList;
import java.util.List;
......@@ -6,11 +6,11 @@ import java.util.List;
import org.apache.commons.lang3.exception.ExceptionUtils;
public class SpiderLog {
private String level;
private String message;
private List<Object> variables;
public SpiderLog(String level,String message, List<Object> variables) {
......
spider-flow-api/src/main/java/org/spiderflow/model/SpiderNode.java → spider-flow-api/src/main/java/zq/spiderflow/model/SpiderNode.java
package org.spiderflow.model;
package zq.spiderflow.model;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
......@@ -90,7 +90,7 @@ public class SpiderNode {
String value = getStringJsonValue(key);
return StringUtils.isNotBlank(value) ? value : defaultValue;
}
public List<Map<String,String>> getListJsonValue(String ... keys){
List<JSONArray> arrays = new ArrayList<>();
int size = -1;
......
spider-flow-api/src/main/java/org/spiderflow/model/SpiderOutput.java → spider-flow-api/src/main/java/zq/spiderflow/model/SpiderOutput.java
package org.spiderflow.model;
package zq.spiderflow.model;
import java.util.ArrayList;
import java.util.List;
public class SpiderOutput {
/**
* 节点名称
*/
private String nodeName;
/**
* 节点Id
*/
private String nodeId;
/**
* 输出项的名
*/
private List<String> outputNames = new ArrayList<>();
/**
* 输出项的值
*/
......@@ -48,7 +48,7 @@ public class SpiderOutput {
public void setValues(List<Object> values) {
this.values = values;
}
public void addOutput(String name,Object value){
this.outputNames.add(name);
this.values.add(value);
......
spider-flow-api/src/main/java/org/spiderflow/utils/Maps.java → spider-flow-api/src/main/java/zq/spiderflow/utils/Maps.java
package org.spiderflow.utils;
package zq.spiderflow.utils;
import java.util.HashMap;
import java.util.List;
......@@ -11,13 +11,13 @@ public class Maps {
destMap.put(k, v);
return destMap;
}
public static <K,V> Map<K,V> newMap(K key,V value){
HashMap<K, V> map = new HashMap<>();
map.put(key, value);
return map;
}
public static <K,V> Map<K,V> add(Map<K,V> srcMap,List<K> ks,List<V> vs){
HashMap<K, V> destMap = new HashMap<>(srcMap);
if(ks != null && vs != null && ks.size() == vs.size()){
......
spider-flow-core/src/main/java/org/spiderflow/core/Spider.java → spider-flow-core/src/main/java/zq/spiderflow/core/Spider.java
package org.spiderflow.core;
package zq.spiderflow.core;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
......@@ -9,25 +9,24 @@ import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.concurrent.*;
import org.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.context.SpiderContextHolder;
import org.spiderflow.core.exception.RpaException;
import org.spiderflow.core.executor.shape.LoopExecutor;
import org.spiderflow.core.model.SpiderFlow;
import org.spiderflow.core.service.FlowNoticeService;
import org.spiderflow.core.utils.ExecutorsUtils;
import org.spiderflow.core.utils.ExpressionUtils;
import org.spiderflow.core.utils.SpiderFlowUtils;
import org.spiderflow.enums.FlowNoticeType;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.listener.SpiderListener;
import org.spiderflow.model.SpiderNode;
import org.spiderflow.model.SpiderOutput;
import zq.spiderflow.concurrent.*;
import zq.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.context.SpiderContextHolder;
import zq.spiderflow.core.exception.RpaException;
import zq.spiderflow.core.executor.shape.LoopExecutor;
import zq.spiderflow.core.model.SpiderFlow;
import zq.spiderflow.core.service.FlowNoticeService;
import zq.spiderflow.core.utils.ExecutorsUtils;
import zq.spiderflow.core.utils.ExpressionUtils;
import zq.spiderflow.core.utils.SpiderFlowUtils;
import zq.spiderflow.enums.FlowNoticeType;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.listener.SpiderListener;
import zq.spiderflow.model.SpiderNode;
import zq.spiderflow.model.SpiderOutput;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
......@@ -60,7 +59,7 @@ public class Spider {
@Value("${spider.detect.dead-cycle:5000}")
private Integer deadCycle;
@Autowired
private FlowNoticeService flowNoticeService;
......
spider-flow-core/src/main/java/org/spiderflow/core/exception/BusinessException.java → spider-flow-core/src/main/java/zq/spiderflow/core/exception/BusinessException.java
package org.spiderflow.core.exception;
package zq.spiderflow.core.exception;
public class BusinessException extends RuntimeException {
......@@ -21,4 +21,4 @@ public class BusinessException extends RuntimeException {
public int getCode() {
return code;
}
}
\ No newline at end of file
}
spider-flow-core/src/main/java/org/spiderflow/core/exception/RpaException.java → spider-flow-core/src/main/java/zq/spiderflow/core/exception/RpaException.java
package org.spiderflow.core.exception;
package zq.spiderflow.core.exception;
public class RpaException extends RuntimeException{
public RpaException(String message){
super(message);
}
}
\ No newline at end of file
}
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/Base64FunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/Base64FunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import org.apache.commons.codec.binary.Base64;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
/**
......@@ -14,72 +14,72 @@ import org.springframework.stereotype.Component;
@Component
@Comment("base64常用方法")
public class Base64FunctionExecutor implements FunctionExecutor{
@Override
public String getFunctionPrefix() {
return "base64";
}
@Comment("根据byte[]进行base64加密")
@Example("${base64.encode(resp.bytes)}")
public static String encode(byte[] bytes){
return bytes != null ? Base64.encodeBase64String(bytes) : null;
}
@Comment("根据String进行base64加密")
@Example("${base64.encode(resp.bytes,'UTF-8')}")
public static String encode(String content,String charset){
return encode(StringFunctionExecutor.bytes(content,charset));
}
@Comment("根据String进行base64加密")
@Example("${base64.encode(resp.html)}")
public static String encode(String content){
return encode(StringFunctionExecutor.bytes(content));
}
@Comment("根据byte[]进行base64加密")
@Example("${base64.encodeBytes(resp.bytes)}")
public static byte[] encodeBytes(byte[] bytes){
return bytes != null ? Base64.encodeBase64(bytes) : null;
}
@Comment("根据String进行base64加密")
@Example("${base64.encodeBytes(resp.html,'UTF-8')}")
public static byte[] encodeBytes(String content,String charset){
return encodeBytes(StringFunctionExecutor.bytes(content,charset));
}
@Comment("根据String进行base64加密")
@Example("${base64.encodeBytes(resp.html)}")
public static byte[] encodeBytes(String content){
return encodeBytes(StringFunctionExecutor.bytes(content));
}
@Comment("根据String进行base64解密")
@Example("${base64.decode(resp.html)}")
public static byte[] decode(String base64){
return base64 != null ? Base64.decodeBase64(base64) :null;
}
@Comment("根据byte[]进行base64解密")
@Example("${base64.decode(resp.bytes)}")
public static byte[] decode(byte[] base64){
return base64 != null ? Base64.decodeBase64(base64) :null;
}
@Comment("根据String进行base64解密")
@Example("${base64.decodeString(resp.html)}")
public static String decodeString(String base64){
return base64 != null ? new String(Base64.decodeBase64(base64)) :null;
}
@Comment("根据byte[]进行base64解密")
@Example("${base64.decodeString(resp.bytes)}")
public static String decodeString(byte[] base64){
return base64 != null ? new String(Base64.decodeBase64(base64)) :null;
}
@Comment("根据byte[]进行base64解密")
@Example("${base64.decodeString(resp.bytes,'UTF-8')}")
public static String decodeString(byte[] base64,String charset){
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/DateFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/DateFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import java.text.ParseException;
import java.util.Date;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
/**
......@@ -17,13 +17,13 @@ import org.springframework.stereotype.Component;
*/
@Component
@Comment("日期常用方法")
public class DateFunctionExecutor implements FunctionExecutor{
public class DateFunctionExecutor implements FunctionExecutor {
@Override
public String getFunctionPrefix() {
return "date";
}
private static final String DEFAULT_PATTERN = "yyyy-MM-dd HH:mm:ss";
@Comment("格式化日期")
......@@ -49,13 +49,13 @@ public class DateFunctionExecutor implements FunctionExecutor{
public static String format(Long millis, String pattern) {
return millis != null ? DateFormatUtils.format(millis, pattern) : null;
}
@Comment("字符串转为日期类型")
@Example("${date.parse('2019-01-01 00:00:00')}")
public static Date parse(String date) throws ParseException{
return date != null ? DateUtils.parseDate(date, DEFAULT_PATTERN) : null;
}
@Comment("字符串转为日期类型")
@Example("${date.parse('2019-01-01','yyyy-MM-dd')}")
public static Date parse(String date,String pattern) throws ParseException{
......@@ -67,43 +67,43 @@ public class DateFunctionExecutor implements FunctionExecutor{
public static Date parse(Long millis){
return new Date(millis);
}
@Comment("获取当前时间")
@Example("${date.now()}")
public static Date now(){
return new Date();
}
@Comment("获取指定日期n年后的日期")
@Example("${date.addYears(date.now(),2)}")
public static Date addYears(Date date,int amount){
return DateUtils.addYears(date, amount);
}
@Comment("获取指定日期n月后的日期")
@Example("${date.addMonths(date.now(),2)}")
public static Date addMonths(Date date,int amount){
return DateUtils.addMonths(date, amount);
}
@Comment("获取指定日期n天后的日期")
@Example("${date.addDays(date.now(),2)}")
public static Date addDays(Date date,int amount){
return DateUtils.addDays(date, amount);
}
@Comment("获取指定日期n小时后的日期")
@Example("${date.addHours(date.now(),2)}")
public static Date addHours(Date date,int amount){
return DateUtils.addHours(date, amount);
}
@Comment("获取指定日期n分钟后的日期")
@Example("${date.addMinutes(date.now(),2)}")
public static Date addMinutes(Date date,int amount){
return DateUtils.addMinutes(date, amount);
}
@Comment("获取指定日期n秒后的日期")
@Example("${date.addSeconds(date.now(),2)}")
public static Date addSeconds(Date date,int amount){
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/ExceptionFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/ExceptionFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.core.exception.RpaException;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.core.exception.RpaException;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
@Component
......@@ -20,4 +20,4 @@ public class ExceptionFunctionExecutor implements FunctionExecutor {
public static void throwx(String info){
throw new RpaException(info);
}
}
\ No newline at end of file
}
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/ExtractFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/ExtractFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.core.utils.ExtractUtils;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.core.utils.ExtractUtils;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
@Component
......@@ -18,79 +18,79 @@ public class ExtractFunctionExecutor implements FunctionExecutor{
public String getFunctionPrefix() {
return "extract";
}
@Comment("根据jsonpath提取内容")
@Example("${extract.jsonpath(resp.json,'$.code')}")
public static Object jsonpath(Object root,String jsonpath){
return ExtractUtils.getValueByJsonPath(root, jsonpath);
}
@Comment("根据正则表达式提取内容")
@Example("${extract.regx(resp.html,'<title>(.*?)</title>')}")
public static String regx(String content,String pattern){
return ExtractUtils.getFirstMatcher(content, pattern, true);
}
@Comment("根据正则表达式提取内容")
@Example("${extract.regx(resp.html,'<title>(.*?)</title>',1)}")
public static String regx(String content,String pattern,int groupIndex){
return ExtractUtils.getFirstMatcher(content, pattern, groupIndex);
}
@Comment("根据正则表达式提取内容")
@Example("${extract.regx(resp.html,'<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
public static List<String> regx(String content,String pattern,List<Integer> groups){
return ExtractUtils.getFirstMatcher(content, pattern, groups);
}
@Comment("根据正则表达式提取内容")
@Example("${extract.regxs(resp.html,'<h2>(.*?)</h2>')}")
public static List<String> regxs(String content,String pattern){
return ExtractUtils.getMatchers(content, pattern, true);
}
@Comment("根据正则表达式提取内容")
@Example("${extract.regxs(resp.html,'<h2>(.*?)</h2>',1)}")
public static List<String> regxs(String content,String pattern,int groupIndex){
return ExtractUtils.getMatchers(content, pattern, groupIndex);
}
@Comment("根据正则表达式提取内容")
@Example("${extract.regxs(resp.html,'<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
public static List<List<String>> regxs(String content,String pattern,List<Integer> groups){
return ExtractUtils.getMatchers(content, pattern, groups);
}
@Comment("根据xpath提取内容")
@Example("${extract.xpath(resp.element(),'//title/text()')}")
public static String xpath(Element element,String xpath){
return ExtractUtils.getValueByXPath(element, xpath);
}
@Comment("根据xpath提取内容")
@Example("${extract.xpath(resp.html,'//title/text()')}")
public static String xpath(String content,String xpath){
return xpath(Jsoup.parse(content),xpath);
}
@Comment("根据xpaths提取内容")
@Example("${extract.xpaths(resp.element(),'//h2/text()')}")
public static List<String> xpaths(Element element,String xpath){
return ExtractUtils.getValuesByXPath(element, xpath);
}
@Comment("根据xpaths提取内容")
@Example("${extract.xpaths(resp.html,'//h2/text()')}")
public static List<String> xpaths(String content,String xpath){
return xpaths(Jsoup.parse(content),xpath);
}
@Comment("根据css选择器提取内容")
@Example("${extract.selectors(resp.html,'div > a')}")
public static List<String> selectors(Object object,String selector){
return ExtractUtils.getHTMLBySelector(getElement(object), selector);
}
@Comment("根据css选择器提取内容")
@Example("${extract.selector(resp.html,'div > a','text')}")
public static Object selector(Object object,String selector,String type){
......@@ -103,7 +103,7 @@ public class ExtractFunctionExecutor implements FunctionExecutor{
}
return null;
}
@Comment("根据css选择器提取内容")
@Example("${extract.selector(resp.html,'div > a','attr','href')}")
public static String selector(Object object,String selector,String type,String attrValue){
......@@ -112,13 +112,13 @@ public class ExtractFunctionExecutor implements FunctionExecutor{
}
return null;
}
@Comment("根据css选择器提取内容")
@Example("${extract.selector(resp.html,'div > a')}")
public static String selector(Object object,String selector){
return ExtractUtils.getFirstHTMLBySelector(getElement(object), selector);
}
@Comment("根据css选择器提取内容")
@Example("${extract.selectors(resp.html,'div > a','element')}")
public static Object selectors(Object object,String selector,String type){
......@@ -131,7 +131,7 @@ public class ExtractFunctionExecutor implements FunctionExecutor{
}
return null;
}
@Comment("根据css选择器提取内容")
@Example("${extract.selectors(resp.html,'div > a','attr','href')}")
public static Object selectors(Object object,String selector,String type,String attrValue){
......@@ -140,7 +140,7 @@ public class ExtractFunctionExecutor implements FunctionExecutor{
}
return null;
}
private static Element getElement(Object object){
if(object != null){
return object instanceof Element ? (Element)object:Jsoup.parse((String) object);
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/FileFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/FileFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import java.io.*;
import java.nio.charset.Charset;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.core.utils.FileUtils;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.core.utils.FileUtils;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
/**
* 文件读写 工具类 防止NPE
* 文件读写 工具类 防止NPE
* @author Administrator
*
*/
@Component
@Comment("file常用方法")
public class FileFunctionExecutor implements FunctionExecutor{
@Override
public String getFunctionPrefix() {
return "file";
}
/**
*
*
* @param path 文件路径/名
* @param createDirectory 是否需要创建
* @return File 文件
......@@ -45,13 +45,13 @@ public class FileFunctionExecutor implements FunctionExecutor{
public static void write(String path,String content,boolean append) throws IOException{
write(path,content,Charset.defaultCharset().name(),append);
}
@Comment("写出文件")
@Example("${file.write('e:/result.html',resp.html,'UTF-8',false)}")
public static void write(String path,String content,String charset,boolean append) throws IOException{
write(path,StringFunctionExecutor.bytes(content, charset),append);
}
@Comment("写出文件")
@Example("${file.write('e:/result.html',resp.bytes,false)}")
public static void write(String path,byte[] bytes,boolean append) throws IOException{
......@@ -71,19 +71,19 @@ public class FileFunctionExecutor implements FunctionExecutor{
public static void write(String path, InputStream stream) throws IOException {
write(path, stream,false);
}
@Comment("写出文件")
@Example("${file.write('e:/result.html',resp.html)}")
public static void write(String path,String content) throws IOException{
write(path,content,false);
}
@Comment("写出文件")
@Example("${file.write('e:/result.html',resp.html,'UTF-8')}")
public static void write(String path,String content,String charset) throws IOException{
write(path,content,charset,false);
}
@Comment("写出文件")
@Example("${file.write('e:/result.html',resp.bytes)}")
public static void write(String path,byte[] bytes) throws IOException{
......@@ -107,25 +107,25 @@ public class FileFunctionExecutor implements FunctionExecutor{
FileUtils.downloadFile(path, url, true);
}
}
@Comment("读取文件")
@Example("${file.bytes('e:/result.html')}")
public static byte[] bytes(String path) throws IOException{
try(FileInputStream fis = new FileInputStream(getFile(path, false))){
return IOUtils.toByteArray(fis);
return IOUtils.toByteArray(fis);
}
}
@Comment("读取文件")
@Example("${file.string('e:/result.html','UTF-8')}")
public static String string(String path,String charset) throws IOException{
return StringFunctionExecutor.newString(bytes(path), charset);
}
@Comment("读取文件")
@Example("${file.string('e:/result.html')}")
public static String string(String path) throws IOException{
return StringFunctionExecutor.newString(bytes(path), Charset.defaultCharset().name());
}
}
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/JsonFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/JsonFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSON;
/**
* Json和String互相转换 工具类 防止NPE
* Json和String互相转换 工具类 防止NPE
* @author Administrator
*
*/
@Component
@Comment("json常用方法")
public class JsonFunctionExecutor implements FunctionExecutor{
public class JsonFunctionExecutor implements FunctionExecutor {
@Override
public String getFunctionPrefix() {
return "json";
......@@ -26,7 +26,7 @@ public class JsonFunctionExecutor implements FunctionExecutor{
public static Object parse(String jsonString){
return jsonString != null ? JSON.parse(jsonString) : null;
}
@Comment("将对象转为json字符串")
@Example("${json.stringify(objVar)}")
public static String stringify(Object object){
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/ListFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/ListFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
/**
* List 工具类 防止NPE 添加了类似python的split()方法
* List 工具类 防止NPE 添加了类似python的split()方法
* @author Administrator
*
*/
@Component
@Comment("list常用方法")
public class ListFunctionExecutor implements FunctionExecutor{
public class ListFunctionExecutor implements FunctionExecutor {
@Override
public String getFunctionPrefix() {
return "list";
......@@ -28,9 +28,9 @@ public class ListFunctionExecutor implements FunctionExecutor{
public static int length(List<?> list){
return list != null ? list.size() : 0;
}
/**
*
*
* @param list 原List
* @param len 按多长进行分割
* @return List<List<?>> 分割后的数组
......@@ -50,7 +50,7 @@ public class ListFunctionExecutor implements FunctionExecutor{
}
return result;
}
@Comment("截取List")
@Example("${list.sublist(listVar,fromIndex,toIndex)}")
public static List<?> sublist(List<?> list,int fromIndex,int toIndex){
......@@ -71,5 +71,5 @@ public class ListFunctionExecutor implements FunctionExecutor{
}
return result;
}
}
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/LogFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/LogFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import lombok.extern.slf4j.Slf4j;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
@Slf4j
......@@ -33,4 +33,4 @@ public class LogFunctionExecutor implements FunctionExecutor {
public static void debug(String info){
log.debug(info);
}
}
\ No newline at end of file
}
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/MD5FunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/MD5FunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import org.apache.commons.codec.digest.DigestUtils;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
import java.io.IOException;
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/RandomFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/RandomFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import org.apache.commons.lang3.RandomUtils;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
/**
* 随机数/字符串 生成方法
* 随机数/字符串 生成方法
* @author Administrator
*
*/
@Component
public class RandomFunctionExecutor implements FunctionExecutor{
@Override
public String getFunctionPrefix() {
return "random";
}
@Comment("随机获取int")
@Example("${random.randomInt(1,10)}")
public static int randomInt(int min,int max){
return RandomUtils.nextInt(min, max);
}
@Comment("随机获取double")
@Example("${random.randomDouble(1,10)}")
public static double randomDouble(double min,double max){
return RandomUtils.nextDouble(min, max);
}
@Comment("随机获取long")
@Example("${random.randomLong(1,10)}")
public static long randomLong(long min,long max){
return RandomUtils.nextLong(min, max);
}
/**
*
*
* @param chars 字符个数
* @param length 字符范围
* @return String 随机字符串
......@@ -56,5 +56,5 @@ public class RandomFunctionExecutor implements FunctionExecutor{
}
return null;
}
}
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/StringFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/StringFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
......@@ -6,20 +6,20 @@ import java.util.Arrays;
import java.util.List;
import java.util.UUID;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
/**
* String 工具类 防止NPE
* String 工具类 防止NPE
* @author Administrator
*
*/
@Component
@Comment("string常用方法")
public class StringFunctionExecutor implements FunctionExecutor{
@Override
public String getFunctionPrefix() {
return "string";
......@@ -66,13 +66,13 @@ public class StringFunctionExecutor implements FunctionExecutor{
public static int indexOf(String content, String str, int fromIndex) {
return content != null ? content.indexOf(str, fromIndex) : -1;
}
@Comment("将字符串转为int")
@Example("${string.toInt(value)}")
public static int toInt(String value){
return Integer.parseInt(value);
}
@Comment("将字符串转为Integer")
@Example("${string.toInt(value,defaultValue)}")
public static Integer toInt(String value,Integer defaultValue){
......@@ -82,49 +82,49 @@ public class StringFunctionExecutor implements FunctionExecutor{
return defaultValue;
}
}
@Comment("字符串替换")
@Example("${string.replace(content,source,target)}")
public static String replace(String content,String source,String target){
return content != null ? content.replace(source, target): null;
}
@Comment("正则替换字符串")
@Example("${string.replaceAll(content,regx,target)}")
public static String replaceAll(String content,String regx,String target){
return content != null ? content.replaceAll(regx, target): null;
}
@Comment("正则替换字符串")
@Example("${string.replaceFirst(content,regx,target)}")
public static String replaceFirst(String content,String regx,String target){
return content != null ? content.replaceFirst(regx, target): null;
}
@Comment("正则替换字符串")
@Example("${string.length(content)}")
public static int length(String content){
return content != null ? content.length() : -1;
}
@Comment("去除字符串两边的空格")
@Example("${string.trim(content)}")
public static String trim(String content){
return content != null ? content.trim() : null;
}
@Comment("分割字符串")
@Example("${string.split(content,regx)}")
public static List<String> split(String content,String regx){
return content != null ? Arrays.asList(content.split(regx)) : new ArrayList<>(0);
}
@Comment("获取字符串的byte[]")
@Example("${string.bytes(content)}")
public static byte[] bytes(String content){
return content != null ? content.getBytes() : null;
}
@Comment("获取字符串的byte[]")
@Example("${string.bytes(content,charset)}")
public static byte[] bytes(String content,String charset){
......@@ -134,13 +134,13 @@ public class StringFunctionExecutor implements FunctionExecutor{
return null;
}
}
@Comment("byte[]转String")
@Example("${string.newString(bytes)}")
public static String newString(byte[] bytes){
return bytes != null ? new String(bytes) : null;
}
@Comment("byte[]转String")
@Example("${string.newString(bytes,charset)}")
public static String newString(byte[] bytes,String charset){
......@@ -150,19 +150,19 @@ public class StringFunctionExecutor implements FunctionExecutor{
return null;
}
}
@Comment("判断两个字符串是否相同")
@Example("${string.newString(bytes,charset)}")
public static boolean equals(String str1,String str2){
return str1 == null ? str2 == null : str1.equals(str2);
}
@Comment("生成UUID")
@Example("${string.uuid()}")
public static String uuid() {
return UUID.randomUUID().toString().replace("-", "");
}
@Comment("生成多个UUID")
@Example("${string.uuid(size)}")
public static List<String> uuids(Integer size) {
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/ThreadFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/ThreadFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
/**
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/UrlFunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/UrlFunctionExecutor.java
package org.spiderflow.core.executor.function;
package zq.spiderflow.core.executor.function;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
......@@ -8,28 +8,28 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExecutor;
import org.springframework.stereotype.Component;
/**
* url 按指定字符集进行编码/解码 默认字符集(UTF-8) 工具类 防止NPE
* url 按指定字符集进行编码/解码 默认字符集(UTF-8) 工具类 防止NPE
*/
@Component
public class UrlFunctionExecutor implements FunctionExecutor{
@Override
public String getFunctionPrefix() {
return "url";
}
@Comment("获取url参数")
@Example("${url.parameter('http://www.baidu.com/s?wd=spider-flow','wd')}")
public static String parameter(String url,String key){
return parameterMap(url).get(key);
}
@Comment("获取url全部参数")
@Example("${url.parameterMap('http://www.baidu.com/s?wd=spider-flow&abbr=sf')}")
public static Map<String,String> parameterMap(String url){
......@@ -60,13 +60,13 @@ public class UrlFunctionExecutor implements FunctionExecutor{
}
return map;
}
@Comment("url编码")
@Example("${url.encode('http://www.baidu.com/s?wd=spider-flow')}")
public static String encode(String url){
return encode(url,Charset.defaultCharset().name());
}
@Comment("url编码")
@Example("${url.encode('http://www.baidu.com/s?wd=spider-flow','UTF-8')}")
public static String encode(String url,String charset){
......@@ -76,13 +76,13 @@ public class UrlFunctionExecutor implements FunctionExecutor{
return null;
}
}
@Comment("url解码")
@Example("${url.decode(strVar)}")
public static String decode(String url){
return decode(url,Charset.defaultCharset().name());
}
@Comment("url解码")
@Example("${url.decode(strVar,'UTF-8')}")
public static String decode(String url,String charset){
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/ArrayFunctionExtension.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/extension/ArrayFunctionExtension.java
package org.spiderflow.core.executor.function.extension;
package zq.spiderflow.core.executor.function.extension;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExtension;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExtension;
import org.springframework.stereotype.Component;
@Component
public class ArrayFunctionExtension implements FunctionExtension{
@Override
public Class<?> support() {
return Object[].class;
}
@Comment("获取数组的长度")
@Example("${arrayVar.size()}")
public static int size(Object[] objs){
return objs.length;
}
@Comment("将数组拼接起来")
@Example("${arrayVar.join()}")
public static String join(Object[] objs,String separator){
return StringUtils.join(objs,separator);
}
@Comment("将数组用separator拼接起来")
@Example("${arrayVar.join('-')}")
public static String join(Object[] objs){
return StringUtils.join(objs);
}
@Comment("将数组转为List")
@Example("${arrayVar.toList()}")
public static List<?> toList(Object[] objs){
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/DateFunctionExtension.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/extension/DateFunctionExtension.java
package org.spiderflow.core.executor.function.extension;
package zq.spiderflow.core.executor.function.extension;
import java.util.Date;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExtension;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExtension;
import org.springframework.stereotype.Component;
@Component
......@@ -15,13 +15,13 @@ public class DateFunctionExtension implements FunctionExtension{
public Class<?> support() {
return Date.class;
}
@Comment("格式化日期")
@Example("${dateVar.format()}")
public static String format(Date date){
return format(date, "yyyy-MM-dd HH:mm:ss");
}
@Comment("格式化日期")
@Example("${dateVar.format('yyyy-MM-dd HH:mm:ss')}")
public static String format(Date date,String pattern){
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/ElementFunctionExtension.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/extension/ElementFunctionExtension.java
package org.spiderflow.core.executor.function.extension;
package zq.spiderflow.core.executor.function.extension;
import java.util.List;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.annotation.Return;
import org.spiderflow.core.utils.ExtractUtils;
import org.spiderflow.executor.FunctionExtension;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.annotation.Return;
import zq.spiderflow.core.utils.ExtractUtils;
import zq.spiderflow.executor.FunctionExtension;
import org.springframework.stereotype.Component;
@Component
......@@ -18,14 +18,14 @@ public class ElementFunctionExtension implements FunctionExtension{
public Class<?> support() {
return Element.class;
}
@Comment("根据xpath提取内容")
@Example("${elementVar.xpath('//title/text()')}")
@Return({Element.class,String.class})
public static String xpath(Element element,String xpath){
return ExtractUtils.getValueByXPath(element, xpath);
}
@Comment("根据xpath提取内容")
@Example("${elementVar.xpaths('//h2/text()')}")
......@@ -33,49 +33,49 @@ public class ElementFunctionExtension implements FunctionExtension{
public static List<String> xpaths(Element element,String xpath){
return ExtractUtils.getValuesByXPath(element, xpath);
}
@Comment("根据正则表达式提取内容")
@Example("${elementVar.regx('<title>(.*?)</title>')}")
public static String regx(Element element,String regx){
return ExtractUtils.getFirstMatcher(element.html(), regx, true);
}
@Comment("根据正则表达式提取内容")
@Example("${elementVar.regx('<title>(.*?)</title>',1)}")
public static String regx(Element element,String regx,int groupIndex){
return ExtractUtils.getFirstMatcher(element.html(), regx, groupIndex);
}
@Comment("根据正则表达式提取内容")
@Example("${elementVar.regx('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
public static List<String> regx(Element element,String regx,List<Integer> groups){
return ExtractUtils.getFirstMatcher(element.html(), regx, groups);
}
@Comment("根据正则表达式提取内容")
@Example("${elementVar.regxs('<h2>(.*?)</h2>')}")
public static List<String> regxs(Element element,String regx){
return ExtractUtils.getMatchers(element.html(), regx, true);
}
@Comment("根据正则表达式提取内容")
@Example("${elementVar.regxs('<h2>(.*?)</h2>',1)}")
public static List<String> regxs(Element element,String regx,int groupIndex){
return ExtractUtils.getMatchers(element.html(), regx, groupIndex);
}
@Comment("根据正则表达式提取内容")
@Example("${elementVar.regxs('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
public static List<List<String>> regxs(Element element,String regx,List<Integer> groups){
return ExtractUtils.getMatchers(element.html(), regx, groups);
}
@Comment("根据css选择器提取内容")
@Example("${elementVar.selector('div > a')}")
public static Element selector(Element element,String cssQuery){
return element.selectFirst(cssQuery);
}
@Comment("根据css选择器提取内容")
@Example("${elementVar.selectors('div > a')}")
public static Elements selectors(Element element,String cssQuery){
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/ElementsFunctionExtension.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/extension/ElementsFunctionExtension.java
package org.spiderflow.core.executor.function.extension;
package zq.spiderflow.core.executor.function.extension;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.core.utils.ExtractUtils;
import org.spiderflow.executor.FunctionExtension;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.core.utils.ExtractUtils;
import zq.spiderflow.executor.FunctionExtension;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
......@@ -18,55 +18,55 @@ public class ElementsFunctionExtension implements FunctionExtension{
public Class<?> support() {
return Elements.class;
}
@Comment("根据xpath提取内容")
@Example("${elementsVar.xpath('//title/text()')}")
public static String xpath(Elements elements,String xpath){
return ExtractUtils.getValueByXPath(elements, xpath);
}
@Comment("根据xpath提取内容")
@Example("${elementsVar.xpaths('//h2/text()')}")
public static List<String> xpaths(Elements elements,String xpath){
return ExtractUtils.getValuesByXPath(elements, xpath);
}
@Comment("根据正则表达式提取内容")
@Example("${elementsVar.regx('<title>(.*?)</title>')}")
public static String regx(Elements elements,String regx){
return ExtractUtils.getFirstMatcher(elements.html(), regx, true);
}
@Comment("根据正则表达式提取内容")
@Example("${elementsVar.regx('<title>(.*?)</title>',1)}")
public static String regx(Elements elements,String regx,int groupIndex){
return ExtractUtils.getFirstMatcher(elements.html(), regx, groupIndex);
}
@Comment("根据正则表达式提取内容")
@Example("${elementsVar.regx('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
public static List<String> regx(Elements elements,String regx,List<Integer> groups){
return ExtractUtils.getFirstMatcher(elements.html(), regx, groups);
}
@Comment("根据正则表达式提取内容")
@Example("${elementsVar.regxs('<h2>(.*?)</h2>')}")
public static List<String> regxs(Elements elements,String regx){
return ExtractUtils.getMatchers(elements.html(), regx, true);
}
@Comment("根据正则表达式提取内容")
@Example("${elementsVar.regxs('<h2>(.*?)</h2>',1)}")
public static List<String> regxs(Elements elements,String regx,int groupIndex){
return ExtractUtils.getMatchers(elements.html(), regx, groupIndex);
}
@Comment("根据正则表达式提取内容")
@Example("${elementsVar.regxs('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
public static List<List<String>> regxs(Elements elements,String regx,List<Integer> groups){
return ExtractUtils.getMatchers(elements.html(), regx, groups);
}
@Comment("根据css选择器提取内容")
@Example("${elementsVar.selector('div > a')}")
public static Element selector(Elements elements,String selector){
......@@ -146,7 +146,7 @@ public class ElementsFunctionExtension implements FunctionExtension{
}
return list;
}
@Comment("根据css选择器提取内容")
@Example("${elementsVar.selectors('div > a')}")
public static Elements selectors(Elements elements,String selector){
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/ListFunctionExtension.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/extension/ListFunctionExtension.java
package org.spiderflow.core.executor.function.extension;
package zq.spiderflow.core.executor.function.extension;
import org.apache.commons.lang3.StringUtils;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExtension;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExtension;
import org.springframework.stereotype.Component;
import java.util.Collections;
......@@ -17,19 +17,19 @@ public class ListFunctionExtension implements FunctionExtension{
public Class<?> support() {
return List.class;
}
@Comment("获取list的长度")
@Example("${listVar.length()}")
public static int length(List<?> list){
return list.size();
}
@Comment("将list拼接起来")
@Example("${listVar.join()}")
public static String join(List<?> list){
return StringUtils.join(list.toArray());
}
@Comment("将list用separator拼接起来")
@Example("${listVar.join('-')}")
public static String join(List<?> list,String separator){
......@@ -53,5 +53,5 @@ public class ListFunctionExtension implements FunctionExtension{
Collections.shuffle(list);
return list;
}
}
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/MapFunctionExtension.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/extension/MapFunctionExtension.java
package org.spiderflow.core.executor.function.extension;
package zq.spiderflow.core.executor.function.extension;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExtension;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExtension;
import org.springframework.stereotype.Component;
import java.util.List;
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/ObjectFunctionExtension.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/extension/ObjectFunctionExtension.java
package org.spiderflow.core.executor.function.extension;
package zq.spiderflow.core.executor.function.extension;
import java.util.Objects;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.core.utils.ExtractUtils;
import org.spiderflow.executor.FunctionExtension;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.core.utils.ExtractUtils;
import zq.spiderflow.executor.FunctionExtension;
import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSON;
@Component
public class ObjectFunctionExtension implements FunctionExtension{
@Override
public Class<?> support() {
return Object.class;
}
@Comment("将对象转为string类型")
@Example("${objVar.string()}")
public static String string(Object obj){
......@@ -26,7 +26,7 @@ public class ObjectFunctionExtension implements FunctionExtension{
}
return Objects.toString(obj);
}
@Comment("根据jsonpath提取内容")
@Example("${objVar.jsonpath('$.code')}")
public static Object jsonpath(Object obj,String path){
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/ResponseFunctionExtension.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/extension/ResponseFunctionExtension.java
package org.spiderflow.core.executor.function.extension;
package zq.spiderflow.core.executor.function.extension;
import java.util.List;
import java.util.regex.Pattern;
......@@ -8,12 +8,12 @@ import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.annotation.Return;
import org.spiderflow.core.utils.ExtractUtils;
import org.spiderflow.executor.FunctionExtension;
import org.spiderflow.io.SpiderResponse;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.annotation.Return;
import zq.spiderflow.core.utils.ExtractUtils;
import zq.spiderflow.executor.FunctionExtension;
import zq.spiderflow.io.SpiderResponse;
import org.springframework.stereotype.Component;
@Component
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/SqlRowSetExtension.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/extension/SqlRowSetExtension.java
package org.spiderflow.core.executor.function.extension;
package zq.spiderflow.core.executor.function.extension;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.spiderflow.annotation.Example;
import org.spiderflow.executor.FunctionExtension;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.executor.FunctionExtension;
import org.springframework.jdbc.support.rowset.SqlRowSet;
import org.springframework.stereotype.Component;
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/StringFunctionExtension.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/function/extension/StringFunctionExtension.java
package org.spiderflow.core.executor.function.extension;
package zq.spiderflow.core.executor.function.extension;
import com.alibaba.fastjson.JSON;
import org.apache.commons.lang3.math.NumberUtils;
......@@ -6,12 +6,12 @@ import org.apache.commons.text.StringEscapeUtils;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Parser;
import org.jsoup.select.Elements;
import org.spiderflow.annotation.Comment;
import org.spiderflow.annotation.Example;
import org.spiderflow.annotation.Return;
import org.spiderflow.core.executor.function.DateFunctionExecutor;
import org.spiderflow.core.utils.ExtractUtils;
import org.spiderflow.executor.FunctionExtension;
import zq.spiderflow.annotation.Comment;
import zq.spiderflow.annotation.Example;
import zq.spiderflow.annotation.Return;
import zq.spiderflow.core.executor.function.DateFunctionExecutor;
import zq.spiderflow.core.utils.ExtractUtils;
import zq.spiderflow.executor.FunctionExtension;
import org.springframework.stereotype.Component;
import java.text.ParseException;
......@@ -24,69 +24,69 @@ public class StringFunctionExtension implements FunctionExtension{
@Override
public Class<?> support() {
return String.class;
}
}
@Comment("根据正则表达式提取String中的内容")
@Example("${strVar.regx('<title>(.*?)</title>')}")
public static String regx(String source,String pattern){
return ExtractUtils.getFirstMatcher(source, pattern, true);
}
@Comment("根据正则表达式提取String中的内容")
@Example("${strVar.regx('<title>(.*?)</title>',1)}")
public static String regx(String source,String pattern,int groupIndex){
return ExtractUtils.getFirstMatcher(source, pattern, groupIndex);
}
@Comment("根据正则表达式提取String中的内容")
@Example("${strVar.regx('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
public static List<String> regx(String source,String pattern,List<Integer> groups){
return ExtractUtils.getFirstMatcher(source, pattern, groups);
}
@Comment("根据正则表达式提取String中的内容")
@Example("${strVar.regxs('<h2>(.*?)</h2>')}")
public static List<String> regxs(String source,String pattern){
return ExtractUtils.getMatchers(source, pattern, true);
}
@Comment("根据正则表达式提取String中的内容")
@Example("${strVar.regxs('<h2>(.*?)</h2>',1)}")
public static List<String> regxs(String source,String pattern,int groupIndex){
return ExtractUtils.getMatchers(source, pattern, groupIndex);
}
@Comment("根据正则表达式提取String中的内容")
@Example("${strVar.regxs('<a href=\"(.*?)\">(.*?)</a>',[1,2])}")
public static List<List<String>> regxs(String source,String pattern,List<Integer> groups){
return ExtractUtils.getMatchers(source, pattern, groups);
}
@Comment("根据xpath在String变量中查找")
@Example("${strVar.xpath('//title/text()')}")
@Return({Element.class,String.class})
public static String xpath(String source,String xpath){
return ExtractUtils.getValueByXPath(element(source), xpath);
}
@Comment("根据xpath在String变量中查找")
@Example("${strVar.xpaths('//a/@href')}")
public static List<String> xpaths(String source,String xpath){
return ExtractUtils.getValuesByXPath(element(source), xpath);
}
@Comment("将String变量转为Element对象")
@Example("${strVar.element()}")
public static Element element(String source){
return Parser.xmlParser().parseInput(source,"");
}
@Comment("根据css选择器提取")
@Example("${strVar.selector('div > a')}")
public static Element selector(String source,String cssQuery){
return element(source).selectFirst(cssQuery);
}
@Comment("根据css选择器提取")
@Example("${strVar.selector('div > a')}")
public static Elements selectors(String source,String cssQuery){
......@@ -98,37 +98,37 @@ public class StringFunctionExtension implements FunctionExtension{
public static Object json(String source){
return JSON.parse(source);
}
@Comment("根据jsonpath提取内容")
@Example("${strVar.jsonpath('$.code')}")
public static Object jsonpath(String source,String jsonPath){
return ExtractUtils.getValueByJsonPath(json(source), jsonPath);
}
@Comment("将字符串转为int类型")
@Example("${strVar.toInt(0)}")
public static Integer toInt(String source,int defaultValue){
return NumberUtils.toInt(source, defaultValue);
}
@Comment("将字符串转为int类型")
@Example("${strVar.toInt()}")
public static Integer toInt(String source){
return NumberUtils.toInt(source);
}
@Comment("将字符串转为double类型")
@Example("${strVar.toDouble()}")
public static Double toDouble(String source){
return NumberUtils.toDouble(source);
}
@Comment("将字符串转为long类型")
@Example("${strVar.toLong()}")
public static Long toLong(String source){
return NumberUtils.toLong(source);
}
@Comment("将字符串转为date类型")
@Example("${strVar.toDate('yyyy-MM-dd HH:mm:ss')}")
public static Date toDate(String source,String pattern) throws ParseException{
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/CommentExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/shape/CommentExecutor.java
package org.spiderflow.core.executor.shape;
package zq.spiderflow.core.executor.shape;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.model.SpiderNode;
import org.springframework.stereotype.Component;
import java.util.Map;
......@@ -12,7 +12,7 @@ public class CommentExecutor implements ShapeExecutor{
@Override
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
}
@Override
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/ExecuteSQLExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/shape/ExecuteSQLExecutor.java
package org.spiderflow.core.executor.shape;
package zq.spiderflow.core.executor.shape;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.Grammerable;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.core.utils.DataSourceUtils;
import org.spiderflow.core.utils.ExpressionUtils;
import org.spiderflow.core.utils.ExtractUtils;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.model.Grammer;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.Grammerable;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.core.utils.DataSourceUtils;
import zq.spiderflow.core.utils.ExpressionUtils;
import zq.spiderflow.core.utils.ExtractUtils;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.model.Grammer;
import zq.spiderflow.model.SpiderNode;
import org.springframework.jdbc.core.ArgumentPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.support.GeneratedKeyHolder;
......@@ -50,7 +50,7 @@ public class ExecuteSQLExecutor implements ShapeExecutor, Grammerable {
public static final String STATEMENT_DELETE = "delete";
public static final String SELECT_RESULT_STREAM = "isStream";
public static final String STATEMENT_INSERT_PK = "insertofPk";
private static final Logger logger = LoggerFactory.getLogger(ExecuteSQLExecutor.class);
@Override
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/ForkJoinExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/shape/ForkJoinExecutor.java
package org.spiderflow.core.executor.shape;
package zq.spiderflow.core.executor.shape;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.model.SpiderNode;
import org.springframework.stereotype.Component;
import java.util.HashMap;
......@@ -10,7 +10,7 @@ import java.util.Map;
/**
* 等待执行结束执行器
*
*
*/
@Component
public class ForkJoinExecutor implements ShapeExecutor {
......@@ -19,7 +19,7 @@ public class ForkJoinExecutor implements ShapeExecutor {
* 缓存已完成节点的变量
*/
private Map<String, Map<String, Object>> cachedVariables = new HashMap<>();
@Override
public void execute(SpiderNode node, SpiderContext context, Map<String, Object> variables) {
}
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/FunctionExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/shape/FunctionExecutor.java
package org.spiderflow.core.executor.shape;
package zq.spiderflow.core.executor.shape;
import java.util.List;
import java.util.Map;
......@@ -7,11 +7,11 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.core.exception.RpaException;
import org.spiderflow.core.utils.ExpressionUtils;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.core.exception.RpaException;
import zq.spiderflow.core.utils.ExpressionUtils;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.model.SpiderNode;
import org.springframework.stereotype.Component;
/**
......@@ -21,11 +21,11 @@ import org.springframework.stereotype.Component;
*/
@Component
public class FunctionExecutor implements ShapeExecutor{
public static final String FUNCTION = "function";
private static final Logger logger = LoggerFactory.getLogger(FunctionExecutor.class);
@Override
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
List<Map<String, String>> functions = node.getListJsonValue(FUNCTION);
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/LoopExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/shape/LoopExecutor.java
package org.spiderflow.core.executor.shape;
package zq.spiderflow.core.executor.shape;
import java.util.Map;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.model.SpiderNode;
import org.springframework.stereotype.Component;
/**
......@@ -14,13 +14,13 @@ import org.springframework.stereotype.Component;
*/
@Component
public class LoopExecutor implements ShapeExecutor{
public static final String LOOP_ITEM = "loopItem";
public static final String LOOP_START = "loopStart";
public static final String LOOP_END = "loopEnd";
@Override
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
}
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/OutputExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/shape/OutputExecutor.java
package org.spiderflow.core.executor.shape;
package zq.spiderflow.core.executor.shape;
import com.alibaba.fastjson.JSON;
import org.apache.commons.csv.CSVFormat;
......@@ -8,15 +8,15 @@ import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.ibatis.jdbc.SQL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.core.serializer.FastJsonSerializer;
import org.spiderflow.core.utils.DataSourceUtils;
import org.spiderflow.core.utils.ExpressionUtils;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.io.SpiderResponse;
import org.spiderflow.listener.SpiderListener;
import org.spiderflow.model.SpiderNode;
import org.spiderflow.model.SpiderOutput;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.core.serializer.FastJsonSerializer;
import zq.spiderflow.core.utils.DataSourceUtils;
import zq.spiderflow.core.utils.ExpressionUtils;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.io.SpiderResponse;
import zq.spiderflow.listener.SpiderListener;
import zq.spiderflow.model.SpiderNode;
import zq.spiderflow.model.SpiderOutput;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/ProcessExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/shape/ProcessExecutor.java
package org.spiderflow.core.executor.shape;
package zq.spiderflow.core.executor.shape;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.core.Spider;
import org.spiderflow.core.model.SpiderFlow;
import org.spiderflow.core.service.SpiderFlowService;
import org.spiderflow.core.utils.SpiderFlowUtils;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.core.Spider;
import zq.spiderflow.core.model.SpiderFlow;
import zq.spiderflow.core.service.SpiderFlowService;
import zq.spiderflow.core.utils.SpiderFlowUtils;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.model.SpiderNode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
......@@ -21,17 +21,17 @@ import org.springframework.stereotype.Component;
*/
@Component
public class ProcessExecutor implements ShapeExecutor{
public static final String FLOW_ID = "flowId";
private static Logger logger = LoggerFactory.getLogger(ProcessExecutor.class);
@Autowired
private SpiderFlowService spiderFlowService;
@Autowired
private Spider spider;
@Override
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
String flowId = node.getStringJsonValue("flowId");
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/RequestExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/shape/RequestExecutor.java
package org.spiderflow.core.executor.shape;
package zq.spiderflow.core.executor.shape;
import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnel;
......@@ -9,18 +9,18 @@ import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.Grammerable;
import org.spiderflow.context.CookieContext;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.core.executor.function.MD5FunctionExecutor;
import org.spiderflow.core.io.HttpRequest;
import org.spiderflow.core.io.HttpResponse;
import org.spiderflow.core.utils.ExpressionUtils;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.io.SpiderResponse;
import org.spiderflow.listener.SpiderListener;
import org.spiderflow.model.Grammer;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.Grammerable;
import zq.spiderflow.context.CookieContext;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.core.executor.function.MD5FunctionExecutor;
import zq.spiderflow.core.io.HttpRequest;
import zq.spiderflow.core.io.HttpResponse;
import zq.spiderflow.core.utils.ExpressionUtils;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.io.SpiderResponse;
import zq.spiderflow.listener.SpiderListener;
import zq.spiderflow.model.Grammer;
import zq.spiderflow.model.SpiderNode;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
......@@ -36,17 +36,17 @@ import java.util.*;
*/
@Component
public class RequestExecutor implements ShapeExecutor,Grammerable, SpiderListener {
public static final String SLEEP = "sleep";
public static final String URL = "url";
public static final String PROXY = "proxy";
public static final String REQUEST_METHOD = "method";
public static final String PARAMETER_NAME = "parameter-name";
public static final String PARAMETER_VALUE = "parameter-value";
public static final String COOKIE_NAME = "cookie-name";
......@@ -54,23 +54,23 @@ public class RequestExecutor implements ShapeExecutor,Grammerable, SpiderListene
public static final String COOKIE_VALUE = "cookie-value";
public static final String PARAMETER_FORM_NAME = "parameter-form-name";
public static final String PARAMETER_FORM_VALUE = "parameter-form-value";
public static final String PARAMETER_FORM_FILENAME = "parameter-form-filename";
public static final String PARAMETER_FORM_TYPE = "parameter-form-type";
public static final String BODY_TYPE = "body-type";
public static final String BODY_CONTENT_TYPE = "body-content-type";
public static final String REQUEST_BODY = "request-body";
public static final String HEADER_NAME = "header-name";
public static final String HEADER_VALUE = "header-value";
public static final String TIMEOUT = "timeout";
public static final String RETRY_COUNT = "retryCount";
......@@ -78,9 +78,9 @@ public class RequestExecutor implements ShapeExecutor,Grammerable, SpiderListene
public static final String RETRY_INTERVAL = "retryInterval";
public static final String RESPONSE_CHARSET = "response-charset";
public static final String FOLLOW_REDIRECT = "follow-redirect";
public static final String TLS_VALIDATE = "tls-validate";
public static final String LAST_EXECUTE_TIME = "__last_execute_time_";
......@@ -101,7 +101,7 @@ public class RequestExecutor implements ShapeExecutor,Grammerable, SpiderListene
private Double errorRate;
private static final Logger logger = LoggerFactory.getLogger(RequestExecutor.class);
@Override
public String supportShape() {
return "request";
......@@ -315,7 +315,7 @@ public class RequestExecutor implements ShapeExecutor,Grammerable, SpiderListene
}
}
}
private List<InputStream> setRequestFormParameter(SpiderNode node, HttpRequest request,List<Map<String, String>> parameters,SpiderContext context,Map<String,Object> variables){
List<InputStream> streams = new ArrayList<>();
if(parameters != null){
......@@ -351,7 +351,7 @@ public class RequestExecutor implements ShapeExecutor,Grammerable, SpiderListene
context.pause(node.getNodeId(),"request-body",parameterName,value);
logger.info("设置请求参数：{}={}",parameterName,value);
}
} catch (Exception e) {
logger.error("设置请求参数：{}出错,异常信息:{}",parameterName,e);
}
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/StartExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/shape/StartExecutor.java
package org.spiderflow.core.executor.shape;
package zq.spiderflow.core.executor.shape;
import java.util.Map;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.model.SpiderNode;
import org.springframework.stereotype.Component;
/**
......@@ -17,7 +17,7 @@ public class StartExecutor implements ShapeExecutor{
@Override
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
}
@Override
......
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/VariableExecutor.java → spider-flow-core/src/main/java/zq/spiderflow/core/executor/shape/VariableExecutor.java
package org.spiderflow.core.executor.shape;
package zq.spiderflow.core.executor.shape;
import java.util.List;
import java.util.Map;
......@@ -6,10 +6,10 @@ import java.util.Map;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.core.utils.ExpressionUtils;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.model.SpiderNode;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.core.utils.ExpressionUtils;
import zq.spiderflow.executor.ShapeExecutor;
import zq.spiderflow.model.SpiderNode;
import org.springframework.stereotype.Component;
/**
......@@ -19,13 +19,13 @@ import org.springframework.stereotype.Component;
*/
@Component
public class VariableExecutor implements ShapeExecutor{
private static final String VARIABLE_NAME = "variable-name";
private static final String VARIABLE_VALUE = "variable-value";
private static final Logger logger = LoggerFactory.getLogger(VariableExecutor.class);
@Override
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
List<Map<String, String>> variableList = node.getListJsonValue(VARIABLE_NAME,VARIABLE_VALUE);
......@@ -49,7 +49,7 @@ public class VariableExecutor implements ShapeExecutor{
public String supportShape() {
return "variable";
}
@Override
public boolean isThread() {
return false;
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/DefaultExpressionEngine.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/DefaultExpressionEngine.java
package org.spiderflow.core.expression;
package zq.spiderflow.core.expression;
import java.util.List;
import java.util.Map;
......@@ -6,29 +6,29 @@ import java.util.Map;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.spiderflow.ExpressionEngine;
import org.spiderflow.core.expression.interpreter.Reflection;
import org.spiderflow.executor.FunctionExecutor;
import org.spiderflow.executor.FunctionExtension;
import zq.spiderflow.ExpressionEngine;
import zq.spiderflow.core.expression.interpreter.Reflection;
import zq.spiderflow.executor.FunctionExecutor;
import zq.spiderflow.executor.FunctionExtension;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class DefaultExpressionEngine implements ExpressionEngine{
@Autowired
private List<FunctionExecutor> functionExecutors;
@Autowired
private List<FunctionExtension> functionExtensions;
@PostConstruct
private void init(){
for (FunctionExtension extension : functionExtensions) {
Reflection.getInstance().registerExtensionClass(extension.support(), extension.getClass());
}
}
@Override
public Object execute(String expression, Map<String, Object> variables) {
if(StringUtils.isBlank(expression)){
......@@ -48,5 +48,5 @@ public class DefaultExpressionEngine implements ExpressionEngine{
ExpressionTemplateContext.remove();
}
}
}
spider-flow-core/src/main/java/org/spiderflow/core/expression/ExpressionError.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/ExpressionError.java
package org.spiderflow.core.expression;
package zq.spiderflow.core.expression;
import org.spiderflow.core.expression.parsing.Span;
import org.spiderflow.core.expression.parsing.Span.Line;
import org.spiderflow.core.expression.parsing.TokenStream;
import zq.spiderflow.core.expression.parsing.Span;
import zq.spiderflow.core.expression.parsing.Span.Line;
import zq.spiderflow.core.expression.parsing.TokenStream;
/** All errors reported by the library go through the static functions of this class. */
public class ExpressionError {
......@@ -109,10 +109,10 @@ public class ExpressionError {
return builder.toString();
}
}
public static class StringLiteralException extends RuntimeException {
private static final long serialVersionUID = 1L;
}
}
spider-flow-core/src/main/java/org/spiderflow/core/expression/ExpressionTemplate.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/ExpressionTemplate.java
package org.spiderflow.core.expression;
package zq.spiderflow.core.expression;
import java.io.OutputStream;
import java.util.List;
import org.spiderflow.core.expression.interpreter.AstInterpreter;
import org.spiderflow.core.expression.parsing.Ast;
import org.spiderflow.core.expression.parsing.Ast.Node;
import org.spiderflow.core.expression.parsing.Parser;
import zq.spiderflow.core.expression.interpreter.AstInterpreter;
import zq.spiderflow.core.expression.parsing.Ast;
import zq.spiderflow.core.expression.parsing.Ast.Node;
import zq.spiderflow.core.expression.parsing.Parser;
/** A template is loaded by a {@link TemplateLoader} from a file marked up with the basis-template language. The template can be
......@@ -21,11 +21,11 @@ public class ExpressionTemplate {
private ExpressionTemplate (List<Node> nodes) {
this.nodes = nodes;
}
public static ExpressionTemplate create(String source){
return new ExpressionTemplate(Parser.parse(source));
}
/** Internal. The AST nodes representing this template after parsing. See {@link Ast}. Used by {@link AstInterpreter}. **/
public List<Node> getNodes () {
return nodes;
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/ExpressionTemplateContext.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/ExpressionTemplateContext.java
package org.spiderflow.core.expression;
package zq.spiderflow.core.expression;
import java.util.ArrayList;
import java.util.HashMap;
......@@ -8,7 +8,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.spiderflow.core.expression.interpreter.AstInterpreter;
import zq.spiderflow.core.expression.interpreter.AstInterpreter;
/**
......@@ -46,7 +46,7 @@ public class ExpressionTemplateContext {
public ExpressionTemplateContext () {
push();
}
public ExpressionTemplateContext(Map<String,Object> variables) {
this();
if(variables != null){
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/interpreter/AstInterpreter.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/interpreter/AstInterpreter.java
package org.spiderflow.core.expression.interpreter;
package zq.spiderflow.core.expression.interpreter;
import java.io.IOException;
import java.util.List;
import org.spiderflow.core.exception.RpaException;
import org.spiderflow.core.expression.ExpressionError;
import org.spiderflow.core.expression.ExpressionError.TemplateException;
import org.spiderflow.core.expression.ExpressionTemplate;
import org.spiderflow.core.expression.ExpressionTemplateContext;
import org.spiderflow.core.expression.parsing.Ast;
import org.spiderflow.core.expression.parsing.Ast.Node;
import org.spiderflow.core.expression.parsing.Ast.Text;
import zq.spiderflow.core.exception.RpaException;
import zq.spiderflow.core.expression.ExpressionError;
import zq.spiderflow.core.expression.ExpressionTemplate;
import zq.spiderflow.core.expression.ExpressionTemplateContext;
import zq.spiderflow.core.expression.parsing.Ast;
/**
* <p>
......@@ -33,21 +30,21 @@ public class AstInterpreter {
} catch (RpaException e){
throw e;
}catch (Throwable t) {
if (t instanceof TemplateException)
throw (TemplateException)t;
if (t instanceof ExpressionError.TemplateException)
throw (ExpressionError.TemplateException)t;
else {
ExpressionError.error("执行表达式出错 " + t.getMessage(), template.getNodes().get(0).getSpan(),t);
return null; // never reached
}
}
}
}
public static Object interpretNodeList (List<Node> nodes, ExpressionTemplate template, ExpressionTemplateContext context) throws IOException {
public static Object interpretNodeList (List<Ast.Node> nodes, ExpressionTemplate template, ExpressionTemplateContext context) throws IOException {
String result = "";
for (int i = 0, n = nodes.size(); i < n; i++) {
Node node = nodes.get(i);
Ast.Node node = nodes.get(i);
Object value = node.evaluate(template, context);
if(node instanceof Text){
if(node instanceof Ast.Text){
result += node.getSpan().getText();
}else if(value == null){
if(i == 0 && i + 1 == n){
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/interpreter/JavaReflection.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/interpreter/JavaReflection.java
package org.spiderflow.core.expression.interpreter;
package zq.spiderflow.core.expression.interpreter;
import lombok.extern.slf4j.Slf4j;
import org.spiderflow.core.exception.RpaException;
import zq.spiderflow.core.exception.RpaException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
......@@ -68,7 +68,7 @@ public class JavaReflection extends Reflection {
throw new RuntimeException("Couldn't get value of field '" + javaField.getName() + "' from object of type '" + obj.getClass().getSimpleName() + "'");
}
}
@Override
public void registerExtensionClass(Class<?> target,Class<?> clazz){
Method[] methods = clazz.getDeclaredMethods();
......@@ -90,7 +90,7 @@ public class JavaReflection extends Reflection {
}
}
}
@Override
public Object getExtensionMethod(Object obj, String name, Object... arguments) {
Class<?> cls = obj instanceof Class ? (Class<?>)obj : obj.getClass();
......@@ -99,7 +99,7 @@ public class JavaReflection extends Reflection {
}
return getExtensionMethod(cls,name,arguments);
}
private Object getExtensionMethod(Class<?> cls, String name, Object... arguments) {
if(cls == null){
cls = Object.class;
......@@ -239,7 +239,7 @@ public class JavaReflection extends Reflection {
}
return foundMethod;
}
/** Returns the method best matching the given signature, including type coercion, or null. **/
private static Method findMethod (Class<?> cls, String name, Class<?>[] parameterTypes) {
Method[] methods = cls.getDeclaredMethods();
......@@ -269,13 +269,13 @@ public class JavaReflection extends Reflection {
if ((from == Character.class || from == char.class) && (to == char.class || to == Character.class)) return true;
return false;
}
public static String[] getStringTypes(Object[] objects){
String[] parameterTypes = new String[objects == null ? 0: objects.length];
if(objects != null){
for(int i=0,len = objects.length;i<len;i++){
Object value = objects[i];
parameterTypes[i] = value == null ? "null" : value.getClass().getSimpleName();
parameterTypes[i] = value == null ? "null" : value.getClass().getSimpleName();
}
}
return parameterTypes;
......@@ -314,11 +314,11 @@ public class JavaReflection extends Reflection {
if (from == Long.class || from == long.class) {
return to == float.class || to == Float.class || to == double.class || to == Double.class;
}
if(from == int[].class || from == Integer[].class){
return to == Object[].class || to == float[].class || to == Float[].class || to == double[].class || to == Double[].class || to == long[].class || to == Long[].class;
}
return false;
}
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/interpreter/Reflection.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/interpreter/Reflection.java
package org.spiderflow.core.expression.interpreter;
package zq.spiderflow.core.expression.interpreter;
/** Used by {@link AstInterpreter} to access fields and methods of objects. This is a singleton class used by all
* {@link AstInterpreter} instances. Replace the default implementation via {@link #setInstance(Reflection)}. The implementation
......@@ -24,9 +24,9 @@ public abstract class Reflection {
* null if the method could not be found. If obj is an instance of Class, the matching static method is returned. If the name
* is null and the object is a {@link FunctionalInterface}, the first declared method on the object is returned. **/
public abstract Object getMethod (Object obj, String name, Object... arguments);
public abstract Object getExtensionMethod (Object obj, String name,Object ... arguments);
public abstract void registerExtensionClass(Class<?> target,Class<?> clazz);
/** Returns the value of the field from the object. The field must have been previously retrieved via
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/parsing/Ast.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/parsing/Ast.java
package org.spiderflow.core.expression.parsing;
package zq.spiderflow.core.expression.parsing;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.spiderflow.core.exception.RpaException;
import org.spiderflow.core.expression.ExpressionError;
import org.spiderflow.core.expression.ExpressionError.TemplateException;
import org.spiderflow.core.expression.ExpressionTemplate;
import org.spiderflow.core.expression.ExpressionTemplateContext;
import org.spiderflow.core.expression.interpreter.AstInterpreter;
import org.spiderflow.core.expression.interpreter.JavaReflection;
import org.spiderflow.core.expression.interpreter.Reflection;
import org.spiderflow.core.script.ScriptManager;
import org.spiderflow.expression.DynamicMethod;
import zq.spiderflow.core.exception.RpaException;
import zq.spiderflow.core.expression.ExpressionError;
import zq.spiderflow.core.expression.ExpressionTemplate;
import zq.spiderflow.core.expression.ExpressionTemplateContext;
import zq.spiderflow.core.expression.interpreter.AstInterpreter;
import zq.spiderflow.core.expression.interpreter.JavaReflection;
import zq.spiderflow.core.expression.interpreter.Reflection;
import zq.spiderflow.core.script.ScriptManager;
import zq.spiderflow.expression.DynamicMethod;
import javax.xml.transform.Source;
import java.io.IOException;
......@@ -950,7 +949,7 @@ public abstract class Ast {
MethodCall methodCall = new MethodCall(getName(),access, Collections.emptyList());
try {
return methodCall.evaluate(template, context);
} catch (TemplateException e) {
} catch (ExpressionError.TemplateException e) {
if(ExceptionUtils.indexOfThrowable(e, InvocationTargetException.class) > -1){
ExpressionError.error(String.format("在%s中调用方法get%s发生异常"
,object.getClass()
......@@ -961,7 +960,7 @@ public abstract class Ast {
methodCall = new MethodCall(getName(),access, Collections.emptyList());
try {
return methodCall.evaluate(template, context);
} catch (TemplateException e1) {
} catch (ExpressionError.TemplateException e1) {
if(ExceptionUtils.indexOfThrowable(e1, InvocationTargetException.class) > -1){
ExpressionError.error(String.format("在%s中调用方法is%s发生异常"
,object.getClass()
......@@ -1181,9 +1180,9 @@ public abstract class Ast {
} catch (Throwable t) {
ExpressionError.error(t.getMessage(), getSpan(), t);
return null; // never reached
}
}
}
// Otherwise try to find a corresponding method or field pointing to a lambda.
Object method = getCachedMethod();
if (method != null) {
......@@ -1193,7 +1192,7 @@ public abstract class Ast {
// fall through
}
}
method = Reflection.getInstance().getMethod(object, getMethod().getName().getText(), argumentValues);
if (method != null) {
// found the method on the object, call it
......@@ -1206,7 +1205,7 @@ public abstract class Ast {
ExpressionError.error(t.getMessage(), getSpan(), t);
return null; // never reached
}
}
}
method = Reflection.getInstance().getExtensionMethod(object, getMethod().getName().getText(), argumentValues);
if(method != null){
try {
......@@ -1243,7 +1242,7 @@ public abstract class Ast {
if (method == null){
ExpressionError.error("在'" + object.getClass() + "'中找不到方法 " + getMethod().getName().getText() + "("+ StringUtils.join(JavaReflection.getStringTypes(argumentValues),",") +")",
getSpan());
}
}
try {
return Reflection.getInstance().callMethod(function, method, argumentValues);
} catch (Throwable t) {
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/parsing/Parser.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/parsing/Parser.java
package org.spiderflow.core.expression.parsing;
package zq.spiderflow.core.expression.parsing;
......@@ -7,31 +7,31 @@ import java.util.List;
import javax.xml.transform.Source;
import org.spiderflow.core.expression.ExpressionError;
import org.spiderflow.core.expression.ExpressionTemplate;
import org.spiderflow.core.expression.parsing.Ast.BinaryOperation;
import org.spiderflow.core.expression.parsing.Ast.BooleanLiteral;
import org.spiderflow.core.expression.parsing.Ast.ByteLiteral;
import org.spiderflow.core.expression.parsing.Ast.CharacterLiteral;
import org.spiderflow.core.expression.parsing.Ast.DoubleLiteral;
import org.spiderflow.core.expression.parsing.Ast.Expression;
import org.spiderflow.core.expression.parsing.Ast.FloatLiteral;
import org.spiderflow.core.expression.parsing.Ast.FunctionCall;
import org.spiderflow.core.expression.parsing.Ast.IntegerLiteral;
import org.spiderflow.core.expression.parsing.Ast.ListLiteral;
import org.spiderflow.core.expression.parsing.Ast.LongLiteral;
import org.spiderflow.core.expression.parsing.Ast.MapLiteral;
import org.spiderflow.core.expression.parsing.Ast.MapOrArrayAccess;
import org.spiderflow.core.expression.parsing.Ast.MemberAccess;
import org.spiderflow.core.expression.parsing.Ast.MethodCall;
import org.spiderflow.core.expression.parsing.Ast.Node;
import org.spiderflow.core.expression.parsing.Ast.NullLiteral;
import org.spiderflow.core.expression.parsing.Ast.ShortLiteral;
import org.spiderflow.core.expression.parsing.Ast.StringLiteral;
import org.spiderflow.core.expression.parsing.Ast.TernaryOperation;
import org.spiderflow.core.expression.parsing.Ast.Text;
import org.spiderflow.core.expression.parsing.Ast.UnaryOperation;
import org.spiderflow.core.expression.parsing.Ast.VariableAccess;
import zq.spiderflow.core.expression.ExpressionError;
import zq.spiderflow.core.expression.ExpressionTemplate;
import zq.spiderflow.core.expression.parsing.Ast.BinaryOperation;
import zq.spiderflow.core.expression.parsing.Ast.BooleanLiteral;
import zq.spiderflow.core.expression.parsing.Ast.ByteLiteral;
import zq.spiderflow.core.expression.parsing.Ast.CharacterLiteral;
import zq.spiderflow.core.expression.parsing.Ast.DoubleLiteral;
import zq.spiderflow.core.expression.parsing.Ast.Expression;
import zq.spiderflow.core.expression.parsing.Ast.FloatLiteral;
import zq.spiderflow.core.expression.parsing.Ast.FunctionCall;
import zq.spiderflow.core.expression.parsing.Ast.IntegerLiteral;
import zq.spiderflow.core.expression.parsing.Ast.ListLiteral;
import zq.spiderflow.core.expression.parsing.Ast.LongLiteral;
import zq.spiderflow.core.expression.parsing.Ast.MapLiteral;
import zq.spiderflow.core.expression.parsing.Ast.MapOrArrayAccess;
import zq.spiderflow.core.expression.parsing.Ast.MemberAccess;
import zq.spiderflow.core.expression.parsing.Ast.MethodCall;
import zq.spiderflow.core.expression.parsing.Ast.Node;
import zq.spiderflow.core.expression.parsing.Ast.NullLiteral;
import zq.spiderflow.core.expression.parsing.Ast.ShortLiteral;
import zq.spiderflow.core.expression.parsing.Ast.StringLiteral;
import zq.spiderflow.core.expression.parsing.Ast.TernaryOperation;
import zq.spiderflow.core.expression.parsing.Ast.Text;
import zq.spiderflow.core.expression.parsing.Ast.UnaryOperation;
import zq.spiderflow.core.expression.parsing.Ast.VariableAccess;
/** Parses a {@link Source} into a {@link ExpressionTemplate}. The implementation is a simple recursive descent parser with a lookahead of
......@@ -132,7 +132,7 @@ public class Parser {
}
stream.prev();
}
return new StringLiteral(stream.expect(TokenType.StringLiteral).getSpan());
} else if (stream.match(TokenType.BooleanLiteral, false)) {
return new BooleanLiteral(stream.expect(TokenType.BooleanLiteral).getSpan());
......@@ -169,7 +169,7 @@ public class Parser {
}else{
keys.add(stream.expect(TokenType.Identifier));
}
stream.expect(":");
values.add(parseExpression(stream));
if (!stream.match("}", false)) stream.expect(TokenType.Comma);
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/parsing/Span.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/parsing/Span.java
package org.spiderflow.core.expression.parsing;
package zq.spiderflow.core.expression.parsing;
/** A span within a source string denoted by start and end index, with the latter being exclusive. */
public class Span {
......@@ -22,7 +22,7 @@ public class Span {
public Span (String source, int start, int end) {
if (start > end) throw new IllegalArgumentException("Start must be <= end.");
if (start < 0) throw new IndexOutOfBoundsException("Start must be >= 0.");
if (start > source.length() - 1)
if (start > source.length() - 1)
throw new IndexOutOfBoundsException("Start outside of string.");
if (end >source.length()) throw new IndexOutOfBoundsException("End outside of string.");
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/parsing/Token.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/parsing/Token.java
package org.spiderflow.core.expression.parsing;
package zq.spiderflow.core.expression.parsing;
/** A token produced by the {@link Tokenizer}. */
public class Token {
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/parsing/TokenStream.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/parsing/TokenStream.java
package org.spiderflow.core.expression.parsing;
package zq.spiderflow.core.expression.parsing;
import java.util.List;
import javax.xml.transform.Source;
import org.spiderflow.core.expression.ExpressionError;
import zq.spiderflow.core.expression.ExpressionError;
/** Iterates over a list of {@link Token} instances, provides methods to match expected tokens and throw errors in case of a
......@@ -25,11 +25,11 @@ public class TokenStream {
public boolean hasMore () {
return index < end;
}
public boolean hasNext(){
return index + 1 < end;
}
public boolean hasPrev(){
return index > 0;
}
......@@ -39,12 +39,12 @@ public class TokenStream {
if (!hasMore()) throw new RuntimeException("Reached the end of the source.");
return tokens.get(index++);
}
public Token next(){
if (!hasMore()) throw new RuntimeException("Reached the end of the source.");
return tokens.get(++index);
}
public Token prev(){
if(index == 0){
throw new RuntimeException("Reached the end of the source.");
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/parsing/TokenType.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/parsing/TokenType.java
package org.spiderflow.core.expression.parsing;
package zq.spiderflow.core.expression.parsing;
import java.util.Arrays;
import java.util.Comparator;
......
spider-flow-core/src/main/java/org/spiderflow/core/expression/parsing/Tokenizer.java → spider-flow-core/src/main/java/zq/spiderflow/core/expression/parsing/Tokenizer.java
package org.spiderflow.core.expression.parsing;
package zq.spiderflow.core.expression.parsing;
import java.util.ArrayList;
import java.util.List;
import org.spiderflow.core.expression.ExpressionError;
import org.spiderflow.core.expression.ExpressionError.StringLiteralException;
import org.spiderflow.core.expression.ExpressionError.TemplateException;
import zq.spiderflow.core.expression.ExpressionError;
public class Tokenizer {
......@@ -35,13 +33,13 @@ public class Tokenizer {
tokens.addAll(tokenizeCodeSpan(stream.endSpan()));
isContinue = false;
re = null;
}catch(TemplateException e){
}catch(ExpressionError.TemplateException e){
re = e;
if(e.getCause() != null || stream.hasMore()){
isContinue = true;
}
}
}while(isContinue);
if(re != null){
throw re;
......@@ -114,7 +112,7 @@ public class Tokenizer {
}
stream.consume();
}
if (!matchedEndQuote) ExpressionError.error("字符串没有结束符\'", stream.endSpan(),new StringLiteralException());
if (!matchedEndQuote) ExpressionError.error("字符串没有结束符\'", stream.endSpan(),new ExpressionError.StringLiteralException());
Span stringSpan = stream.endSpan();
stringSpan = new Span(stringSpan.getSource(), stringSpan.getStart() - 1, stringSpan.getEnd());
tokens.add(new Token(TokenType.StringLiteral, stringSpan));
......@@ -136,13 +134,13 @@ public class Tokenizer {
}
stream.consume();
}
if (!matchedEndQuote) ExpressionError.error("字符串没有结束符\"", stream.endSpan(),new StringLiteralException());
if (!matchedEndQuote) ExpressionError.error("字符串没有结束符\"", stream.endSpan(),new ExpressionError.StringLiteralException());
Span stringSpan = stream.endSpan();
stringSpan = new Span(stringSpan.getSource(), stringSpan.getStart() - 1, stringSpan.getEnd());
tokens.add(new Token(TokenType.StringLiteral, stringSpan));
continue;
}
// Identifier, keyword, boolean literal, or null literal
if (stream.matchIdentifierStart(true)) {
stream.startSpan();
......
spider-flow-core/src/main/java/org/spiderflow/core/handlle/GlobalExceptionHandler.java → spider-flow-core/src/main/java/zq/spiderflow/core/handlle/GlobalExceptionHandler.java
package org.spiderflow.core.handlle;
package zq.spiderflow.core.handlle;
import lombok.extern.slf4j.Slf4j;
import org.spiderflow.core.exception.RpaException;
import org.spiderflow.core.vo.ResultVo;
import zq.spiderflow.core.exception.RpaException;
import zq.spiderflow.core.vo.ResultVo;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import java.util.concurrent.ExecutionException;
@Slf4j
@RestControllerAdvice
public class GlobalExceptionHandler {
......@@ -23,4 +21,4 @@ public class GlobalExceptionHandler {
e.printStackTrace();
return ResultVo.fail("操作失败，请重试或联系管理员");
}
}
\ No newline at end of file
}
spider-flow-core/src/main/java/org/spiderflow/core/io/HttpRequest.java → spider-flow-core/src/main/java/zq/spiderflow/core/io/HttpRequest.java
package org.spiderflow.core.io;
package zq.spiderflow.core.io;
import java.io.IOException;
import java.io.InputStream;
......@@ -15,30 +15,30 @@ import org.jsoup.Jsoup;
*
*/
public class HttpRequest {
private Connection connection = null;
public static HttpRequest create(){
return new HttpRequest();
}
public HttpRequest url(String url){
this.connection = Jsoup.connect(url);
this.connection.method(Method.GET);
this.connection.timeout(60000);
return this;
}
public HttpRequest headers(Map<String,String> headers){
this.connection.headers(headers);
return this;
}
public HttpRequest header(String key,String value){
this.connection.header(key, value);
return this;
}
public HttpRequest header(String key,Object value){
if(value != null){
this.connection.header(key,value.toString());
......@@ -57,67 +57,67 @@ public class HttpRequest {
}
return this;
}
public HttpRequest contentType(String contentType){
this.connection.header("Content-Type", contentType);
return this;
}
public HttpRequest data(String key,String value){
this.connection.data(key, value);
return this;
}
public HttpRequest data(String key,Object value){
if(value != null){
this.connection.data(key, value.toString());
}
return this;
}
public HttpRequest data(String key,String filename,InputStream is){
this.connection.data(key, filename, is);
return this;
}
public HttpRequest data(Object body){
if(body != null){
this.connection.requestBody(body.toString());
this.connection.requestBody(body.toString());
}
return this;
}
public HttpRequest data(Map<String,String> data){
this.connection.data(data);
return this;
}
public HttpRequest method(String method){
this.connection.method(Method.valueOf(method));
return this;
}
public HttpRequest followRedirect(boolean followRedirects){
this.connection.followRedirects(followRedirects);
return this;
}
public HttpRequest timeout(int timeout){
this.connection.timeout(timeout);
return this;
}
public HttpRequest proxy(String host,int port){
this.connection.proxy(host, port);
return this;
}
@SuppressWarnings("deprecation")
public HttpRequest validateTLSCertificates(boolean value){
this.connection.validateTLSCertificates(value);
return this;
}
public HttpResponse execute() throws IOException{
this.connection.ignoreContentType(true);
this.connection.ignoreHttpErrors(true);
......
spider-flow-core/src/main/java/org/spiderflow/core/io/HttpResponse.java → spider-flow-core/src/main/java/zq/spiderflow/core/io/HttpResponse.java
package org.spiderflow.core.io;
package zq.spiderflow.core.io;
import com.alibaba.fastjson.JSON;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.spiderflow.io.SpiderResponse;
import zq.spiderflow.io.SpiderResponse;
import java.io.InputStream;
import java.util.Map;
......@@ -33,7 +33,7 @@ public class HttpResponse implements SpiderResponse{
this.statusCode = response.statusCode();
this.urlLink = response.url().toExternalForm();
}
@Override
public int getStatusCode(){
return statusCode;
......@@ -58,7 +58,7 @@ public class HttpResponse implements SpiderResponse{
}
return htmlValue;
}
@Override
public Object getJson(){
if(jsonValue == null){
......@@ -66,22 +66,22 @@ public class HttpResponse implements SpiderResponse{
}
return jsonValue;
}
@Override
public Map<String,String> getCookies(){
return response.cookies();
}
@Override
public Map<String,String> getHeaders(){
return response.headers();
}
@Override
public byte[] getBytes(){
return response.bodyAsBytes();
}
@Override
public String getContentType(){
return response.contentType();
......
spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJob.java → spider-flow-core/src/main/java/zq/spiderflow/core/job/SpiderJob.java
package org.spiderflow.core.job;
package zq.spiderflow.core.job;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.context.SpiderContextHolder;
import org.spiderflow.core.Spider;
import org.spiderflow.core.exception.RpaException;
import org.spiderflow.core.model.SpiderFlow;
import org.spiderflow.core.model.Task;
import org.spiderflow.core.service.SpiderFlowService;
import org.spiderflow.core.service.TaskService;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.context.SpiderContextHolder;
import zq.spiderflow.core.Spider;
import zq.spiderflow.core.exception.RpaException;
import zq.spiderflow.core.model.SpiderFlow;
import zq.spiderflow.core.model.Task;
import zq.spiderflow.core.service.SpiderFlowService;
import zq.spiderflow.core.service.TaskService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.quartz.QuartzJobBean;
......
spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJobContext.java → spider-flow-core/src/main/java/zq/spiderflow/core/job/SpiderJobContext.java
package org.spiderflow.core.job;
package zq.spiderflow.core.job;
import java.io.File;
import java.io.FileOutputStream;
......@@ -8,15 +8,15 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.model.SpiderOutput;
import zq.spiderflow.context.SpiderContext;
import zq.spiderflow.model.SpiderOutput;
public class SpiderJobContext extends SpiderContext{
private static final long serialVersionUID = 9099787449108938453L;
private static Logger logger = LoggerFactory.getLogger(SpiderJobContext.class);
private OutputStream outputstream;
private List<SpiderOutput> outputs = new ArrayList<>();
......@@ -28,7 +28,7 @@ public class SpiderJobContext extends SpiderContext{
this.outputstream = outputstream;
this.output = output;
}
public void close(){
try {
this.outputstream.close();
......@@ -53,7 +53,7 @@ public class SpiderJobContext extends SpiderContext{
public OutputStream getOutputstream(){
return this.outputstream;
}
public static SpiderJobContext create(String directory,String id,Integer taskId,boolean output){
OutputStream os = null;
try {
......
spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJobManager.java → spider-flow-core/src/main/java/zq/spiderflow/core/job/SpiderJobManager.java
package org.spiderflow.core.job;
package zq.spiderflow.core.job;
import org.quartz.CronScheduleBuilder;
import org.quartz.CronTrigger;
......@@ -11,8 +11,7 @@ import org.quartz.TriggerBuilder;
import org.quartz.TriggerKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.core.Spider;
import org.spiderflow.core.model.SpiderFlow;
import zq.spiderflow.core.model.SpiderFlow;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
......@@ -25,30 +24,30 @@ import java.util.Date;
*/
@Component
public class SpiderJobManager {
private static Logger logger = LoggerFactory.getLogger(SpiderJobManager.class);
private final static String JOB_NAME = "SPIDER_TASK_";
public final static String JOB_PARAM_NAME = "SPIDER_FLOW";
@Autowired
private SpiderJob spiderJob;
/**
* 调度器
*/
@Autowired
private Scheduler scheduler;
private JobKey getJobKey(String id){
return JobKey.jobKey(JOB_NAME + id);
}
private TriggerKey getTriggerKey(String id){
return TriggerKey.triggerKey(JOB_NAME + id);
}
/**
* 新建定时任务
* @param spiderFlow 爬虫流程图
......@@ -58,11 +57,11 @@ public class SpiderJobManager {
try {
JobDetail job = JobBuilder.newJob(SpiderJob.class).withIdentity(getJobKey(spiderFlow.getId())).build();
job.getJobDataMap().put(JOB_PARAM_NAME, spiderFlow);
CronScheduleBuilder cronScheduleBuilder = CronScheduleBuilder.cronSchedule(spiderFlow.getCron()).withMisfireHandlingInstructionDoNothing();
CronTrigger trigger = TriggerBuilder.newTrigger().withIdentity(getTriggerKey(spiderFlow.getId())).withSchedule(cronScheduleBuilder).build();
return scheduler.scheduleJob(job,trigger);
} catch (SchedulerException e) {
logger.error("创建定时任务出错",e);
......@@ -79,7 +78,7 @@ public class SpiderJobManager {
public void run(String id, String body){
spiderJob.run(id, body);
}
public boolean remove(String id){
try {
scheduler.deleteJob(getJobKey(id));
......
spider-flow-core/src/main/java/org/spiderflow/core/mapper/DataSourceMapper.java → spider-flow-core/src/main/java/zq/spiderflow/core/mapper/DataSourceMapper.java
package org.spiderflow.core.mapper;
package zq.spiderflow.core.mapper;
import java.util.List;
import org.apache.ibatis.annotations.Select;
import org.spiderflow.core.model.DataSource;
import zq.spiderflow.core.model.DataSource;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
public interface DataSourceMapper extends BaseMapper<DataSource>{
@Select("select id,name from sp_datasource")
List<DataSource> selectAll();
......
spider-flow-core/src/main/java/org/spiderflow/core/mapper/FlowNoticeMapper.java → spider-flow-core/src/main/java/zq/spiderflow/core/mapper/FlowNoticeMapper.java
package org.spiderflow.core.mapper;
package zq.spiderflow.core.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
import org.spiderflow.core.model.FlowNotice;
import zq.spiderflow.core.model.FlowNotice;
@Mapper
public interface FlowNoticeMapper extends BaseMapper<FlowNotice> {
......
spider-flow-core/src/main/java/org/spiderflow/core/mapper/FunctionMapper.java → spider-flow-core/src/main/java/zq/spiderflow/core/mapper/FunctionMapper.java
package org.spiderflow.core.mapper;
package zq.spiderflow.core.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
import org.spiderflow.core.model.Function;
import zq.spiderflow.core.model.Function;
@Mapper
public interface FunctionMapper extends BaseMapper<Function> {
......
spider-flow-core/src/main/java/org/spiderflow/core/mapper/SpiderFlowMapper.java → spider-flow-core/src/main/java/zq/spiderflow/core/mapper/SpiderFlowMapper.java
package org.spiderflow.core.mapper;
package zq.spiderflow.core.mapper;
import java.util.Date;
import java.util.List;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import org.spiderflow.core.model.SpiderFlow;
import org.apache.ibatis.annotations.*;
import org.springframework.stereotype.Repository;
import zq.spiderflow.core.model.SpiderFlow;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
......@@ -18,6 +16,7 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
* @author Administrator
*
*/
@Repository
public interface SpiderFlowMapper extends BaseMapper<SpiderFlow>{
@Select({
......@@ -36,22 +35,22 @@ public interface SpiderFlowMapper extends BaseMapper<SpiderFlow>{
@Insert("insert into sp_flow(id,name,xml,enabled) values(#{id},#{name},#{xml},'0')")
int insertSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml);
@Update("update sp_flow set name = #{name},xml = #{xml} where id = #{id}")
int updateSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml);
@Update("update sp_flow set execute_count = 0 where id = #{id}")
int resetExecuteCount(@Param("id") String id);
@Update("update sp_flow set execute_count = ifnull(execute_count,0) + 1,last_execute_time = #{lastExecuteTime},next_execute_time = #{nextExecuteTime} where id = #{id}")
int executeCountIncrementAndExecuteTime(@Param("id") String id, @Param("lastExecuteTime") Date lastExecuteTime, @Param("nextExecuteTime") Date nextExecuteTime);
@Update("update sp_flow set execute_count = ifnull(execute_count,0) + 1,last_execute_time = #{lastExecuteTime} where id = #{id}")
int executeCountIncrement(@Param("id") String id, @Param("lastExecuteTime") Date lastExecuteTime);
@Update("update sp_flow set cron = #{cron},next_execute_time = #{nextExecuteTime} where id = #{id}")
int resetCornExpression(@Param("id") String id, @Param("cron") String cron, @Param("nextExecuteTime") Date nextExecuteTime);
@Update("update sp_flow set enabled = #{enabled} where id = #{id}")
int resetSpiderStatus(@Param("id") String id, @Param("enabled") String enabled);
......@@ -60,16 +59,16 @@ public interface SpiderFlowMapper extends BaseMapper<SpiderFlow>{
@Update("update sp_flow set next_execute_time = null")
int resetNextExecuteTime();
@Select("select id,name from sp_flow")
List<SpiderFlow> selectFlows();
@Select("select id,name from sp_flow where id != #{id}")
List<SpiderFlow> selectOtherFlows(@Param("id") String id);
@Select("select max(a.id) from `sp_task` a left join sp_flow b on a.flow_id = b.id where b.id = #{id}")
Integer getFlowMaxTaskId(@Param("id")String id);
@Select("select COUNT(id) from sp_flow where id = #{id}")
Integer getCountById(@Param("id")String id);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment