Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
N
nnjcy-data-model
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
梁家彪
nnjcy-data-model
Commits
b181e091
Commit
b181e091
authored
Nov 03, 2023
by
梁家彪
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
保存汇聚日志
parent
41dc65d0
Show whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
110 additions
and
26 deletions
+110
-26
common-client/src/main/java/com/zq/common/utils/EasyExcelUtil.java
+58
-14
data-collect-server/src/main/java/com/zq/datacollect/controller/DataCollectDbController.java
+1
-1
data-collect-server/src/main/java/com/zq/datacollect/entity/DataCollectTaskLog.java
+10
-1
data-collect-server/src/main/java/com/zq/datacollect/service/DataCollectFileService.java
+2
-1
data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectApiServiceImpl.java
+10
-5
data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectDbServiceImpl.java
+9
-0
data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectFileServiceImpl.java
+20
-4
No files found.
common-client/src/main/java/com/zq/common/utils/EasyExcelUtil.java
View file @
b181e091
...
...
@@ -3,31 +3,68 @@ package com.zq.common.utils;
import
com.alibaba.excel.EasyExcel
;
import
com.alibaba.excel.context.AnalysisContext
;
import
com.alibaba.excel.event.AnalysisEventListener
;
import
com.zq.common.exception.BusinessException
;
import
java.io.File
;
import
java.io.FileInputStream
;
import
java.io.FileNotFoundException
;
import
java.io.InputStream
;
import
java.util.LinkedHashMap
;
import
java.util.LinkedList
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.*
;
public
class
EasyExcelUtil
{
public
static
List
<
Map
<
String
,
Object
>>
read
(
File
file
){
/**
* 读取全部数据
* @param file
* @return
*/
public
static
List
<
Map
<
String
,
Object
>>
readAll
(
File
file
){
try
{
return
read
(
new
FileInputStream
(
file
));
return
read
All
(
new
FileInputStream
(
file
));
}
catch
(
FileNotFoundException
e
)
{
throw
new
RuntimeException
(
e
);
e
.
printStackTrace
();
throw
new
BusinessException
(
"文件读取失败,请检查后重试"
);
}
}
public
static
List
<
Map
<
String
,
Object
>>
read
(
InputStream
is
){
List
<
Map
<
String
,
Object
>>
list
=
new
LinkedList
<>();
EasyExcel
.
read
(
is
,
new
AnalysisEventListener
<
Map
<
Integer
,
Object
>>()
{
private
Map
<
Integer
,
String
>
headMap
;
/**
* 读取全部数据
* @param is
* @return
*/
public
static
List
<
Map
<
String
,
Object
>>
readAll
(
InputStream
is
){
return
(
List
<
Map
<
String
,
Object
>>)
read
(
is
,
0
).
get
(
"rows"
);
}
/**
* 预览前100条数据
* @param file
* @return
*/
public
static
Map
<
String
,
Object
>
preview100
(
File
file
){
try
{
return
preview100
(
new
FileInputStream
(
file
));
}
catch
(
FileNotFoundException
e
)
{
e
.
printStackTrace
();
throw
new
BusinessException
(
"文件解析失败"
);
}
}
/**
* 预览前100条数据
* @param is
* @return
*/
public
static
Map
<
String
,
Object
>
preview100
(
InputStream
is
){
return
read
(
is
,
100
);
}
public
static
Map
<
String
,
Object
>
read
(
InputStream
is
,
int
readSize
){
Map
<
String
,
Object
>
map
=
new
HashMap
();
EasyExcel
.
read
(
is
,
new
AnalysisEventListener
<
Map
<
Integer
,
Object
>>()
{
Map
<
Integer
,
String
>
headMap
;
List
<
Map
<
String
,
Object
>>
list
=
new
LinkedList
<>();
int
total
=
0
;
@Override
public
void
invokeHeadMap
(
Map
<
Integer
,
String
>
headMap
,
AnalysisContext
context
)
{
this
.
headMap
=
headMap
;
...
...
@@ -35,16 +72,22 @@ public class EasyExcelUtil {
@Override
public
void
invoke
(
Map
<
Integer
,
Object
>
valueMap
,
AnalysisContext
analysisContext
)
{
LinkedHashMap
map
=
new
LinkedHashMap
(
headMap
.
size
());
total
++;
if
(
readSize
<=
0
||
list
.
size
()<
readSize
){
LinkedHashMap
<
String
,
Object
>
map
=
new
LinkedHashMap
(
headMap
.
size
());
for
(
int
i
=
0
;
i
<
headMap
.
size
();
i
++){
map
.
put
(
headMap
.
get
(
i
),
valueMap
.
get
(
i
));
}
list
.
add
(
map
);
}
}
@Override
public
void
doAfterAllAnalysed
(
AnalysisContext
analysisContext
)
{}
public
void
doAfterAllAnalysed
(
AnalysisContext
analysisContext
)
{
map
.
put
(
"rows"
,
list
);
map
.
put
(
"total"
,
total
);
}
}).
sheet
().
doReadSync
();
return
list
;
return
map
;
}
}
\ No newline at end of file
data-collect-server/src/main/java/com/zq/datacollect/controller/DataCollectDbController.java
View file @
b181e091
...
...
@@ -12,7 +12,7 @@ import javax.annotation.Resource;
@RestController
@Api
(
tags
=
"数据库汇聚"
)
@RequestMapping
(
value
=
"/data/collect"
)
@RequestMapping
(
value
=
"/data/collect
/db
"
)
public
class
DataCollectDbController
{
@Resource
...
...
data-collect-server/src/main/java/com/zq/datacollect/entity/DataCollectTaskLog.java
View file @
b181e091
...
...
@@ -16,7 +16,7 @@ import java.util.Date;
public
class
DataCollectTaskLog
{
/**
*
数据汇聚表
id
*
汇聚设置
id
*/
private
Integer
dataCollectSettingId
;
...
...
@@ -65,5 +65,14 @@ public class DataCollectTaskLog {
*/
@TableField
(
fill
=
FieldFill
.
INSERT_UPDATE
)
private
Date
updateTime
;
public
DataCollectTaskLog
(
Integer
dataCollectSettingId
,
Date
startTime
,
Date
endTime
,
Integer
duration
,
Integer
status
,
String
error
)
{
this
.
dataCollectSettingId
=
dataCollectSettingId
;
this
.
startTime
=
startTime
;
this
.
endTime
=
endTime
;
this
.
duration
=
duration
;
this
.
status
=
status
;
this
.
error
=
error
;
}
}
data-collect-server/src/main/java/com/zq/datacollect/service/DataCollectFileService.java
View file @
b181e091
...
...
@@ -15,7 +15,7 @@ public interface DataCollectFileService {
void
update
(
DataCollectFileAddReq
dataCollectFileAddReq
);
List
<
Map
<
String
,
Object
>
>
analysisExcel
(
MultipartFile
multipartFile
);
Map
<
String
,
Object
>
analysisExcel
(
MultipartFile
multipartFile
);
void
execute
(
Integer
dataCollectSettingId
,
MultipartFile
file
);
}
\ No newline at end of file
data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectApiServiceImpl.java
View file @
b181e091
...
...
@@ -12,10 +12,12 @@ import com.zq.common.utils.AssertUtils;
import
com.zq.datacollect.entity.DataCollectSetting
;
import
com.zq.datacollect.entity.DataCollectSettingApi
;
import
com.zq.datacollect.entity.DataCollectSettingApiPara
;
import
com.zq.datacollect.entity.DataCollectTaskLog
;
import
com.zq.datacollect.holder.DatabaseHolder
;
import
com.zq.datacollect.mapper.DataCollectMapper
;
import
com.zq.datacollect.mapper.DataCollectSettingApiMapper
;
import
com.zq.datacollect.mapper.DataCollectSettingApiParaMapper
;
import
com.zq.datacollect.mapper.DataCollectTaskLogMapper
;
import
com.zq.datacollect.service.DataCollectApiService
;
import
com.zq.datacollect.util.SqlUtil
;
import
com.zq.datacollect.vo.DataCollectApiAddReq
;
...
...
@@ -29,10 +31,7 @@ import org.springframework.transaction.annotation.Transactional;
import
javax.annotation.Resource
;
import
java.sql.PreparedStatement
;
import
java.sql.SQLException
;
import
java.util.LinkedList
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Set
;
import
java.util.*
;
@Service
public
class
DataCollectApiServiceImpl
extends
ServiceImpl
<
DataCollectSettingApiMapper
,
DataCollectSettingApi
>
implements
DataCollectApiService
{
...
...
@@ -43,6 +42,9 @@ public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApi
@Resource
private
DataCollectSettingApiParaMapper
dataCollectSettingApiParaMapper
;
@Resource
private
DataCollectTaskLogMapper
dataCollectTaskLogMapper
;
@Override
public
DataCollectSettingApiResp
get
(
Integer
dataCollectSettingId
)
{
DataCollectSettingApi
settingApi
=
lambdaQuery
().
eq
(
DataCollectSettingApi:
:
getDataCollectSettingId
,
dataCollectSettingId
).
one
();
...
...
@@ -73,6 +75,7 @@ public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApi
AssertUtils
.
isTrue
(
updateById
(
dataCollectApiAddReq
.
getDataCollectSettingApi
()),
"修改失败"
);
for
(
DataCollectSettingApiPara
dataCollectSettingApiPara
:
dataCollectApiAddReq
.
getDataCollectSettingApiParas
())
{
if
(
dataCollectSettingApiPara
.
getId
()
==
null
){
dataCollectSettingApiPara
.
setDataCollectSettingId
(
dataCollectApiAddReq
.
getDataCollectSetting
().
getId
());
AssertUtils
.
isTrue
(
dataCollectSettingApiParaMapper
.
insert
(
dataCollectSettingApiPara
)==
1
,
"修改失败"
);
}
else
{
AssertUtils
.
isTrue
(
dataCollectSettingApiParaMapper
.
updateById
(
dataCollectSettingApiPara
)==
1
,
"修改失败"
);
...
...
@@ -84,6 +87,7 @@ public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApi
@Override
@Transactional
public
void
execute
(
Integer
dataCollectSettingId
,
DataCollectApiExecuteReq
dataCollectApiExecuteReq
)
{
long
start
=
System
.
currentTimeMillis
();
DataCollectSettingApi
settingApi
=
lambdaQuery
().
eq
(
DataCollectSettingApi:
:
getDataCollectSettingId
,
dataCollectSettingId
).
one
();
List
<
DataCollectSettingApiPara
>
apiParas
;
if
(
dataCollectApiExecuteReq
.
getParas
()
!=
null
&&
dataCollectApiExecuteReq
.
getParas
().
size
()
>
0
){
...
...
@@ -120,7 +124,6 @@ public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApi
}
String
body
=
request
.
execute
().
body
();
log
.
error
(
body
);
LinkedList
<
Map
<
String
,
Object
>>
list
=
JSONObject
.
parseObject
(
body
).
getJSONObject
(
"data"
).
getJSONArray
(
"rows"
).
to
(
LinkedList
.
class
);
DataCollectSetting
dataCollectSetting
=
dataCollectMapper
.
selectById
(
dataCollectSettingId
);
...
...
@@ -145,6 +148,8 @@ public class DataCollectApiServiceImpl extends ServiceImpl<DataCollectSettingApi
return
list
.
size
();
}
});
dataCollectTaskLogMapper
.
insert
(
new
DataCollectTaskLog
(
dataCollectSettingId
,
new
Date
(
start
),
new
Date
(),
Long
.
valueOf
(((
System
.
currentTimeMillis
()-
start
)/
1000
)).
intValue
(),
2
,
null
));
}
public
String
urlParams
(
List
<
DataCollectSettingApiPara
>
settingApiParas
){
...
...
data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectDbServiceImpl.java
View file @
b181e091
...
...
@@ -6,9 +6,11 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import
com.zq.common.utils.AssertUtils
;
import
com.zq.datacollect.entity.DataCollectSetting
;
import
com.zq.datacollect.entity.DataCollectSettingDb
;
import
com.zq.datacollect.entity.DataCollectTaskLog
;
import
com.zq.datacollect.holder.DatabaseHolder
;
import
com.zq.datacollect.mapper.DataCollectMapper
;
import
com.zq.datacollect.mapper.DataCollectSettingDbMapper
;
import
com.zq.datacollect.mapper.DataCollectTaskLogMapper
;
import
com.zq.datacollect.service.DataCollectDbService
;
import
com.zq.datacollect.util.SqlUtil
;
import
com.zq.datacollect.vo.DataCollectDbAddReq
;
...
...
@@ -20,6 +22,7 @@ import org.springframework.transaction.annotation.Transactional;
import
javax.annotation.Resource
;
import
java.sql.PreparedStatement
;
import
java.sql.SQLException
;
import
java.util.Date
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Set
;
...
...
@@ -30,6 +33,9 @@ public class DataCollectDbServiceImpl extends ServiceImpl<DataCollectSettingDbMa
@Resource
private
DataCollectMapper
dataCollectMapper
;
@Resource
private
DataCollectTaskLogMapper
dataCollectTaskLogMapper
;
@Override
public
DataCollectSettingDb
get
(
Integer
dataCollectSettingId
)
{
return
lambdaQuery
().
eq
(
DataCollectSettingDb:
:
getDataCollectSettingId
,
dataCollectSettingId
).
one
();
...
...
@@ -56,6 +62,7 @@ public class DataCollectDbServiceImpl extends ServiceImpl<DataCollectSettingDbMa
@Override
@Transactional
public
void
execute
(
Integer
dataCollectSettingId
)
{
long
start
=
System
.
currentTimeMillis
();
DataCollectSetting
dataCollectSetting
=
dataCollectMapper
.
selectById
(
dataCollectSettingId
);
DataCollectSettingDb
settingDb
=
getOne
(
new
LambdaQueryWrapper
<
DataCollectSettingDb
>().
eq
(
DataCollectSettingDb:
:
getDataCollectSettingId
,
dataCollectSettingId
));
JdbcTemplate
toDbJdbcTemplate
=
DatabaseHolder
.
getJdbcTemplate
(
dataCollectSetting
.
getToDbId
());
...
...
@@ -81,5 +88,7 @@ public class DataCollectDbServiceImpl extends ServiceImpl<DataCollectSettingDbMa
return
list
.
size
();
}
});
dataCollectTaskLogMapper
.
insert
(
new
DataCollectTaskLog
(
dataCollectSettingId
,
new
Date
(
start
),
new
Date
(),
Long
.
valueOf
(((
System
.
currentTimeMillis
()-
start
)/
1000
)).
intValue
(),
2
,
null
));
}
}
data-collect-server/src/main/java/com/zq/datacollect/service/impl/DataCollectFileServiceImpl.java
View file @
b181e091
package
com
.
zq
.
datacollect
.
service
.
impl
;
import
cn.hutool.core.io.FileUtil
;
import
cn.hutool.core.util.StrUtil
;
import
com.baomidou.mybatisplus.extension.service.impl.ServiceImpl
;
import
com.zq.common.exception.BusinessException
;
import
com.zq.common.utils.AssertUtils
;
import
com.zq.common.utils.EasyExcelUtil
;
import
com.zq.datacollect.entity.DataCollectSetting
;
import
com.zq.datacollect.entity.DataCollectSettingFile
;
import
com.zq.datacollect.entity.DataCollectTaskLog
;
import
com.zq.datacollect.holder.DatabaseHolder
;
import
com.zq.datacollect.mapper.DataCollectMapper
;
import
com.zq.datacollect.mapper.DataCollectSettingFileMapper
;
import
com.zq.datacollect.mapper.DataCollectTaskLogMapper
;
import
com.zq.datacollect.service.DataCollectFileService
;
import
com.zq.datacollect.util.SqlUtil
;
import
com.zq.datacollect.vo.DataCollectFileAddReq
;
...
...
@@ -21,8 +22,10 @@ import org.springframework.transaction.annotation.Transactional;
import
org.springframework.web.multipart.MultipartFile
;
import
javax.annotation.Resource
;
import
java.io.IOException
;
import
java.sql.PreparedStatement
;
import
java.sql.SQLException
;
import
java.util.Date
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Set
;
...
...
@@ -33,6 +36,9 @@ public class DataCollectFileServiceImpl extends ServiceImpl<DataCollectSettingFi
@Resource
private
DataCollectMapper
dataCollectMapper
;
@Resource
private
DataCollectTaskLogMapper
dataCollectTaskLogMapper
;
@Override
public
DataCollectSettingFile
get
(
Integer
dataCollectSettingId
)
{
return
lambdaQuery
().
eq
(
DataCollectSettingFile:
:
getDataCollectSettingId
,
dataCollectSettingId
).
one
();
...
...
@@ -57,11 +63,11 @@ public class DataCollectFileServiceImpl extends ServiceImpl<DataCollectSettingFi
}
@Override
public
List
<
Map
<
String
,
Object
>
>
analysisExcel
(
MultipartFile
file
)
{
public
Map
<
String
,
Object
>
analysisExcel
(
MultipartFile
file
)
{
String
suffix
=
FileUtil
.
extName
(
file
.
getOriginalFilename
());
AssertUtils
.
isTrue
(
suffix
.
equals
(
"xls"
)
||
suffix
.
equals
(
"xlsx"
)
||
suffix
.
equals
(
"et"
),
"不支持的文件类型"
);
try
{
return
EasyExcelUtil
.
read
(
file
.
getInputStream
());
return
EasyExcelUtil
.
preview100
(
file
.
getInputStream
());
}
catch
(
Exception
e
)
{
throw
new
BusinessException
(
"文件读取失败,请检查后重试"
);
}
...
...
@@ -70,7 +76,11 @@ public class DataCollectFileServiceImpl extends ServiceImpl<DataCollectSettingFi
@Override
@Transactional
public
void
execute
(
Integer
dataCollectSettingId
,
MultipartFile
file
)
{
List
<
Map
<
String
,
Object
>>
list
=
analysisExcel
(
file
);
long
start
=
System
.
currentTimeMillis
();
String
suffix
=
FileUtil
.
extName
(
file
.
getOriginalFilename
());
AssertUtils
.
isTrue
(
suffix
.
equals
(
"xls"
)
||
suffix
.
equals
(
"xlsx"
)
||
suffix
.
equals
(
"et"
),
"不支持的文件类型"
);
try
{
List
<
Map
<
String
,
Object
>>
list
=
EasyExcelUtil
.
readAll
(
file
.
getInputStream
());
DataCollectSetting
dataCollectSetting
=
dataCollectMapper
.
selectById
(
dataCollectSettingId
);
JdbcTemplate
toDbJdbcTemplate
=
DatabaseHolder
.
getJdbcTemplate
(
dataCollectSetting
.
getToDbId
());
Set
<
String
>
set
=
list
.
get
(
0
).
keySet
();
...
...
@@ -93,5 +103,10 @@ public class DataCollectFileServiceImpl extends ServiceImpl<DataCollectSettingFi
return
list
.
size
();
}
});
dataCollectTaskLogMapper
.
insert
(
new
DataCollectTaskLog
(
dataCollectSettingId
,
new
Date
(
start
),
new
Date
(),
Long
.
valueOf
(((
System
.
currentTimeMillis
()-
start
)/
1000
)).
intValue
(),
2
,
null
));
}
catch
(
IOException
e
)
{
dataCollectTaskLogMapper
.
insert
(
new
DataCollectTaskLog
(
dataCollectSettingId
,
new
Date
(
start
),
new
Date
(),
Long
.
valueOf
(((
System
.
currentTimeMillis
()-
start
)/
1000
)).
intValue
(),
3
,
e
.
getMessage
()));
throw
new
BusinessException
(
"文件读取失败,请检查后重试"
);
}
}
}
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment