Skip to content

I
ibzdst
提交 14ca8aa9 编写于 作者: zhouweidong's avatar zhouweidong
浏览文件
操作

规则执行结果动态存储

上级 329091a8
隐藏空白字符变更
内嵌 并排
正在显示 包含 145 行增加 和 178 行删除
ibzdst-core/src/main/java/cn/ibizlab/core/extensions/service/ExecResultExService.java
浏览文件 @ 14ca8aa9
......@@ -3,19 +3,12 @@ package cn.ibizlab.core.extensions.service;
import cn.ibizlab.core.lite.extensions.service.DbEntityService;
import cn.ibizlab.core.rule.domain.ExecResult;
import cn.ibizlab.core.rule.service.impl.ExecResultServiceImpl;
import com.alibaba.druid.pool.DruidDataSource;
import com.baomidou.dynamic.datasource.DynamicRoutingDataSource;
import com.baomidou.dynamic.datasource.toolkit.DynamicDataSourceContextHolder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;
import javax.sql.DataSource;
import java.util.*;
/**
......@@ -26,16 +19,9 @@ import java.util.*;
@Service("ExecResultExService")
public class ExecResultExService extends ExecResultServiceImpl {
@Autowired
private DstDataSourceExService dstDataSourceService;
@Autowired
private DbEntityService dbEntityService;
@Autowired
@Lazy
private DynamicRoutingDataSource dynamicRoutingDataSource;
private Map<String, List<ExecResult>> resultsSaveAsync = new HashMap<>();
private Object objResultSaveAsyncLock = new Object();
private Timer timerSaveAsync = null;
......@@ -66,7 +52,6 @@ public class ExecResultExService extends ExecResultServiceImpl {
timerSaveAsync = new Timer();
//schedule方法是执行时间定时任务的方法
timerSaveAsync.schedule(new TimerTask() {
//run方法就是具体需要定时执行的任务
@Override
public void run() {
......@@ -74,25 +59,20 @@ public class ExecResultExService extends ExecResultServiceImpl {
synchronized (objResultSaveAsyncLock) {
if (resultsSaveAsync.size() > 0) {
for (Map.Entry<String, List<ExecResult>> entry : resultsSaveAsync.entrySet()) {
String tabName = "";
String dsName = entry.getKey();
List<ExecResult> execResults = entry.getValue();
List<ExecResult> tempExecResults = new ArrayList<>();
if (execResults.size() > 0) {
dstDataSourceService.initDataSource(dsName);
DynamicDataSourceContextHolder.push(dsName);
for (ExecResult execResult : execResults) {
tempExecResults.add(execResult);
Object objTabName = execResult.get(RuleEngineExService.Setting_ResultTableName);
if (StringUtils.isEmpty(tabName) && !ObjectUtils.isEmpty(objTabName)) {
tabName = String.valueOf(objTabName);
}
if (tempExecResults.size() >= 500) {
saveResultBatch(tempExecResults, dsName, tabName);
dbEntityService.saveResultBatch(tempExecResults,dsName);
tempExecResults.clear();
}
}
if (tempExecResults.size() > 0) {
saveResultBatch(tempExecResults, dsName, tabName);
dbEntityService.saveResultBatch(tempExecResults,dsName);
tempExecResults.clear();
}
resultsSaveAsync.remove(dsName);
}
......@@ -101,10 +81,7 @@ public class ExecResultExService extends ExecResultServiceImpl {
}
} catch (Exception ex) {
log.error(ex.getMessage());
} finally {
DynamicDataSourceContextHolder.poll();
}
}
}, 1000, 10000);
}
......@@ -135,25 +112,18 @@ public class ExecResultExService extends ExecResultServiceImpl {
synchronized (objResultDeleteAsyncLock) {
if (resultsDeleteAsync.size() > 0) {
for (Map.Entry<String, List<ExecResult>> entry : resultsDeleteAsync.entrySet()) {
String tabName = "";
String dsName = entry.getKey();
List<ExecResult> execResults = entry.getValue();
List<ExecResult> tempExecResults = new ArrayList<>();
if (execResults.size() > 0) {
dstDataSourceService.initDataSource(dsName);
DynamicDataSourceContextHolder.push(dsName);
for (ExecResult execResult : execResults) {
tempExecResults.add(execResult);
Object objTabName = execResult.get(RuleEngineExService.Setting_ResultTableName);
if (StringUtils.isEmpty(tabName) && !ObjectUtils.isEmpty(objTabName)) {
tabName = String.valueOf(objTabName);
}
if (tempExecResults.size() >= 500) {
dbEntityService.clearResultBatch(tempExecResults, tabName);
dbEntityService.clearResultBatch(tempExecResults,dsName);
}
}
if (tempExecResults.size() > 0) {
dbEntityService.clearResultBatch(tempExecResults, tabName);
dbEntityService.clearResultBatch(tempExecResults,dsName);
}
resultsDeleteAsync.remove(dsName);
}
......@@ -162,10 +132,7 @@ public class ExecResultExService extends ExecResultServiceImpl {
}
} catch (Exception ex) {
log.error(ex.getMessage());
} finally {
DynamicDataSourceContextHolder.poll();
}
}
}, 1000, 10000);
}
......@@ -195,27 +162,5 @@ public class ExecResultExService extends ExecResultServiceImpl {
}});
}
}
/**
* 根据数据源类型保存结果
* @param args
* @param
*/
private void saveResultBatch(List<ExecResult> args, String dsName, String tabName) {
Map<String, DataSource> dynamicDSMap = dynamicRoutingDataSource.getCurrentDataSources();
if (!ObjectUtils.isEmpty(dynamicDSMap) && dynamicDSMap.containsKey(dsName)) {
DataSource ds = dynamicDSMap.get(dsName);
if (ds instanceof DruidDataSource) {
DruidDataSource druidDataSource = (DruidDataSource) dynamicDSMap.get(dsName);
if ("oracle".equals(druidDataSource.getDbType())) {
dbEntityService.saveResultBatchByOracle(args, tabName);
args.clear();
return;
}
}
}
dbEntityService.saveResultBatch(args, tabName);
args.clear();
}
}
ibzdst-core/src/main/java/cn/ibizlab/core/lite/extensions/service/DbEntityService.java
浏览文件 @ 14ca8aa9
......@@ -2,6 +2,7 @@ package cn.ibizlab.core.lite.extensions.service;
import cn.ibizlab.core.extensions.mapper.DbEntityMapper;
import cn.ibizlab.core.extensions.service.DstDataSourceExService;
import cn.ibizlab.core.extensions.service.RuleEngineExService;
import cn.ibizlab.core.lite.extensions.domain.EntityModel;
import cn.ibizlab.core.lite.extensions.domain.EntityObj;
import cn.ibizlab.core.lite.extensions.domain.FieldModel;
......@@ -12,70 +13,71 @@ import cn.ibizlab.core.lite.extensions.model.Property;
import cn.ibizlab.core.lite.service.IMetaModelService;
import cn.ibizlab.core.rule.domain.ExecResult;
import cn.ibizlab.util.filter.QueryFilter;
import com.alibaba.fastjson.JSON;
import com.alibaba.druid.pool.DruidDataSource;
import com.baomidou.dynamic.datasource.DynamicRoutingDataSource;
import com.baomidou.dynamic.datasource.toolkit.DynamicDataSourceContextHolder;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;
import javax.sql.DataSource;
import java.io.BufferedReader;
import java.io.Reader;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Service
@Primary
@Slf4j
public class DbEntityService extends ServiceImpl<DbEntityMapper, EntityObj> implements CommonEntityService{
public class DbEntityService extends ServiceImpl<DbEntityMapper, EntityObj> implements CommonEntityService {
@Autowired
private DstDataSourceExService dstDataSourceService;
@Autowired
@Lazy
private DynamicRoutingDataSource dynamicRoutingDataSource;
@Override
public List<EntityObj> selectBase(EntityModel entityModel, QueryFilter filter) {
return search("BASE",entityModel,filter);
return search("BASE", entityModel, filter);
}
@Override
public List<EntityObj> selectCore(EntityModel entityModel, QueryFilter filter) {
return search("CORE",entityModel,filter);
return search("CORE", entityModel, filter);
}
@Override
public Integer selectCount(EntityModel entityModel, QueryFilter filter) {
String sql=entityModel.getSqlSegment("COUNT");
String sql = entityModel.getSqlSegment("COUNT");
String dsName = entityModel.getDsName();
try
{
try {
dstDataSourceService.initDataSource(dsName);
DynamicDataSourceContextHolder.push(dsName);
DbEntitySearchContext context=new DbEntitySearchContext();
DbEntitySearchContext context = new DbEntitySearchContext();
context.setFilter(filter);
QueryWrapper qw=context.getSelectCond();
if(!StringUtils.isEmpty(filter.getCustSqlSegment()))
QueryWrapper qw = context.getSelectCond();
if (!StringUtils.isEmpty(filter.getCustSqlSegment()))
qw.apply(filter.getCustSqlSegment());
Integer count=baseMapper.searchCount(sql,qw);
Integer count = baseMapper.searchCount(sql, qw);
return count;
}
catch(Exception ex)
{
} catch (Exception ex) {
log.error("详细错误信息：" + ex.getMessage() + ", 执行sql：" + sql);
return null;
}
finally
{
} finally {
DynamicDataSourceContextHolder.poll();
}
}
......@@ -83,38 +85,36 @@ public class DbEntityService extends ServiceImpl<DbEntityMapper, EntityObj> impl
@Override
public List<EntityObj> search(String dataSet, EntityModel entityModel, QueryFilter filter) {
String sql=entityModel.getSqlSegment(dataSet);
if(StringUtils.isEmpty(sql))
String sql = entityModel.getSqlSegment(dataSet);
if (StringUtils.isEmpty(sql))
return new ArrayList<>();
else
return search(entityModel.getDsName(),sql,filter);
return search(entityModel.getDsName(), sql, filter);
}
@Override
public List<EntityObj> search(String dsName, String sql, QueryFilter filter) {
try
{
try {
dstDataSourceService.initDataSource(dsName);
DynamicDataSourceContextHolder.push(dsName);
DbEntitySearchContext context=new DbEntitySearchContext();
DbEntitySearchContext context = new DbEntitySearchContext();
context.setFilter(filter);
QueryWrapper qw=context.getSelectCond();
if(!StringUtils.isEmpty(filter.getCustSqlSegment()))
QueryWrapper qw = context.getSelectCond();
if (!StringUtils.isEmpty(filter.getCustSqlSegment()))
qw.apply(filter.getCustSqlSegment());
List<EntityObj> list=baseMapper.search(sql,qw);
List<EntityObj> list = baseMapper.search(sql, qw);
list.forEach(entityObj -> {
entityObj.keySet().forEach(key->{
entityObj.keySet().forEach(key -> {
Object value=entityObj.get(key);
if(value!=null&&value instanceof java.sql.Clob)
{
Object value = entityObj.get(key);
if (value != null && value instanceof java.sql.Clob) {
// Clob类型 转String
BufferedReader br =null;
BufferedReader br = null;
Reader is = null;
try {
String reString = "";
is = ((java.sql.Clob)value).getCharacterStream();
is = ((java.sql.Clob) value).getCharacterStream();
br = new BufferedReader(is);
String s = br.readLine();
StringBuffer sb = new StringBuffer();
......@@ -124,29 +124,22 @@ public class DbEntityService extends ServiceImpl<DbEntityMapper, EntityObj> impl
}
reString = sb.toString();
entityObj.put(key,reString);
}
catch (Exception ex)
{
entityObj.put(key, reString);
} catch (Exception ex) {
}
finally {
try
{
if(br!=null){
} finally {
try {
if (br != null) {
br.close();
}
} catch (Exception ex) {
}
catch (Exception ex)
{}
try
{
if(is!=null){
try {
if (is != null) {
is.close();
}
} catch (Exception ex) {
}
catch (Exception ex)
{}
}
......@@ -154,14 +147,10 @@ public class DbEntityService extends ServiceImpl<DbEntityMapper, EntityObj> impl
});
});
return list;
}
catch(Exception ex)
{
} catch (Exception ex) {
log.error("详细错误信息：" + ex.getMessage() + ", 执行sql：" + sql);
return null;
}
finally
{
} finally {
DynamicDataSourceContextHolder.poll();
}
}
......@@ -179,18 +168,59 @@ public class DbEntityService extends ServiceImpl<DbEntityMapper, EntityObj> impl
@Autowired
private DbEntityMapper dbEntityMapper;
public int saveResultBatch(List<ExecResult> var1,String tableName)
{
return dbEntityMapper.replaceBatch(var1,tableName);
public int saveResultBatch(List<ExecResult> execResults, String dsName) {
int result = 0;
try {
String tableName = getTableName(execResults);
Map<String, DataSource> dynamicDSMap = dynamicRoutingDataSource.getCurrentDataSources();
dstDataSourceService.initDataSource(dsName);
DynamicDataSourceContextHolder.push(dsName);
if (!ObjectUtils.isEmpty(dynamicDSMap) && dynamicDSMap.containsKey(dsName)) {
DataSource ds = dynamicDSMap.get(dsName);
if (ds instanceof DruidDataSource) {
DruidDataSource druidDataSource = (DruidDataSource) dynamicDSMap.get(dsName);
if ("oracle".equals(druidDataSource.getDbType())) {
return dbEntityMapper.replaceBatchByOracle(execResults, tableName);
}
}
}
result = dbEntityMapper.replaceBatch(execResults, tableName);
} catch (Exception ex) {
log.error("存储规则结果发生异常，详细错误信息：" + ex.getMessage());
} finally {
DynamicDataSourceContextHolder.poll();
}
return result;
}
public int saveResultBatchByOracle(List<ExecResult> var1,String tableName) {
return dbEntityMapper.replaceBatchByOracle(var1,tableName);
/**
* 获取规则结果表名
* @param execResults
* @return
*/
private String getTableName(List<ExecResult> execResults) {
String tableName= RuleEngineExService.Default_ResultTableName;
for(ExecResult execResult : execResults){
if(!ObjectUtils.isEmpty(execResult.get(RuleEngineExService.Setting_ResultTableName))){
return String.valueOf(execResult.get(RuleEngineExService.Setting_ResultTableName));
}
}
return tableName;
}
public int clearResultBatch(List<ExecResult> var1,String tableName)
{
return dbEntityMapper.clearBatch(var1,tableName);
public int clearResultBatch(List<ExecResult> execResults, String dsName) {
int result = 0;
try {
String tableName = getTableName(execResults);
dstDataSourceService.initDataSource(dsName);
DynamicDataSourceContextHolder.push(dsName);
result = dbEntityMapper.clearBatch(execResults, tableName);
} catch (Exception ex) {
log.error("存储规则结果发生异常，详细错误信息：" + ex.getMessage());
} finally {
DynamicDataSourceContextHolder.poll();
}
return result;
}
@Autowired
......@@ -201,84 +231,76 @@ public class DbEntityService extends ServiceImpl<DbEntityMapper, EntityObj> impl
@Override
public List<ModelObj> getModelObjs(String metaModelId, Set<String> fillPropertys, Timestamp lastModify)
{
DataModel dataModel= liteModelService.getDataModel(metaModelId);
EntityModel entityModel=dataModel.getFactEntityModel();
FieldModel lastModifyField=entityModel.getLastModifyField();
QueryFilter filter=new QueryFilter();
if(lastModifyField!=null)
public List<ModelObj> getModelObjs(String metaModelId, Set<String> fillPropertys, Timestamp lastModify) {
DataModel dataModel = liteModelService.getDataModel(metaModelId);
EntityModel entityModel = dataModel.getFactEntityModel();
FieldModel lastModifyField = entityModel.getLastModifyField();
QueryFilter filter = new QueryFilter();
if (lastModifyField != null)
filter.ge(lastModifyField.getColumnName(), lastModify);
List<EntityObj> kEntityObjs=this.selectCore(entityModel, filter);
return getModelObjs(dataModel,fillPropertys,kEntityObjs);
List<EntityObj> kEntityObjs = this.selectCore(entityModel, filter);
return getModelObjs(dataModel, fillPropertys, kEntityObjs);
}
@Override
public ModelObj getModelObjs(String metaModelId, Set<String> fillPropertys, Object factKey)
{
DataModel dataModel= liteModelService.getDataModel(metaModelId);
public ModelObj getModelObjs(String metaModelId, Set<String> fillPropertys, Object factKey) {
DataModel dataModel = liteModelService.getDataModel(metaModelId);
List<EntityObj> kEntityObjs = new ArrayList<>();
kEntityObjs.add(new EntityObj().set(dataModel.getFactEntityModel().getKeyField().getColumnName(),factKey));
List<ModelObj> list = getModelObjs(dataModel,fillPropertys,kEntityObjs);
if(list.size()>0)
kEntityObjs.add(new EntityObj().set(dataModel.getFactEntityModel().getKeyField().getColumnName(), factKey));
List<ModelObj> list = getModelObjs(dataModel, fillPropertys, kEntityObjs);
if (list.size() > 0)
return list.get(0);
return null;
}
@Override
public ModelObj getModelObjs(String metaModelId, Set<String> fillPropertys, Object... factUniKey)
{
DataModel dataModel= liteModelService.getDataModel(metaModelId);
public ModelObj getModelObjs(String metaModelId, Set<String> fillPropertys, Object... factUniKey) {
DataModel dataModel = liteModelService.getDataModel(metaModelId);
List<EntityObj> kEntityObjs = new ArrayList<>();
EntityObj entityObj = new EntityObj();
List<FieldModel> uniKeys = dataModel.getFactEntityModel().getUnionKeyFields();
for(int i=0;i<uniKeys.size();i++)
entityObj.set(uniKeys.get(i).getColumnName(),factUniKey[i]);
for (int i = 0; i < uniKeys.size(); i++)
entityObj.set(uniKeys.get(i).getColumnName(), factUniKey[i]);
kEntityObjs.add(entityObj);
List<ModelObj> list = getModelObjs(dataModel,fillPropertys,kEntityObjs);
if(list.size()>0)
List<ModelObj> list = getModelObjs(dataModel, fillPropertys, kEntityObjs);
if (list.size() > 0)
return list.get(0);
return null;
}
@Override
public List<ModelObj> getModelObjs(String metaModelId, Set<String> fillPropertys, List<EntityObj> kEntityObjs)
{
DataModel dataModel= liteModelService.getDataModel(metaModelId);
return getModelObjs(dataModel,fillPropertys,kEntityObjs);
public List<ModelObj> getModelObjs(String metaModelId, Set<String> fillPropertys, List<EntityObj> kEntityObjs) {
DataModel dataModel = liteModelService.getDataModel(metaModelId);
return getModelObjs(dataModel, fillPropertys, kEntityObjs);
}
public List<ModelObj> getModelObjs(DataModel dataModel, Set<String> fillPropertys, List<EntityObj> kEntityObjs)
{
EntityModel entityModel=dataModel.getFactEntityModel();
List<ModelObj> rt=new ArrayList<>();
public List<ModelObj> getModelObjs(DataModel dataModel, Set<String> fillPropertys, List<EntityObj> kEntityObjs) {
EntityModel entityModel = dataModel.getFactEntityModel();
List<ModelObj> rt = new ArrayList<>();
List<EntityObj> factEntityList = new ArrayList<>();
if(fillPropertys==null||fillPropertys.size()==0||fillPropertys.contains(dataModel.getFactPorperty().getPropertyName()))
splitList(kEntityObjs,1000).forEach(list -> factEntityList.addAll(this.selectBase(entityModel, QueryFilter.createQuery().cust(dataModel.lookup(list)))));
factEntityList.forEach(entityObj->
if (fillPropertys == null || fillPropertys.size() == 0 || fillPropertys.contains(dataModel.getFactPorperty().getPropertyName()))
splitList(kEntityObjs, 1000).forEach(list -> factEntityList.addAll(this.selectBase(entityModel, QueryFilter.createQuery().cust(dataModel.lookup(list)))));
factEntityList.forEach(entityObj ->
rt.add(new ModelObj().setDataModel(dataModel).setFactEntity(entityObj.setProperty(dataModel.getFactPorperty()))));
fillEntityObj(dataModel,fillPropertys,rt);
dataModel.getNestedDataModels().forEach(subModel->
this.getModelObjs(subModel,fillPropertys, factEntityList).forEach(subObj->subObj.findParent(rt)));
fillEntityObj(dataModel, fillPropertys, rt);
dataModel.getNestedDataModels().forEach(subModel ->
this.getModelObjs(subModel, fillPropertys, factEntityList).forEach(subObj -> subObj.findParent(rt)));
rt.forEach(modelObj -> modelObj.getTimestamp());
return rt;
}
public void fillEntityObj(DataModel dataModel, Set<String> fillPropertys, List<ModelObj> modelObjs)
{
if(dataModel.getObjectProperties().size()>1)
{
for(Property property:dataModel.getObjectProperties())
{
if(property.getPropertyName().equals(dataModel.getFactPorperty().getPropertyName()))
public void fillEntityObj(DataModel dataModel, Set<String> fillPropertys, List<ModelObj> modelObjs) {
if (dataModel.getObjectProperties().size() > 1) {
for (Property property : dataModel.getObjectProperties()) {
if (property.getPropertyName().equals(dataModel.getFactPorperty().getPropertyName()))
continue;
EntityModel entityModel=property.getEntityModel();
EntityModel entityModel = property.getEntityModel();
List<EntityObj> entityObjs = new ArrayList<>();
if(fillPropertys==null||fillPropertys.size()==0||fillPropertys.contains(property.getPropertyName()))
splitList(modelObjs,1000).forEach(list -> entityObjs.addAll(this.selectBase(entityModel, QueryFilter.createQuery().cust(property.lookup(list)))));
if (fillPropertys == null || fillPropertys.size() == 0 || fillPropertys.contains(property.getPropertyName()))
splitList(modelObjs, 1000).forEach(list -> entityObjs.addAll(this.selectBase(entityModel, QueryFilter.createQuery().cust(property.lookup(list)))));
entityObjs.forEach(entityObj -> {
entityObj.setProperty(property);
......@@ -289,19 +311,19 @@ public class DbEntityService extends ServiceImpl<DbEntityMapper, EntityObj> impl
}
private <T> List<List<T>> splitList(List<T> list, int groupSize){
private <T> List<List<T>> splitList(List<T> list, int groupSize) {
int length = list.size();
// 计算可以分成多少组
int num = ( length + groupSize - 1 )/groupSize ; // TODO
int num = (length + groupSize - 1) / groupSize; // TODO
List<List<T>> newList = new ArrayList<>(num);
for (int i = 0; i < num; i++) {
// 开始位置
int fromIndex = i * groupSize;
// 结束位置
int toIndex = (i+1) * groupSize < length ? ( i+1 ) * groupSize : length ;
newList.add(list.subList(fromIndex,toIndex)) ;
int toIndex = (i + 1) * groupSize < length ? (i + 1) * groupSize : length;
newList.add(list.subList(fromIndex, toIndex));
}
return newList ;
return newList;
}
......
Markdown 格式
0% or
您添加了 0 人 到此讨论。请谨慎行事。
先完成此消息的编辑！
想要评论请 注册 或