Skip to content

D
data-service
提交 535adc44 authored 作者: songchuancai's avatar songchuancai
浏览文件
操作

优化代码

上级 5d11c0d7
全部展开 隐藏空白字符变更
内嵌 并排
正在显示 包含 584 行增加 和 97 行删除
src/main/java/com/hisense/dataservice/bo/QueryDataTask.java
浏览文件 @ 535adc44
......@@ -29,16 +29,12 @@ public class QueryDataTask<T> implements Callable<T> {
public T call() throws Exception {
if (StringUtils.isEmpty(querySql)) {
Result<Long> dataResult = JdbcUtil.executeCountSql(dataApiDataSource, countSql, jdbcParamValues);
return (T) dataResult.getData();
// return (T) JdbcUtil.testCountSql().getData(); // TODO FOR TEST
return (T) JdbcUtil.executeCountSql(dataApiDataSource, countSql, jdbcParamValues);
} else {
List<List<Object>> result = new ArrayList<>();
List<List<Object>> data = new ArrayList<>();
Result<List<JSONObject>> dataResult = JdbcUtil.executeSql(dataApiDataSource, querySql, jdbcParamValues, queryDataTotalPerThread);
Result<List<List<Object>>> result = new Result<>();
List<JSONObject> dataJsonList = dataResult.getData();
// TODO 针对错误信息保存到es
for (JSONObject dataJson : dataJsonList) {
List<Object> lineData = new ArrayList<>();
for (List<String> head : columnHeads) {
......@@ -46,11 +42,12 @@ public class QueryDataTask<T> implements Callable<T> {
Object columnValue = dataJson.get(columnName);
lineData.add(columnValue); // 获取列名对应的值
}
result.add(lineData);
data.add(lineData);
}
result.setMsg(dataResult.getMsg());
result.setData(data);
result.setMsg(dataResult.getMsg());
return (T) result;
// return (T) JdbcUtil.testExecuteSql(pageNumber).getData(); // TODO FOR TEST
}
}
}
......
src/main/java/com/hisense/dataservice/controller/DataServiceLogManagementController.java 0 → 100644
浏览文件 @ 535adc44
package com.hisense.dataservice.controller;
import com.hisense.dataservice.library.model.page.PageData;
import com.hisense.dataservice.library.model.page.PageLink;
import com.hisense.dataservice.service.DataServiceLogManagementService;
import com.hisense.dataservice.vo.CallLogVo;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
/**
* @author : scc
* @date : 2023/03/20
**/
@Api(tags = "数据api服务-日志管理")
@RestController
@RequestMapping("/api/v1/dataService/logs")
public class DataServiceLogManagementController extends BaseController{
@Autowired
private DataServiceLogManagementService dataServiceLogManagementService;
@ApiImplicitParams({
@ApiImplicitParam(name = "environment", value = "环境", required = true),
@ApiImplicitParam(name = "systemId", value = "系统ID", required = true),
@ApiImplicitParam(name = "nextToken", value = "下一页token", required = false),
@ApiImplicitParam(name = "pageSize", value = "指定返回结果中每页显示的记录数量", defaultValue = "10", required = false),
@ApiImplicitParam(name = "page", value = "指定显示返回结果中的第几页", defaultValue = "1", required = false),
@ApiImplicitParam(name = "sortProperty", value = "根据哪个字段排序", required = false, allowableValues = "timestamp, status", defaultValue = "timestamp"),
@ApiImplicitParam(name = "sortOrder", value = "排序方向", required = false, allowableValues = "ASC,DESC")
})
@GetMapping("/callLog/{environment}/{systemId}")
public PageData<CallLogVo> getCallLog(@PathVariable String environment,
@PathVariable Long systemId,
@RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize,
@RequestParam(value = "page", required = false, defaultValue = "1") Integer page,
@RequestParam(value = "sortProperty", required = false) String sortProperty,
@RequestParam(value = "sortOrder", required = false) String sortOrder){
PageLink pageLink = createPageLink(pageSize, page, "", sortProperty, sortOrder);
return dataServiceLogManagementService.getCallLog(environment, systemId, pageLink);
}
}
src/main/java/com/hisense/dataservice/controller/DataServiceManagementController.java
浏览文件 @ 535adc44
package com.hisense.dataservice.controller;
import com.alibaba.fastjson.JSONObject;
import com.hisense.dataservice.dto.DataApiModelDto;
import com.hisense.dataservice.enums.ModelConfigType;
import com.hisense.dataservice.library.model.Result;
import com.hisense.dataservice.library.model.page.PageData;
import com.hisense.dataservice.library.model.page.PageLink;
......@@ -94,5 +96,19 @@ public class DataServiceManagementController extends BaseController {
return null;
}
@ApiOperation(value = "数据服务管理-服务调试")
@ApiImplicitParams({
@ApiImplicitParam(name = "environment", value = "环境", required = true),
@ApiImplicitParam(name = "dataModelId", value = "数据服务ID", required = true),
@ApiImplicitParam(name = "type", value = "调试类型（发布/订阅）", required = true, allowableValues = "PUBLISH/SUBSCRIBE")
})
@GetMapping("/debugService/{environment}/{dataModelId}/{type}")
public Result<JSONObject> debugService(@PathVariable String environment,
@PathVariable Long dataModelId,
@PathVariable ModelConfigType type) {
return dataApiServiceManagementService.debugDataApiService(environment, dataModelId,type);
}
}
src/main/java/com/hisense/dataservice/entity/DataApiModel.java
浏览文件 @ 535adc44
......@@ -50,6 +50,8 @@ public class DataApiModel extends BaseEntity{
@Column(nullable = false)
private Long publishSystemId;
// 发布系统名称
private String publishSystemName;
// 内外网
@Enumerated(EnumType.STRING)
@Column(name = "network_env", columnDefinition = "varchar2(10)", nullable = false)
......
src/main/java/com/hisense/dataservice/enums/OperateEnum.java
浏览文件 @ 535adc44
......@@ -7,12 +7,7 @@ package com.hisense.dataservice.enums;
**/
public enum OperateEnum {
GREATER_THAN("大于"),
LESS_THAN("小于"),
EQUAL("等于"),
NOT_EQUAL("不等于"),
GT_EQ("大于等于"),
LT_EQ("小于等于"),
LIKE("模糊匹配"),
IN("集合"),
BETWEEN_AND("范围");
......
src/main/java/com/hisense/dataservice/es/config/ElasticsearchConfig.java 0 → 100644
浏览文件 @ 535adc44
package com.hisense.dataservice.es.config;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.util.Assert;
import java.util.Arrays;
import java.util.Objects;
/**
* @author : scc
* @date : 2023/01/05
**/
@Configuration
@ConfigurationProperties(prefix = "es")
@Slf4j
@Data
public class ElasticsearchConfig {
private String hosts;
private String username;
private String password;
private int connectTimeout;
private int socketTimeout;
private int connectionRequestTimeout;
@Bean
public RestHighLevelClient client() {
log.info("elasticsearch init start ");
Assert.hasLength(hosts, "elasticsearch host is null");
HttpHost[] httpHosts = Arrays.stream(hosts.split(",")).map(host -> {
Assert.hasLength(host, "elasticsearch host is null");
String[] h = host.split(":");
if (h.length != 2) throw new RuntimeException("host ip or port formate is error");
return new HttpHost(h[0], Integer.parseInt(h[1]));
}).filter(Objects::nonNull).toArray(HttpHost[]::new);
RestClientBuilder builder = RestClient.builder(httpHosts);
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
builder.setHttpClientConfigCallback(f -> f.setDefaultCredentialsProvider(credentialsProvider))
.setRequestConfigCallback(r -> {
r.setConnectTimeout(connectTimeout);
r.setSocketTimeout(socketTimeout);
r.setConnectionRequestTimeout(connectionRequestTimeout);
return r;
});
return new RestHighLevelClient(builder);
}
}
\ No newline at end of file
src/main/java/com/hisense/dataservice/es/data/EsBaseData.java 0 → 100644
浏览文件 @ 535adc44
package com.hisense.dataservice.es.data;
import lombok.Data;
/**
* @author : scc
* @date : 2023/01/05
**/
@Data
public class EsBaseData {
//索引
public String indexName;
//id
public String esId;
}
\ No newline at end of file
src/main/java/com/hisense/dataservice/es/data/EsField.java 0 → 100644
浏览文件 @ 535adc44
package com.hisense.dataservice.es.data;
import com.hisense.dataservice.es.enums.FieldTypeEnum;
import lombok.Data;
/**
* @author : scc
* @date : 2023/01/05
**/
@Data
public class EsField {
//字段对应操作功能
private FieldTypeEnum fieldTypeEnum;
//查询字段名称
private String field;
//查询字段对应的值
private String value;
public EsField(){
}
public EsField(String field ,String value , FieldTypeEnum fieldTypeEnum){
this.value = value;
this.field = field;
this.fieldTypeEnum = fieldTypeEnum;
}
}
\ No newline at end of file
src/main/java/com/hisense/dataservice/es/data/EsPage.java 0 → 100644
浏览文件 @ 535adc44
package com.hisense.dataservice.es.data;
import org.springframework.util.CollectionUtils;
import java.util.List;
/**
* @author : scc
* @date : 2023/01/05
**/
public class EsPage<T> {
//当前页的数量
private int size;
/**
* 当前页
*/
private int pageNum;
/**
* 每页显示多少条
*/
private int pageSize;
/**
* 总记录数
*/
private int total;
/**
* 本页的数据列表
*/
private List<T> list;
/**
* 总页数
*/
private int pages;
/**
* 页码列表的开始索引（包含）
*/
private int beginPageIndex;
/**
* 页码列表的结束索引（包含）
*/
private int endPageIndex;
/**
* 只接受前4个必要的属性，会自动的计算出其他3个属性的值
*
* @param pageNum
* @param pageSize
* @param total
* @param list
*/
public EsPage(int pageNum, int pageSize, int total, List<T> list) {
this.pageNum = pageNum;
this.pageSize = pageSize;
this.total = total;
this.list = list;
this.size = CollectionUtils.isEmpty(list)?0:list.size();
// 计算总页码
pages =pageSize==0?0:(total + pageSize - 1) / pageSize;
// 计算 beginPageIndex 和 endPageIndex
// >> 总页数不多于10页，则全部显示
if (pages <= 10) {
beginPageIndex = 1;
endPageIndex = pages;
}
// >> 总页数多于10页，则显示当前页附近的共10个页码
else {
// 当前页附近的共10个页码（前4个 + 当前页 + 后5个）
beginPageIndex = pageNum - 4;
endPageIndex = pageNum + 5;
// 当前面的页码不足4个时，则显示前10个页码
if (beginPageIndex < 1) {
beginPageIndex = 1;
endPageIndex = 10;
}
// 当后面的页码不足5个时，则显示后10个页码
if (endPageIndex > pages) {
endPageIndex = pages;
beginPageIndex = pages - 10 + 1;
}
}
}
public int getPageSize() {
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
public int getPages() {
return pages;
}
public void setPages(int pages) {
this.pages = pages;
}
public int getBeginPageIndex() {
return beginPageIndex;
}
public void setBeginPageIndex(int beginPageIndex) {
this.beginPageIndex = beginPageIndex;
}
public int getEndPageIndex() {
return endPageIndex;
}
public void setEndPageIndex(int endPageIndex) {
this.endPageIndex = endPageIndex;
}
public List<T> getList() {
return list;
}
public void setList(List<T> list) {
this.list = list;
}
public int getSize() {
return size;
}
public void setSize(int size) {
this.size = size;
}
public int getPageNum() {
return pageNum;
}
public void setPageNum(int pageNum) {
this.pageNum = pageNum;
}
public int getTotal() {
return total;
}
public void setTotal(int total) {
this.total = total;
}
}
src/main/java/com/hisense/dataservice/es/data/EsSearchData.java 0 → 100644
浏览文件 @ 535adc44
package com.hisense.dataservice.es.data;
import lombok.AllArgsConstructor;
import lombok.Data;
/**
* @author : scc
* @date : 2023/01/10
**/
@Data
@AllArgsConstructor
public class EsSearchData {
private String tenantId;
private String deviceId;
private String productId;
private String deviceType;
private String topic;
private String ts;
private String data;
private Long nextToken;
}
src/main/java/com/hisense/dataservice/es/enums/FieldTypeEnum.java 0 → 100644
浏览文件 @ 535adc44
package com.hisense.dataservice.es.enums;
/**
* @author : scc
* @date : 2023/01/05
**/
public enum FieldTypeEnum {
//精准查询
PRECISE_QUERY,
//模糊查询
VAGUE_QUERY,
//排序_升序
ORDER_ASC,
//排序_降序
ORDER_ESC,
//时间范围查询
TS_RANGE_QUERY
;
}
src/main/java/com/hisense/dataservice/repository/DataApiModelRepository.java
浏览文件 @ 535adc44
......@@ -6,6 +6,7 @@ import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
import java.util.Optional;
......@@ -24,4 +25,7 @@ public interface DataApiModelRepository extends JpaRepository<DataApiModel,Long>
Optional<DataApiModel> findByIdAndSystemEnv(Long id, SystemEnv systemEnv);
boolean existsByNameAndSystemEnv(String name, SystemEnv env);
@Query(value = "select name from system_info where id = :systemId", nativeQuery = true)
String findSystemNameBySystemId(Long systemId);
}
src/main/java/com/hisense/dataservice/service/DataApiServiceManagementService.java
浏览文件 @ 535adc44
......@@ -2,6 +2,7 @@ package com.hisense.dataservice.service;
import com.alibaba.fastjson.JSONObject;
import com.hisense.dataservice.dto.DataApiModelDto;
import com.hisense.dataservice.enums.ModelConfigType;
import com.hisense.dataservice.library.model.Result;
import com.hisense.dataservice.library.model.page.PageData;
import com.hisense.dataservice.library.model.page.PageLink;
......@@ -22,7 +23,7 @@ public interface DataApiServiceManagementService extends BaseService {
// 创建或修改未发布的数据api服务（草稿状态）
Result<Long> createDataApi(String environment, DataApiModelDto dataApiModelDto);
Result<JSONObject> debugDataApiService(String environment, Long dataModelId);
Result<JSONObject> debugDataApiService(String environment, Long dataModelId, ModelConfigType type);
Result<PageData<DataApiModelItemVo>> queryDataApiList(String environment, Integer status, PageLink pageLink);
......
src/main/java/com/hisense/dataservice/service/DataServiceLogManagementService.java 0 → 100644
浏览文件 @ 535adc44
package com.hisense.dataservice.service;
import com.hisense.dataservice.library.model.page.PageData;
import com.hisense.dataservice.library.model.page.PageLink;
import com.hisense.dataservice.vo.CallLogVo;
/**
* @author : scc
* @date : 2023/03/20
**/
public interface DataServiceLogManagementService {
PageData<CallLogVo> getCallLog(String environment, Long systemId, PageLink pageLink);
}
src/main/java/com/hisense/dataservice/service/impl/DataApiServiceManagementServiceImpl.java
浏览文件 @ 535adc44
......@@ -5,6 +5,7 @@ import com.hisense.dataservice.bo.*;
import com.hisense.dataservice.dto.DataApiModelDto;
import com.hisense.dataservice.entity.*;
import com.hisense.dataservice.enums.*;
import com.hisense.dataservice.es.utils.ElasticsearchUtils;
import com.hisense.dataservice.library.model.Result;
import com.hisense.dataservice.library.model.page.PageData;
import com.hisense.dataservice.library.model.page.PageLink;
......@@ -58,6 +59,9 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
private ThreadPoolManagerUtil debugQueryDataThreadPool = new ThreadPoolManagerUtil("debugQueryDataThread");
@Autowired
private ElasticsearchUtils elasticsearchUtils;
@Transactional
public Result<Boolean> createOrUpdateDataApi(DataApiModelDto dataApiModelDto) {
// 获取数据源
......@@ -170,6 +174,7 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
if(null == dataApiModelDto.getPublishSystemId()){
return failure("发布系统ID不能为空", null);
}
dataApiModel.setPublishSystemName(dataApiModelRepository.findSystemNameBySystemId(dataApiModelDto.getPublishSystemId()));
dataApiModel.setNetworkEnv(NetworkEnv.valueOf(dataApiModelDto.getNetworkEnv()));
dataApiModel.setSystemEnv(SystemEnv.valueOf(dataApiModelDto.getSystemEnv()));
dataApiModel.setPublishSystemId(dataApiModelDto.getPublishSystemId());
......@@ -450,7 +455,7 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
}
return tableIdAndColumnIdToSelfMap;
}
public Result<JSONObject> debugDataApiService(String environment, Long dataModelId){
public Result<JSONObject> debugDataApiService(String environment, Long dataModelId, ModelConfigType type){
JSONObject result = new JSONObject();
Optional<DataApiModel> dataApiModelOptional = dataApiModelRepository.findById(dataModelId);
......@@ -467,37 +472,44 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
}
DataApiDataSource dataApiDataSource = dataApiDataSourceOptional.get();
List<TableJoinRelationConfig> tableJoinRelationConfigs = dataApiModel.getTableJoinRelationConfig();
if(CollectionUtils.isEmpty(tableJoinRelationConfigs)){
return failure("调试数据api服务失败--数据表关系配置不存在", null);
}
DataApiModelConfig dataApiModelConfig;
Optional<DataApiModelConfig> dataApiModelConfigOptional = dataApiModelConfigRepository.findByTypeIdAndType(dataApiModel.getId(), ModelConfigType.PUBLISH);
Optional<DataApiModelConfig> dataApiModelConfigOptional = dataApiModelConfigRepository.findByTypeIdAndType(dataApiModel.getId(), type);
if (!dataApiModelConfigOptional.isPresent()) {
return failure("调试数据api服务失败--行列配置不存在", null);
}
dataApiModelConfig = dataApiModelConfigOptional.get();
List<TableJoinRelationConfig> tableJoinRelationConfigs = dataApiModel.getTableJoinRelationConfig();
if(CollectionUtils.isEmpty(tableJoinRelationConfigs)){
return failure("调试数据api服务失败--数据表关系配置不存在", null);
}
String fullSql = dataApiModelConfig.generateAndSetFullSql(tableJoinRelationConfigs, dataApiDataSource.getType());
if(StringUtils.isEmpty(fullSql)){
return failure("调试数据api服务失败--生成sql失败", null);
}
// 查询出前10条数据进行返回
Result<List<List<Object>>> executeSqlResult = new Result<>();
List<List<Object>> data = new ArrayList<>();
List<Object> jdbcParams = getJdbcParams(dataApiModelConfig, dataApiDataSource.getType());
try{
data = executeDataQueryTask(fullSql, dataApiModelConfig, dataApiDataSource);
executeSqlResult = executeDataQueryTask(fullSql, dataApiModelConfig, dataApiDataSource,jdbcParams);
dataApiModel.setDebugStatus(DebugStatusEnum.SUCCESS);
data = executeSqlResult.getData();
}catch (Exception e){
log.error("执行数据查询失败：", e);
dataApiModel.setDebugStatus(DebugStatusEnum.FAIL);
}
// 设置调试结果
// 保存调试结果
dataApiModelRepository.save(dataApiModel);
// 获取数据的列名
List<List<String>> heads = ColumnDataRangeFilter.generateExcelHead(dataApiModelConfig.getColumnDataRangeConfig());
result.put("data", data);
result.put("heads", heads);
return success("数据api服务调试", result);
if(DebugStatusEnum.FAIL.equals(dataApiModel.getDebugStatus())){
return failure(executeSqlResult.getMsg());
}
return success("数据api服务调试成功", result);
}
@Override
......@@ -537,13 +549,25 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
dataApiModelDetailVo.setColumnDataRangeConfig(item.getColumnDataRangeConfig());
dataApiModelDetailVo.setLineDataRangeConfig(item.getLineDataRangeConfig());
});
// TODO 设置发布系统相关信息
// dataApiModelDetailVo.setPublishSystem();
// dataApiModelDetailVo.setSystemLeader();
dataApiModelDetailVo.setSystemLeader(dataApiModel.getCreator());
return success(dataApiModelDetailVo);
}
private List<List<Object>> executeDataQueryTask(String dataSql, DataApiModelConfig dataApiModelConfig, DataApiDataSource dataSource) throws Exception{
private Result<List<List<Object>>> executeDataQueryTask(String dataSql, DataApiModelConfig dataApiModelConfig, DataApiDataSource dataSource, List<Object> jdbcParamValues) throws Exception{
QueryDataTask queryDataTask = new QueryDataTask();
queryDataTask.setDataApiDataSource(dataSource);
queryDataTask.setQuerySql(dataSql);
queryDataTask.setPageNumber(1L);
queryDataTask.setColumnHeads(ColumnDataRangeFilter.generateExcelHead(dataApiModelConfig.getColumnDataRangeConfig()));
queryDataTask.setJdbcParamValues(jdbcParamValues);
Future<?> future = debugQueryDataThreadPool.addExecuteTask(queryDataTask);
Result<List<List<Object>>> result = new Result<>();
result = (Result<List<List<Object>>>) future.get();
return result;
}
private List<Object> getJdbcParams(DataApiModelConfig dataApiModelConfig, DataSourceTypeEnum dataSourceType){
// 获取sql的填充参数
List<Object> jdbcParamValues = new ArrayList<>();
List<LineDataRangeFilter> lineDataRangeConfigs = dataApiModelConfig.getLineDataRangeConfig();
......@@ -554,8 +578,6 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
}
}
}
DataSourceTypeEnum dataSourceType = dataSource.getType();
// 封装查询数据任务
Long limitFirstValue = 0L;
Long limitSecondValue = 10L;
......@@ -571,17 +593,7 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
}
jdbcParamValues.add(limitFirstValue);
jdbcParamValues.add(limitSecondValue);
QueryDataTask queryDataTask = new QueryDataTask();
queryDataTask.setDataApiDataSource(dataSource);
queryDataTask.setQuerySql(dataSql);
queryDataTask.setPageNumber(1L);
queryDataTask.setColumnHeads(ColumnDataRangeFilter.generateExcelHead(dataApiModelConfig.getColumnDataRangeConfig()));
queryDataTask.setJdbcParamValues(jdbcParamValues);
Future<?> future = debugQueryDataThreadPool.addExecuteTask(queryDataTask);
List<List<Object>> data = new ArrayList<>();
data = (List<List<Object>>) future.get();
return data;
return jdbcParamValues;
}
}
src/main/java/com/hisense/dataservice/service/impl/DataServiceLogManagementServiceImpl.java 0 → 100644
浏览文件 @ 535adc44
package com.hisense.dataservice.service.impl;
import com.hisense.dataservice.es.data.EsField;
import com.hisense.dataservice.es.data.EsPage;
import com.hisense.dataservice.es.enums.FieldTypeEnum;
import com.hisense.dataservice.es.utils.ElasticsearchUtils;
import com.hisense.dataservice.library.model.page.PageData;
import com.hisense.dataservice.library.model.page.PageLink;
import com.hisense.dataservice.library.model.page.SortOrder;
import com.hisense.dataservice.service.DataServiceLogManagementService;
import com.hisense.dataservice.vo.CallLogVo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
/**
* @author : scc
* @date : 2023/03/20
**/
@Service
@Slf4j
public class DataServiceLogManagementServiceImpl implements DataServiceLogManagementService {
@Autowired
private ElasticsearchUtils elasticsearchUtils;
@Override
public PageData<CallLogVo> getCallLog(String environment, Long systemId, PageLink pageLink) {
List<EsField> conditions = new ArrayList<>();
// 排序
EsField esField = new EsField();
esField.setField(pageLink.getSortOrder().getProperty());
esField.setFieldTypeEnum(pageLink.getSortOrder().getDirection().equals(SortOrder.Direction.ASC) ? FieldTypeEnum.ORDER_ASC :FieldTypeEnum.ORDER_ESC);
conditions.add(esField);
// 索引名称 ： data_serv_call_log_+环境+订阅系统ID
String indexName = "data_serv_call_log_" + environment + "_" +systemId;
PageData result = new PageData();
result.setData(new ArrayList());
EsPage<CallLogVo> callLogVoEsPage;
try{
callLogVoEsPage = elasticsearchUtils.selectDataPage(indexName, pageLink.getPage(), pageLink.getPageSize(), conditions, CallLogVo.class);
result.setData(callLogVoEsPage.getList());
result.setCurrentPage(callLogVoEsPage.getPageNum());
result.setTotalPages(callLogVoEsPage.getPages());
result.setTotalElements(callLogVoEsPage.getTotal());
}catch (Exception e){
log.error("查询数据服务调用日志失败", e.getMessage());
}
return result;
}
}
src/main/java/com/hisense/dataservice/util/JdbcUtil.java
浏览文件 @ 535adc44
......@@ -12,49 +12,9 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@Slf4j
public class JdbcUtil {
// for test
public static Result<Long> testCountSql() {
Integer result = 1100;
String threadName = Thread.currentThread().getName();
log.info("thread: {}, start...", threadName);
try {
Thread.sleep(1000 * new Random().nextInt(5));
} catch (InterruptedException e) {
e.printStackTrace();
}
log.info("thread: {}, end...", threadName);
return new Result<>(result.longValue());
}
// for test
public static Result<List<List<Object>>> testExecuteSql(Long pageNumber) {
List<List<Object>> jsonObjects = new ArrayList<>();
String threadName = Thread.currentThread().getName();
log.info("thread: {}, start...", threadName);
try {
Thread.sleep(1000 * new Random().nextInt(5));
} catch (InterruptedException e) {
e.printStackTrace();
}
for (int i = 0; i < 10; i++) {
List<Object> jsonObject = new ArrayList<>();
jsonObject.add("number:" + pageNumber + "scc_" + i);
jsonObject.add(i + 10);
jsonObject.add(i + 11);
jsonObject.add(i + 12);
jsonObject.add(i + 13);
jsonObject.add(i + 14);
jsonObjects.add(jsonObject);
}
log.info("thread: {}, end...", threadName);
return new Result<>(jsonObjects);
}
public static Result<List<JSONObject>> executeSql(DataApiDataSource datasource, String sql, List<Object> jdbcParamValues, Integer batchSize) throws Exception {
String threadName = Thread.currentThread().getName();
......@@ -102,13 +62,10 @@ public class JdbcUtil {
return new Result("200", "查询成功", result);
} catch (SQLException sqlException) {
log.error("[{}] 数据库连接失败：", datasource.getSourceName(), sqlException);
// TODO 记录到ES
throw new SQLException("数据库连接失败");
// return new Result("600", sqlException.getMessage(), result);
return new Result("300", "数据源异常：" + sqlException.getMessage(), result);
} catch (Exception e) {
log.error("thread: {}, sql: {}, params: {}, 分页查询数据异常：", threadName, sql, jdbcParamValues, e);
throw new SQLException("分页查询数据异常");
// return new Result("500", e.getMessage(), result);
return new Result("400", "系统异常：" + e.getMessage(), result);
} finally {
try {
if (connection != null)
......
src/main/java/com/hisense/dataservice/vo/CallLogVo.java 0 → 100644
浏览文件 @ 535adc44
package com.hisense.dataservice.vo;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.Date;
import java.util.List;
/**
* @author : scc
* @date : 2023/03/20
**/
@ApiModel("数据服务调用日志vo")
@Data
public class CallLogVo {
@ApiModelProperty(name = "订阅ID")
private Long subscribeSystemId;
@ApiModelProperty(name = "环境")
private String environment;
@ApiModelProperty(name = "数据服务名称")
private String serviceName;
@ApiModelProperty(name = "调用系统")
private String systemName;
@ApiModelProperty(name = "调用时间")
@JsonFormat(shape = JsonFormat.Shape.STRING,pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private String timestamp;
@ApiModelProperty(name = "调用状态")
private String status;
@ApiModelProperty(name = "原因")
private String reason;
@ApiModelProperty(name = "sql语句")
private String sql;
@ApiModelProperty(name = "sql过滤值")
private List<String> jdbcParams;
}
src/main/java/com/hisense/dataservice/vo/DataApiModelDetailVo.java
浏览文件 @ 535adc44
......@@ -39,7 +39,7 @@ public class DataApiModelDetailVo {
private String systemLeader;
@ApiModelProperty(name = "发布系统")
private String publishSystem;
private String publishSystemName;
@ApiModelProperty(name = "发布时间")
private Date createdTime;
......
src/main/java/com/hisense/dataservice/vo/DataApiModelItemVo.java
浏览文件 @ 535adc44
......@@ -13,9 +13,6 @@ import lombok.Data;
@Data
public class DataApiModelItemVo {
@ApiModelProperty(name = "数据源ID")
private Long dataSourceId;
@ApiModelProperty(name = "数据服务ID")
private Long id;
......@@ -31,4 +28,10 @@ public class DataApiModelItemVo {
@ApiModelProperty(name = "系统环境")
private SystemEnv systemEnv;
@ApiModelProperty(name = "发布人")
private String creator;
@ApiModelProperty(name = "发布系统名")
private String systemName;
}
src/main/resources/application.yml
浏览文件 @ 535adc44
......@@ -15,4 +15,10 @@ spring:
server:
port=8080 # 应用服务 WEB 访问端口
es:
hosts: ${ES_HOST:192.168.78.132:9200}
username: ${ES_USERNAME:elastic}
password: ${ES_PASSWORD:elastic}
connectTimeout: ${ES_CONNECT_TIMEOUT:30000}
socketTimeout: ${ES_SOCKET_TIMEOUT:30000}
connectionRequestTimeout: ${CONNECTION_REQUEST_TIMEOUT:30000}
src/test/java/com/hisense/dataservice/service/DataApiServiceManagementServiceTest.java
浏览文件 @ 535adc44
......@@ -4,10 +4,16 @@ import com.alibaba.fastjson.JSONObject;
import com.hisense.dataservice.DataServiceApplication;
import com.hisense.dataservice.bo.*;
import com.hisense.dataservice.dto.DataApiModelDto;
import com.hisense.dataservice.enums.ModelConfigType;
import com.hisense.dataservice.enums.NetworkEnv;
import com.hisense.dataservice.enums.OperateEnum;
import com.hisense.dataservice.enums.SystemEnv;
import com.hisense.dataservice.es.utils.ElasticsearchUtils;
import com.hisense.dataservice.library.model.Result;
import com.hisense.dataservice.library.model.page.PageData;
import com.hisense.dataservice.library.model.page.PageLink;
import com.hisense.dataservice.library.model.page.SortOrder;
import com.hisense.dataservice.vo.CallLogVo;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Test;
......@@ -29,6 +35,10 @@ import java.util.List;
public class DataApiServiceManagementServiceTest {
@Autowired
private DataServiceLogManagementService dataServiceLogManagementService;
@Autowired
private ElasticsearchUtils elasticsearchUtils;
@Autowired
private DataApiServiceManagementService dataApiServiceManagementService;
@Test
......@@ -52,7 +62,7 @@ public class DataApiServiceManagementServiceTest {
log.info("执行第四步结果，发布api：{}", result4);
Assert.assertEquals(false, result4.getData());
Result<JSONObject> result5 = dataApiServiceManagementService.debugDataApiService("test", result3.getData());
Result<JSONObject> result5 = dataApiServiceManagementService.debugDataApiService("test", result3.getData(), ModelConfigType.PUBLISH);
log.info("执行第五步结果，调试api：{}", result5);
Assert.assertEquals(false, result5.getData());
......@@ -85,7 +95,7 @@ public class DataApiServiceManagementServiceTest {
log.info("执行发布api；第四步结果，发布api：{}", result4);
Assert.assertEquals(false, result4.getData());
Result<JSONObject> result5 = dataApiServiceManagementService.debugDataApiService("test", result3.getData());
Result<JSONObject> result5 = dataApiServiceManagementService.debugDataApiService("test", result3.getData(), ModelConfigType.PUBLISH);
log.info("执行调试；第五步结果，调试api：{}", result5);
Assert.assertEquals(null, result5.getData());
......@@ -93,7 +103,7 @@ public class DataApiServiceManagementServiceTest {
@Test
public void test(){
Result<JSONObject> result5 = dataApiServiceManagementService.debugDataApiService("DEV", 541L);
Result<JSONObject> result5 = dataApiServiceManagementService.debugDataApiService("DEV", 541L, ModelConfigType.PUBLISH);
log.info("执行调试；第五步结果，调试api：{}", result5);
Assert.assertEquals(null, result5.getData());
}
......@@ -267,4 +277,31 @@ public class DataApiServiceManagementServiceTest {
filterCondition.setFilterValues(filterValues);
return filterCondition;
}
@Test
public void testEsLog(){
// 保存调用记录到ES中
JSONObject callSqlEsLogJson = new JSONObject();
callSqlEsLogJson.put("sql", "select * from A where 1=1 and namel = ?");
callSqlEsLogJson.put("jdbcParams", "['a']");
callSqlEsLogJson.put("serviceName", "测试服务名");
callSqlEsLogJson.put("systemName", "测试系统名");
callSqlEsLogJson.put("status", "失败");
callSqlEsLogJson.put("reason", "数据库连接失败");
callSqlEsLogJson.put("timestamp", System.currentTimeMillis());
callSqlEsLogJson.put("environment", "staging");
callSqlEsLogJson.put("subscribeSystemId", "1");
String indexName = "call_log_" + "staging_" + "2";
boolean addLogResult = elasticsearchUtils.addData(indexName, callSqlEsLogJson.toJSONString());
log.info("添加记录到es，结果：{}", addLogResult);
PageLink pageLink = new PageLink(10,1,"", new SortOrder("timestamp", SortOrder.Direction.DESC));
PageData<CallLogVo> pageData1 = dataServiceLogManagementService.getCallLog("staging", 2L, pageLink);
log.info("pageData1: {}", pageData1);
PageData<CallLogVo> pageData2 = dataServiceLogManagementService.getCallLog("staging", 1L, pageLink);
log.info("pageData2: {}", pageData2);
}
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论