Skip to content

D
data-service
提交 ef00bcd9 authored 作者: songchuancai's avatar songchuancai
浏览文件
操作

修改数据api服务创建逻辑

上级 7ac5d815
隐藏空白字符变更
内嵌 并排
正在显示 包含 536 行增加 和 168 行删除
src/main/java/com/hisense/dataservice/bo/ColumnDataRangeFilter.java
浏览文件 @ ef00bcd9
package com.hisense.dataservice.bo;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
......@@ -11,21 +15,47 @@ import java.util.List;
* @author : scc
* @date : 2023/02/20
* 列数据范围过滤
* TODO tableName tableId 和columnId columnName不为空的设计
* TODO tableName tableId 和columnId columnName不为空的设计
**/
@ApiModel("列数据范围过滤")
@Data
@AllArgsConstructor
@NoArgsConstructor
public class ColumnDataRangeFilter {
// 表名
@JsonIgnore
@ApiModelProperty(name = "表名", required = false)
private String tableName;
// 表名ID
@ApiModelProperty(name = "表名ID", required = true)
private Long tableId;
// 字段
private List<DataColumn> fields;
private String generateSql(){
private StringBuilder generateSql(StringBuilder resultBuilder) {
if (this.getFields().isEmpty()) {
resultBuilder.append(this.getTableName())
.append(".*")
.append(", ");
return resultBuilder;
}
for (DataColumn field : this.getFields()) {
resultBuilder.append(this.getTableName())
.append(".")
.append(field.getColumnName())
.append(" as ")
.append(this.getTableName())
.append("_")
.append(field.getColumnName())
.append(",");
}
return resultBuilder;
}
private String generateSql() {
StringBuilder resultBuilder = new StringBuilder();
if(this.getFields().isEmpty()){
resultBuilder.append(this.getTableName())
......@@ -46,7 +76,8 @@ public class ColumnDataRangeFilter {
return resultBuilder.toString();
}
private List<List<String>> generateExcelHead(){
private List<List<String>> generateExcelHead() {
List<List<String>> heads = new ArrayList<>();
StringBuilder headBuilder;
List<String> head;
......@@ -67,24 +98,26 @@ public class ColumnDataRangeFilter {
/**
* 生成sql-select
* 例如： select * 或者 select t1.a, t2.b, t2.c
*
* @param columnDataRangeFilters
* @return
*/
public static String generateFilterSelectSql(List<ColumnDataRangeFilter> columnDataRangeFilters){
public static String generateFilterSelectSql(List<ColumnDataRangeFilter> columnDataRangeFilters) {
StringBuilder result = new StringBuilder();
if (columnDataRangeFilters.isEmpty()) {
result.append(" select * ");
if (CollectionUtils.isEmpty(columnDataRangeFilters)) {
result.append(" * ");
return result.toString();
}
for (ColumnDataRangeFilter columnDataRangeFilter : columnDataRangeFilters) {
result.append(columnDataRangeFilter.generateSql());
// result.append(columnDataRangeFilter.generateSql());
result = columnDataRangeFilter.generateSql(result);
}
return result.substring(0, result.lastIndexOf(","));
}
public static List<List<String>> generateExcelHead(List<ColumnDataRangeFilter> columnDataRangeFilters){
public static List<List<String>> generateExcelHead(List<ColumnDataRangeFilter> columnDataRangeFilters) {
List<List<String>> result = new ArrayList<>();
for (ColumnDataRangeFilter columnDataRangeFilter : columnDataRangeFilters) {
result.addAll(columnDataRangeFilter.generateExcelHead());
......
src/main/java/com/hisense/dataservice/bo/DataColumn.java
浏览文件 @ ef00bcd9
package com.hisense.dataservice.bo;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.hisense.dataservice.enums.FieldTypeEnum;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@ApiModel("数据字段描述")
@Data
@AllArgsConstructor
@NoArgsConstructor
public class DataColumn {
// 字段名
@JsonIgnore
@ApiModelProperty(name = "字段名", required = false)
private String columnName;
// 字段ID
@ApiModelProperty(name = "字段ID", required = true)
private Long id;
// 字段类型
@ApiModelProperty(name = "字段类型", required = false)
private FieldTypeEnum fieldType;
// 字段描述
@ApiModelProperty(name = "字段描述", required = false)
private String desc;
}
\ No newline at end of file
src/main/java/com/hisense/dataservice/bo/FilterCondition.java
浏览文件 @ ef00bcd9
package com.hisense.dataservice.bo;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.hisense.dataservice.enums.OperateEnum;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
......@@ -13,21 +16,33 @@ import java.util.List;
* @author : scc
* @date : 2023/02/27
**/
@ApiModel("行过滤条件")
@Data
@AllArgsConstructor
@NoArgsConstructor
public class FilterCondition {
// 表名
@JsonIgnore
@ApiModelProperty(name = "表名", required = false)
private String tableName;
// 表ID
@JsonIgnore
@ApiModelProperty(name = "表ID", required = false)
private Long tableId;
// 字段
@ApiModelProperty(name = "字段", required = true)
@NonNull
private DataColumn field;
// 过滤条件
@ApiModelProperty(name = "过滤条件", required = true)
@NonNull
private OperateEnum operateEnumType;
// 过滤值
@ApiModelProperty(name = "过滤值", required = true)
@NonNull
private List<Object> filterValues;
......
src/main/java/com/hisense/dataservice/bo/LineDataRangeFilter.java
浏览文件 @ ef00bcd9
package com.hisense.dataservice.bo;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
......@@ -11,16 +14,25 @@ import java.util.List;
* @date : 2023/02/20
* 行数据范围过滤
**/
@ApiModel("行数据范围过滤")
@Data
@AllArgsConstructor
@NoArgsConstructor
public class LineDataRangeFilter {
// 条件全部满足还是满足一个: true:全部匹配， false: 只需要满足一个即可
@ApiModelProperty(name = "条件是否全部满足", required = false)
private Boolean fullMatched = Boolean.TRUE;
// 过滤条件
@ApiModelProperty(name = "过滤条件", required = false)
private List<FilterCondition> filterConditions;
@ApiModelProperty(name = "表ID", required = true)
private Long tableId;
@JsonIgnore
@ApiModelProperty(name = "表名", required = false)
private String tableName;
/**
......@@ -35,6 +47,7 @@ public class LineDataRangeFilter {
StringBuilder result = new StringBuilder();
// 拼接初始条件
if (LineDataRangeFilters.isEmpty()) {
result.append(" 1=1 ");
return result.toString();
}
......
src/main/java/com/hisense/dataservice/bo/QueryDataTask.java 0 → 100644
浏览文件 @ ef00bcd9
package com.hisense.dataservice.bo;
import com.alibaba.fastjson.JSONObject;
import com.hisense.dataservice.entity.DataApiDataSource;
import com.hisense.dataservice.library.model.Result;
import com.hisense.dataservice.util.JdbcUtil;
import lombok.Data;
import org.springframework.util.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
/**
* @author : scc
* @date : 2023/03/14
**/
@Data
public class QueryDataTask<T> implements Callable<T> {
private DataApiDataSource dataApiDataSource;
private String querySql;
private String countSql;
private List<Object> jdbcParamValues;
private Long pageNumber;
private Integer queryDataTotalPerThread = 2000; // 默认2000
private List<List<String>> columnHeads;
@Override
public T call() throws Exception {
if (StringUtils.isEmpty(querySql)) {
Result<Long> dataResult = JdbcUtil.executeCountSql(dataApiDataSource, countSql, jdbcParamValues);
return (T) dataResult.getData();
// return (T) JdbcUtil.testCountSql().getData(); // TODO FOR TEST
} else {
List<List<Object>> result = new ArrayList<>();
Result<List<JSONObject>> dataResult = JdbcUtil.executeSql(dataApiDataSource, querySql, jdbcParamValues, queryDataTotalPerThread);
List<JSONObject> dataJsonList = dataResult.getData();
for (JSONObject dataJson : dataJsonList) {
List<Object> lineData = new ArrayList<>();
for (List<String> head : columnHeads) {
String columnName = head.get(0);
Object columnValue = dataJson.get(columnName);
lineData.add(columnValue); // 获取列名对应的值
}
result.add(lineData);
}
return (T) result;
// return (T) JdbcUtil.testExecuteSql(pageNumber).getData(); // TODO FOR TEST
}
}
}
src/main/java/com/hisense/dataservice/bo/TableJoinRelationConfig.java
浏览文件 @ ef00bcd9
package com.hisense.dataservice.bo;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
......@@ -10,12 +12,17 @@ import lombok.NoArgsConstructor;
* @date : 2023/02/27
* 多表之间join关系配置
**/
@ApiModel("数据表连接关系")
@Data
@AllArgsConstructor
@NoArgsConstructor
public class TableJoinRelationConfig {
@ApiModelProperty(name = "表名", required = true)
private String tableName;
@ApiModelProperty(name = "列名", required = true)
private String columnName;
@ApiModelProperty(name = "表ID", required = true)
private Long tableId;
@ApiModelProperty(name = "列ID", required = true)
private Long columnId;
}
src/main/java/com/hisense/dataservice/controller/DataApiCommonServiceController.java
浏览文件 @ ef00bcd9
......@@ -6,7 +6,12 @@ import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
/**
* @author : scc
......@@ -27,7 +32,7 @@ public class DataApiCommonServiceController {
@ApiImplicitParam(name = "pattern", value = "请求路径", required = true)
})
@GetMapping("/{environment}/{pattern}")
public Object dataQuery(@PathVariable String environment, @PathVariable String pattern) {
return dataApiCommonService.queryData(environment, pattern);
public Object dataQuery(@PathVariable String environment, @PathVariable String pattern, HttpServletRequest request) {
return dataApiCommonService.queryData(environment, pattern, request);
}
}
src/main/java/com/hisense/dataservice/controller/DataServiceManagementController.java
浏览文件 @ ef00bcd9
......@@ -6,6 +6,7 @@ import com.hisense.dataservice.service.DataApiCommonService;
import com.hisense.dataservice.service.DataApiDataSourceManagementService;
import com.hisense.dataservice.service.DataApiServiceManagementService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
......@@ -26,9 +27,22 @@ public class DataServiceManagementController {
@Autowired
private DataApiServiceManagementService dataApiServiceManagementService;
@PostMapping("/")
public Result<Boolean> createOrUpdateDataApiService(DataApiModelDto dataApiModelDto){
return dataApiServiceManagementService.createOrUpdateDataApi(dataApiModelDto);
// @PostMapping("/")
// public Result<Boolean> createOrUpdateDataApiService(DataApiModelDto dataApiModelDto){
// return dataApiServiceManagementService.createOrUpdateDataApi(dataApiModelDto);
// }
@ApiOperation(value = "数据服务管理-创建数据服务（草稿状态）",notes = "返回数据模型ID")
@PostMapping("/{environment}")
public Result<Long> createDataApiService(@RequestBody DataApiModelDto dataApiModelDto, @PathVariable String environment){
return dataApiServiceManagementService.createDataApi(environment,dataApiModelDto);
}
@ApiOperation(value = "数据服务管理-发布数据服务")
@PostMapping("/{environment}/{modelId}")
public Result<Boolean> publishDataApi(@PathVariable String environment,
@PathVariable Long modelId) {
return dataApiServiceManagementService.publishDataApi(environment, modelId);
}
@GetMapping("/{environment}")
......@@ -36,11 +50,7 @@ public class DataServiceManagementController {
return null;
}
@PostMapping("/{environment}")
public Object publishDataApi(@PathVariable String environment,
@RequestBody DataApiModelDto dataApiModelDto) {
return null;
}
@PatchMapping("/{environment}")
public Object updateDataApi(@PathVariable String environment) {
......
src/main/java/com/hisense/dataservice/dto/DataApiModelDto.java
浏览文件 @ ef00bcd9
......@@ -33,9 +33,6 @@ public class DataApiModelDto {
@ApiModelProperty(name = "行数据范围", required = true)
private List<LineDataRangeFilter> lineDataRangeConfig;
@ApiModelProperty(name = "apiID", required = true)
private Integer apiID;
@ApiModelProperty(name = "发布系统ID", required = false)
private Long publishSystemId;
......@@ -48,4 +45,7 @@ public class DataApiModelDto {
@ApiModelProperty(name = "系统环境", required = true)
private String systemEnv;
@ApiModelProperty(name = "数据服务描述", required = true)
private String desc;
}
src/main/java/com/hisense/dataservice/entity/DataApiModel.java
浏览文件 @ ef00bcd9
package com.hisense.dataservice.entity;
import com.hisense.dataservice.bo.TableJoinRelationConfig;
import com.hisense.dataservice.enums.DebugStatusEnum;
import com.hisense.dataservice.enums.NetworkEnv;
import com.hisense.dataservice.enums.SystemEnv;
import com.vladmihalcea.hibernate.type.json.JsonBlobType;
......@@ -42,7 +43,7 @@ public class DataApiModel extends BaseEntity{
// apiId
@Column(nullable = false)
private Integer apiID;
private String apiID;
// 发布系统ID
@Column(nullable = false)
......@@ -58,8 +59,14 @@ public class DataApiModel extends BaseEntity{
@Column(name = "system_env", columnDefinition = "varchar2(10)", nullable = true)
private SystemEnv systemEnv;
// 发布：1、草稿:0
// 草稿:0、发布：1
@Column(name = "status", columnDefinition = "CHAR(1) default 0", nullable = false)
private Integer status;
// 未调试：NO_DEBUG、调试成功:SUCCESS、调试失败：FAIL
@Enumerated(EnumType.ORDINAL)
@Column(name = "debug_status", columnDefinition = "varchar2(10)", nullable = false)
private DebugStatusEnum debugStatus;
}
src/main/java/com/hisense/dataservice/entity/DataApiModelConfig.java
浏览文件 @ ef00bcd9
......@@ -110,7 +110,7 @@ public class DataApiModelConfig extends BaseEntity{
}
// 生成数据量大小查询sql
public String generateDataCountSql(DataSourceTypeEnum dataSourceTypeEnum) {
public String generateDataCountSql() {
// 已经通过校验行列筛选条件
StringBuilder fullSqlBuilder = new StringBuilder();
......
src/main/java/com/hisense/dataservice/enums/DebugStatusEnum.java 0 → 100644
浏览文件 @ ef00bcd9
package com.hisense.dataservice.enums;
/**
* @author : scc
* @date : 2023/03/14
**/
public enum DebugStatusEnum {
NO_DEBUG,
FAIL,
SUCCESS,
}
src/main/java/com/hisense/dataservice/repository/DataApiColumnRepository.java
浏览文件 @ ef00bcd9
......@@ -9,6 +9,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.Optional;
/**
* @author : scc
......@@ -20,6 +21,8 @@ public interface DataApiColumnRepository extends JpaRepository<DataApiColumn,Lon
// 根据数据表ID查询所有的列信息
List<DataApiColumn> findAllByDataTableIdAndDeletedIsFalse(Long dataTableId);
Optional<DataApiColumn> findByDataTableIdAndIdDeletedIsFalse(Long dataTableId, Long id);
Page<DataApiColumn> findAll(Specification specification, Pageable pageable);
......
src/main/java/com/hisense/dataservice/repository/DataApiModelRepository.java
浏览文件 @ ef00bcd9
......@@ -13,5 +13,5 @@ import java.util.Optional;
@Repository
public interface DataApiModelRepository extends JpaRepository<DataApiModel,Long> {
Optional<DataApiModel> findByApiID(Integer apiId);
Optional<DataApiModel> findByApiID(String apiId);
}
src/main/java/com/hisense/dataservice/repository/DataApiTableRepository.java
浏览文件 @ ef00bcd9
......@@ -13,6 +13,7 @@ import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.Optional;
/**
* @author : scc
......@@ -24,6 +25,8 @@ public interface DataApiTableRepository extends JpaRepository<DataApiTable,Long>
// 根据数据源ID查询所有的数据表
List<DataApiTable> findAllByDataSourceIdAndDeletedIsFalse(Long dataSourceId);
Optional<DataApiTable> findAllByDataSourceIdAndIdDeletedIsFalse(Long dataSourceId, Long id);
Page<DataApiTable> findAll(Specification specification, Pageable pageable);
// 根据数据源ID查询所有的数据表
......
src/main/java/com/hisense/dataservice/repository/PublishApplicationRepository.java 0 → 100644
浏览文件 @ ef00bcd9
/*
* Licensed Materials - Property of tenxcloud.com
* (C) Copyright 2019 TenxCloud. All Rights Reserved.
*
* @auther douyanwei
* @date 2020/2/19
*/
package com.hisense.dataservice.repository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface PublishApplicationRepository{
@Query(value ="SELECT " +
"app.system" +
"FROM publish_application app " +
"WHERE app.USER_KEY = :userKey", nativeQuery=true)
List<Integer> findAllByUserKey(@Param("userKey") String userKey);
@Query(value ="SELECT " +
"app.system" +
"FROM publish_application app " +
"WHERE app.app_id = :appId and app.app_key = :appkey", nativeQuery=true)
List<Integer> findAllByAppIdAndAppKey(@Param("appId") String appId, @Param("appKey") String appKey);
}
src/main/java/com/hisense/dataservice/service/DataApiCommonService.java
浏览文件 @ ef00bcd9
......@@ -2,11 +2,13 @@ package com.hisense.dataservice.service;
import org.springframework.web.bind.annotation.PathVariable;
import javax.servlet.http.HttpServletRequest;
/**
* @author : scc
* @date : 2023/02/24
**/
public interface DataApiCommonService {
Object queryData(@PathVariable String environment, @PathVariable String pattern);
Object queryData(String environment, String pattern, HttpServletRequest request);
}
src/main/java/com/hisense/dataservice/service/DataApiServiceManagementService.java
浏览文件 @ ef00bcd9
......@@ -9,5 +9,9 @@ import com.hisense.dataservice.library.model.Result;
**/
public interface DataApiServiceManagementService extends BaseService {
Result<Boolean> createOrUpdateDataApi(DataApiModelDto dataApiModelDto);
// 发布数据api服务
Result<Boolean> publishDataApi(String environment, Long modelId);
// 创建或修改未发布的数据api服务（草稿状态）
Result<Long> createDataApi(String environment, DataApiModelDto dataApiModelDto);
}
src/main/java/com/hisense/dataservice/service/impl/DataApiCommonServiceImpl.java
浏览文件 @ ef00bcd9
......@@ -25,6 +25,7 @@ import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import javax.servlet.http.HttpServletRequest;
import javax.transaction.Transactional;
import java.util.*;
import java.util.concurrent.Callable;
......@@ -53,12 +54,30 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
@Autowired
private DataApiDataSourceRepository dataApiDataSourceRepository;
@Autowired
private PublishApplicationRepository publishApplicationRepository;
private Integer queryDataTotalPerThread = 2000;
private ThreadPoolManagerUtil batchQueryDataThreadPool = new ThreadPoolManagerUtil("batchQueryDataThread");
private Integer findSystemByUserKeyOrAppId(HttpServletRequest request) {
String userKey = request.getParameter("user_key");
String appId = request.getParameter("app_id");
String appKey = request.getParameter("app_key");
List<Integer> systemIds;
if (!StringUtils.isEmpty(userKey)) {
systemIds = publishApplicationRepository.findAllByUserKey(userKey);
return systemIds.isEmpty() ? null : systemIds.get(0);
} else if (StringUtils.isEmpty(userKey) && !StringUtils.isEmpty(appId) && !StringUtils.isEmpty(appKey)) {
systemIds = publishApplicationRepository.findAllByAppIdAndAppKey(appId, appKey);
return systemIds.isEmpty() ? null : systemIds.get(0);
}
return null;
}
@Override
public Object queryData(String environment, String pattern) {
public Object queryData(String environment, String pattern, HttpServletRequest request) {
// TODO 区分数据库连接异常和服务本身异常，保存到ES中
......@@ -87,8 +106,13 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
// 4.根据订阅系统ID获取自定义的配置
// 4.1根据apiID查询创建的数据api模板
Integer apiId = 1; // test
Integer systemId = 1; // test
Integer systemId = findSystemByUserKeyOrAppId(request);
if (systemId == null) {
jsonResult.put("status", HttpStatus.INTERNAL_SERVER_ERROR.value());
jsonResult.put("message", "获取订阅系统失败!");
return jsonResult;
}
String apiId = request.getParameter("apiId");
Optional<DataApiModel> dataApiModelOptional = dataApiModelRepository.findByApiID(apiId);
if (!dataApiModelOptional.isPresent()) {
......@@ -98,7 +122,7 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
}
DataApiModel dataApiModel = dataApiModelOptional.get();
// 4.2获取自定义的配置
// 4.2获取订阅的相关配置
Optional<DataApiSubscribeConfig> dataApiSubscribeConfigOptional = dataApiSubscribeRepository.findBySubscribeSystemIdAndDataApiModelIdAndDeletedIsFalse(systemId, dataApiModel.getId());
if (!dataApiSubscribeConfigOptional.isPresent()) {
jsonResult.put("status", HttpStatus.INTERNAL_SERVER_ERROR.value());
......@@ -107,7 +131,7 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
}
DataApiSubscribeConfig dataApiSubscribeConfig = dataApiSubscribeConfigOptional.get();
// 生成sql并保存
// 获取行列筛选配置
Optional<DataApiModelConfig> bySubscribeConfigIdAndType = dataApiModelConfigRepository.findByTypeIdAndType(dataApiSubscribeConfig.getId(), ModelConfigType.SUBSCRIBE);
if (!bySubscribeConfigIdAndType.isPresent()) {
jsonResult.put("status", HttpStatus.INTERNAL_SERVER_ERROR.value());
......@@ -116,7 +140,7 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
}
DataApiModelConfig dataApiModelConfig = bySubscribeConfigIdAndType.get();
String fullSql = dataApiModelConfig.getFullSql();
String countSql = "";
String countSql = dataApiModelConfig.generateDataCountSql();
// 获取数据源
Optional<DataApiDataSource> dataApiDataSourceOptional = dataApiDataSourceRepository.findById(dataApiModel.getDataSourceId());
......@@ -127,18 +151,6 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
}
DataApiDataSource dataApiDataSource = dataApiDataSourceOptional.get();
DataSourceTypeEnum dataSourceType = dataApiDataSource.getType();
if (StringUtils.isEmpty(fullSql)) {
// 拼接sql（TODO 正常会在订阅的时候会自动生成的）
fullSql = dataApiModelConfig.generateAndSetFullSql(dataApiModel.getTableJoinRelationConfig(), dataSourceType);
dataApiModelConfigRepository.save(dataApiModelConfig); // 保存生成的fullSql数据
countSql = dataApiModelConfig.generateDataCountSql(dataSourceType);
log.info("fullSql: {}, countSql: {}", fullSql, countSql);
if (StringUtils.isEmpty(fullSql) || StringUtils.isEmpty(countSql)) {
jsonResult.put("status", HttpStatus.INTERNAL_SERVER_ERROR.value());
jsonResult.put("message", "获取订阅系统sql失败!");
return jsonResult;
}
}
// 获取sql的填充参数
List<Object> jdbcParamValues = new ArrayList<>();
......@@ -228,6 +240,7 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
queryDataTask.setPageNumber(pageTotal);// FOR TEST
queryDataTask.setColumnHeads(heads);
queryDataTask.setJdbcParamValues(values);
queryDataTask.setQueryDataTotalPerThread(queryDataTotalPerThread);
queryDataTasks.add(queryDataTask);
--pageTotal; // 页码减一
}
......@@ -266,110 +279,6 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
return result;
}
@Data
class QueryDataTask<T> implements Callable<T> {
private DataApiDataSource dataApiDataSource;
private String querySql;
private String countSql;
private List<Object> jdbcParamValues;
private Long pageNumber;
private List<List<String>> columnHeads;
@Override
public T call() throws Exception {
if (StringUtils.isEmpty(querySql)) {
// Result<Long> dataResult = JdbcUtil.executeCountSql(dataApiDataSource, countSql, jdbcParamValues);
// return (T) dataResult.getData();
return (T) JdbcUtil.testCountSql().getData(); // TODO FOR TEST
} else {
// List<List<Object>> result = new ArrayList<>();
// Result<List<JSONObject>> dataResult = JdbcUtil.executeSql(dataApiDataSource, querySql, jdbcParamValues, queryDataTotalPerThread);
// List<JSONObject> dataJsonList = dataResult.getData();
// for (JSONObject dataJson : dataJsonList) {
// List<Object> lineData = new ArrayList<>();
// for (List<String> head : columnHeads) {
// String columnName = head.get(0);
// Object columnValue = dataJson.get(columnName);
// lineData.add(columnValue); // 获取列名对应的值
// }
// result.add(lineData);
// }
// return (T) result;
return (T) JdbcUtil.testExecuteSql(pageNumber).getData(); // TODO FOR TEST
}
}
}
// 创建或修改数据api
@Transactional
public Boolean createOrUpdateDataApi(DataApiModelDto dataApiModelDto) throws Exception {
// 获取数据源
Long dataSourceId = dataApiModelDto.getDataSourceId();
if (dataSourceId == null) {
return false;
}
Optional<DataApiDataSource> dataApiDataSourceOptional = dataApiDataSourceRepository.findById(dataSourceId);
if (!dataApiDataSourceOptional.isPresent()) {
return false;
}
DataApiDataSource dataApiDataSource = dataApiDataSourceOptional.get();
// 创建或修改数据服务模板
Long modelId = dataApiModelDto.getModelId();
DataApiModel dataApiModel;
if (null == modelId) {
dataApiModel = new DataApiModel();
} else {
Optional<DataApiModel> dataApiModelOptional = dataApiModelRepository.findById(modelId);
if (!dataApiModelOptional.isPresent()) {
return false;
}
dataApiModel = dataApiModelOptional.get();
}
List<TableJoinRelationConfig> tableJoinRelationConfig = dataApiModelDto.getTableJoinRelationConfig();
if (!CollectionUtils.isEmpty(tableJoinRelationConfig)) {
dataApiModel.setTableJoinRelationConfig(tableJoinRelationConfig);
}
dataApiModel.setDataSourceId(dataSourceId);
dataApiModel.setNetworkEnv(NetworkEnv.valueOf(dataApiModelDto.getNetworkEnv()));
dataApiModel.setSystemEnv(SystemEnv.valueOf(dataApiModelDto.getSystemEnv()));
dataApiModel.setCreatedTime(new Date());
dataApiModel.setName(dataApiModelDto.getModelName());
dataApiModel.setPublishSystemId(dataApiModelDto.getPublishSystemId());
dataApiModel.setApiID(dataApiModelDto.getApiID());
dataApiModel = dataApiModelRepository.saveAndFlush(dataApiModel);
modelId = dataApiModel.getId();
// 创建或修改数据服务行列筛选条件
DataApiModelConfig dataApiModelConfig;
Optional<DataApiModelConfig> dataApiModelConfigOptional = dataApiModelConfigRepository.findByTypeIdAndType(modelId, ModelConfigType.PUBLISH);
if (!dataApiModelConfigOptional.isPresent()) {
dataApiModelConfig = new DataApiModelConfig();
} else {
dataApiModelConfig = dataApiModelConfigOptional.get();
}
dataApiModelConfig.setTypeId(dataApiModel.getId());
dataApiModelConfig.setType(ModelConfigType.PUBLISH);
List<ColumnDataRangeFilter> columnDataRangeConfig = dataApiModelDto.getColumnDataRangeConfig();
if (!CollectionUtils.isEmpty(columnDataRangeConfig)) {
dataApiModelConfig.setColumnDataRangeConfig(columnDataRangeConfig);
}
List<LineDataRangeFilter> lineDataRangeConfig = dataApiModelDto.getLineDataRangeConfig();
if (!CollectionUtils.isEmpty(lineDataRangeConfig)) {
dataApiModelConfig.setLineDataRangeConfig(lineDataRangeConfig);
}
// 生成完整sql
String fullSql = dataApiModelConfig.generateAndSetFullSql(tableJoinRelationConfig, dataApiDataSource.getType());
if (StringUtils.isEmpty(fullSql)) {
throw new Exception("生成完成sql失败");
}
dataApiModelConfigRepository.saveAndFlush(dataApiModelConfig);
return true;
}
// 订阅数据api
@Transactional
public Boolean subscribeDataApi(DataApiModelDto dataApiModelDto) {
......
src/main/java/com/hisense/dataservice/service/impl/DataApiServiceManagementServiceImpl.java
浏览文件 @ ef00bcd9
package com.hisense.dataservice.service.impl;
import com.hisense.dataservice.bo.ColumnDataRangeFilter;
import com.hisense.dataservice.bo.LineDataRangeFilter;
import com.hisense.dataservice.bo.TableJoinRelationConfig;
import com.alibaba.fastjson.JSONObject;
import com.hisense.dataservice.bo.*;
import com.hisense.dataservice.dto.DataApiModelDto;
import com.hisense.dataservice.entity.DataApiDataSource;
import com.hisense.dataservice.entity.DataApiModel;
import com.hisense.dataservice.entity.DataApiModelConfig;
import com.hisense.dataservice.enums.ModelConfigType;
import com.hisense.dataservice.enums.NetworkEnv;
import com.hisense.dataservice.enums.SystemEnv;
import com.hisense.dataservice.entity.*;
import com.hisense.dataservice.enums.*;
import com.hisense.dataservice.library.model.Result;
import com.hisense.dataservice.repository.DataApiDataSourceRepository;
import com.hisense.dataservice.repository.DataApiModelConfigRepository;
import com.hisense.dataservice.repository.DataApiModelRepository;
import com.hisense.dataservice.repository.*;
import com.hisense.dataservice.service.DataApiServiceManagementService;
import com.hisense.dataservice.util.ThreadPoolManagerUtil;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.common.UUIDs;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.interceptor.TransactionAspectSupport;
......@@ -23,9 +18,9 @@ import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import javax.transaction.Transactional;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.*;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
/**
* @author : scc
......@@ -44,8 +39,16 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
@Autowired
private DataApiModelConfigRepository dataApiModelConfigRepository;
@Autowired
private DataApiTableRepository dataApiTableRepository;
@Autowired
private DataApiColumnRepository dataApiColumnRepository;
private ThreadPoolManagerUtil debugQueryDataThreadPool = new ThreadPoolManagerUtil("debugQueryDataThread");
@Transactional
@Override
public Result<Boolean> createOrUpdateDataApi(DataApiModelDto dataApiModelDto) {
// 获取数据源
Long dataSourceId = dataApiModelDto.getDataSourceId();
......@@ -82,7 +85,7 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
dataApiModel.setCreatedTime(new Date());
dataApiModel.setName(dataApiModelDto.getModelName());
dataApiModel.setPublishSystemId(dataApiModelDto.getPublishSystemId());
dataApiModel.setApiID(dataApiModelDto.getApiID());
dataApiModel.setApiID(UUID.randomUUID().toString().replaceAll("-", ""));
dataApiModel = dataApiModelRepository.saveAndFlush(dataApiModel);
modelId = dataApiModel.getId();
......@@ -117,4 +120,252 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
return success("创建数据api服务成功", true);
}
@Transactional
@Override
public Result<Boolean> publishDataApi(String environment, Long modelId) {
// 修改数据api服务状态
Optional<DataApiModel> dataApiModelOptional = dataApiModelRepository.findById(modelId);
if (!dataApiModelOptional.isPresent()) {
return failure("发布数据api服务失败，数据api服务不存在", false);
}
DataApiModel dataApiModel = dataApiModelOptional.get();
if (DebugStatusEnum.SUCCESS.equals(dataApiModel.getDebugStatus())) {
// 调试成功后才能发布该数据api服务
dataApiModel.setStatus(1);
dataApiModelRepository.saveAndFlush(dataApiModel);
return success("发布数据api服务成功", true);
}else{
return failure("发布数据api服务失败，请先进行服务的调试", false);
}
}
// 创建或修改未发布的数据api服务（草稿状态）
@Transactional
@Override
public Result<Long> createDataApi(String environment, DataApiModelDto dataApiModelDto) {
// 创建或修改数据服务模板
Long modelId = dataApiModelDto.getModelId();
DataApiModel dataApiModel;
if (null == modelId) {
// 新建--设置数据服务相关基础配置
dataApiModel = new DataApiModel();
dataApiModel.setNetworkEnv(NetworkEnv.valueOf(dataApiModelDto.getNetworkEnv()));
dataApiModel.setSystemEnv(SystemEnv.valueOf(dataApiModelDto.getSystemEnv()));
dataApiModel.setPublishSystemId(dataApiModelDto.getPublishSystemId());
dataApiModel.setApiID(UUID.randomUUID().toString().replaceAll("-", ""));
dataApiModel.setCreatedTime(new Date());
dataApiModel.setStatus(0); // 草稿状态
} else {
// 修改
Optional<DataApiModel> dataApiModelOptional = dataApiModelRepository.findById(modelId);
if (!dataApiModelOptional.isPresent()) {
return failure("数据api服务不存在", null);
}
dataApiModel = dataApiModelOptional.get();
}
if(!StringUtils.isEmpty(dataApiModelDto.getDesc())){
dataApiModel.setDescription(dataApiModelDto.getDesc());
}
if (!StringUtils.isEmpty(dataApiModelDto.getModelName())) {
dataApiModel.setName(dataApiModelDto.getModelName());
}
// 数据源配置
Long dataSourceId = dataApiModelDto.getDataSourceId();
if (dataSourceId != null) {
Optional<DataApiDataSource> dataApiDataSourceOptional = dataApiDataSourceRepository.findById(dataSourceId);
if (!dataApiDataSourceOptional.isPresent()) {
return failure("数据源不存在", null);
}
dataApiModel.setDataSourceId(dataSourceId);
}
// 设置表之间关联关系
List<TableJoinRelationConfig> tableJoinRelationConfig = dataApiModelDto.getTableJoinRelationConfig();
if (!CollectionUtils.isEmpty(tableJoinRelationConfig)) {
dataApiModel.setTableJoinRelationConfig(tableJoinRelationConfig);
}
// 保存数据api服务
dataApiModel = dataApiModelRepository.saveAndFlush(dataApiModel);
// 获取数据api服务创建后的ID
modelId = dataApiModel.getId();
// 获取或新建行列筛选配置
DataApiModelConfig dataApiModelConfig;
Optional<DataApiModelConfig> dataApiModelConfigOptional = dataApiModelConfigRepository.findByTypeIdAndType(modelId, ModelConfigType.PUBLISH);
if (!dataApiModelConfigOptional.isPresent()) {
dataApiModelConfig = new DataApiModelConfig();
dataApiModelConfig.setTypeId(dataApiModel.getId());
dataApiModelConfig.setType(ModelConfigType.PUBLISH);
} else {
dataApiModelConfig = dataApiModelConfigOptional.get();
}
Boolean checkFilterResult;
// 保存列筛选条件
List<ColumnDataRangeFilter> columnDataRangeConfigs = dataApiModelDto.getColumnDataRangeConfig();
if (!CollectionUtils.isEmpty(columnDataRangeConfigs)) {
checkFilterResult = isRightColumnConfigAndSetNameField(dataSourceId, columnDataRangeConfigs);
if(!checkFilterResult){
return failure("数据列过滤配置不正确", modelId);
}
dataApiModelConfig.setColumnDataRangeConfig(columnDataRangeConfigs);
}
// 保存行筛选条件
List<LineDataRangeFilter> lineDataRangeConfigs = dataApiModelDto.getLineDataRangeConfig();
if (!CollectionUtils.isEmpty(lineDataRangeConfigs)) {
checkFilterResult = isRightLineConfigAndSetNameField(dataSourceId, lineDataRangeConfigs);
if(!checkFilterResult){
return failure("数据行过滤配置不正确", modelId);
}
dataApiModelConfig.setLineDataRangeConfig(lineDataRangeConfigs);
}
dataApiModelConfigRepository.saveAndFlush(dataApiModelConfig);
return success("创建数据api服务成功", modelId);
}
// 列配置检查
private Boolean isRightColumnConfigAndSetNameField(Long dataSourceId, List<ColumnDataRangeFilter> columnDataRangeFilters){
for (ColumnDataRangeFilter columnDataRangeFilter : columnDataRangeFilters) {
List<DataColumn> fields = columnDataRangeFilter.getFields();
Long tableId = columnDataRangeFilter.getTableId();
Optional<DataApiTable> dataApiTableOptional = dataApiTableRepository.findAllByDataSourceIdAndIdDeletedIsFalse(dataSourceId, tableId);
if(!dataApiTableOptional.isPresent()){
return Boolean.FALSE;
}
DataApiTable dataApiTable = dataApiTableOptional.get();
columnDataRangeFilter.setTableName(dataApiTable.getTableName()); // 设置表名
List<DataApiColumn> dataApiColumns = dataApiColumnRepository.findAllByDataTableIdAndDeletedIsFalse(tableId);
Map<Long, String> columnIdToNameMap = dataApiColumns.stream().collect(Collectors.toMap(DataApiColumn::getId, DataApiColumn::getFieldName));
for (DataColumn field : fields) {
Long id = field.getId();
String columnName = columnIdToNameMap.get(id);
if(StringUtils.isEmpty(columnName)){
log.info("数据列：【id：{}，name:{}】不存在表：【{}】中", id, columnName, tableId);
return Boolean.FALSE;
}
field.setColumnName(columnName); // 设置表名
}
}
return Boolean.TRUE;
}
// 行配置检查
private Boolean isRightLineConfigAndSetNameField(Long dataSourceId, List<LineDataRangeFilter> lineDataRangeFilters){
for (LineDataRangeFilter lineDataRangeFilter : lineDataRangeFilters) {
Long tableId = lineDataRangeFilter.getTableId();
Optional<DataApiTable> dataApiTableOptional = dataApiTableRepository.findAllByDataSourceIdAndIdDeletedIsFalse(dataSourceId, tableId);
if(!dataApiTableOptional.isPresent()){
return Boolean.FALSE;
}
DataApiTable dataApiTable = dataApiTableOptional.get();
lineDataRangeFilter.setTableName(dataApiTable.getTableName()); // 设置表名
for (FilterCondition filterCondition : lineDataRangeFilter.getFilterConditions()) {
filterCondition.setTableName(dataApiTable.getTableName()); // 设置表名
filterCondition.setTableId(dataApiTable.getId()); // 设置表ID
DataColumn field = filterCondition.getField();
Optional<DataApiColumn> dataApiColumnOptional = dataApiColumnRepository.findByDataTableIdAndIdDeletedIsFalse(dataApiTable.getId(), field.getId());
if(!dataApiColumnOptional.isPresent()){
return Boolean.FALSE;
}
DataApiColumn dataApiColumn = dataApiColumnOptional.get();
field.setColumnName(dataApiColumn.getFieldName()); // 设置字段名
field.setFieldType(dataApiColumn.getFiledType()); // 设置字段类型
}
}
return Boolean.TRUE;
}
public Result<JSONObject> debugDataApiService(String environment, Long dataModelId){
JSONObject result = new JSONObject();
Optional<DataApiModel> dataApiModelOptional = dataApiModelRepository.findById(dataModelId);
if (!dataApiModelOptional.isPresent()) {
return failure("调试数据api服务失败--数据api服务不存在", null);
}
DataApiModel dataApiModel = dataApiModelOptional.get();
Optional<DataApiDataSource> dataApiDataSourceOptional = dataApiDataSourceRepository.findById(dataApiModel.getDataSourceId());
if (dataApiDataSourceOptional.isPresent()) {
return failure("调试数据api服务失败--数据源不存在", null);
}
DataApiDataSource dataApiDataSource = dataApiDataSourceOptional.get();
DataSourceTypeEnum dataSourceType = dataApiDataSource.getType();
DataApiModelConfig dataApiModelConfig;
Optional<DataApiModelConfig> dataApiModelConfigOptional = dataApiModelConfigRepository.findByTypeIdAndType(dataApiModel.getId(), ModelConfigType.PUBLISH);
if (!dataApiModelConfigOptional.isPresent()) {
return failure("调试数据api服务失败--行列配置不存在", null);
}
dataApiModelConfig = dataApiModelConfigOptional.get();
List<TableJoinRelationConfig> tableJoinRelationConfig = dataApiModel.getTableJoinRelationConfig();
String fullSql = dataApiModelConfig.generateAndSetFullSql(tableJoinRelationConfig, dataApiDataSource.getType());
if(StringUtils.isEmpty(fullSql)){
return failure("调试数据api服务失败--生成sql失败", null);
}
// 查询出前10条数据进行返回
List<List<Object>> data = new ArrayList<>();
try{
data = executeDataQueryTask(fullSql, dataApiModelConfig, dataApiDataSource);
}catch (Exception e){
log.error("执行数据查询失败：", e);
dataApiModel.setDebugStatus(DebugStatusEnum.FAIL);
}
// 设置调试结果
dataApiModel.setDebugStatus(DebugStatusEnum.SUCCESS);
dataApiModelRepository.save(dataApiModel);
List<List<String>> heads = ColumnDataRangeFilter.generateExcelHead(dataApiModelConfig.getColumnDataRangeConfig());
result.put("data", data);
result.put("heads", heads);
return success("数据api服务调试成功", result);
}
private List<List<Object>> executeDataQueryTask(String dataSql, DataApiModelConfig dataApiModelConfig, DataApiDataSource dataSource) throws Exception{
// 获取sql的填充参数
List<Object> jdbcParamValues = new ArrayList<>();
for (LineDataRangeFilter lineDataRangeFilter : dataApiModelConfig.getLineDataRangeConfig()) {
for (FilterCondition filterCondition : lineDataRangeFilter.getFilterConditions()) {
jdbcParamValues.addAll(filterCondition.getFilterValues());
}
}
DataSourceTypeEnum dataSourceType = dataSource.getType();
// 封装查询数据任务
Long limitFirstValue = 0L;
Long limitSecondValue = 10L;
if (DataSourceTypeEnum.ORACLE.equals(dataSourceType)) {
limitFirstValue = 10L;
limitSecondValue = 0L;
} else if (DataSourceTypeEnum.POSTGRESQL.equals(dataSourceType)) {
limitFirstValue = 10L;
limitSecondValue = 0L;
} else if (DataSourceTypeEnum.MYSQL.equals(dataSourceType)) {
limitFirstValue = 0L;
limitSecondValue = 10L;
}
jdbcParamValues.add(limitFirstValue);
jdbcParamValues.add(limitSecondValue);
QueryDataTask queryDataTask = new QueryDataTask();
queryDataTask.setDataApiDataSource(dataSource);
queryDataTask.setQuerySql(dataSql);
queryDataTask.setPageNumber(1L);
queryDataTask.setJdbcParamValues(jdbcParamValues);
Future<?> future = debugQueryDataThreadPool.addExecuteTask(queryDataTask);
List<List<Object>> data = new ArrayList<>();
data = (List<List<Object>>) future.get();
return data;
}
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论