Skip to content

D
data-service
提交 b2f8a7ef authored 作者: songchuancai's avatar songchuancai
浏览文件
操作

修改数据api验证逻辑并进行测试

上级 ef00bcd9
隐藏空白字符变更
内嵌 并排
正在显示 包含 355 行增加 和 212 行删除
src/main/java/com/hisense/dataservice/bo/ColumnDataRangeFilter.java
浏览文件 @ b2f8a7ef
......@@ -15,7 +15,6 @@ import java.util.List;
* @author : scc
* @date : 2023/02/20
* 列数据范围过滤
* TODO tableName tableId 和columnId columnName不为空的设计
**/
@ApiModel("列数据范围过滤")
@Data
......@@ -23,11 +22,10 @@ import java.util.List;
@NoArgsConstructor
public class ColumnDataRangeFilter {
// 表名
@JsonIgnore
@ApiModelProperty(name = "表名", required = false)
@ApiModelProperty(value = "表名", required = false)
private String tableName;
// 表名ID
@ApiModelProperty(name = "表名ID", required = true)
@ApiModelProperty(value = "表名ID", required = true)
private Long tableId;
// 字段
......
src/main/java/com/hisense/dataservice/bo/DataColumn.java
浏览文件 @ b2f8a7ef
package com.hisense.dataservice.bo;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.hisense.dataservice.enums.FieldTypeEnum;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
......@@ -14,16 +13,15 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class DataColumn {
// 字段名
@JsonIgnore
@ApiModelProperty(name = "字段名", required = false)
@ApiModelProperty(value = "字段名", required = false)
private String columnName;
// 字段ID
@ApiModelProperty(name = "字段ID", required = true)
@ApiModelProperty(value = "字段ID", required = true)
private Long id;
// 字段类型
@ApiModelProperty(name = "字段类型", required = false)
private FieldTypeEnum fieldType;
@ApiModelProperty(value = "字段类型", required = false)
private String fieldType;
// 字段描述
@ApiModelProperty(name = "字段描述", required = false)
@ApiModelProperty(value = "字段描述", required = false)
private String desc;
}
\ No newline at end of file
src/main/java/com/hisense/dataservice/bo/FilterCondition.java
浏览文件 @ b2f8a7ef
......@@ -22,27 +22,25 @@ import java.util.List;
@NoArgsConstructor
public class FilterCondition {
// 表名
@JsonIgnore
@ApiModelProperty(name = "表名", required = false)
@ApiModelProperty(value = "表名", required = false)
private String tableName;
// 表ID
@JsonIgnore
@ApiModelProperty(name = "表ID", required = false)
@ApiModelProperty(value = "表ID", required = false)
private Long tableId;
// 字段
@ApiModelProperty(name = "字段", required = true)
@ApiModelProperty(value = "字段", required = true)
@NonNull
private DataColumn field;
// 过滤条件
@ApiModelProperty(name = "过滤条件", required = true)
@ApiModelProperty(value = "过滤条件", required = true)
@NonNull
private OperateEnum operateEnumType;
// 过滤值
@ApiModelProperty(name = "过滤值", required = true)
@ApiModelProperty(value = "过滤值", required = true)
@NonNull
private List<Object> filterValues;
......
src/main/java/com/hisense/dataservice/bo/LineDataRangeFilter.java
浏览文件 @ b2f8a7ef
......@@ -6,6 +6,7 @@ import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.util.CollectionUtils;
import java.util.List;
......@@ -21,43 +22,42 @@ import java.util.List;
public class LineDataRangeFilter {
// 条件全部满足还是满足一个: true:全部匹配， false: 只需要满足一个即可
@ApiModelProperty(name = "条件是否全部满足", required = false)
@ApiModelProperty(value = "条件是否全部满足", required = false)
private Boolean fullMatched = Boolean.TRUE;
// 过滤条件
@ApiModelProperty(name = "过滤条件", required = false)
@ApiModelProperty(value = "过滤条件", required = false)
private List<FilterCondition> filterConditions;
@ApiModelProperty(name = "表ID", required = true)
@ApiModelProperty(value = "表ID", required = true)
private Long tableId;
@JsonIgnore
@ApiModelProperty(name = "表名", required = false)
@ApiModelProperty(value = "表名", required = false)
private String tableName;
/**
* TODO 优化stringbuilder只创建一份并且进行引用传递
* 生成where条件
* 例如： where 1=1 and (v1 or v2) and (v3 and v4)
*
* @param LineDataRangeFilters
* @param lineDataRangeFilters
* @return
*/
public static String generateFilterWhereSql(List<LineDataRangeFilter> LineDataRangeFilters) {
public static String generateFilterWhereSql(List<LineDataRangeFilter> lineDataRangeFilters) {
StringBuilder result = new StringBuilder();
// 拼接初始条件
if (LineDataRangeFilters.isEmpty()) {
if (CollectionUtils.isEmpty(lineDataRangeFilters)) {
result.append(" 1=1 ");
return result.toString();
}
for (int i = 0; i < LineDataRangeFilters.size(); i++) {
if (i == LineDataRangeFilters.size() - 1) {
result = LineDataRangeFilters.get(i).generateFilterSql(result);
for (int i = 0; i < lineDataRangeFilters.size(); i++) {
if (i == lineDataRangeFilters.size() - 1) {
result = lineDataRangeFilters.get(i).generateFilterSql(result);
result.append(" ");
} else {
result = LineDataRangeFilters.get(i).generateFilterSql(result);
result = lineDataRangeFilters.get(i).generateFilterSql(result);
result.append(" and ");
}
}
......
src/main/java/com/hisense/dataservice/bo/QueryDataTask.java
浏览文件 @ b2f8a7ef
......@@ -38,6 +38,7 @@ public class QueryDataTask<T> implements Callable<T> {
List<List<Object>> result = new ArrayList<>();
Result<List<JSONObject>> dataResult = JdbcUtil.executeSql(dataApiDataSource, querySql, jdbcParamValues, queryDataTotalPerThread);
List<JSONObject> dataJsonList = dataResult.getData();
// TODO 针对错误信息保存到es
for (JSONObject dataJson : dataJsonList) {
List<Object> lineData = new ArrayList<>();
for (List<String> head : columnHeads) {
......
src/main/java/com/hisense/dataservice/bo/TableJoinRelationConfig.java
浏览文件 @ b2f8a7ef
......@@ -17,12 +17,12 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
@NoArgsConstructor
public class TableJoinRelationConfig {
@ApiModelProperty(name = "表名", required = true)
@ApiModelProperty(value = "表名", required = false)
private String tableName;
@ApiModelProperty(name = "列名", required = true)
@ApiModelProperty(value = "列名", required = false)
private String columnName;
@ApiModelProperty(name = "表ID", required = true)
@ApiModelProperty(value = "表ID", required = true)
private Long tableId;
@ApiModelProperty(name = "列ID", required = true)
@ApiModelProperty(value = "列ID", required = true)
private Long columnId;
}
src/main/java/com/hisense/dataservice/controller/DataApiCommonServiceController.java
浏览文件 @ b2f8a7ef
......@@ -29,10 +29,14 @@ public class DataApiCommonServiceController {
@ApiOperation(value = "公共服务-数据查询")
@ApiImplicitParams({
@ApiImplicitParam(name = "environment", value = "环境", required = true),
@ApiImplicitParam(name = "pattern", value = "请求路径", required = true)
@ApiImplicitParam(name = "pattern", value = "请求路径", required = true),
@ApiImplicitParam(name = "apiId", value = "api订阅的唯一标识", required = true)
})
@GetMapping("/{environment}/{pattern}")
public Object dataQuery(@PathVariable String environment, @PathVariable String pattern, HttpServletRequest request) {
return dataApiCommonService.queryData(environment, pattern, request);
@GetMapping("/{environment}/{pattern}/{apiId}")
public Object dataQuery(@PathVariable String environment,
@PathVariable String pattern,
@PathVariable String apiId,
HttpServletRequest request) {
return dataApiCommonService.queryData(environment, pattern, apiId, request);
}
}
src/main/java/com/hisense/dataservice/controller/DataServiceManagementController.java
浏览文件 @ b2f8a7ef
......@@ -6,6 +6,8 @@ import com.hisense.dataservice.service.DataApiCommonService;
import com.hisense.dataservice.service.DataApiDataSourceManagementService;
import com.hisense.dataservice.service.DataApiServiceManagementService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -33,12 +35,19 @@ public class DataServiceManagementController {
// }
@ApiOperation(value = "数据服务管理-创建数据服务（草稿状态）",notes = "返回数据模型ID")
@ApiImplicitParams({
@ApiImplicitParam(name = "environment", value = "环境", required = true)
})
@PostMapping("/{environment}")
public Result<Long> createDataApiService(@RequestBody DataApiModelDto dataApiModelDto, @PathVariable String environment){
public Result<Long> createDataApiService(@PathVariable String environment, @RequestBody DataApiModelDto dataApiModelDto){
return dataApiServiceManagementService.createDataApi(environment,dataApiModelDto);
}
@ApiOperation(value = "数据服务管理-发布数据服务")
@ApiImplicitParams({
@ApiImplicitParam(name = "environment", value = "环境", required = true),
@ApiImplicitParam(name = "modelId", value = "数据服务ID", required = true)
})
@PostMapping("/{environment}/{modelId}")
public Result<Boolean> publishDataApi(@PathVariable String environment,
@PathVariable Long modelId) {
......
src/main/java/com/hisense/dataservice/dto/DataApiModelDto.java
浏览文件 @ b2f8a7ef
......@@ -3,6 +3,7 @@ package com.hisense.dataservice.dto;
import com.hisense.dataservice.bo.ColumnDataRangeFilter;
import com.hisense.dataservice.bo.LineDataRangeFilter;
import com.hisense.dataservice.bo.TableJoinRelationConfig;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
......@@ -12,40 +13,41 @@ import java.util.List;
* @author : scc
* @date : 2023/02/24
**/
@ApiModel("数据api服务dto")
@Data
public class DataApiModelDto {
@ApiModelProperty(name = "数据模板ID", required = false)
@ApiModelProperty(value = "数据模板ID", required = false)
private Long modelId;
@ApiModelProperty(name = "数据服务名", required = true)
@ApiModelProperty(value = "数据服务名", required = true)
private String modelName;
@ApiModelProperty(name = "数据源ID", required = true)
@ApiModelProperty(value = "数据源ID", required = true)
private Long dataSourceId;
@ApiModelProperty(name = "列数据范围", required = true)
@ApiModelProperty(value = "列数据范围", required = true)
private List<ColumnDataRangeFilter> columnDataRangeConfig;
@ApiModelProperty(name = "数据表关系", required = false)
@ApiModelProperty(value = "数据表关系", required = false)
private List<TableJoinRelationConfig> tableJoinRelationConfig;
@ApiModelProperty(name = "行数据范围", required = true)
@ApiModelProperty(value = "行数据范围", required = true)
private List<LineDataRangeFilter> lineDataRangeConfig;
@ApiModelProperty(name = "发布系统ID", required = false)
@ApiModelProperty(value = "发布系统ID", required = false)
private Long publishSystemId;
@ApiModelProperty(name = "订阅系统ID", required = false)
@ApiModelProperty(value = "订阅系统ID", required = false)
private Integer subscribeSystemId;
@ApiModelProperty(name = "网络环境", required = true)
@ApiModelProperty(value = "网络环境", required = true)
private String networkEnv;
@ApiModelProperty(name = "系统环境", required = true)
@ApiModelProperty(value = "系统环境", required = true)
private String systemEnv;
@ApiModelProperty(name = "数据服务描述", required = true)
@ApiModelProperty(value = "数据服务描述", required = true)
private String desc;
}
src/main/java/com/hisense/dataservice/entity/DataApiColumn.java
浏览文件 @ b2f8a7ef
package com.hisense.dataservice.entity;
import com.hisense.dataservice.enums.FieldTypeEnum;
import com.hisense.dataservice.vo.DataFieldVo;
import lombok.AllArgsConstructor;
import lombok.Data;
......@@ -24,15 +23,14 @@ public class DataApiColumn extends BaseEntity {
// 字段名
private String fieldName;
// 字段类型
@Enumerated(EnumType.STRING)
@Column(name = "field_type", columnDefinition = "varchar2(32)")
private FieldTypeEnum filedType;
private String filedType;
public DataFieldVo toItem(){
DataFieldVo dataFieldVo = new DataFieldVo();
dataFieldVo.setFieldId(this.getId());
dataFieldVo.setFieldName(this.getFieldName());
dataFieldVo.setFiledType(this.getFiledType().name());
dataFieldVo.setFiledType(this.getFiledType());
dataFieldVo.setDataTableId(this.getDataTableId());
dataFieldVo.setDesc(this.getDescription());
return dataFieldVo;
......
src/main/java/com/hisense/dataservice/entity/DataApiModel.java
浏览文件 @ b2f8a7ef
......@@ -30,7 +30,6 @@ public class DataApiModel extends BaseEntity{
// 名称
private String name;
// 数据源ID
@Column(nullable = false)
private Long dataSourceId;
// 数据表和对应列集合 Map<数据表ID,列ID列表> 提供订阅时使用
......@@ -64,9 +63,9 @@ public class DataApiModel extends BaseEntity{
private Integer status;
// 未调试：NO_DEBUG、调试成功:SUCCESS、调试失败：FAIL
@Enumerated(EnumType.ORDINAL)
@Column(name = "debug_status", columnDefinition = "varchar2(10)", nullable = false)
private DebugStatusEnum debugStatus;
@Enumerated(EnumType.STRING)
@Column(name = "debug_status", columnDefinition = "varchar2(10) default 'NO_DEBUG' ", nullable = false)
private DebugStatusEnum debugStatus = DebugStatusEnum.NO_DEBUG;
}
src/main/java/com/hisense/dataservice/entity/PublishApplication.java 0 → 100644
浏览文件 @ b2f8a7ef
package com.hisense.dataservice.entity;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import javax.persistence.*;
import java.util.Date;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Entity
@Table(name = "publish_application")
public class PublishApplication {
@Id
@GeneratedValue(strategy = GenerationType.SEQUENCE)
private Integer id;
private String name;// 订阅名称
private String creator;
private Integer system;// 订阅系统所属ID
private String systemName;
private String scaleApplicationId;
/**
* application类型：0-创建api；1-订阅api
*/
private Integer type;
private String userKey;
/**
* 订阅状态: 0-取消订阅,1-正常-创建,2-审批通过,3-审批取消，4-重置密匙前的数据
*/
private Integer status;
private Date createTime;
private Date updateTime;
private String description;
private String state;
private String appId;
private String appKey;
private String mqTags;//订阅时选择的topic下的tag 的id 逗号分割
private String consumerGroup;//订阅时 消费者组
private String subscribedParams;//订阅参数
private String webSubscribedParams;//前端订阅参数原数据,供回显
private Integer originalApiId;//参数订阅生成的api存放 原apiId
}
src/main/java/com/hisense/dataservice/entity/convert/ColumnDataRangeFilterAttributeConvert.java deleted 100644 → 0
浏览文件 @ ef00bcd9
package com.hisense.dataservice.entity.convert;
import com.alibaba.fastjson.JSONObject;
import com.hisense.dataservice.bo.ColumnDataRangeFilter;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import javax.persistence.AttributeConverter;
import java.util.ArrayList;
import java.util.List;
/**
* @author : scc
* @date : 2023/02/20
* 数据范围过滤条件转换器
**/
public class ColumnDataRangeFilterAttributeConvert implements AttributeConverter<List<ColumnDataRangeFilter>, String> {
@Override
public String convertToDatabaseColumn(List<ColumnDataRangeFilter> columnDataRangeFilters) {
return CollectionUtils.isEmpty(columnDataRangeFilters) ? "" : JSONObject.toJSONString(columnDataRangeFilters);
}
@Override
public List<ColumnDataRangeFilter> convertToEntityAttribute(String dataRangeFilterJsonStr) {
return StringUtils.isEmpty(dataRangeFilterJsonStr) ? new ArrayList<>() : JSONObject.parseObject(dataRangeFilterJsonStr, List.class);
}
}
src/main/java/com/hisense/dataservice/entity/convert/DataTableIdsAttributeConvert.java deleted 100644 → 0
浏览文件 @ ef00bcd9
package com.hisense.dataservice.entity.convert;
import com.alibaba.fastjson.JSONObject;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import javax.persistence.AttributeConverter;
import java.util.ArrayList;
import java.util.List;
/**
* @author : scc
* @date : 2023/02/20
* 关联表ids转换器
**/
public class DataTableIdsAttributeConvert implements AttributeConverter<List<Integer>, String> {
@Override
public String convertToDatabaseColumn(List<Integer> dataTableIds) {
return CollectionUtils.isEmpty(dataTableIds) ? "" : JSONObject.toJSONString(dataTableIds);
}
@Override
public List<Integer> convertToEntityAttribute(String dataTableIdJsonStr) {
return StringUtils.isEmpty(dataTableIdJsonStr) ? new ArrayList<>() : JSONObject.parseObject(dataTableIdJsonStr, List.class);
}
}
src/main/java/com/hisense/dataservice/entity/convert/LineDataRangeFilterAttributeConvert.java deleted 100644 → 0
浏览文件 @ ef00bcd9
package com.hisense.dataservice.entity.convert;
import com.alibaba.fastjson.JSONObject;
import com.hisense.dataservice.bo.LineDataRangeFilter;
import org.springframework.util.StringUtils;
import javax.persistence.AttributeConverter;
/**
* @author : scc
* @date : 2023/02/20
* 数据范围过滤条件转换器
**/
public class LineDataRangeFilterAttributeConvert implements AttributeConverter<LineDataRangeFilter, String> {
@Override
public String convertToDatabaseColumn(LineDataRangeFilter LineDataRangeFilter) {
return LineDataRangeFilter == null ? "" : JSONObject.toJSONString(LineDataRangeFilter);
}
@Override
public LineDataRangeFilter convertToEntityAttribute(String dataRangeFilterJsonStr) {
return StringUtils.isEmpty(dataRangeFilterJsonStr) ? new LineDataRangeFilter() : JSONObject.parseObject(dataRangeFilterJsonStr, LineDataRangeFilter.class);
}
}
src/main/java/com/hisense/dataservice/entity/convert/TableRelationsAttributeConvert.java deleted 100644 → 0
浏览文件 @ ef00bcd9
package com.hisense.dataservice.entity.convert;
import com.alibaba.fastjson.JSONObject;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import javax.persistence.AttributeConverter;
/**
* @author : scc
* @date : 2023/02/20
* 关联表关系转换器
**/
public class TableRelationsAttributeConvert implements AttributeConverter<JSONObject, String> {
@Override
public String convertToDatabaseColumn(JSONObject tableRelations) {
return CollectionUtils.isEmpty(tableRelations) ? "" : JSONObject.toJSONString(tableRelations);
}
@Override
public JSONObject convertToEntityAttribute(String tableRelationJsonStr) {
return StringUtils.isEmpty(tableRelationJsonStr) ? new JSONObject() : JSONObject.parseObject(tableRelationJsonStr);
}
}
src/main/java/com/hisense/dataservice/enums/FieldTypeEnum.java deleted 100644 → 0
浏览文件 @ ef00bcd9
package com.hisense.dataservice.enums;
/**
* @author : scc
* @date : 2023/02/20
* 字段类型
* TODO 根据数据源配置不同的数据类型
**/
public enum FieldTypeEnum {
STRING,
INTEGER,
LONG,
DATE,
TIMESTAMP,
DOUBLE;
}
src/main/java/com/hisense/dataservice/repository/DataApiColumnRepository.java
浏览文件 @ b2f8a7ef
......@@ -21,7 +21,7 @@ public interface DataApiColumnRepository extends JpaRepository<DataApiColumn,Lon
// 根据数据表ID查询所有的列信息
List<DataApiColumn> findAllByDataTableIdAndDeletedIsFalse(Long dataTableId);
Optional<DataApiColumn> findByDataTableIdAndIdDeletedIsFalse(Long dataTableId, Long id);
Optional<DataApiColumn> findByDataTableIdAndIdAndDeletedIsFalse(Long dataTableId, Long id);
Page<DataApiColumn> findAll(Specification specification, Pageable pageable);
......
src/main/java/com/hisense/dataservice/repository/DataApiTableRepository.java
浏览文件 @ b2f8a7ef
......@@ -25,7 +25,7 @@ public interface DataApiTableRepository extends JpaRepository<DataApiTable,Long>
// 根据数据源ID查询所有的数据表
List<DataApiTable> findAllByDataSourceIdAndDeletedIsFalse(Long dataSourceId);
Optional<DataApiTable> findAllByDataSourceIdAndIdDeletedIsFalse(Long dataSourceId, Long id);
Optional<DataApiTable> findAllByDataSourceIdAndIdAndDeletedIsFalse(Long dataSourceId, Long id);
Page<DataApiTable> findAll(Specification specification, Pageable pageable);
......
src/main/java/com/hisense/dataservice/repository/PublishApplicationRepository.java
浏览文件 @ b2f8a7ef
......@@ -7,24 +7,27 @@
*/
package com.hisense.dataservice.repository;
import com.hisense.dataservice.entity.PublishApplication;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface PublishApplicationRepository{
public interface PublishApplicationRepository extends JpaRepository<PublishApplication, Integer> {
@Query(value ="SELECT " +
"app.system" +
@Query(value = "SELECT " +
"app.system " +
"FROM publish_application app " +
"WHERE app.USER_KEY = :userKey", nativeQuery=true)
"WHERE app.user_key = :userKey", nativeQuery = true)
List<Integer> findAllByUserKey(@Param("userKey") String userKey);
@Query(value ="SELECT " +
"app.system" +
@Query(value = "SELECT " +
"app.system " +
"FROM publish_application app " +
"WHERE app.app_id = :appId and app.app_key = :appkey", nativeQuery=true)
"WHERE app.app_id = :appId and app.app_key = :appKey", nativeQuery = true)
List<Integer> findAllByAppIdAndAppKey(@Param("appId") String appId, @Param("appKey") String appKey);
}
src/main/java/com/hisense/dataservice/service/DataApiCommonService.java
浏览文件 @ b2f8a7ef
......@@ -10,5 +10,5 @@ import javax.servlet.http.HttpServletRequest;
**/
public interface DataApiCommonService {
Object queryData(String environment, String pattern, HttpServletRequest request);
Object queryData(String environment, String pattern, String apiId, HttpServletRequest request);
}
src/main/java/com/hisense/dataservice/service/DataApiServiceManagementService.java
浏览文件 @ b2f8a7ef
package com.hisense.dataservice.service;
import com.alibaba.fastjson.JSONObject;
import com.hisense.dataservice.dto.DataApiModelDto;
import com.hisense.dataservice.library.model.Result;
......@@ -14,4 +15,7 @@ public interface DataApiServiceManagementService extends BaseService {
// 创建或修改未发布的数据api服务（草稿状态）
Result<Long> createDataApi(String environment, DataApiModelDto dataApiModelDto);
Result<JSONObject> debugDataApiService(String environment, Long dataModelId);
}
src/main/java/com/hisense/dataservice/service/impl/DataApiCommonServiceImpl.java
浏览文件 @ b2f8a7ef
......@@ -77,7 +77,7 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
return null;
}
@Override
public Object queryData(String environment, String pattern, HttpServletRequest request) {
public Object queryData(String environment, String pattern, String apiId, HttpServletRequest request) {
// TODO 区分数据库连接异常和服务本身异常，保存到ES中
......@@ -112,7 +112,6 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
jsonResult.put("message", "获取订阅系统失败!");
return jsonResult;
}
String apiId = request.getParameter("apiId");
Optional<DataApiModel> dataApiModelOptional = dataApiModelRepository.findByApiID(apiId);
if (!dataApiModelOptional.isPresent()) {
......
src/main/java/com/hisense/dataservice/service/impl/DataApiServiceManagementServiceImpl.java
浏览文件 @ b2f8a7ef
......@@ -10,7 +10,6 @@ import com.hisense.dataservice.repository.*;
import com.hisense.dataservice.service.DataApiServiceManagementService;
import com.hisense.dataservice.util.ThreadPoolManagerUtil;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.common.UUIDs;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.interceptor.TransactionAspectSupport;
......@@ -151,6 +150,15 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
if (null == modelId) {
// 新建--设置数据服务相关基础配置
dataApiModel = new DataApiModel();
if(null == dataApiModelDto.getNetworkEnv()){
return failure("网络环境不能为空", null);
}
if(null == dataApiModelDto.getSystemEnv()){
return failure("系统环境不能为空", null);
}
if(null == dataApiModelDto.getPublishSystemId()){
return failure("发布系统ID不能为空", null);
}
dataApiModel.setNetworkEnv(NetworkEnv.valueOf(dataApiModelDto.getNetworkEnv()));
dataApiModel.setSystemEnv(SystemEnv.valueOf(dataApiModelDto.getSystemEnv()));
dataApiModel.setPublishSystemId(dataApiModelDto.getPublishSystemId());
......@@ -188,6 +196,9 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
// 设置表之间关联关系
List<TableJoinRelationConfig> tableJoinRelationConfig = dataApiModelDto.getTableJoinRelationConfig();
if (!CollectionUtils.isEmpty(tableJoinRelationConfig)) {
if(!isRightTableRelationAndSetName(dataSourceId, tableJoinRelationConfig)){
return failure("多个数据表关系配置不正确", modelId);
}
dataApiModel.setTableJoinRelationConfig(tableJoinRelationConfig);
}
// 保存数据api服务
......@@ -208,6 +219,7 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
}
Boolean checkFilterResult;
dataSourceId = dataApiModel.getDataSourceId(); // 在次获取最新的数据源ID
// 保存列筛选条件
List<ColumnDataRangeFilter> columnDataRangeConfigs = dataApiModelDto.getColumnDataRangeConfig();
if (!CollectionUtils.isEmpty(columnDataRangeConfigs)) {
......@@ -236,8 +248,9 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
for (ColumnDataRangeFilter columnDataRangeFilter : columnDataRangeFilters) {
List<DataColumn> fields = columnDataRangeFilter.getFields();
Long tableId = columnDataRangeFilter.getTableId();
Optional<DataApiTable> dataApiTableOptional = dataApiTableRepository.findAllByDataSourceIdAndIdDeletedIsFalse(dataSourceId, tableId);
Optional<DataApiTable> dataApiTableOptional = dataApiTableRepository.findAllByDataSourceIdAndIdAndDeletedIsFalse(dataSourceId, tableId);
if(!dataApiTableOptional.isPresent()){
log.warn("数据表：【tableId：{}】不存在", tableId);
return Boolean.FALSE;
}
DataApiTable dataApiTable = dataApiTableOptional.get();
......@@ -249,7 +262,7 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
Long id = field.getId();
String columnName = columnIdToNameMap.get(id);
if(StringUtils.isEmpty(columnName)){
log.info("数据列：【id：{}，name:{}】不存在表：【{}】中", id, columnName, tableId);
log.warn("数据列：【id：{}，name:{}】不存在表：【{}】中", id, columnName, tableId);
return Boolean.FALSE;
}
field.setColumnName(columnName); // 设置表名
......@@ -262,8 +275,9 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
private Boolean isRightLineConfigAndSetNameField(Long dataSourceId, List<LineDataRangeFilter> lineDataRangeFilters){
for (LineDataRangeFilter lineDataRangeFilter : lineDataRangeFilters) {
Long tableId = lineDataRangeFilter.getTableId();
Optional<DataApiTable> dataApiTableOptional = dataApiTableRepository.findAllByDataSourceIdAndIdDeletedIsFalse(dataSourceId, tableId);
Optional<DataApiTable> dataApiTableOptional = dataApiTableRepository.findAllByDataSourceIdAndIdAndDeletedIsFalse(dataSourceId, tableId);
if(!dataApiTableOptional.isPresent()){
log.warn("数据表：【tableId：{}】不存在", tableId);
return Boolean.FALSE;
}
DataApiTable dataApiTable = dataApiTableOptional.get();
......@@ -272,8 +286,9 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
filterCondition.setTableName(dataApiTable.getTableName()); // 设置表名
filterCondition.setTableId(dataApiTable.getId()); // 设置表ID
DataColumn field = filterCondition.getField();
Optional<DataApiColumn> dataApiColumnOptional = dataApiColumnRepository.findByDataTableIdAndIdDeletedIsFalse(dataApiTable.getId(), field.getId());
Optional<DataApiColumn> dataApiColumnOptional = dataApiColumnRepository.findByDataTableIdAndIdAndDeletedIsFalse(dataApiTable.getId(), field.getId());
if(!dataApiColumnOptional.isPresent()){
log.warn("数据表：【tableId：{}】不存在数据字段：【columnId: {}】", tableId, field.getId());
return Boolean.FALSE;
}
DataApiColumn dataApiColumn = dataApiColumnOptional.get();
......@@ -284,6 +299,36 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
return Boolean.TRUE;
}
// 表关系配置检查
private Boolean isRightTableRelationAndSetName(Long dataSourceId, List<TableJoinRelationConfig> tableJoinRelationConfigs){
DataApiTable dataApiTable;
DataApiColumn dataApiColumn;
Long tableId;
Long columnId;
Optional<DataApiTable> dataApiTableOptional;
Optional<DataApiColumn> dataApiColumnOptional;
for (TableJoinRelationConfig tableJoinRelationConfig : tableJoinRelationConfigs) {
tableId = tableJoinRelationConfig.getTableId();
columnId = tableJoinRelationConfig.getColumnId();
dataApiTableOptional = dataApiTableRepository.findAllByDataSourceIdAndIdAndDeletedIsFalse(dataSourceId, tableId);
if(!dataApiTableOptional.isPresent()){
log.warn("数据表：【tableId：{}】不存在", tableId);
return Boolean.FALSE;
}
dataApiTable = dataApiTableOptional.get();
tableJoinRelationConfig.setTableName(dataApiTable.getTableName());
dataApiColumnOptional = dataApiColumnRepository.findByDataTableIdAndIdAndDeletedIsFalse(dataApiTable.getId(), columnId);
if(!dataApiColumnOptional.isPresent()){
log.warn("数据表：【tableId：{}】不存在数据字段：【columnId：{}】", tableId, columnId);
return Boolean.FALSE;
}
dataApiColumn = dataApiColumnOptional.get();
tableJoinRelationConfig.setColumnName(dataApiColumn.getFieldName());
}
return Boolean.TRUE;
}
public Result<JSONObject> debugDataApiService(String environment, Long dataModelId){
JSONObject result = new JSONObject();
......@@ -292,13 +337,14 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
return failure("调试数据api服务失败--数据api服务不存在", null);
}
DataApiModel dataApiModel = dataApiModelOptional.get();
if(null == dataApiModel.getDataSourceId()){
return failure("调试数据api服务失败--数据源未配置", null);
}
Optional<DataApiDataSource> dataApiDataSourceOptional = dataApiDataSourceRepository.findById(dataApiModel.getDataSourceId());
if (dataApiDataSourceOptional.isPresent()) {
if (!dataApiDataSourceOptional.isPresent()) {
return failure("调试数据api服务失败--数据源不存在", null);
}
DataApiDataSource dataApiDataSource = dataApiDataSourceOptional.get();
DataSourceTypeEnum dataSourceType = dataApiDataSource.getType();
DataApiModelConfig dataApiModelConfig;
Optional<DataApiModelConfig> dataApiModelConfigOptional = dataApiModelConfigRepository.findByTypeIdAndType(dataApiModel.getId(), ModelConfigType.PUBLISH);
......@@ -306,9 +352,12 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
return failure("调试数据api服务失败--行列配置不存在", null);
}
dataApiModelConfig = dataApiModelConfigOptional.get();
List<TableJoinRelationConfig> tableJoinRelationConfig = dataApiModel.getTableJoinRelationConfig();
List<TableJoinRelationConfig> tableJoinRelationConfigs = dataApiModel.getTableJoinRelationConfig();
if(CollectionUtils.isEmpty(tableJoinRelationConfigs)){
return failure("调试数据api服务失败--数据表关系配置不存在", null);
}
String fullSql = dataApiModelConfig.generateAndSetFullSql(tableJoinRelationConfig, dataApiDataSource.getType());
String fullSql = dataApiModelConfig.generateAndSetFullSql(tableJoinRelationConfigs, dataApiDataSource.getType());
if(StringUtils.isEmpty(fullSql)){
return failure("调试数据api服务失败--生成sql失败", null);
}
......@@ -316,15 +365,14 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
List<List<Object>> data = new ArrayList<>();
try{
data = executeDataQueryTask(fullSql, dataApiModelConfig, dataApiDataSource);
dataApiModel.setDebugStatus(DebugStatusEnum.SUCCESS);
}catch (Exception e){
log.error("执行数据查询失败：", e);
dataApiModel.setDebugStatus(DebugStatusEnum.FAIL);
}
// 设置调试结果
dataApiModel.setDebugStatus(DebugStatusEnum.SUCCESS);
dataApiModelRepository.save(dataApiModel);
List<List<String>> heads = ColumnDataRangeFilter.generateExcelHead(dataApiModelConfig.getColumnDataRangeConfig());
result.put("data", data);
result.put("heads", heads);
......@@ -334,11 +382,15 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
private List<List<Object>> executeDataQueryTask(String dataSql, DataApiModelConfig dataApiModelConfig, DataApiDataSource dataSource) throws Exception{
// 获取sql的填充参数
List<Object> jdbcParamValues = new ArrayList<>();
for (LineDataRangeFilter lineDataRangeFilter : dataApiModelConfig.getLineDataRangeConfig()) {
for (FilterCondition filterCondition : lineDataRangeFilter.getFilterConditions()) {
jdbcParamValues.addAll(filterCondition.getFilterValues());
List<LineDataRangeFilter> lineDataRangeConfigs = dataApiModelConfig.getLineDataRangeConfig();
if(!CollectionUtils.isEmpty(lineDataRangeConfigs)){
for (LineDataRangeFilter lineDataRangeFilter : lineDataRangeConfigs) {
for (FilterCondition filterCondition : lineDataRangeFilter.getFilterConditions()) {
jdbcParamValues.addAll(filterCondition.getFilterValues());
}
}
}
DataSourceTypeEnum dataSourceType = dataSource.getType();
// 封装查询数据任务
Long limitFirstValue = 0L;
......
src/main/java/com/hisense/dataservice/util/JdbcUtil.java
浏览文件 @ b2f8a7ef
......@@ -55,7 +55,7 @@ public class JdbcUtil {
return new Result<>(jsonObjects);
}
public static Result<List<JSONObject>> executeSql(DataApiDataSource datasource, String sql, List<Object> jdbcParamValues, Integer batchSize) {
public static Result<List<JSONObject>> executeSql(DataApiDataSource datasource, String sql, List<Object> jdbcParamValues, Integer batchSize) throws Exception {
String threadName = Thread.currentThread().getName();
......@@ -102,10 +102,13 @@ public class JdbcUtil {
return new Result("200", "查询成功", result);
} catch (SQLException sqlException) {
log.error("[{}] 数据库连接失败：", datasource.getSourceName(), sqlException);
return new Result("600", sqlException.getMessage(), result);
// TODO 记录到ES
throw new SQLException("数据库连接失败");
// return new Result("600", sqlException.getMessage(), result);
} catch (Exception e) {
log.error("thread: {}, sql: {}, params: {}, 分页查询数据异常：", threadName, sql, jdbcParamValues, e);
return new Result("500", e.getMessage(), result);
throw new SQLException("分页查询数据异常");
// return new Result("500", e.getMessage(), result);
} finally {
try {
if (connection != null)
......
src/test/java/com/hisense/dataservice/repository/DataApiColumnRepositoryTest.java
浏览文件 @ b2f8a7ef
......@@ -2,7 +2,6 @@ package com.hisense.dataservice.repository;
import com.hisense.dataservice.DataServiceApplication;
import com.hisense.dataservice.entity.DataApiColumn;
import com.hisense.dataservice.enums.FieldTypeEnum;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -25,7 +24,7 @@ public class DataApiColumnRepositoryTest {
DataApiColumn dataApiColumn = new DataApiColumn();
dataApiColumn.setDataTableId(1L);
dataApiColumn.setFieldName("id");
dataApiColumn.setFiledType(FieldTypeEnum.LONG);
dataApiColumn.setFiledType("LONG");
dataApiColumnRepository.save(dataApiColumn);
}
}
src/test/java/com/hisense/dataservice/service/DataApiCommonServiceTest.java
浏览文件 @ b2f8a7ef
......@@ -56,7 +56,7 @@ public class DataApiCommonServiceTest {
Long dataApiSubscribeConfigId;
DataApiModel dataApiModel = new DataApiModel();
dataApiModel.setApiID(1);
dataApiModel.setApiID("1");
dataApiModel.setNetworkEnv(NetworkEnv.INNER);
dataApiModel.setSystemEnv(SystemEnv.DEV);
dataApiModel.setDataSourceId(1L);
......@@ -165,7 +165,7 @@ public class DataApiCommonServiceTest {
dataApiModelConfigIds.add(dataApiModelConfigRepository.save(dataApiModelConfig2).getId());
// 执行业务
JSONObject jsonObject = (JSONObject) dataApiModelService.queryData("test", "tttt");
JSONObject jsonObject = (JSONObject) dataApiModelService.queryData("test", "tttt","1",null);
// 清理数据
cleanTestData(dataApiModelId, dataApiModelConfigIds, dataApiSubscribeConfigId);
......@@ -340,7 +340,6 @@ public class DataApiCommonServiceTest {
lineDataRangeFilters.add(lineDataRangeFilter2);
dataApiModelDto.setLineDataRangeConfig(lineDataRangeFilters);
dataApiModelDto.setApiID(1);
dataApiModelDto.setModelName("testModelName");
dataApiModelDto.setNetworkEnv(NetworkEnv.INNER.name());
dataApiModelDto.setSystemEnv(SystemEnv.DEV.name());
......@@ -348,7 +347,7 @@ public class DataApiCommonServiceTest {
dataApiModelDto.setDataSourceId(1L);
try {
dataApiModelService.createOrUpdateDataApi(dataApiModelDto);
// dataApiModelService.publishDataApi(dataApiModelDto);
} catch (Exception e) {
log.error("创建或更新数据服务api失败：" , e);
}
......@@ -472,7 +471,6 @@ public class DataApiCommonServiceTest {
lineDataRangeFilters.add(lineDataRangeFilter2);
dataApiModelDto.setLineDataRangeConfig(lineDataRangeFilters);
dataApiModelDto.setApiID(1);
dataApiModelDto.setModelId(429l);
dataApiModelDto.setNetworkEnv(NetworkEnv.INNER.name());
dataApiModelDto.setSystemEnv(SystemEnv.DEV.name());
......
src/test/java/com/hisense/dataservice/service/DataApiServiceManagementServiceTest.java 0 → 100644
浏览文件 @ b2f8a7ef
package com.hisense.dataservice.service;
import com.alibaba.fastjson.JSONObject;
import com.hisense.dataservice.DataServiceApplication;
import com.hisense.dataservice.bo.ColumnDataRangeFilter;
import com.hisense.dataservice.bo.DataColumn;
import com.hisense.dataservice.bo.TableJoinRelationConfig;
import com.hisense.dataservice.dto.DataApiModelDto;
import com.hisense.dataservice.enums.NetworkEnv;
import com.hisense.dataservice.enums.SystemEnv;
import com.hisense.dataservice.library.model.Result;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.List;
/**
* @author : scc
* @date : 2023/03/15
**/
@Slf4j
@SpringBootTest(classes = DataServiceApplication.class)
@RunWith(SpringRunner.class)
public class DataApiServiceManagementServiceTest {
@Autowired
private DataApiServiceManagementService dataApiServiceManagementService;
@Test
public void testCreateDataApi(){
Result<Long> result1 = dataApiServiceManagementService.createDataApi("test", buildModelDtoForServiceCreate());
Assert.assertNotNull(result1.getData());
log.info("执行第一步结果：{}", result1);
DataApiModelDto dataApiModelDto = buildModelDtoForSourceConfigWithSingleTable();
dataApiModelDto.setModelId(result1.getData());
Result<Long> result2 = dataApiServiceManagementService.createDataApi("test", dataApiModelDto);
Assert.assertNotNull(result2.getData());
log.info("执行第二步结果：{}", result2);
dataApiModelDto = buildModelDtoDataColumnConfigWithSingleTable();
dataApiModelDto.setModelId(result2.getData());
Result<Long> result3 = dataApiServiceManagementService.createDataApi("test", dataApiModelDto);
log.info("执行第三步结果：{}", result3);
Result<Boolean> result4 = dataApiServiceManagementService.publishDataApi("test", result3.getData());
log.info("执行第四步结果，发布api：{}", result4);
Assert.assertEquals(false, result4.getData());
Result<JSONObject> result5 = dataApiServiceManagementService.debugDataApiService("test", result3.getData());
log.info("执行第五步结果，调试api：{}", result5);
Assert.assertEquals(false, result5.getData());
}
private DataApiModelDto buildModelDtoForServiceCreate(){
DataApiModelDto dataApiModelDto = new DataApiModelDto();
dataApiModelDto.setModelName("测试模式名称");
dataApiModelDto.setDesc("测试模式描述");
dataApiModelDto.setNetworkEnv(NetworkEnv.INNER.name());
dataApiModelDto.setSystemEnv(SystemEnv.DEV.name());
dataApiModelDto.setPublishSystemId(1L);
return dataApiModelDto;
}
// 单表
private DataApiModelDto buildModelDtoForSourceConfigWithSingleTable(){
DataApiModelDto dataApiModelDto = new DataApiModelDto();
dataApiModelDto.setDataSourceId(1L);
TableJoinRelationConfig tableJoinRelationConfig1 = new TableJoinRelationConfig();
tableJoinRelationConfig1.setTableId(1L);
tableJoinRelationConfig1.setColumnId(2L);
List<TableJoinRelationConfig> tableJoinRelationConfigs = new ArrayList<>();
tableJoinRelationConfigs.add(tableJoinRelationConfig1);
dataApiModelDto.setTableJoinRelationConfig(tableJoinRelationConfigs);
return dataApiModelDto;
}
// 多表配置关系
private DataApiModelDto buildModelDtoForSourceConfigWithMutilTable(){
DataApiModelDto dataApiModelDto = new DataApiModelDto();
dataApiModelDto.setDataSourceId(1L);
TableJoinRelationConfig tableJoinRelationConfig1 = new TableJoinRelationConfig();
tableJoinRelationConfig1.setTableId(1L);
tableJoinRelationConfig1.setColumnId(2L);
TableJoinRelationConfig tableJoinRelationConfig2 = new TableJoinRelationConfig();
tableJoinRelationConfig2.setTableId(2L);
tableJoinRelationConfig2.setColumnId(6L); // TODO 设置不存在的Id测试
List<TableJoinRelationConfig> tableJoinRelationConfigs = new ArrayList<>();
tableJoinRelationConfigs.add(tableJoinRelationConfig1);
tableJoinRelationConfigs.add(tableJoinRelationConfig2);
dataApiModelDto.setTableJoinRelationConfig(tableJoinRelationConfigs);
return dataApiModelDto;
}
private DataApiModelDto buildModelDtoDataColumnConfigWithSingleTable(){
DataApiModelDto dataApiModelDto = new DataApiModelDto();
List<ColumnDataRangeFilter> columnDataRangeFilters = new ArrayList<>();
ColumnDataRangeFilter columnDataRangeFilter = new ColumnDataRangeFilter();
List<DataColumn> dataColumns = new ArrayList<>();
DataColumn dataColumn = new DataColumn();
dataColumn.setId(1L);
dataColumns.add(dataColumn);
dataColumn = new DataColumn();
dataColumn.setId(2L);
dataColumns.add(dataColumn);
dataColumn = new DataColumn();
dataColumn.setId(3L);
dataColumns.add(dataColumn);
columnDataRangeFilter.setTableId(1L);
columnDataRangeFilter.setFields(dataColumns);
columnDataRangeFilters.add(columnDataRangeFilter);
dataApiModelDto.setColumnDataRangeConfig(columnDataRangeFilters);
return dataApiModelDto;
}
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论