优化代码

pom.xml

@@ -145,6 +145,11 @@
             <artifactId>elasticsearch-rest-client</artifactId>
             <version>7.2.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.5.2</version>
+        </dependency>
         <!-- end es-->
     </dependencies>
 

src/main/java/com/hisense/dataservice/controller/DataServiceLogManagementController.java

 package com.hisense.dataservice.controller;
 
+import com.hisense.dataservice.library.model.Result;
 import com.hisense.dataservice.library.model.page.PageData;
 import com.hisense.dataservice.library.model.page.PageLink;
 import com.hisense.dataservice.service.DataServiceLogManagementService;
@@ -7,6 +8,7 @@ import com.hisense.dataservice.vo.CallLogVo;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiImplicitParam;
 import io.swagger.annotations.ApiImplicitParams;
+import io.swagger.annotations.ApiOperation;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 
@@ -22,6 +24,7 @@ public class DataServiceLogManagementController extends BaseController{
     @Autowired
     private DataServiceLogManagementService dataServiceLogManagementService;
 
+    @ApiOperation("日志列表")
     @ApiImplicitParams({
             @ApiImplicitParam(name = "environment", value = "环境", required = true),
             @ApiImplicitParam(name = "systemId", value = "系统ID", required = true),
@@ -31,15 +34,30 @@ public class DataServiceLogManagementController extends BaseController{
             @ApiImplicitParam(name = "sortOrder", value = "排序方向", required = false, allowableValues = "ASC,DESC")
     })
     @GetMapping("/callLog/{environment}/{systemId}")
-    public PageData<CallLogVo> getCallLog(@PathVariable String environment,
-                                          @PathVariable Long systemId,
-                                          @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize,
-                                          @RequestParam(value = "page", required = false, defaultValue = "1") Integer page,
-                                          @RequestParam(value = "sortProperty", required = false) String sortProperty,
-                                          @RequestParam(value = "sortOrder", required = false) String sortOrder){
+    public Result<PageData<CallLogVo>> getCallLog(@PathVariable String environment,
+                                                  @PathVariable Long systemId,
+                                                  @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize,
+                                                  @RequestParam(value = "page", required = false, defaultValue = "1") Integer page,
+                                                  @RequestParam(value = "sortProperty", required = false) String sortProperty,
+                                                  @RequestParam(value = "sortOrder", required = false) String sortOrder){
 
         PageLink pageLink = createPageLink(pageSize, page, "", sortProperty, sortOrder);
         return dataServiceLogManagementService.getCallLog(environment, systemId, pageLink);
 
     }
+
+    @ApiOperation("查询重试")
+    @ApiImplicitParams({
+            @ApiImplicitParam(name = "environment", value = "环境", required = true),
+            @ApiImplicitParam(name = "systemId", value = "系统ID", required = true),
+            @ApiImplicitParam(name = "subscribeConfigId", value = "配置ID", required = true)
+    })
+    @GetMapping("/callLog/{environment}/{systemId}/{subscribeConfigId}")
+    public Result<Boolean> retryQuery(@PathVariable String environment,
+                                                  @PathVariable Integer systemId,
+                                                  @PathVariable Long subscribeConfigId){
+
+        return dataServiceLogManagementService.retryQuery(environment, systemId, subscribeConfigId);
+
+    }
 }

src/main/java/com/hisense/dataservice/entity/DataApiModelConfig.java

@@ -64,11 +64,11 @@ public class DataApiModelConfig extends BaseEntity{
     private String whereSql;
 
     // SQL语句，记录生成的sql
-    @Column(name = "sql", columnDefinition = "varchar2(1000)", nullable = true)
+    @Column(name = "sql", columnDefinition = "varchar2", nullable = true)
     private String fullSql;
 
     // SQL语句，记录生成的countSql
-    @Column(name = "count_sql", columnDefinition = "varchar2(1000)", nullable = true)
+    @Column(name = "count_sql", columnDefinition = "varchar2", nullable = true)
     private String countSql;
 
 

src/main/java/com/hisense/dataservice/es/config/ElasticsearchConfig.java

@@ -54,6 +54,7 @@ public class ElasticsearchConfig {
                     r.setConnectionRequestTimeout(connectionRequestTimeout);
                     return r;
                 });
+        log.info("elasticsearch init stop ");
         return new RestHighLevelClient(builder);
     }
 }
\ No newline at end of file

src/main/java/com/hisense/dataservice/service/DataServiceLogManagementService.java

 package com.hisense.dataservice.service;
 
+import com.hisense.dataservice.library.model.Result;
 import com.hisense.dataservice.library.model.page.PageData;
 import com.hisense.dataservice.library.model.page.PageLink;
 import com.hisense.dataservice.vo.CallLogVo;
@@ -8,6 +9,8 @@ import com.hisense.dataservice.vo.CallLogVo;
  * @author : scc
  * @date : 2023/03/20
  **/
-public interface DataServiceLogManagementService {
-    PageData<CallLogVo> getCallLog(String environment, Long systemId, PageLink pageLink);
+public interface DataServiceLogManagementService extends BaseService {
+    Result<PageData<CallLogVo>> getCallLog(String environment, Long systemId, PageLink pageLink);
+
+    Result<Boolean> retryQuery(String environment, Integer systemId, Long subscribeConfigId);
 }

src/main/java/com/hisense/dataservice/service/FileManagementService.java

+package com.hisense.dataservice.service;
+
+import com.hisense.dataservice.library.model.Result;
+
+/**
+ * @author : scc
+ * @date : 2023/03/30
+ **/
+public interface FileManagementService extends BaseService{
+    Result<String> uploadFile(String filePath);
+}

src/main/java/com/hisense/dataservice/service/impl/DataApiCommonServiceImpl.java

@@ -16,6 +16,7 @@ import com.hisense.dataservice.exceptions.DataQueryException;
 import com.hisense.dataservice.library.model.Result;
 import com.hisense.dataservice.repository.*;
 import com.hisense.dataservice.service.DataApiCommonService;
+import com.hisense.dataservice.service.FileManagementService;
 import com.hisense.dataservice.service.WorkFlowService;
 import com.hisense.dataservice.util.FileUtil;
 import com.hisense.dataservice.util.SpringBeanUtil;
@@ -30,6 +31,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.interceptor.TransactionAspectSupport;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
+import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.context.request.RequestContextHolder;
 import org.springframework.web.context.request.ServletRequestAttributes;
 
@@ -92,6 +94,8 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
     private ThreadPoolManagerUtil subscribeApiTaskThreadPool = new ThreadPoolManagerUtil("subscribeApiTaskThreadPool");
 
     @Autowired
+    private FileManagementService fileManagementService;
+    @Autowired
     private ElasticsearchUtils elasticsearchUtils;
     @Value("${es.index-pre}")
     private String ES_INDEX;
@@ -242,6 +246,13 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
         return jsonResult;
     }
 
+    public Result<JSONObject> queryDataLatest2(String environment,Boolean isDebug,ModelConfigType type,Long dataModelId){
+        Result<JSONObject> result = new Result<>();
+
+
+
+        return result;
+    }
     public Result<JSONObject> queryDataLatest(String environment, String apiId) {
 
         Result<JSONObject> result = new Result<>();
@@ -282,7 +293,7 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
         DataApiDataSource dataApiDataSource = dataApiDataSourceOptional.get();
 
         // 数据临时文件名
-        String tmpFileName = FileUtil.getPath() + System.currentTimeMillis() + ".xlsx";
+        String tmpFilePath = FileUtil.getPath() + System.currentTimeMillis() + ".xlsx";
 
         // excel的列名
         List<List<String>> heads = ColumnDataRangeFilter.generateExcelHead(dataApiModelConfig.getColumnDataRangeConfig());
@@ -295,22 +306,27 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
         List<List<List<Object>>> data = dataResult.getData();
 
         // 保存数据到文件
-        if (!saveTmpDataFile(tmpFileName, data, heads)) {
+        if (!saveTmpDataFile(tmpFilePath, data, heads)) {
             result.setMsg("系统异常：保存数据到文件中错误");
             return result;
         }
-        // 上传文件到存储服务器中 TODO
-
+        Result<String> fileUploadResult = fileManagementService.uploadFile(tmpFilePath);
+        if(fileUploadResult.getCode().equals(Result.FAIL)){
+            result.setMsg("系统异常：上传文件失败");
+            return result;
+        }
+        String fileDownloadUrl = fileUploadResult.getData();
         // 根据数据量返回数据,文件上传并获取下载地址（异步）并响应客户端数据 TODO
         if (data.size() <= 10000) {
             // 直接返回数据 TODO
             log.info("数据少于10000条，返回数据");
             object.put("data", data);
+            object.put("fileUrl", fileDownloadUrl);
             object.put("heads", heads);
         } else {
             //  返回文件下载地址 TODO
             log.info("数据大于10000条，返回数据文件下载地址");
-            object.put("fileUrl", "http://test.com");
+            object.put("fileUrl", fileDownloadUrl);
         }
         return result;
     }
@@ -452,9 +468,16 @@ public class DataApiCommonServiceImpl implements DataApiCommonService {
             }
             ThreadPoolExecutor threadPool = batchQueryDataThreadPool.getThreadPool();
             futures = threadPool.invokeAll(queryDataTasks);
-
-            for (int i = 0; i < futures.size(); i++) {
-                List<List<Object>> dataJsonObjectList = futures.get(i).get();
+            // 获取所有的执行结果
+            Iterator<Future<List<List<Object>>>> iterators = futures.iterator();
+            while(iterators.hasNext()) {
+                Future<List<List<Object>>> f = iterators.next();
+                if (!f.isDone()) {
+                    continue;
+                } else {
+                    iterators.remove();
+                }
+                List<List<Object>> dataJsonObjectList = f.get();
                 results.add(dataJsonObjectList);
             }
             done = Boolean.TRUE;

src/main/java/com/hisense/dataservice/service/impl/DataApiServiceManagementServiceImpl.java

@@ -568,6 +568,11 @@ public class DataApiServiceManagementServiceImpl implements DataApiServiceManage
             dataApiModelDetailVo.setLineDataRangeConfig(item.getLineDataRangeConfig());
         });
         dataApiModelDetailVo.setSystemLeader(dataApiModel.getCreator());
+        Optional<DataApiDataSource> dataApiDataSourceOptional = dataApiDataSourceRepository.findById(dataApiModel.getDataSourceId());
+        dataApiDataSourceOptional.ifPresent(item->{
+            dataApiModelDetailVo.setDataSourceType(item.getType().name());
+            dataApiModelDetailVo.setDataSourceName(item.getSourceName());
+        });
         return success(dataApiModelDetailVo);
     }
 

src/main/java/com/hisense/dataservice/service/impl/DataServiceLogManagementServiceImpl.java

 package com.hisense.dataservice.service.impl;
 
+import com.hisense.dataservice.entity.DataApiApplication;
+import com.hisense.dataservice.entity.DataApiSubscribeConfig;
+import com.hisense.dataservice.enums.SystemEnv;
 import com.hisense.dataservice.es.data.EsField;
 import com.hisense.dataservice.es.data.EsPage;
 import com.hisense.dataservice.es.enums.FieldTypeEnum;
 import com.hisense.dataservice.es.utils.ElasticsearchUtils;
+import com.hisense.dataservice.library.model.Result;
 import com.hisense.dataservice.library.model.page.PageData;
 import com.hisense.dataservice.library.model.page.PageLink;
 import com.hisense.dataservice.library.model.page.SortOrder;
+import com.hisense.dataservice.repository.DataApiApplicationRepository;
+import com.hisense.dataservice.repository.DataApiSubscribeRepository;
 import com.hisense.dataservice.service.DataServiceLogManagementService;
+import com.hisense.dataservice.util.HttpUtil;
 import com.hisense.dataservice.vo.CallLogVo;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import org.springframework.util.StringUtils;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * @author : scc
@@ -30,8 +39,17 @@ public class DataServiceLogManagementServiceImpl implements DataServiceLogManage
     @Value("${es.index-pre}")
     private String ES_INDEX;
 
+    @Value("${data-server.base-url}")
+    private String DATA_SERVER_BASE_URL;
+
+    @Autowired
+    private DataApiApplicationRepository dataApiApplicationRepository;
+
+    @Autowired
+    private DataApiSubscribeRepository dataApiSubscribeRepository;
+
     @Override
-    public PageData<CallLogVo> getCallLog(String environment, Long systemId, PageLink pageLink) {
+    public Result<PageData<CallLogVo>> getCallLog(String environment, Long systemId, PageLink pageLink) {
         List<EsField> conditions = new ArrayList<>();
         // 环境
         EsField esField = new EsField();
@@ -68,6 +86,32 @@ public class DataServiceLogManagementServiceImpl implements DataServiceLogManage
             log.error("查询数据服务调用日志失败", e.getMessage());
         }
 
-        return result;
+        return success(result);
+    }
+
+    @Override
+    public Result<Boolean> retryQuery(String environment, Integer systemId, Long subscribeConfigId){
+        Optional<DataApiSubscribeConfig> dataApiSubscribeConfigOptional = dataApiSubscribeRepository.findByIdAndSystemEnvAndSubscribeSystemId(subscribeConfigId, SystemEnv.valueOf(environment), systemId);
+        if (!dataApiSubscribeConfigOptional.isPresent()) {
+            failure("重试的订阅配置不存在");
+        }
+        DataApiSubscribeConfig subscribeConfig = dataApiSubscribeConfigOptional.get();
+        String hashStr = subscribeConfig.getHashStr();
+        DataApiApplication dataApiApplication = dataApiApplicationRepository.findDataApiApplicationByDataModelSubConfigId(subscribeConfigId);
+        if(dataApiApplication == null){
+            failure("订阅应用信息不存在");
+        }
+        String appKey = dataApiApplication.getAppKey();
+        String appId = dataApiApplication.getAppId();
+        String userKey = dataApiApplication.getUserKey();
+        String queryUrlForUserKey = String.format("%s/%s?userKey=%s", DATA_SERVER_BASE_URL, hashStr, userKey);
+        String queryUrlForAppIdAndKey = String.format("%s/%s?appId=%s&appKey=%s", DATA_SERVER_BASE_URL, hashStr, appId, appKey);
+        String result = HttpUtil.get(queryUrlForUserKey);
+        if(StringUtils.isEmpty(result)){
+            failure("重试失败");
+        }else{
+            success(Boolean.TRUE);
+        }
+        return success(Boolean.TRUE);
     }
 }

src/main/java/com/hisense/dataservice/service/impl/FileManagementServiceImpl.java

+package com.hisense.dataservice.service.impl;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.hisense.dataservice.library.model.Result;
+import com.hisense.dataservice.service.FileManagementService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import java.io.File;
+import org.apache.http.entity.mime.HttpMultipartMode;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
+
+
+/**
+ * @author : scc
+ * @date : 2023/03/30
+ **/
+@Service
+@Slf4j
+public class FileManagementServiceImpl implements FileManagementService {
+
+    @Value("${fileServer.sysCode}")
+    private String sysCode;
+    @Value("${fileServer.bucketName}")
+    private String bucketName;
+    @Value("${fileServer.accessKey}")
+    private String accessKey;
+    @Value("${fileServer.secretKey}")
+    private String secretKey;
+    @Value("${fileServer.expires}")
+    private String expires;
+    @Value("${fileServer.url}")
+    private String uploadUrl;
+
+    @Override
+    public Result<String> uploadFile(String filePath) {
+
+        String fileDownUrl = "";
+
+        // 定义要上传的文件
+        File file = new File(filePath);
+
+        // 创建HTTP POST请求对象
+        HttpPost httpPost = new HttpPost(uploadUrl);
+
+        // 构建请求参数
+        HttpEntity entity = MultipartEntityBuilder.create()
+                .setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
+                .addBinaryBody("file", file, ContentType.DEFAULT_BINARY, file.getName())
+                .addTextBody("sysCode", sysCode)
+                .addTextBody("bucketName", bucketName)
+                .addTextBody("accessKey", accessKey)
+                .addTextBody("secretKey", secretKey)
+                .addTextBody("expires", expires)
+                .build();
+
+        // 将HTTP实体添加到POST请求中
+        httpPost.setEntity(entity);
+
+        try (CloseableHttpClient httpClient = HttpClients.createDefault();
+             CloseableHttpResponse response = httpClient.execute(httpPost)) {
+            // 处理响应结果
+            StatusLine statusLine = response.getStatusLine();
+            int statusCode = statusLine.getStatusCode();
+            if(statusCode == 200){
+                HttpEntity entityRes = response.getEntity();
+                String ret = EntityUtils.toString(entityRes);
+                log.info("文件上传结果: {}", ret);
+                JSONObject retJson = JSON.parseObject(ret);
+                String code = retJson.getString("code");
+                String msg = retJson.getString("msg");
+                if("0".equals(code)){
+                    fileDownUrl = retJson.getString("filePath");
+                }else{
+                    return failure(msg);
+                }
+            }else{
+                return failure("上传文件失败，状态码：" + statusCode);
+            }
+        }catch (Exception e){
+            log.error("file upload fail: ", e);
+            return failure("文件下载失败，原因: "+ e.getMessage());
+        }
+        return success(fileDownUrl);
+    }
+}

src/main/java/com/hisense/dataservice/util/JdbcUtil.java

@@ -63,10 +63,12 @@ public class JdbcUtil {
             return new Result("200", "查询成功", result);
         } catch (SQLException sqlException) {
             log.error("sql: {}, params: {},数据查询异常：", sql, jdbcParamValues, sqlException);
-            throw new DataQueryException(sqlException.getMessage(), sqlException.getCause());
+            List<String> errors = collectExceptionMessages(sqlException, 5, 1);
+            throw new DataQueryException(errors.toString(), sqlException.getCause());
         } catch (Exception e) {
             log.error("sql: {}, params: {}, 分页查询数据异常：", sql, jdbcParamValues, e);
-            throw new RuntimeException("系统异常：" + e.getMessage());
+            List<String> errors = collectExceptionMessages(e, 5, 1);
+            throw new RuntimeException("系统异常：" + errors);
         } finally {
             try {
                 if (connection != null)
@@ -101,9 +103,11 @@ public class JdbcUtil {
             log.info("返回的数据：{}", count);
             return new Result("200", "查询成功", count);
         } catch (SQLException e) {
-            throw new DataQueryException(e.getMessage());
+            List<String> errors = collectExceptionMessages(e, 5, 1);
+            throw new DataQueryException(errors.toString());
         }catch (Exception e){
-            throw new RuntimeException("系统异常：" + e.getMessage());
+            List<String> errors = collectExceptionMessages(e, 5, 1);
+            throw new RuntimeException("系统异常：" + errors);
         } finally {
             try {
                 if (connection != null)
@@ -114,4 +118,12 @@ public class JdbcUtil {
         }
     }
 
+    public static List<String> collectExceptionMessages(Throwable t, int maxDepth, int currentDepth) {
+        List<String> messages = new ArrayList<>();
+        if (t != null && currentDepth <= maxDepth) {
+            messages.add(t.getMessage());
+            messages.addAll(collectExceptionMessages(t.getCause(), maxDepth, currentDepth + 1));
+        }
+        return messages;
+    }
 }

src/main/java/com/hisense/dataservice/vo/DataApiModelDetailVo.java

@@ -28,9 +28,15 @@ public class DataApiModelDetailVo {
     @ApiModelProperty(name = "数据服务描述")
     private String description;
 
-    @ApiModelProperty(name = "数据源类型")
+    @ApiModelProperty(name = "数据源ID")
     private String dataSourceId;
 
+    @ApiModelProperty(name = "数据源名称")
+    private String dataSourceName;
+
+    @ApiModelProperty(name = "数据源类型")
+    private String dataSourceType;
+
     @ApiModelProperty(name = "发布/创建人")
     private String creator;
 

src/main/resources/application.yml

@@ -38,3 +38,10 @@ flow:
     im-link: https://kk-proxy.kubenote.com/hiphichat/#/approvalAgency
 data-server:
   base-url: ${DATA_SERVER_BASE_URL:https://data-server.com}
+fileServer:
+  url: ${FILE_SERVER_URL:}
+  sysCode: ${FILE_SERVER_SYS_CODE:S0369}
+  bucketName: ${FILE_SERVER_BUCKET_NAME:hiptest-bt1}
+  accessKey: ${FILE_SERVER_ACCESS_kEY:A1H9P431IG2D00Y59E53}
+  secretKey: ${FILE_SERVER_SECRET_kEY:s5AyjO69eGxMxZ7OE50uvprxfjpBFkGnOn7BCd4R}
+  expires: ${FILE_SERVER_FILE_EXPIRES:7}

src/test/java/com/hisense/dataservice/service/DataApiServiceManagementServiceTest.java

@@ -299,12 +299,12 @@ public class DataApiServiceManagementServiceTest {
         boolean addLogResult = elasticsearchUtils.addData(indexName, callSqlEsLogJson.toJSONString());
         log.info("添加记录到es，结果：{}", addLogResult);
 
-        PageLink pageLink = new PageLink(10,1,"", new SortOrder("timestamp", SortOrder.Direction.DESC));
-        PageData<CallLogVo> pageData1 = dataServiceLogManagementService.getCallLog("staging", 1L, pageLink);
-        log.info("pageData1: {}", pageData1);
-
-        PageData<CallLogVo> pageData2 = dataServiceLogManagementService.getCallLog("staging", 2L, pageLink);
-        log.info("pageData2: {}", pageData2);
+//        PageLink pageLink = new PageLink(10,1,"", new SortOrder("timestamp", SortOrder.Direction.DESC));
+//        PageData<CallLogVo> pageData1 = dataServiceLogManagementService.getCallLog("staging", 1L, pageLink);
+//        log.info("pageData1: {}", pageData1);
+//
+//        PageData<CallLogVo> pageData2 = dataServiceLogManagementService.getCallLog("staging", 2L, pageLink);
+//        log.info("pageData2: {}", pageData2);
 
     }