Skip to content

M
mlops-op-service
提交 c3b7e6e5 authored 作者: songchuancai's avatar songchuancai
浏览文件
操作

init

上级
隐藏空白字符变更
内嵌 并排
正在显示 包含 502 行增加 和 0 行删除
.gitignore 0 → 100644
浏览文件 @ c3b7e6e5
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/
Dockerfile 0 → 100644
浏览文件 @ c3b7e6e5
FROM java:8
ENV TZ=Asia/Shanghai SPRING_OUTPUT_ANSI_ENABLED=ALWAYS JAVA_OPTS=""
ENV WORK_PATH /home/mlops
ENV APP_NAME mlops-op-service-1.0.jar
ENV HEAP_DUMP_PATH /home/mlops/operator
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezonemvn
EXPOSE 8080
VOLUME ["/home/mlops", "/tmp/data"]
COPY target/$APP_NAME $WORK_PATH/
WORKDIR $WORK_PATH
ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom"]
CMD ["-Xms1024m", "-Xmx1024m","-XX:+HeapDumpOnOutOfMemoryError","-XX:HeapDumpPath=${HEAP_DUMP_PATH}/heapdump.hprof","-jar", "mlops-op-service-1.0.jar"]
# docker build -t registry.iytcloud.com/mlops/mlops-op-service:1.1 .
# docker push registry.iytcloud.com/mlops/mlops-op-service:1.1
pom.xml 0 → 100644
浏览文件 @ c3b7e6e5
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.5.RELEASE</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.notemap</groupId>
<artifactId>mlops-op-service</artifactId>
<version>1.0</version>
<name>mlops-op-service</name>
<packaging>jar</packaging>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-boot.version>2.6.13</spring-boot.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>8.0.1</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.30</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.15</version>
</dependency>
</dependencies>
<build>
<finalName>mlops-op-service-${version}</finalName>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<!-- 不执行单元测试，但会编译测试类并在target/test-classes目录下生成相应的class -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
</plugins>
</build>
</project>
src/main/java/com/notemap/mlopsoperatorservice/MlopsOperatorServiceApplication.java 0 → 100644
浏览文件 @ c3b7e6e5
package com.notemap.mlopsoperatorservice;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class MlopsOperatorServiceApplication {
public static void main(String[] args) {
SpringApplication.run(MlopsOperatorServiceApplication.class, args);
}
}
src/main/java/com/notemap/mlopsoperatorservice/configs/MinIOClientConfig.java 0 → 100644
浏览文件 @ c3b7e6e5
package com.notemap.mlopsoperatorservice.configs;
import io.minio.MinioClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
@Component
public class MinIOClientConfig {
@Autowired
private Environment environment;
@Bean
public MinioClient minioClient(){
String minioEndpoint = environment.getProperty("minioEndpoint");
String minioAccessKey = environment.getProperty("minioAccessKey");
String minioSecretKey = environment.getProperty("minioSecretKey");
return MinioClient.builder()
.endpoint(minioEndpoint)
.credentials(minioAccessKey, minioSecretKey)
.build();
}
}
src/main/java/com/notemap/mlopsoperatorservice/configs/MinioService.java 0 → 100644
浏览文件 @ c3b7e6e5
package com.notemap.mlopsoperatorservice.configs;
import io.minio.*;
import io.minio.messages.DeleteError;
import io.minio.messages.DeleteObject;
import io.minio.messages.Item;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.apache.commons.lang3.StringUtils;
import java.io.InputStream;
import java.util.*;
@Slf4j
@Component
public class MinioService {
@Autowired(required = false)
private MinioClient minioClient;
private boolean clientIsAvailable(){
if (minioClient == null) {
log.error("minio客户端未初始化成功");
return false;
}
return true;
}
/**
* 判断bucket是否存在
* @param bucketName
* @return
*/
public boolean bucketExists(String bucketName) {
if (StringUtils.isEmpty(bucketName) || !clientIsAvailable()) {
return false;
}
try {
return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
} catch (Exception e) {
log.error("查询bucket是否存在错误：", e);
}
return false;
}
/**
* 创建bucket
* @param bucketName
* @return
*/
public boolean createBucket(String bucketName) {
if (StringUtils.isEmpty(bucketName) || !clientIsAvailable()) {
return false;
}
if(bucketExists(bucketName)){
return true;
}
try {
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
return true;
} catch (Exception e) {
log.error("创建bucket错误：", e);
}
return false;
}
/**
* 上传文件object
* @param bucketName
* @param objectName
* @param filePath
* @return
*/
public boolean uploadObject(String bucketName, String objectName, String filePath) {
if (StringUtils.isEmpty(bucketName) || StringUtils.isEmpty(objectName) || !clientIsAvailable()) {
return false;
}
try {
Map<String,String> tag = new HashMap<>();
ObjectWriteResponse objectWriteResponse = minioClient.uploadObject(
UploadObjectArgs.builder()
.bucket(bucketName)
.object(objectName)
.filename(filePath)
.tags(tag)
.build());
log.info("上传对象文件的结果：", objectWriteResponse);
return true;
} catch (Exception e) {
log.error("上传对象文件失败：",e);
}
return false;
}
/**
* 移除bucket下的文件object
* @param bucketName
* @param objectName
* @return
*/
public boolean removeObject(String bucketName, String objectName){
if (StringUtils.isEmpty(bucketName) || StringUtils.isEmpty(objectName) || !clientIsAvailable()) {
return false;
}
try{
minioClient.removeObject(
RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build());
return true;
}catch (Exception e){
log.error("删除文件对象失败：",e);
}
return false;
}
/**
* 移除一个bucketName下多个文件object
* @param bucketName
* @param objectNames
* @return
*/
public boolean removeObjects(String bucketName, List<String> objectNames){
if (StringUtils.isEmpty(bucketName) || !clientIsAvailable()) {
return false;
}
try{
List<DeleteObject> objects = new LinkedList<>();
for (String objectName : objectNames) {
objects.add(new DeleteObject(objectName));
}
Iterable<Result<DeleteError>> results =
minioClient.removeObjects(
RemoveObjectsArgs.builder().bucket(bucketName).objects(objects).build());
boolean success = true;
for (Result<DeleteError> result : results) {
DeleteError error = result.get();
log.info("Error code: {}, object: {}, message: {}", error.code(), error.objectName(), error.message());
success = false;
}
return success;
}catch (Exception e){
log.error("删除文件对象:{},失败：{}",objectNames,e);
}
return false;
}
/**
* 下载文件object到指定的文件路径下（fileNamePath)
* @param bucketName
* @param objectName
* @param fileNamePath
* @return
*/
public boolean downloadObject(String bucketName, String objectName, String fileNamePath){
if(StringUtils.isEmpty(bucketName) || StringUtils.isEmpty(objectName) || !clientIsAvailable() || StringUtils.isEmpty(fileNamePath)){
return false;
}
try{
minioClient.downloadObject(
DownloadObjectArgs.builder()
.bucket(bucketName)
.object(objectName)
.filename(fileNamePath)
.build());
return true;
}catch (Exception e){
log.error("下载文件对象失败：", e);
}
return false;
}
/**
* 下载对象文件
* @param bucketName
* @param objectName
* @return 返回文件流
*/
public InputStream downloadObject(String bucketName, String objectName){
InputStream result = null;
if(StringUtils.isEmpty(bucketName) || StringUtils.isEmpty(objectName) || !clientIsAvailable()){
return result;
}
try{
result = minioClient.getObject(GetObjectArgs.builder().bucket(bucketName).object(objectName).build());
}catch (Exception e){
log.error("下载文件对象失败：", e);
}
return result;
}
public List<Map<String,String>> listObjectsByPrefix(String bucketName, String prefix){
List<Map<String,String>> results = new ArrayList<>();
if(StringUtils.isEmpty(bucketName)){
return results;
}
try{
Iterable<Result<Item>> listObjects =
minioClient.listObjects(ListObjectsArgs.builder().bucket(bucketName).prefix(prefix).build());
for (Result<Item> itemResult : listObjects) {
Item item = itemResult.get();
HashMap map = new HashMap();
map.put("fileName", StringUtils.substringAfterLast(item.objectName(), "/"));
map.put("size", String.valueOf(item.size()));
log.info("minio：fileName:{},size:{}",item.objectName(), item.size());
results.add(map);
}
}catch (Exception e){
log.error("获取文件对象列表失败：",e);
}
return results;
}
/**
* 复制文件object
* @param targetBucketName
* @param targetObjectName
* @param sourceBucketName
* @param sourceObjectName
* @return
*/
public boolean copyObject(String targetBucketName, String targetObjectName, String sourceBucketName, String sourceObjectName){
if (StringUtils.isEmpty(targetBucketName) || StringUtils.isEmpty(targetObjectName)
|| StringUtils.isEmpty(sourceBucketName) || StringUtils.isEmpty(sourceObjectName) || !clientIsAvailable()) {
return false;
}
try{
minioClient.copyObject(CopyObjectArgs.builder()
.bucket(targetBucketName)
.object(targetObjectName)
.source(
CopySource.builder()
.bucket(sourceBucketName)
.object(sourceObjectName)
.build())
.build());
return true;
}catch (Exception e){
log.error("复制文件对象失败：",e);
}
return false;
}
}
src/main/java/com/notemap/mlopsoperatorservice/web/BasicController.java 0 → 100644
浏览文件 @ c3b7e6e5
package com.notemap.mlopsoperatorservice.web;
import cn.hutool.core.lang.Assert;
import cn.hutool.core.util.CharsetUtil;
import cn.hutool.core.util.ZipUtil;
import com.notemap.mlopsoperatorservice.configs.MinioService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
@Controller
@Slf4j
public class BasicController {
private final String FILE_PATH = "/opt/ams/app";
//private final String FILE_PATH = "D:\\opt\\ams\\app";
private final String FILE_SUFFIX = ".zip";
@Autowired
private MinioService minioService;
@RequestMapping("/health")
@ResponseBody
public String health() {
log.info("health");
return "health";
}
@RequestMapping("/loadCodeFiles")
@ResponseBody
public String loadCodeFiles(@RequestParam String bucketName, @RequestParam String objectName, @RequestParam String opId, HttpServletResponse response) {
String msg = "success";
String descFileDirPath = FILE_PATH + File.separator + opId;
String descFileName = FILE_PATH + File.separator + objectName;
try {
new File(descFileDirPath).mkdirs();
Assert.isTrue(objectName.endsWith(FILE_SUFFIX), "不支持的压缩文件格式");
Boolean result = minioService.downloadObject(bucketName, objectName, descFileName);
Assert.isTrue(result, "下载minio服务器文件失败");
File file = ZipUtil.unzip(descFileName, descFileDirPath);
Assert.isTrue(file != null, "解压文件失败");
} catch (Exception e) {
log.error("从minio服务器获取文件异常：minio中不存在文件，bucketName：{}，objectName：{}，fileRealPath：{}，reason：", bucketName, objectName, descFileName, e);
msg = e.getMessage();
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
}
return msg;
}
@RequestMapping("/uploadCodeFiles")
@ResponseBody
public String uploadCodeFiles(@RequestParam String bucketName, @RequestParam String objectName, HttpServletResponse response) {
String sourceFileName = objectName.substring(0, objectName.lastIndexOf("."));
String srcFileRealPath = FILE_PATH + File.separator + sourceFileName;
String descFileRealPath = FILE_PATH + File.separator + sourceFileName + FILE_SUFFIX;
String msg = "success";
try {
ZipUtil.zip(srcFileRealPath, descFileRealPath,CharsetUtil.defaultCharset(),true);
Boolean result = minioService.uploadObject(bucketName, objectName, descFileRealPath);
Assert.isTrue(result, "上传文件到minio服务器失败！");
} catch (Exception e) {
log.error("从minio服务器获取文件异常：minio中不存在文件，bucketName：{}，objectName：{}，fileRealPath：{}，reason：", bucketName, objectName, srcFileRealPath, e);
msg = e.getMessage();
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
}
return msg;
}
}
src/main/resources/application.yml 0 → 100644
浏览文件 @ c3b7e6e5
# 应用服务 WEB 访问端口
server:
port: 8080
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论