OSS文件上传、MinIO分布式文件存储系统

2026/8/16 5:23:16 拓冰建站浏览量

阿里云OSS上传图片

实现步骤：

1. 定义OSS相关配置

application-dev.yml

sky:alioss:endpoint: oss-cn-hangzhou.aliyuncs.comaccess-key-id: LTAI5tPeFLzsPPT8gG3LPW64access-key-secret: U6k1brOZ8gaOIXv3nXbulGTUzy6Pd7bucket-name: sky-take-out

application.yml

spring:profiles:active: dev    #设置环境
sky:alioss:endpoint: ${sky.alioss.endpoint}access-key-id: ${sky.alioss.access-key-id}access-key-secret: ${sky.alioss.access-key-secret}bucket-name: ${sky.alioss.bucket-name}

2. 读取OSS配置

在sky-common模块中，已定义

package com.sky.properties;import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;@Component
@ConfigurationProperties(prefix = "sky.alioss")
@Data
public class AliOssProperties {private String endpoint;private String accessKeyId;private String accessKeySecret;private String bucketName;}

3. 生成OSS工具类对象

在sky-server模块

package com.sky.config;import com.sky.properties.AliOssProperties;
import com.sky.utils.AliOssUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;/*** 配置类，用于创建AliOssUtil对象*/
@Configuration
@Slf4j
public class OssConfiguration {@Bean@ConditionalOnMissingBeanpublic AliOssUtil aliOssUtil(AliOssProperties aliOssProperties){log.info("开始创建阿里云文件上传工具类对象：{}",aliOssProperties);return new AliOssUtil(aliOssProperties.getEndpoint(),aliOssProperties.getAccessKeyId(),aliOssProperties.getAccessKeySecret(),aliOssProperties.getBucketName());}
}

其中，AliOssUtil.java已在sky-common模块中定义

package com.sky.utils;import com.aliyun.oss.ClientException;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.OSSException;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import java.io.ByteArrayInputStream;@Data
@AllArgsConstructor
@Slf4j
public class AliOssUtil {private String endpoint;private String accessKeyId;private String accessKeySecret;private String bucketName;/*** 文件上传** @param bytes* @param objectName* @return*/public String upload(byte[] bytes, String objectName) {// 创建OSSClient实例。OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);try {// 创建PutObject请求。ossClient.putObject(bucketName, objectName, new ByteArrayInputStream(bytes));} catch (OSSException oe) {System.out.println("Caught an OSSException, which means your request made it to OSS, "+ "but was rejected with an error response for some reason.");System.out.println("Error Message:" + oe.getErrorMessage());System.out.println("Error Code:" + oe.getErrorCode());System.out.println("Request ID:" + oe.getRequestId());System.out.println("Host ID:" + oe.getHostId());} catch (ClientException ce) {System.out.println("Caught an ClientException, which means the client encountered "+ "a serious internal problem while trying to communicate with OSS, "+ "such as not being able to access the network.");System.out.println("Error Message:" + ce.getMessage());} finally {if (ossClient != null) {ossClient.shutdown();}}//文件访问路径规则 https://BucketName.Endpoint/ObjectNameStringBuilder stringBuilder = new StringBuilder("https://");stringBuilder.append(bucketName).append(".").append(endpoint).append("/").append(objectName);log.info("文件上传到:{}", stringBuilder.toString());return stringBuilder.toString();}
}

4. 定义文件上传接口

在sky-server模块中定义接口

package com.sky.controller.admin;import com.sky.constant.MessageConstant;
import com.sky.result.Result;
import com.sky.utils.AliOssUtil;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.UUID;/*** 通用接口*/
@RestController
@RequestMapping("/admin/common")
@Api(tags = "通用接口")
@Slf4j
public class CommonController {@Autowiredprivate AliOssUtil aliOssUtil;/*** 文件上传* @param file* @return*/@PostMapping("/upload")@ApiOperation("文件上传")public Result<String> upload(MultipartFile file){log.info("文件上传：{}",file);try {//原始文件名String originalFilename = file.getOriginalFilename();//截取原始文件名的后缀   dfdfdf.pngString extension = originalFilename.substring(originalFilename.lastIndexOf("."));//构造新文件名称String objectName = UUID.randomUUID().toString() + extension;//文件的请求路径String filePath = aliOssUtil.upload(file.getBytes(), objectName);return Result.success(filePath);} catch (IOException e) {log.error("文件上传失败：{}", e);}return Result.error(MessageConstant.UPLOAD_FAILED);}
}

MinIO分布式文件存储系统

MinIO集群采用去中心化共享架构，每个结点是对等关系，通过Nginx可对MinIO进行负载均衡访问。

它将分布在不同服务器上的多块硬盘组成一个对象存储服务。由于硬盘分布在不同的节点上，分布式Minio避免了单点故障。

Minio使用纠删码技术来保护数据，它是一种恢复丢失和损坏数据的数学算法，它将数据分块冗余的分散存储在各各节点的磁盘上，所有的可用磁盘组成一个集合，上图由8块硬盘组成一个集合，当上传一个文件时会通过纠删码算法计算对文件进行分块存储，除了将文件本身分成4个数据块，还会生成4个校验块，数据块和校验块会分散的存储在这8块硬盘上。

使用纠删码的好处是即便丢失一半数量（N/2）的硬盘，仍然可以恢复数据。比如上边集合中有4个以内的硬盘损害仍可保证数据恢复，不影响上传和下载，如果多于一半的硬盘坏了则无法恢复。

准备环境

MinIO提供多个语言版本SDK的支持，下边找到java版本的文档：

地址：https://docs.min.io/docs/java-client-quickstart-guide.html

引入依赖

<dependency><groupId>io.minio</groupId><artifactId>minio</artifactId><version>8.4.3</version>
</dependency>
<dependency><groupId>com.squareup.okhttp3</groupId><artifactId>okhttp</artifactId><version>4.8.1</version>
</dependency>

配置相关信息

minio:endpoint: http://192.168.101.65:9000accessKey: minioadminsecretKey: minioadminbucket:files: mediafilesvideofiles: video

编写minio的配置类

@Configuration
public class MinioConfig {@Value("${minio.endpoint}")private String endpoint;@Value("${minio.accessKey}")private String accessKey;@Value("${minio.secretKey}")private String secretKey;@Beanpublic MinioClient minioClient() {MinioClient minioClient =MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build();return minioClient;}
}

上传文件

设置contentType可以通过com.j256.simplemagic.ContentType枚举类查看常用的mimeType（媒体类型）
通过扩展名得到mimeType，代码如下：

//根据扩展名取出mimeType
ContentInfo extensionMatch = ContentInfoUtil.findExtensionMatch(".mp4");
String mimeType = MediaType.APPLICATION_OCTET_STREAM_VALUE;//通用mimeType，字节流

public class MinioTest {static MinioClient minioClient =MinioClient.builder().endpoint("http://192.168.101.65:9000").credentials("minioadmin", "minioadmin").build();//上传文件@Testpublic  void upload() {//根据扩展名取出mimeTypeContentInfo extensionMatch = ContentInfoUtil.findExtensionMatch(".mp4");String mimeType = MediaType.APPLICATION_OCTET_STREAM_VALUE;//通用mimeType，字节流if(extensionMatch!=null){mimeType = extensionMatch.getMimeType();}try {UploadObjectArgs testbucket = UploadObjectArgs.builder().bucket("testbucket")
//                    .object("test001.mp4").object("001/test001.mp4")//添加子目录.filename("D:\\develop\\upload\\1mp4.temp").contentType(mimeType)//默认根据扩展名确定文件内容类型，也可以指定.build();minioClient.uploadObject(testbucket);System.out.println("上传成功");} catch (Exception e) {e.printStackTrace();System.out.println("上传失败");}}
}

定义接口：需要指定消费者类型 consumes = MediaType.MULTIPART_FORM_DATA_VALUE

@RequestMapping(value = "/upload/coursefile", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public UploadFileResultDto upload(@RequestPart("filedata") MultipartFile upload) throws IOException {return null;
}

删除文件

@Test
public void delete(){try {minioClient.removeObject(RemoveObjectArgs.builder().bucket("testbucket").object("001/test001.mp4").build());System.out.println("删除成功");} catch (Exception e) {e.printStackTrace();System.out.println("删除失败");}
}

查询文件

//查询文件
@Test
public void getFile() {GetObjectArgs getObjectArgs = GetObjectArgs.builder().bucket("testbucket").object("test001.mp4").build();try(FilterInputStream inputStream = minioClient.getObject(getObjectArgs);FileOutputStream outputStream = new FileOutputStream(new File("D:\\develop\\upload\\1_2.mp4"));) {IOUtils.copy(inputStream,outputStream);} catch (Exception e) {e.printStackTrace();}
}

校验一致性

校验文件的完整性，对文件计算出md5值，比较原始文件的md5和目标文件的md5，一致则说明完整

//校验文件的完整性对文件的内容进行md5
FileInputStream fileInputStream1 = new FileInputStream(new File("D:\\develop\\upload\\1.mp4"));
String source_md5 = DigestUtils.md5Hex(fileInputStream1);
FileInputStream fileInputStream = new FileInputStream(new File("D:\\develop\\upload\\1a.mp4"));
String local_md5 = DigestUtils.md5Hex(fileInputStream);
if(source_md5.equals(local_md5)){System.out.println("下载成功");
}

MinIO断点续传

1、前端对文件进行分块。

2、前端上传分块文件前请求媒资服务检查文件是否存在，如果已经存在则不再上传。

3、如果分块文件不存在则前端开始上传

4、前端请求媒资服务上传分块。

5、媒资服务将分块上传至MinIO。

6、前端将分块上传完毕请求媒资服务合并分块。

7、媒资服务判断分块上传完成则请求MinIO合并文件。

8、合并完成校验合并后的文件是否完整，如果完整则上传完成，否则删除文件。

1.上传分块开发

检查文件和分块

接口完成进行接口实现，首先实现检查文件方法和检查分块方法。

在MediaFileService中定义service接口如下：

/*** @description 检查文件是否存在* @param fileMd5 文件的md5
*/
public RestResponse<Boolean> checkFile(String fileMd5);/*** @description 检查分块是否存在* @param fileMd5  文件的md5* @param chunkIndex  分块序号
*/
public RestResponse<Boolean> checkChunk(String fileMd5, int chunkIndex);

service接口实现方法：

@Override
public RestResponse<Boolean> checkFile(String fileMd5) {//查询文件信息MediaFiles mediaFiles = mediaFilesMapper.selectById(fileMd5);if (mediaFiles != null) {//桶String bucket = mediaFiles.getBucket();//存储目录String filePath = mediaFiles.getFilePath();//文件流InputStream stream = null;try {stream = minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(filePath).build());if (stream != null) {//文件已存在return RestResponse.success(true);}} catch (Exception e) {}}//文件不存在return RestResponse.success(false);
}@Override
public RestResponse<Boolean> checkChunk(String fileMd5, int chunkIndex) {//得到分块文件目录String chunkFileFolderPath = getChunkFileFolderPath(fileMd5);//得到分块文件的路径String chunkFilePath = chunkFileFolderPath + chunkIndex;//文件流InputStream fileInputStream = null;try {fileInputStream = minioClient.getObject(GetObjectArgs.builder().bucket(bucket_videoFiles).object(chunkFilePath).build());if (fileInputStream != null) {//分块已存在return RestResponse.success(true);}} catch (Exception e) {}//分块未存在return RestResponse.success(false);
}//得到分块文件的目录
private String getChunkFileFolderPath(String fileMd5) {return fileMd5.substring(0, 1) + "/" + fileMd5.substring(1, 2) + "/" + fileMd5 + "/" + "chunk" + "/";
}

上传分块

定义service接口

/*** @description 上传分块* @param fileMd5  文件md5* @param chunk  分块序号* @param localChunkFilePath  分块文件本地路径*/
public RestResponse uploadChunk(String fileMd5,int chunk,String localChunkFilePath);

接口实现：

@Override
public RestResponse uploadChunk(String fileMd5, int chunk, String localChunkFilePath) {//得到分块文件的目录路径String chunkFileFolderPath = getChunkFileFolderPath(fileMd5);//得到分块文件的路径String chunkFilePath = chunkFileFolderPath + chunk;//mimeTypeString mimeType = getMimeType(null);//将文件存储至minIOboolean b = addMediaFilesToMinIO(localChunkFilePath, mimeType, bucket_videoFiles, chunkFilePath);if (!b) {log.debug("上传分块文件失败:{}", chunkFilePath);return RestResponse.validfail(false, "上传分块失败");}log.debug("上传分块文件成功:{}",chunkFilePath);return RestResponse.success(true);}

上传分块测试

完善接口层：

@ApiOperation(value = "文件上传前检查文件")
@PostMapping("/upload/checkfile")
public RestResponse<Boolean> checkfile(@RequestParam("fileMd5") String fileMd5
) throws Exception {return mediaFileService.checkFile(fileMd5);
}@ApiOperation(value = "分块文件上传前的检测")
@PostMapping("/upload/checkchunk")
public RestResponse<Boolean> checkchunk(@RequestParam("fileMd5") String fileMd5,@RequestParam("chunk") int chunk) throws Exception {return mediaFileService.checkChunk(fileMd5,chunk);
}@ApiOperation(value = "上传分块文件")
@PostMapping("/upload/uploadchunk")
public RestResponse uploadchunk(@RequestParam("file") MultipartFile file,@RequestParam("fileMd5") String fileMd5,@RequestParam("chunk") int chunk) throws Exception {//创建临时文件File tempFile = File.createTempFile("minio", "temp");//上传的文件拷贝到临时文件file.transferTo(tempFile);//文件路径String absolutePath = tempFile.getAbsolutePath();return mediaFileService.uploadChunk(fileMd5,chunk,absolutePath);
}

前端对文件分块的大小为5MB，SpringBoot web默认上传文件的大小限制为1MB，这里需要在media-api工程修改配置如下：

spring:servlet:multipart:max-file-size: 50MBmax-request-size: 50MB

max-file-size：单个文件的大小限制

Max-request-size: 单次请求的大小限制

2.合并分块开发

service开发

定义service接口：

/*** @description 合并分块* @param companyId  机构id* @param fileMd5  文件md5* @param chunkTotal 分块总和* @param uploadFileParamsDto 文件信息*/
public RestResponse mergechunks(Long companyId,String fileMd5,int chunkTotal,UploadFileParamsDto uploadFileParamsDto);

service实现：

@Override
public RestResponse mergechunks(Long companyId, String fileMd5, int chunkTotal, UploadFileParamsDto uploadFileParamsDto) {//=====获取分块文件路径=====String chunkFileFolderPath = getChunkFileFolderPath(fileMd5);//组成将分块文件路径组成 List<ComposeSource>List<ComposeSource> sourceObjectList = Stream.iterate(0, i -> ++i).limit(chunkTotal).map(i -> ComposeSource.builder().bucket(bucket_videoFiles).object(chunkFileFolderPath.concat(Integer.toString(i))).build()).collect(Collectors.toList());//=====合并=====//文件名称String fileName = uploadFileParamsDto.getFilename();//文件扩展名String extName = fileName.substring(fileName.lastIndexOf("."));//合并文件路径String mergeFilePath = getFilePathByMd5(fileMd5, extName);try {//合并文件ObjectWriteResponse response = minioClient.composeObject(ComposeObjectArgs.builder().bucket(bucket_videoFiles).object(mergeFilePath).sources(sourceObjectList).build());log.debug("合并文件成功:{}",mergeFilePath);} catch (Exception e) {log.debug("合并文件失败,fileMd5:{},异常:{}",fileMd5,e.getMessage(),e);return RestResponse.validfail(false, "合并文件失败。");}// ====验证md5====//下载合并后的文件File minioFile = downloadFileFromMinIO(bucket_videoFiles,mergeFilePath);if(minioFile == null){log.debug("下载合并后文件失败,mergeFilePath:{}",mergeFilePath);return RestResponse.validfail(false, "下载合并后文件失败。");}try (InputStream newFileInputStream = new FileInputStream(minioFile)) {//minio上文件的md5值String md5Hex = DigestUtils.md5Hex(newFileInputStream);//比较md5值，不一致则说明文件不完整if(!fileMd5.equals(md5Hex)){return RestResponse.validfail(false, "文件合并校验失败，最终上传失败。");}//文件大小uploadFileParamsDto.setFileSize(minioFile.length());}catch (Exception e){log.debug("校验文件失败,fileMd5:{},异常:{}",fileMd5,e.getMessage(),e);return RestResponse.validfail(false, "文件合并校验失败，最终上传失败。");}finally {if(minioFile!=null){minioFile.delete();}}//文件入库currentProxy.addMediaFilesToDb(companyId,fileMd5,uploadFileParamsDto,bucket_videoFiles,mergeFilePath);//=====清除分块文件=====clearChunkFiles(chunkFileFolderPath,chunkTotal);return RestResponse.success(true);
}/*** 从minio下载文件* @param bucket 桶* @param objectName 对象名称* @return 下载后的文件*/
public File downloadFileFromMinIO(String bucket,String objectName){//临时文件File minioFile = null;FileOutputStream outputStream = null;try{InputStream stream = minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(objectName).build());//创建临时文件minioFile=File.createTempFile("minio", ".merge");outputStream = new FileOutputStream(minioFile);IOUtils.copy(stream,outputStream);return minioFile;} catch (Exception e) {e.printStackTrace();}finally {if(outputStream!=null){try {outputStream.close();} catch (IOException e) {e.printStackTrace();}}}return null;
}
/*** 得到合并后的文件的地址* @param fileMd5 文件id即md5值* @param fileExt 文件扩展名* @return*/
private String getFilePathByMd5(String fileMd5,String fileExt){return   fileMd5.substring(0,1) + "/" + fileMd5.substring(1,2) + "/" + fileMd5 + "/" +fileMd5 +fileExt;
}/*** 清除分块文件* @param chunkFileFolderPath 分块文件路径* @param chunkTotal 分块文件总数*/
private void clearChunkFiles(String chunkFileFolderPath,int chunkTotal){try {List<DeleteObject> deleteObjects = Stream.iterate(0, i -> ++i).limit(chunkTotal).map(i -> new DeleteObject(chunkFileFolderPath.concat(Integer.toString(i)))).collect(Collectors.toList());RemoveObjectsArgs removeObjectsArgs = RemoveObjectsArgs.builder().bucket("video").objects(deleteObjects).build();Iterable<Result<DeleteError>> results = minioClient.removeObjects(removeObjectsArgs);results.forEach(r->{DeleteError deleteError = null;try {deleteError = r.get();} catch (Exception e) {e.printStackTrace();log.error("清除分块文件失败,objectname:{}",deleteError.objectName(),e);}});} catch (Exception e) {e.printStackTrace();log.error("清除分块文件失败,chunkFileFolderPath:{}",chunkFileFolderPath,e);}
}

接口层完善

@ApiOperation(value = "合并文件")
@PostMapping("/upload/mergechunks")
public RestResponse mergechunks(@RequestParam("fileMd5") String fileMd5,@RequestParam("fileName") String fileName,@RequestParam("chunkTotal") int chunkTotal) throws Exception {Long companyId = 1232141425L;UploadFileParamsDto uploadFileParamsDto = new UploadFileParamsDto();uploadFileParamsDto.setFileType("001002");uploadFileParamsDto.setTags("课程视频");uploadFileParamsDto.setRemark("");uploadFileParamsDto.setFilename(fileName);return mediaFileService.mergechunks(companyId,fileMd5,chunkTotal,uploadFileParamsDto);}

建站实战干货