Spring Boot 集成 AmazonS3 存储服务教程

喜欢ヅ旅行 2021-06-22 15:38 1045阅读 0赞

参考文档：[https://docs.ceph.com/en/latest/radosgw/s3/java/\#download-an-object-to-a-file][https_docs.ceph.com_en_latest_radosgw_s3_java_download-an-object-to-a-file]

## 环境准备 ##

1.  Java环境：JDK6以上版本
2.  SpringBoot 2.2.5
3.  存储服务的 accessKey、secreKey 以及 地址:端口

## 使用 ##

### 1. 引入依赖 ###

在 `pom.xml` 文件中添加下列依赖：

<dependency>
        <groupId>com.amazonaws</groupId>
        <artifactId>aws-java-sdk</artifactId>
        <version>1.11.433</version>
    </dependency>

### 2. 初始化 ###

> 简单模式

String accessKey = "your-accesskey";
    String secretKey = "your-secretKey";
    
    AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 conn = AmazonS3ClientBuilder.standard()
                    .withCredentials(new AWSStaticCredentialsProvider(credentials))
                    .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("endpoint","region"))
                    .build();

示例：

String accessKey = "TPDDEA5PCT9C8RUPKAWW";
    String secretKey = "ryesPMSiSfOzaP1TkFe9TIOBnxpdrA2sw6isFwDZ";
    
    AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 conn = AmazonS3ClientBuilder.standard()
                    .withCredentials(new AWSStaticCredentialsProvider(credentials))
                    .withEndpointConfiguration(new   AwsClientBuilder.EndpointConfiguration("http://10.200.100.37:7480","region"))
                    .build();

> 配置模式

示例：

> Amazon3 是线程安全的，因此可以将其注入到 Spring 容器中进行管理。

@AmazonS3Config.java

import com.amazonaws.ClientConfiguration;
    import com.amazonaws.Protocol;
    import com.amazonaws.auth.AWSCredentials;
    import com.amazonaws.auth.AWSStaticCredentialsProvider;
    import com.amazonaws.auth.BasicAWSCredentials;
    import com.amazonaws.client.builder.AwsClientBuilder;
    import com.amazonaws.services.s3.AmazonS3;
    import com.amazonaws.services.s3.AmazonS3ClientBuilder;
    import com.amazonaws.services.s3.model.CreateBucketRequest;
    import com.amazonaws.services.s3.transfer.TransferManager;
    import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.boot.context.properties.EnableConfigurationProperties;
    import org.springframework.context.annotation.Bean;
    import org.springframework.context.annotation.Configuration;
    
    /**
     * @Author: ChangXuan
     * @Decription: 初始化AmazonS3实例
     * @Date: 17:33 2020/6/3
     **/
    @Configuration
    @EnableConfigurationProperties(UploadConfig.class)
    public class AmazonS3Config {
    
        @Autowired
        private UploadConfig uploadConfig;
    
        @Bean(name = "amazonS3")
        public AmazonS3 getAmazonS3(){
            String accessKey = uploadConfig.getCeph().getAccessKey();
            String secretKey = uploadConfig.getCeph().getSecretKey();
            AWSCredentials credentials = new BasicAWSCredentials(accessKey,secretKey);
            ClientConfiguration conf = new ClientConfiguration();
            // 设置AmazonS3使用的最大连接数
            conf.setMaxConnections(uploadConfig.getCeph().getAmazonS3MaxConnections());
            // 设置socket超时时间
            conf.setSocketTimeout(uploadConfig.getCeph().getAmazonS3SocketTimeout());
            // 设置失败请求重试次数
            conf.setMaxErrorRetry(uploadConfig.getCeph().getAmazonS3MaxErrorRetry());
            // 设置协议
            if (!"blank".equals(uploadConfig.getCeph().getAmazonS3Protocol())){
                switch (uploadConfig.getCeph().getAmazonS3Protocol()){
                    case "https":
                        conf.setProtocol(Protocol.HTTPS);
                        break;
                    case "http":
                        conf.setProtocol(Protocol.HTTP);
                        break;
                    default:
                        break;
                }
            }
            AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                    .withCredentials(new AWSStaticCredentialsProvider(credentials))
                    .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(uploadConfig.getCeph().getHosts().get(0),uploadConfig.getCeph().getRegion()))
                    .withClientConfiguration(conf)
                    .build();
            checkAndCreateBucket(s3Client);
            return s3Client;
        }
    
    
        @Bean(name = "transferManager")
        public TransferManager getTransferManager(){
            return TransferManagerBuilder.standard().withS3Client(getAmazonS3()).build();
        }
    
    
        /**
         * 检查桶是否存在，不存在则创建创建
         * @param s3Client
         */
        private void checkAndCreateBucket(AmazonS3 s3Client){
            boolean exists = s3Client.doesBucketExistV2(uploadConfig.getCeph().getBucketName());
            if (!exists){
                CreateBucketRequest request = new CreateBucketRequest(uploadConfig.getCeph().getBucketName());
                s3Client.createBucket(request);
            }
        }

@UploadConfig.java

import lombok.Data;
    import org.springframework.boot.context.properties.ConfigurationProperties;
    
    /**
     * @Author: ChangXuan
     * @Decription: 上传 配置
     * @Date: 17:25 2020/6/3
     **/
    @ConfigurationProperties(prefix = "upload")
    @Data
    public class UploadConfig {
        private CephConfig ceph;
    }

@CephConfig.java

import com.google.common.collect.Lists;
    import lombok.Data;
    
    import java.util.List;
    
    /**
     * @Author: ChangXuan
     * @Decription:配置
     * @Date: 17:28 2020/6/3
     **/
    @Data
    public class CephConfig {
    
        /**
         * 集群ip:port
         */
        private List<String> hosts = Lists.newArrayList();
    
        private String accessKey;
    
        private String secretKey;
    
        private String region;
    
        /**
         * 连接协议
         */
        private String amazonS3Protocol;
    
        /**
         * 失败请求重试次数
         */
        private Integer amazonS3MaxErrorRetry;
    
        /**
         * 超时时间
         */
        private Integer amazonS3SocketTimeout;
    
        /**
         * 最大连接数
         */
        private Integer amazonS3MaxConnections;
    
        /**
         * 桶名称
         */
        private String bucketName;
    
    }

@application.yml

upload:
      ceph:
        bucketName: ic-storage-dev
        region: region
        hosts:
          - "10.200.100.37:7480"
          - "10.200.100.38:7480"
          - "10.200.100.39:7480"
        accessKey: TPDDEA5PCT9C8RUPKAWW
        secretKey: ryesPMSiSfOzaP1TkFe9TIWQnxpdrA2sw6isFwDZ
        amazonS3MaxConnections: 200
        amazonS3SocketTimeout: 10000
        amazonS3MaxErrorRetry: 2
        amazonS3Protocol: http

### 3. 依赖注入 ###

@Autowired
    private AmazonS3 amazonS3;
    
    //或
    
    @Autowired
    private TransferManager transferManager;

### 4. 文件操作 ###

文件上传

1. 直接内容上传

//要上传文件内容
    String content = "Object content";
    try {
       amazonS3.putObject("your-bucketname","your-objectname",content);
    }catch (Exception e){
       System.out.println(e.getMessage());
    }

2. 本地文件普通上传

对于小对象可以使用putObject接口进行上传，putObject上传支持的最大文件大小为100M，如果上传大于100M的文件需要使用分块上传。本地文件普通上传的示例代码如下：

//要上传文件的路径
    String filePath = "your-local-file-path";
    try {
       amazonS3.putObject("your-bucketname","your-objectname", new File(filePath));
    }catch (Exception e){
       System.out.println(e.getMessage());
    }

3. 上传文件时设置文件元数据信息

String filePath = "your-local-file-path";
    ObjectMetadata objectMetadata = new ObjectMetadata();
    //设置Content-Type
    objectMetadata.setContentType("application/xml");
    //设置标准http消息头（元数据）
    objectMetadata.setHeader("Cache-Control", "no-cache");
    //设置用户自定义元数据信息
    Map<String, String> userMeta = new HashMap<String, String>();
    userMeta.put("ud", "test");
    objectMetadata.setUserMetadata(userMeta);
    PutObjectRequest putObjectRequest = new PutObjectRequest("your-bucketname","your-objectname", new File(filePath));
    putObjectRequest.setMetadata(objectMetadata);
    amazonS3.putObject(putObjectRequest);

4. 流式上传

try {
       ObjectMetadata objectMetadata = new ObjectMetadata();
       //设置流的长度，您还可以设置其他文件元数据信息
       objectMetadata.setContentLength(streamLength);
       amazonS3.putObject("your-bucketname","your-objectname", inputStream, objectMetadata)
    }catch (Exception e){
       System.out.println(e.getMessage());
    }

文件下载

1. 流式下载

S3Object fileObject = amazonS3.getObject("your-bucketname","your-objectname");
    //可以通过getObjectMetadata方法获取对象的ContentType等元数据信息
    String contentType = fileObject.getObjectMetadata().getContentType();
    //流式获取文件内容
    InputStream in = fileObject.getObjectContent();
    BufferedReader reader = new BufferedReader(new InputStreamReader(in));
    while (true) {
      String line;
      try {
        line = reader.readLine();
        if (line == null) break;
        System.out.println("\n" + line);
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
    try {
      reader.close();
    } catch (IOException e) {
      e.printStackTrace();
    }

2. 下载到本地文件

String destinationFile = "your-local-filepath";
    GetObjectRequest getObjectRequest = new GetObjectRequest("your-bucketname","your-objectname");
    ObjectMetadata objectMetadata = amazonS3.getObject(getObjectRequest, new File(destinationFile));

3. Range 下载

GetObjectRequest getObjectRequest = new GetObjectRequest("your-bucketname","your-objectname");
    getObjectRequest.setRange(0, 100);
    S3Object nosObject = amazonS3.getObject(getObjectRequest);
    BufferedReader reader = new BufferedReader(new InputStreamReader(in));
    while (true) {
      String line;
      try {
        line = reader.readLine();
        if (line == null) break;
          System.out.println("\n" + line);
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
    try {
      reader.close();
    } catch (IOException e) {
      e.printStackTrace();
    }

文件管理

判断文件是否存在

您可以通过AmazonS3.doesObjectExist判断文件是否存在。

boolean isExist = amazonS3.doesObjectExist("your-bucketname","your-objectname");

文件删除

您可以通过AmazonS3.deleteObject删除单个文件

amazonS3.deleteObject("your-bucketname","your-objectname");

您还可以通过AmazonS3.deleteObjects一次删除多个文件

try {
        DeleteObjectsResult result = amazonS3.deleteObjects(deleteObjectsRequest);
        List<DeletedObject>  deleteObjects = result.getDeletedObjects();
        //print the delete results
        for (DeletedObject items: deleteObjects){
          System.out.println(items.getKey());
        }
    // 部分对象删除失败
    } catch (MultiObjectDeleteException e) { 
        List<DeleteError> deleteErrors = e.getErrors();
        for (DeleteError error : deleteErrors) {
            System.out.println(error.getKey());
        }
    } catch (AmazonServiceException  e) {
            //捕捉服务器异常错误
    } catch (AmazonClientException ace) {
           //捕捉客户端错误
    }

获取文件元数据信息

您可以通过AmazonS3.getObjectMetadata获取文件元数据信息

amazonS3.getObjectMetadata("your-bucketname","your-objectname");

文件复制(copy)

您可以通过AmazonS3.copyObject接口实现文件拷贝功能。

amazonS3.copyObject("source-bucket", "source-object", "dst-bucket", "dst-object");

列举桶内文件

您可以通过AmazonS3.listObjects列出桶里的文件。listObjects接口如果调用成功，会返回一个ObjectListing对象，列举的结果保持在该对象中。

ObjectListing的具体信息如下表所示：

<table> 
 <thead> 
  <tr> 
   <th>方法</th> 
   <th>含义</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>List getObjectSummaries()</td> 
   <td>返回的文件列表（包含文件的名称、Etag的元数据信息）</td> 
  </tr> 
  <tr> 
   <td>String getPrefix()</td> 
   <td>本次查询的文件名前缀</td> 
  </tr> 
  <tr> 
   <td>String getDelimiter()</td> 
   <td>文件分界符</td> 
  </tr> 
  <tr> 
   <td>String getMarker()</td> 
   <td>这次List Objects的起点</td> 
  </tr> 
  <tr> 
   <td>int getMaxKeys()</td> 
   <td>响应请求内返回结果的最大数目</td> 
  </tr> 
  <tr> 
   <td>String getNextMarker()</td> 
   <td>下一次List Object的起点</td> 
  </tr> 
  <tr> 
   <td>boolean isTruncated()</td> 
   <td>是否截断，如果因为设置了limit导致不是所有的数据集都返回，则该值设置为true</td> 
  </tr> 
  <tr> 
   <td>List getCommonPrefixes()</td> 
   <td>如果请求中指定了delimiter参数，则返回的包含CommonPrefixes元素。该元素标明以delimiter结尾，并有共同前缀的对象的集合</td> 
  </tr> 
 </tbody> 
</table>

AmazonS3.listObjects接口提供两种调用方式：简单列举、通过ListObjectsRequest列举

简单列举

简单列举只需指定需要列举的桶名，最多返回100条对象记录，建议桶内对象数较少时（小于100）使用。

ObjectListing objectListing = amazonS3.listObjects("your-bucketname");
    List<S3ObjectSummary> sums = objectListing.getObjectSummaries();
    for (S3ObjectSummary s : sums) {
      System.out.println("\t" + s.getKey());
    }

通过ListObjectsRequest列举

您还可以通过设置ListObjectsReques参数实现各种灵活的查询功能。ListObjectsReques的可设置的参数如下：

<table> 
 <thead> 
  <tr> 
   <th>设置方法</th> 
   <th>作用</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>setPrefix(String prefix)</td> 
   <td>限定返回的object key必须以prefix作为前缀</td> 
  </tr> 
  <tr> 
   <td>setDelimiter(String delimiter)</td> 
   <td>是一个用于对Object名字进行分组的字符。所有名字包含指定的前缀且第一次出现delimiter字符之间的object作为一组元素——CommonPrefixes</td> 
  </tr> 
  <tr> 
   <td>setMarker(String marker)</td> 
   <td>字典序的起始标记，只列出该标记之后的部分</td> 
  </tr> 
  <tr> 
   <td>setMaxKeys(Integer maxKeys)</td> 
   <td>限定返回的数量，返回的结果小于或等于该值(默认值为100)</td> 
  </tr> 
 </tbody> 
</table>

1、分页列举桶内的所有文件：

List<S3ObjectSummary> listResult = new ArrayList<S3ObjectSummary>();
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName("your-bucketname");
    listObjectsRequest.setMaxKeys(50);
    ObjectListing listObjects = amazonS3.listObjects(listObjectsRequest);
    do {
      listResult.addAll(listObjects.getObjectSummaries());
      if (listObjects.isTruncated()) {
        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(listObjectsRequest.getBucketName());
        request.setMarker(listObjects.getNextMarker());
        listObjects =  amazonS3.listObjects(request);
      } else {
        break;
      }
    } while (listObjects != null);

2、使用Delimiter模拟文件夹功能

假设桶内有如下对象：a/1.jpg、a/2.jpg、a/b/1.txt、a/b/2.txt，列举a文件夹下的文件及子文件夹的示例代码如下：

ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName("your-bucketname");
    listObjectsRequest.setDelimiter("/");
    listObjectsRequest.setPrefix("a/");
    ObjectListing listing = amazonS3.listObjects(listObjectsRequest);
    // 遍历所有Object
    System.out.println("Objects:");
    for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
        System.out.println(objectSummary.getKey());
    }
    // 遍历所有CommonPrefix
    System.out.println("CommonPrefixs:");
    for (String commonPrefix : listing.getCommonPrefixes()) {
        System.out.println(commonPrefix);
    }

示例代码的输出如下：

> Objects:
> 
> a/1.jpg
> 
> a/2.jpg
> 
> CommonPrefixs:
> 
> a/b/

生成私有对象可下载的URL链接

AWS Java SDK支持生成可下载私有对象的URL连接，您可以将该链接提供给第三方进行文件下载：

GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key);
    // 设置可下载URL的过期时间为1天后
    generatePresignedUrlRequest.setExpiration(new Date(System.currentTimeMillis()+3600*1000*24));
    URL url = amazonS3.generatePresignedUrl(generatePresignedUrlRequest);//生成URL
    System.out.println(url);//可以用这个url来下载文件

文件上传下载工具类 TransferManager

前文提到的是 Java SDK提供的基础接口，为方便用户进行文件上传下载，Java SDK提供了封装更好、使用更方便的工具类：TransferManager。

TransferManager的初始化

注：在 SpringBoot 项目中 TransferManager 的初始化 参考上述 \[配置模式\](\#\#\#\# 配置模式)

//先实例化一个AmazonS3
    String accessKey = "your-accesskey";
    String secretKey = "your-secretKey ";
    Credentials credentials = new BasicCredentials(accessKey, secretKey);
    AmazonS3 amazonS3 = new AmazonS3(credentials);
    amazonS3.setEndpoint(endPoint);
    //然后通过AmazonS3对象来初始化TransferManager
    TransferManager transferManager = new TransferManager(amazonS3);
    Download download = transferManager.download(TestConfig.bucketName,key,new File("localFilePath"));
    try {
        download.waitForCompletion();
    } catch (InterruptedException e) {
        e.printStackTrace();
    }

使用TransferManager进行文件上传

TransferManager会根据文件大小，选择是否进行分块上传。当文件小于等于16M时，TransferManager会自动调用PutObject接口，否则TransferManager会自动对文件进行分块上传。

1、上传本地文件：

如果指定上传的本地文件大于16M，TransferManager会自动对文件进行分块，并发调用分块上传接口进行上传，大大提高上传文件的速度。

//上传文件
    Upload upload = transferManager.upload("your-bucketname", "your-objectname", new File("your-file"));
    try {
        upload.waitForUploadResult();
    } catch (InterruptedException e) {
        e.printStackTrace();
    }

2、流式上传：

您也可以使用TransferManager进行流式上传，但是相比本地文件上传，流式上传无法做到多个分块并发上传，只能一个分块一个分块顺序上传。

//流式上传文件
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(file.length());
    Upload upload = transferManager.upload("your-bucketname", "your-objectname", inputStream, objectMetadata);
    UploadResult  result = upload.waitForUploadResult();

3、上传目录

您可以使用TransferManager将某个目录下的文件全部上传到NOS,对象名即文件名

*  3.1 不支持多级目录

MultipleFileUpload result = transferManager.uploadDirectory("your-buckename", null, new File("dirPath"), false);
    result.waitForCompletion();

*  3.2 支持多级目录，会递归的上传目录下的所有文件

MultipleFileUpload result = transferManager.uploadDirectory("your-buckename", null, new File("dirPath"), true);
    result.waitForCompletion();

4、下载文件

File file = new  File("your-destFile");
    Download download = transferManager.download("your-bucketname", "your-objectname", file);
    download.waitForCompletion();

## 参考资料 ##

\[1\] [*网易数帆S3 Java SDK 手册*][S3 Java SDK]

[https_docs.ceph.com_en_latest_radosgw_s3_java_download-an-object-to-a-file]: https://docs.ceph.com/en/latest/radosgw/s3/java/#download-an-object-to-a-file
[S3 Java SDK]: https://www.163yun.com/help/documents/105044625933389824