go操作亚马逊S3教程(aws)

Myth丶恋晨 2024-03-16 19:00 79阅读 0赞

## go操作亚马逊S3教程 ##

> 官网API地址：https://docs.aws.amazon.com/sdk-for-go/api/service/s3/

### 前置：S3介绍 ###

### 一、aws-sdk-go ###

### 1 初始化S3配置 ###

> 初始化session和连接，同时打印错误日志

var (
    	sess *session.Session
    	svc  *s3.S3
    )
    
    func init() {
        
    	access_key := "xxxxxx"
    	secret_key := "xxxxxxxxxxx"
    	end_point := "http://xxx.xxx.xxx.xxx:xxx/" //根据自己endpint修改
    
    	sess, _ = session.NewSession(&aws.Config{
        
    		Credentials:      credentials.NewStaticCredentials(access_key, secret_key, ""),
    		Endpoint:         aws.String(end_point),
    		Region:           aws.String("us-east-1"),
    		DisableSSL:       aws.Bool(true),
    		S3ForcePathStyle: aws.Bool(false), //virtual-host style方式，不要修改
    	})
    
    	svc = s3.New(sess)
    }
    
    func main(){
        
    	....
    }
    func exitErrorf(msg string, args ...interface{
        }) {
        
    	fmt.Fprintf(os.Stderr, msg+"\n", args...)
    	os.Exit(1)
    }

### 2 具体操作 ###

#### 2.1 获取所有桶信息 ####

//获取所有桶
    func ListBuckets() {
        
    	result, err := svc.ListBuckets(nil)
    	if err != nil {
        
    		exitErrorf("Unable to list buckets, %v", err)
    	}
    
    	fmt.Println("Buckets:")
    	for _, b := range result.Buckets {
        
    		fmt.Printf("* %s created on %s\n",
    			aws.StringValue(b.Name), aws.TimeValue(b.CreationDate))
    	}
    
    	for _, b := range result.Buckets {
        
    		fmt.Printf("%s\n", aws.StringValue(b.Name))
    	}
    }

#### 2.2 创建桶 ####

func createBucket(bucket string) {
        
    	params := &s3.CreateBucketInput{
        
    		Bucket: aws.String(bucket),
    	}
    
    	_, err := svc.CreateBucket(params)
    
    	if err != nil {
        
    		exitErrorf("Unable to create bucket %q, %v", bucket, err)
    	}
    
    	// Wait until bucket is created before finishing
    	fmt.Printf("Waiting for bucket %q to be created...\n", bucket)
    
    	err = svc.WaitUntilBucketExists(&s3.HeadBucketInput{
        
    		Bucket: aws.String(bucket),
    	})
    
    	if err != nil {
        
    		exitErrorf("Error occurred while waiting for bucket to be created, %v", bucket)
    	}
    
    	fmt.Printf("Bucket %q successfully created\n", bucket)
    }

#### 2.3 删除桶 ####

func deleteBucket(bucket string) {
        
    	params := &s3.DeleteBucketInput{
        
    		Bucket: aws.String(bucket),
    	}
    
    	_, err := svc.DeleteBucket(params)
    
    	if err != nil {
        
    		exitErrorf("Unable to delete bucket %q, %v", bucket, err)
    	}
    
    	//wait until bucket is deleted
    	err = svc.WaitUntilBucketNotExists(&s3.HeadBucketInput{
        
    		Bucket: aws.String(bucket),
    	})
    
    	if err != nil {
        
    		exitErrorf("Error occurred while waiting for bucket to be deleted, %v", bucket)
    	}
    
    	fmt.Printf("Bucket %q successfully delete\n", bucket)
    }

#### 2.4 下载桶里的文件 ####

func downloadFile(bucket, item string) {
        
    	file, err := os.Create(item)
    	if err != nil {
        
    		exitErrorf("Unable to open file %q, %v", err)
    	}
    
    	defer file.Close()
    
    	downloader := s3manager.NewDownloader(sess)
    
    	numBytes, err := downloader.Download(file,
    		&s3.GetObjectInput{
        
    			Bucket: aws.String(bucket),
    			Key:    aws.String(item),
    		})
    	if err != nil {
        
    		exitErrorf("Unable to download item %q, %v", item, err)
    	}
    
    	fmt.Println("Downloaded", file.Name(), numBytes, "bytes")
    }

#### 2.5 删除桶里的文件 ####

func deleteFile(bucket, obj string) {
        
    	_, err := svc.DeleteObject(&s3.DeleteObjectInput{
        Bucket: aws.String(bucket), Key: aws.String(obj)})
    	if err != nil {
        
    		exitErrorf("Unable to delete object %q from bucket %q, %v", obj, bucket, err)
    	}
    
    	err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
        
    		Bucket: aws.String(bucket),
    		Key:    aws.String(obj),
    	})
    
    	fmt.Printf("Object %q successfully deleted\n", obj)
    }

#### 2.6 上传文件到指定桶 ####

func uploadFile(bucket, filename string) {
        
    	file, err := os.Open(filename)
    	if err != nil {
        
    		exitErrorf("Unable to open file %q, %v", err)
    	}
    
    	defer file.Close()
    
    	uploader := s3manager.NewUploader(sess)
    
    	_, err = uploader.Upload(&s3manager.UploadInput{
        
    		Bucket: aws.String(bucket),
    		Key:    aws.String(filename),
    		Body:   file,
    	})
    	if err != nil {
        
    		// Print the error and exit.
    		exitErrorf("Unable to upload %q to %q, %v", filename, bucket, err)
    	}
    
    	fmt.Printf("Successfully uploaded %q to %q\n", filename, bucket)
    }

### 3 全部代码 ###

package main
    
    import (
    	"fmt"
    	"github.com/aws/aws-sdk-go/aws"
    	"github.com/aws/aws-sdk-go/aws/credentials"
    	"github.com/aws/aws-sdk-go/aws/session"
    	"github.com/aws/aws-sdk-go/service/s3"
    	"github.com/aws/aws-sdk-go/service/s3/s3manager"
    	_ "github.com/aws/aws-sdk-go/service/s3/s3manager"
    	"os"
    )
    
    var (
    	sess *session.Session
    	svc  *s3.S3
    )
    
    func init() {
        
    	access_key := "xxxxxx"
    	secret_key := "xxxxxxxxxxx"
    	end_point := "http://xxx.xxx.xxx.xxx:xxx/" //根据自己endpint修改
    
    	sess, _ = session.NewSession(&aws.Config{
        
    		Credentials:      credentials.NewStaticCredentials(access_key, secret_key, ""),
    		Endpoint:         aws.String(end_point),
    		Region:           aws.String("us-east-1"),
    		DisableSSL:       aws.Bool(true),
    		S3ForcePathStyle: aws.Bool(false), //virtual-host style方式，不要修改
    	})
    
    	svc = s3.New(sess)
    }
    
    func main() {
        
    	//注意！！！桶名称后面一定要根`/`
    	//downloadFile("bucket-ziyi/", "banner.png")
    	//uploadFile("bucket-ziyi/", "test/test1/test2/test.csv")
    	//createBuckt("bucket-demo1/")
    	//deleteBucket("bucket-demo1/")
    
    	deleteFile("bucket-ziyi/test/", "log.txt")
    
    }
    
    func deleteFile(bucket, obj string) {
        
    	_, err := svc.DeleteObject(&s3.DeleteObjectInput{
        Bucket: aws.String(bucket), Key: aws.String(obj)})
    	if err != nil {
        
    		exitErrorf("Unable to delete object %q from bucket %q, %v", obj, bucket, err)
    	}
    
    	err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
        
    		Bucket: aws.String(bucket),
    		Key:    aws.String(obj),
    	})
    
    	fmt.Printf("Object %q successfully deleted\n", obj)
    }
    
    func deleteBucket(bucket string) {
        
    	params := &s3.DeleteBucketInput{
        
    		Bucket: aws.String(bucket),
    	}
    
    	_, err := svc.DeleteBucket(params)
    
    	if err != nil {
        
    		exitErrorf("Unable to delete bucket %q, %v", bucket, err)
    	}
    
    	//wait until bucket is deleted
    	err = svc.WaitUntilBucketNotExists(&s3.HeadBucketInput{
        
    		Bucket: aws.String(bucket),
    	})
    
    	if err != nil {
        
    		exitErrorf("Error occurred while waiting for bucket to be deleted, %v", bucket)
    	}
    
    	fmt.Printf("Bucket %q successfully delete\n", bucket)
    }
    
    func createBucket(bucket string) {
        
    	params := &s3.CreateBucketInput{
        
    		Bucket: aws.String(bucket),
    	}
    
    	_, err := svc.CreateBucket(params)
    
    	if err != nil {
        
    		exitErrorf("Unable to create bucket %q, %v", bucket, err)
    	}
    
    	// Wait until bucket is created before finishing
    	fmt.Printf("Waiting for bucket %q to be created...\n", bucket)
    
    	err = svc.WaitUntilBucketExists(&s3.HeadBucketInput{
        
    		Bucket: aws.String(bucket),
    	})
    
    	if err != nil {
        
    		exitErrorf("Error occurred while waiting for bucket to be created, %v", bucket)
    	}
    
    	fmt.Printf("Bucket %q successfully created\n", bucket)
    }
    
    func uploadFile(bucket, filename string) {
        
    	file, err := os.Open(filename)
    	if err != nil {
        
    		exitErrorf("Unable to open file %q, %v", err)
    	}
    
    	defer file.Close()
    
    	uploader := s3manager.NewUploader(sess)
    
    	_, err = uploader.Upload(&s3manager.UploadInput{
        
    		Bucket: aws.String(bucket),
    		Key:    aws.String(filename),
    		Body:   file,
    	})
    	if err != nil {
        
    		// Print the error and exit.
    		exitErrorf("Unable to upload %q to %q, %v", filename, bucket, err)
    	}
    
    	fmt.Printf("Successfully uploaded %q to %q\n", filename, bucket)
    }
    
    func downloadFile(bucket, item string) {
        
    	file, err := os.Create(item)
    	if err != nil {
        
    		exitErrorf("Unable to open file %q, %v", err)
    	}
    
    	defer file.Close()
    
    	downloader := s3manager.NewDownloader(sess)
    
    	numBytes, err := downloader.Download(file,
    		&s3.GetObjectInput{
        
    			Bucket: aws.String(bucket),
    			Key:    aws.String(item),
    		})
    	if err != nil {
        
    		exitErrorf("Unable to download item %q, %v", item, err)
    	}
    
    	fmt.Println("Downloaded", file.Name(), numBytes, "bytes")
    }
    
    //获取所有桶
    func ListBuckets() {
        
    	result, err := svc.ListBuckets(nil)
    	if err != nil {
        
    		exitErrorf("Unable to list buckets, %v", err)
    	}
    
    	fmt.Println("Buckets:")
    	for _, b := range result.Buckets {
        
    		fmt.Printf("* %s created on %s\n",
    			aws.StringValue(b.Name), aws.TimeValue(b.CreationDate))
    	}
    
    	for _, b := range result.Buckets {
        
    		fmt.Printf("%s\n", aws.StringValue(b.Name))
    	}
    }
    
    func exitErrorf(msg string, args ...interface{
        }) {
        
    	fmt.Fprintf(os.Stderr, msg+"\n", args...)
    	os.Exit(1)
    }

### 二、aws-sdk-go-v2 ###

package main
    
    import (
        "archive/zip"
        "bufio"
        "context"
        "fmt"
        "github.com/aobco/log"
        "github.com/aws/aws-sdk-go-v2/aws"
        "github.com/aws/aws-sdk-go-v2/credentials"
        "github.com/aws/aws-sdk-go-v2/service/s3"
        "io"
        "os"
        "path/filepath"
        "strings"
    )
    
    var (
        ak       = "xxxxxxxxxxxxxxx"
        sk       = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
        endpoint = "http://xxx.xxx.xxx:xx/" //endpoint设置，不要动
        client   *s3.Client
    )
    
    func init() {
        
        // 配置AWS SDK
        var credentialsProvider aws.CredentialsProvider
        credentialsProvider = credentials.StaticCredentialsProvider{
        Value: aws.Credentials{
        AccessKeyID: ak, SecretAccessKey: sk, SessionToken: ""}}
        cfg := aws.Config{
        
           Credentials: credentialsProvider,
           EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(
              func(service string, region string, options ...interface{
        }) (aws.Endpoint, error) {
        
                 return aws.Endpoint{
        
                    URL:               endpoint,
                    HostnameImmutable: true,
                 }, nil
              }),
        }
    
        // 创建S3客户端
        client = s3.NewFromConfig(cfg)
    }
    func main() {
        
        //step1 : download target file
        key1 := "log1.txt"
        key2 := "log/log2.txt"
        bucket1 := "bucket-ziyi"
        bucket2 := "bucket-ziyi"
        tmpDir := "test/test1/test2/tmp"
        DownloadFile(bucket1, key1, fmt.Sprintf("%v/%v", tmpDir, key1))
        DownloadFile(bucket2, key2, fmt.Sprintf("%v/%v", tmpDir, key2))
    
        //step2 : compression
        zipFileName, err := compressionDir(tmpDir)
        if err != nil {
        
           log.Fatal("compression file err=", err)
           return
        }
        //fmt.Println(zipFileName)
    
        //step3 : upload zip
        key3 := "demo.zip"
        bucket3 := "bucket-ziyi"
        UploadFile(bucket3, key3, zipFileName)
    }
    
    // UploadFile 上传文件到S3
    func UploadFile(bucket string, key string, filePath string) {
        
        // 打开本地文件
        file, err := os.Open(filePath)
        if err != nil {
        
           log.Info("upload file err=", err)
        }
        defer file.Close()
        // 上传文件到S3 bucket
        _, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
        
           Bucket: &bucket,
           Key:    &key,
           Body:   file,
        })
        if err != nil {
        
           log.Error("upload file err=", err)
           return
        }
        log.Infof("upload file success, bucket=%v, name=%v", bucket, key)
    }
    
    // DownloadFile 下载S3文件到本地
    func DownloadFile(bucket string, key string, filePath string) {
        
        // 下载文件内容
        resp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
        
           Bucket: &bucket,
           Key:    &key,
        })
        if err != nil {
        
           log.Error("download file err=", err)
           return
        }
        defer resp.Body.Close()
    
        index := strings.LastIndex(filePath, "/")
        dir := filePath[:index]
        err = os.MkdirAll(dir, os.ModePerm)
        if err != nil {
        
           log.Error("download file err=", err)
        }
        // 写入文件
        file, err := os.Create(filePath)
        if err != nil {
        
           log.Error("download file err=", err)
           return
        }
        defer file.Close()
        buffer := make([]byte, 1024*2)
        for {
        
           reader := bufio.NewReader(resp.Body)
           n, err := reader.Read(buffer)
           if err == io.EOF {
        
              break
           }
           if err != nil {
        
              log.Fatal("download file err=", err)
              return
           }
           file.Write(buffer[:n])
        }
        log.Info("download file success, filePath=", filePath)
    }
    
    func compressionDir(baseDir string) (string, error) {
        
        zipFileName := baseDir + ".zip"
    
        // 创建一个新的 zip 文件
        zipFile, err := os.Create(zipFileName)
        if err != nil {
        
           return "", err
        }
        defer zipFile.Close()
    
        // 创建一个 zip.Writer
        zipWriter := zip.NewWriter(zipFile)
        defer zipWriter.Close()
    
        // 遍历目录下的所有文件和子目录
        err = filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
        
           if err != nil {
        
              return err
           }
    
           // 创建一个 zip 文件中的文件或目录
           relativePath := strings.TrimPrefix(path, baseDir)
           zipPath := strings.TrimLeft(filepath.Join("/", relativePath), "/")
    
           // 如果是目录或空目录，则在 zip 文件中创建一个目录
           if info.IsDir() || isEmptyDir(path) {
        
              _, err := zipWriter.Create(zipPath + "/")
              if err != nil {
        
                 return err
              }
           } else {
        
              // 如果是文件，则创建一个 zip 文件中的文件
              zipFile, err := zipWriter.Create(zipPath)
              if err != nil {
        
                 return err
              }
    
              // 打开原始文件
              file, err := os.Open(path)
              if err != nil {
        
                 return err
              }
              defer file.Close()
    
              // 将原始文件的内容拷贝到 zip 文件中
              _, err = io.Copy(zipFile, file)
              if err != nil {
        
                 return err
              }
           }
    
           return nil
        })
    
        if err != nil {
        
           return "", err
        }
        return zipFileName, nil
    }
    
    // 判断目录是否为空目录
    func isEmptyDir(dirPath string) bool {
        
        dir, err := os.Open(dirPath)
        if err != nil {
        
           return false
        }
        defer dir.Close()
    
        _, err = dir.Readdirnames(1)
        return err == io.EOF
    }

##### ① 遍历某个桶目录下的所有文件并下载 #####

> 1.  检查ak、sk、endpoint等是否正确

ctx := context.TODO()
    	// 发送 ListBuckets 请求
    	_, err := client.ListBuckets(ctx, &s3.ListBucketsInput{
        })
    	if err != nil {
        
    		fmt.Println("Connection failed:", err)
    		return
    	}
    	fmt.Println("Connection succeeded")

> 1.  可以先检查桶是否存在

// 指定要验证的存储桶名称
    	bucketName := "bucket-ziyi"
    
    	// 创建输入参数
    	input := &s3.HeadBucketInput{
        
    		Bucket: &bucketName,
    	}
    
    	// 发送 HeadBucket 请求
    	_, err := client.HeadBucket(context.TODO(), input)
    	if err != nil {
        
    		// 如果返回错误，说明存储桶不存在
    		fmt.Println("Bucket does not exist")
    		return
    	}
    
    	// 存储桶存在
    	fmt.Println("Bucket exists")

业务操作：遍历桶并下载：

var(
    	ak = ""
    	sk = ""
    	endpoint = ""
    	client   *s3.Client
    )
    func init() {
        
    	// 配置AWS SDK
    	var credentialsProvider aws.CredentialsProvider
    	credentialsProvider = credentials.StaticCredentialsProvider{
        Value: aws.Credentials{
        AccessKeyID: ak, SecretAccessKey: sk, SessionToken: ""}}
    	cfg := aws.Config{
        
    		Credentials: credentialsProvider,
    		EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(
    			func(service string, region string, options ...interface{
        }) (aws.Endpoint, error) {
        
    				return aws.Endpoint{
        
    					URL:               endpoint,
    					HostnameImmutable: true,
    				}, nil
    			}),
    	}
    
    	// 创建S3客户端
    	client = s3.NewFromConfig(cfg)
    }
    
    func main() {
        
    	// AWS 配置
    	bucket := "bucket-ziyi"
    	prefix := "log/"
    
    	// 获取对象列表
    	resp, err := client.ListObjects(context.TODO(), &s3.ListObjectsInput{
        
    		Bucket: &bucket,
    		Prefix: &prefix,
    	})
    	if err != nil {
        
    		fmt.Println("Failed to list objects:", err)
    		return
    	}
    	// 创建下载根目录
    	downloadRootDir := "./download"
    	if err := os.MkdirAll(downloadRootDir, 0700); err != nil {
        
    		fmt.Println("Failed to create download directory:", err)
    		return
    	}
    	keys := make([]string, 0)
    	// 遍历对象列表并下载
    	for i, obj := range resp.Contents {
        
    		if strings.HasSuffix(*obj.Key, "/") {
        
    			continue
    		}
    		keys = append(keys, *obj.Key)
    
    		// 构建下载路径
    		//downloadPath := filepath.Join(downloadRootDir, *obj.Key)
    		downloadPath := filepath.Join(downloadRootDir, *obj.Key)
    		//fmt.Println("downloadPath=", *obj.Key, i)
    		// 如果对象是目录，则创建对应的目录
    		if strings.HasSuffix(*obj.Key, "/") {
        
    			if err := os.MkdirAll(downloadPath, 0700); err != nil {
        
    				fmt.Printf("Failed to create directory '%s': %v\n", downloadPath, err)
    				continue
    			}
    			fmt.Printf("Created directory '%s'\n", downloadPath)
    			continue
    		}
    
    		// 下载对象
    		err := downloadObject(context.TODO(), client, bucket, *obj.Key, downloadPath)
    		if err != nil {
        
    			fmt.Printf("Failed to download object '%s': %v\n", *obj.Key, err)
    		} else {
        
    			fmt.Printf("Downloaded object '%s' to '%s'\n", *obj.Key, downloadPath)
    		}
    	}
    	fmt.Println(keys)
    }
    
    // 下载对象
    func downloadObject(ctx context.Context, client *s3.Client, bucket, key, filePath string) error {
        
    	// 创建文件
    	file, err := os.Create(filePath)
    	if err != nil {
        
    		return err
    	}
    	defer file.Close()
    
    	// 下载对象
    	resp, err := client.GetObject(ctx, &s3.GetObjectInput{
        
    		Bucket: &bucket,
    		Key:    &key,
    	})
    	if err != nil {
        
    		return err
    	}
    	defer resp.Body.Close()
    
    	// 写入文件
    	_, err = io.Copy(file, resp.Body)
    	if err != nil {
        
    		return err
    	}
    
    	return nil
    }

### 三、aws相关bug ###

##### ①Versions bucket is not support origin. #####

> 对应bucket没有开版本控制

### 四、业务相关 ###

#### 4.1 将下载下来的文件进行压缩 ####

> 将文件下载到临时目录，然后通过zip命令或者golang自带的压缩api进行解压处理
> 
>  *  zip -r -compressionLevel xxx.zip \*
>  *  其中compressionLevel范围是0-9，0表示不压缩；-r表示如果有目录，则也对应压缩目录内的内容

//os.TempDir()：获取临时目录
    downloadDir := filepath.Join(os.TempDir(), fmt.Sprintf("%d-%d", id1, id2))

`注意：`golang的exec.Command()无法支持通配符`*`，因此需要通过filepath.Glob(“\*”)的方式来列出所有的文件

zipName := filepath.Base(dest)
    command := []string{
        
    	"-r",
    	compressionLevel,
    	zipName,
    }
    tmp, err := filepath.Glob("*")
    if len(tmp) == 0 {
        
    	log.Infof("No matching files found")
    }
    log.Infof("* tmp=%v", tmp)
    command = append(command, tmp...)
    if err != nil {
        
    	log.Error(err)
    }
    log.Infof("args=%v", command)
    
    cmd := exec.Command("/usr/bin/zip", command...)
    log.Infof("compression cmd=%v", cmd)
    err = cmd.Run()

#### 4.2 开启多个协程压缩文件，超出大小时暂停 ####

package main
    
    import (
    	"context"
    	"fmt"
    	"github.com/aobco/log"
    	"github.com/aws/aws-sdk-go-v2/aws"
    	"github.com/aws/aws-sdk-go-v2/credentials"
    	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
    	"github.com/aws/aws-sdk-go-v2/service/s3"
    	"os"
    	"strings"
    	"sync"
    	"sync/atomic"
    )
    
    var (
    	access_key = "xxxxxx"
    	secret_key = "xxxx"
    	end_point  = "xxxxx/" //endpoint设置，不要动
    	client     *s3.Client
    	maxWorkers = 3
    	//keys       = []string{"视频/黑鹰坠落.mp4", "视频/多文件打包接口文档.md", "视频/啊1Aa~!@#$%^&()_+{}.txt"}
    	keys                = []string{
        "视频/黑鹰坠落.mp4", "视频/多文件打包接口文档.md", "goroutine/docker-wsl-cli.iso"}
    	fileSizeLimit int64 = 23 * 1024
    	totalFileSize int64 = 0
    )
    
    func init() {
        
    	// 配置AWS SDK
    	var credentialsProvider aws.CredentialsProvider
    	credentialsProvider = credentials.StaticCredentialsProvider{
        Value: aws.Credentials{
        AccessKeyID: access_key, SecretAccessKey: secret_key, SessionToken: ""}}
    	cfg := aws.Config{
        
    		Credentials: credentialsProvider,
    		EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(
    			func(service string, region string, options ...interface{
        }) (aws.Endpoint, error) {
        
    				return aws.Endpoint{
        
    					URL:               end_point,
    					HostnameImmutable: true,
    				}, nil
    			}),
    	}
    
    	// 创建S3客户端
    	client = s3.NewFromConfig(cfg)
    }
    func main() {
        
    	wg := new(sync.WaitGroup)
    	bucket := "bucket-ziyi"
    	path := ""
    	ctx, cancel := context.WithCancel(context.Background())
    	for i, key := range keys {
        
    
    		fileName := strings.Split(key, "/")[1]
    		path = fmt.Sprintf("%s\\%d-%s", "E:\\Go\\GoPro\\src\\go_code\\gouitest\\context\\downloader", i, fileName)
    		wg.Add(1)
    		go func(key, path string, cancel context.CancelFunc) {
        
    			_, _, err := downLoadFile(ctx, client, bucket, key, path, wg, cancel)
    			if err != nil {
        
    				fmt.Println("err=", err)
    			}
    		}(key, path, cancel)
    	}
    	//key = "视频/多文件打包接口文档.md"
    
    	//timer := time.NewTicker(time.Millisecond * 2000)
    	//defer timer.Stop()
    	//go func(timer *time.Ticker, cancel context.CancelFunc) {
        
    	//	<-timer.C
    	//	cancel()
    	//}(timer, cancel)
    	wg.Wait()
    	fmt.Println("all the task is end...........")
    }
    
    func downLoadFile(ctx context.Context, client *s3.Client, bucket, key, path string, wg *sync.WaitGroup, cancel context.CancelFunc) (*os.File, int64, error) {
        
    	downloader := manager.NewDownloader(client)
    	input := &s3.GetObjectInput{
        
    		Bucket: aws.String(bucket),
    		Key:    aws.String(key),
    	}
    	downloadFile, err := os.Create(path)
    	defer func() {
        
    		wg.Done()
    		if err := downloadFile.Close(); err != nil {
        
    			log.Errorf("error when close file %v %v", downloadFile, err)
    			return
    		}
    	}()
    	downloadSize, err := downloader.Download(ctx, downloadFile, input, func(downloader *manager.Downloader) {
        
    		downloader.PartSize = 10 * 1024 * 1024
    		downloader.Concurrency = 5
    	})
    	atomic.AddInt64(&totalFileSize, downloadSize)
    	if totalFileSize > fileSizeLimit {
        
    		cancel()
    	}
    	if err != nil {
        
    		fmt.Errorf("download file err %v", err)
    	}
    	fmt.Println("downloadSize=", downloadSize)
    	return downloadFile, downloadSize, err
    }