Java zip批量下载和踩坑笔记

电玩女神 2023-01-07 03:40 157阅读 0赞

话不多说，直接上代码  
需求是 阿里云 url pfd 文件打包成zip  
代码如下

import com.google.common.collect.Lists;
    import com.google.common.util.concurrent.ThreadFactoryBuilder;
    import org.apache.commons.lang3.StringUtils;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import org.springframework.util.StopWatch;
    
    import javax.annotation.Resource;
    import javax.servlet.http.HttpServletResponse;
    import java.io.BufferedInputStream;
    import java.io.ByteArrayOutputStream;
    import java.io.IOException;
    import java.io.InputStream;
    import java.io.OutputStream;
    import java.net.HttpURLConnection;
    import java.net.URL;
    import java.util.ArrayList;
    import java.util.Collections;
    import java.util.List;
    import java.util.Map;
    import java.util.concurrent.ArrayBlockingQueue;
    import java.util.concurrent.BlockingQueue;
    import java.util.concurrent.CompletableFuture;
    import java.util.concurrent.LinkedBlockingDeque;
    import java.util.concurrent.RejectedExecutionHandler;
    import java.util.concurrent.ThreadPoolExecutor;
    import java.util.concurrent.TimeUnit;
    import java.util.stream.Collectors;
    import java.util.zip.GZIPOutputStream;
    import java.util.zip.ZipEntry;
    import java.util.zip.ZipOutputStream;
    /**
     * 下载工具类.
     * @author  <a href="huangxy@smartebao.com">huang__2</a>
     * @version $Id: DownLoadTool.java  2021/1/19 1:44 下午  huangxy $
     * @since 2.0
     */
    public class DownLoadTool {
    
        /**
         * logger.
         */
        private static Logger log = LoggerFactory.getLogger(DownLoadTool.class);
    
        /**
         * 线程池.
         */
        RejectedExecutionHandler handler = new ThreadPoolExecutor.CallerRunsPolicy();
        private ThreadPoolExecutor executor = new ThreadPoolExecutor(4,
                20,
                30, TimeUnit.SECONDS,
                new LinkedBlockingDeque<>(4),
                new ThreadFactoryBuilder().setNameFormat("pdf-threadPool-%d").build(),
                handler);
    
        private static DownLoadTool instance = new DownLoadTool();
    
        private DownLoadTool() {
        }
    
        /**
         * @return the instance
         */
        public static DownLoadTool getInstance() {
            return instance;
        }
    
    
        /**
         * 下载节点.
         */
        class DownloadNode {
            String fileName;
            InputStream in;
    
            public DownloadNode(String fileName, InputStream in) {
                this.fileName = fileName;
                this.in = in;
            }
    
            /**
             * @return the fileName
             */
            public String getFileName() {
                return fileName;
            }
    
            /**
             * @return the in
             */
            public InputStream getIn() {
                return in;
            }
        }
     private void packageResponse(HttpServletRequest request, HttpServletResponse response, String zipName) throws UnsupportedEncodingException {
            response.setHeader("content-type", "application/octet-stream");
            response.setCharacterEncoding("utf-8");
            final String userAgent = request.getHeader("user-agent");
            boolean flag = null != userAgent && (userAgent.indexOf("Firefox") >= 0
                    || userAgent.indexOf("Chrome") >= 0 || userAgent.indexOf("Safari") >= 0);
            if (flag) {
                zipName = new String(zipName.getBytes(), "ISO8859-1");
            } else {
                zipName = URLEncoder.encode(zipName, "UTF8");
            }
            response.setHeader("Content-disposition",
                    "attachment;filename=" + new String(zipName.getBytes("gbk"), "iso8859-1"));
        }
    
    	// 多线程打包下载
        public void downloadZipByMultiThread(HttpServletResponse response, Map<String, String> urlName, String zipName) {
            StopWatch sw = new StopWatch("downloadZipByMultiThead");
            sw.start("task2");
            OutputStream out = null;
            ZipOutputStream zos = null;
            try {
                out = response.getOutputStream();
                response.reset();
                // 设定输出文件头
                response.setHeader("Content-Disposition", "attachment;fileName="
                        + new String(zipName.getBytes("utf-8"), "iso8859-1"));
                response.setContentType("application/zip");
                zos = new ZipOutputStream(out);
                List<DownloadNode> downloadNodes = Collections.synchronizedList(Lists.newArrayListWithCapacity(urlName.size()));
                CompletableFuture[] futures = urlName.entrySet()
                        .stream()
                        .map(entry -> CompletableFuture.runAsync(() -> downloadNodes.add(new DownloadNode(entry.getValue(), getIn(entry.getKey()))), executor))
                        .toArray(size -> new CompletableFuture[size]);
                CompletableFuture.allOf(futures).get(5, TimeUnit.SECONDS);
                for (DownloadNode node : downloadNodes) {
                    writeZipEntry(zos, node);
                }
                sw.stop();
                System.out.println(sw.prettyPrint());
            } catch (Exception e) {
                log.error("下载失败", e);
            } finally {
                if (zos != null) {
                    try {
                        zos.flush();
                        zos.close();
                    } catch (IOException e) {
                        log.error("zos 异常", e);
                    }
                }
    
                if (out != null) {
                    try {
                        out.flush();
                        out.close();
                    } catch (IOException e) {
                        log.error("out 异常", e);
                    }
                }
            }
    
    
        }
    
        private void writeZipEntry(ZipOutputStream zos, DownloadNode downloadNode) {
            try {
                byte[] buf = new byte[8192];
                int len;
                InputStream in = downloadNode.getIn();
                ZipEntry ze = new ZipEntry(downloadNode.getFileName() + ".pdf");
                zos.putNextEntry(ze);
                BufferedInputStream bis = new BufferedInputStream(in);
                while ((len = bis.read(buf)) != -1) {
                    zos.write(buf, 0, len);
                }
                bis.close();
                zos.closeEntry();
            } catch (IOException e) {
                log.error("io 异常", e);
            }
    
        }
    
    
        /**
         * 压缩文件 打包 pdf
         *
         * @param response response
         * @param urlName  key -> url  , value  -> fileName
         * @param zipName  打包名字
         */
        public void downloadZip(HttpServletResponse response, Map<String, String> urlName, String zipName) {
            OutputStream out = null;
            ZipOutputStream zos = null;
            byte[] buf = new byte[8192];
            int len = 0;
            try {
                out = response.getOutputStream();
                response.reset();
                // 设定输出文件头
                response.setHeader("Content-Disposition", "attachment;fileName="
                        + new String(zipName.getBytes("utf-8"), "iso8859-1"));
                response.setContentType("application/zip");
                zos = new ZipOutputStream(out);
    
                for (Map.Entry<String, String> entry : urlName.entrySet()) {
                    InputStream in = getIn(entry.getKey());
                    ZipEntry ze = new ZipEntry(entry.getValue() + ".pdf");
                    zos.putNextEntry(ze);
                    BufferedInputStream bis = new BufferedInputStream(in);
                    while ((len = bis.read(buf)) != -1) {
                        zos.write(buf, 0, len);
                    }
                    bis.close();
                    zos.closeEntry();
                }
            } catch (Exception e) {
                log.error("下载失败", e);
            } finally {
                // 流关闭的顺序很重要。
                if (zos != null) {
                    try {
                        zos.flush();
                        zos.close();
                    } catch (IOException e) {
                        log.error("zos 异常", e);
                    }
                }
                if (out != null) {
                    try {
                        out.flush();
                        out.close();
                    } catch (IOException e) {
                        log.error("out 异常", e);
                    }
                }
            }
        }
    
        /**
         * 通过 url 打开 一个io 流
         *
         * @param src 阿里云url 地址
         * @return InputStream
         */
        private static InputStream getIn(String src) {
            if (StringUtils.isEmpty(src)) {
                return null;
            }
            InputStream in = null;
            try {
                URL url = new URL(src);
                HttpURLConnection httpUrl = (HttpURLConnection) url.openConnection();
                httpUrl.connect();
                in = httpUrl.getInputStream();
                return in;
            } catch (IOException e) {
                log.error("读取url数据失败", e);
            }
            return null;
        }
    }

## 踩坑 ##

1.  流关闭的顺序很重要。不然你会发现 zip 包打不开。

if(zos != null) {
                    try {
                        zos.flush();
                        zos.close();
                    } catch (IOException e) {
                        //e.printStackTrace();
                    }
                }
                if(out != null) {
                    try {
                        out.flush();
                        out.close();
                    } catch (IOException e) {
                        //e.printStackTrace();
                    }
                }