用 Java 爬小姐姐图片，这个厉害了。。。

分手后的思念是犯贱 2022-09-11 03:26 178阅读 0赞

来源：blog.csdn.net/qq_35402412/article/details/113627625

### 目的 ###

爬取搜狗图片上千张美女图片并下载到本地

### 准备工作 ###

爬取地址：https://pic.sogou.com/pics?query=%E7%BE%8E%E5%A5%B3

![bf3b3a2836580ef35bd8eddc051b5dca.png][]

### 分析 ###

打开上面的地址，按F12开发者工具 - NetWork - XHR - 页面往下滑动XHR栏出现请求信息如下：

Request URL ：https://pic.sogou.com/napi/pc/searchList?mode=1&start=48&xml\_len=48&query=%E7%BE%8E%E5%A5%B3

分析这段请求URL的主要几个参数：

start=48 表示从第48张图片开始检索

xml\_len=48 从地48张往后获取48张图片

query=？搜索关键词（例：美女，这里浏览器自动做了转码，不影响我们使用）

![0ba4caa620543c6be8cf936943a191ea.png][]

点击Respose，找个JSON格式器辅助过去看看。

![3401e2512f48068fae727ecbc6e4d44d.png][]

JSON格式：https://www.bejson.com/

分析Respose返回的信息，可以发现我们想要的图片地址放在 picUrl里，

![dd13ac6c47d24ddd5f387cd7be7967fc.png][]

### 思路    ###

通过以上分析，不难实现下载方法，思路如下：

1.  设置URL请求参数
2.  访问URL请求，获取图片地址
3.  图片地址存入List
4.  遍历List，使用线程池下载到本地

### 代码 ###

SougouImgProcessor.java 爬取图片类

import com.alibaba.fastjson.JSONObject;
    import us.codecraft.webmagic.utils.HttpClientUtils;
    import victor.chang.crawler.pipeline.SougouImgPipeline;
    
    import java.util.ArrayList;
    import java.util.List;
    
    /**
     * A simple PageProcessor.
     * @author code4crafter@gmail.com <br>
     * @since 0.1.0
     */
    public class SougouImgProcessor {
    
        private String url;
        private SougouImgPipeline pipeline;
        private List<JSONObject> dataList;
        private List<String> urlList;
        private String word;
    
        public SougouImgProcessor(String url,String word) {
            this.url = url;
            this.word = word;
            this.pipeline = new SougouImgPipeline();
            this.dataList = new ArrayList<>();
            this.urlList = new ArrayList<>();
        }
    
        public void process(int idx, int size) {
            String res = HttpClientUtils.get(String.format(this.url, idx, size, this.word));
            JSONObject object = JSONObject.parseObject(res);
            List<JSONObject> items = (List<JSONObject>)((JSONObject)object.get("data")).get("items");
            for(JSONObject item : items){
                this.urlList.add(item.getString("picUrl"));
            }
            this.dataList.addAll(items);
        }
    
        // 下载
        public void pipelineData(){
            // 多线程
            pipeline.processSync(this.urlList, this.word);
        }
    
    
        public static void main(String[] args) {
            String url = "https://pic.sogou.com/napi/pc/searchList?mode=1&start=%s&xml_len=%s&query=%s";
            SougouImgProcessor processor = new SougouImgProcessor(url,"美女");
    
            int start = 0, size = 50, limit = 1000; // 定义爬取开始索引、每次爬取数量、总共爬取数量
    
            for(int i=start;i<start+limit;i+=size)
                processor.process(i, size);
    
            processor.pipelineData();
    
        }
    
    }

SougouImgPipeline.java 图片下载类

import java.io.File;
    import java.io.FileOutputStream;
    import java.io.InputStream;
    import java.net.URL;
    import java.net.URLConnection;
    import java.util.List;
    import java.util.Objects;
    import java.util.concurrent.ExecutorService;
    import java.util.concurrent.Executors;
    import java.util.concurrent.TimeUnit;
    import java.util.concurrent.atomic.AtomicInteger;
    
    /**
     * Store results in files.<br>
     * @author code4crafter@gmail.com <br>
     * @since 0.1.0
     */
    public class SougouImgPipeline {
    
        private String extension = ".jpg";
        private String path;
    
        private volatile AtomicInteger suc;
        private volatile AtomicInteger fails;
    
        public SougouImgPipeline() {
            setPath("E:/pipeline/sougou");
            suc = new AtomicInteger();
            fails = new AtomicInteger();
        }
    
        public SougouImgPipeline(String path) {
            setPath(path);
            suc = new AtomicInteger();
            fails = new AtomicInteger();
        }
    
        public SougouImgPipeline(String path, String extension) {
            setPath(path);
            this.extension = extension;
            suc = new AtomicInteger();
            fails = new AtomicInteger();
        }
    
        public void setPath(String path) {
            this.path = path;
        }
    
        /**
         * 下载
         * @param url
         * @param cate
         * @throws Exception
         */
        private void downloadImg(String url, String cate, String name) throws Exception {
            String path = this.path + "/" + cate + "/";
            File dir = new File(path);
            if (!dir.exists()) {    // 目录不存在则创建目录
                dir.mkdirs();
            }
            String realExt = url.substring(url.lastIndexOf("."));   // 获取扩展名
            String fileName = name + realExt;
            fileName = fileName.replace("-", "");
            String filePath = path + fileName;
            File img = new File(filePath);
            if(img.exists()){   // 若文件之前已经下载过，则跳过
                System.out.println(String.format("文件%s已存在本地目录",fileName));
                return;
            }
    
            URLConnection con = new URL(url).openConnection();
            con.setConnectTimeout(5000);
            con.setReadTimeout(5000);
            InputStream inputStream = con.getInputStream();
            byte[] bs = new byte[1024];
    
            File file = new File(filePath);
            FileOutputStream os = new FileOutputStream(file, true);
            // 开始读取 写入
            int len;
            while ((len = inputStream.read(bs)) != -1) {
                os.write(bs, 0, len);
            }
            System.out.println("picUrl: " + url);
            System.out.println(String.format("正在下载第%s张图片", suc.getAndIncrement()));
        }
    
        /**
         * 单线程处理
         *
         * @param data
         * @param word
         */
        public void process(List<String> data, String word) {
            long start = System.currentTimeMillis();
            for (String picUrl : data) {
                if (picUrl == null)
                    continue;
                try {
                    downloadImg(picUrl, word, picUrl);
                } catch (Exception e) {
                    fails.incrementAndGet();
                }
            }
            System.out.println("下载成功: " + suc.get());
            System.out.println("下载失败: " + fails.get());
            long end = System.currentTimeMillis();
            System.out.println("耗时：" + (end - start) / 1000 + "秒");
        }
    
    
        /**
         * 多线程处理
         *
         * @param data
         * @param word
         */
        public void processSync(List<String> data, String word) {
            long start = System.currentTimeMillis();
            int count = 0;
            ExecutorService executorService = Executors.newCachedThreadPool(); // 创建缓存线程池
            for (int i=0;i<data.size();i++) {
                String picUrl = data.get(i);
                if (picUrl == null)
                    continue;
                String name = "";
                if(i<10){
                    name="000"+i;
                }else if(i<100){
                    name="00"+i;
                }else if(i<1000){
                    name="0"+i;
                }
                String finalName = name;
                executorService.execute(() -> {
                    try {
                        downloadImg(picUrl, word, finalName);
                    } catch (Exception e) {
                        fails.incrementAndGet();
                    }
                });
                count++;
            }
            executorService.shutdown();
            try {
                if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
                    // 超时的时候向线程池中所有的线程发出中断(interrupted)。
                    // executorService.shutdownNow();
                }
                System.out.println("AwaitTermination Finished");
                System.out.println("共有URL: "+data.size());
                System.out.println("下载成功: " + suc);
                System.out.println("下载失败: " + fails);
    
                File dir = new File(this.path + "/" + word + "/");
                int len = Objects.requireNonNull(dir.list()).length;
                System.out.println("当前共有文件： "+len);
    
                long end = System.currentTimeMillis();
                System.out.println("耗时：" + (end - start) / 1000.0 + "秒");
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
    
        }
    
    
        /**
         * 多线程分段处理
         *
         * @param data
         * @param word
         * @param threadNum
         */
        public void processSync2(List<String> data, final String word, int threadNum) {
            if (data.size() < threadNum) {
                process(data, word);
            } else {
                ExecutorService executorService = Executors.newCachedThreadPool();
                int num = data.size() / threadNum;    //每段要处理的数量
                for (int i = 0; i < threadNum; i++) {
                    int start = i * num;
                    int end = (i + 1) * num;
                    if (i == threadNum - 1) {
                        end = data.size();
                    }
                    final List<String> cutList = data.subList(start, end);
                    executorService.execute(() -> process(cutList, word));
                }
                executorService.shutdown();
            }
        }
    
    }

HttpClientUtils.java http请求工具类

import org.apache.http.Header;
    import org.apache.http.HttpEntity;
    import org.apache.http.NameValuePair;
    import org.apache.http.client.entity.UrlEncodedFormEntity;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.client.methods.HttpPost;
    import org.apache.http.client.methods.HttpUriRequest;
    import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
    import org.apache.http.conn.ssl.TrustStrategy;
    import org.apache.http.entity.StringEntity;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.message.BasicNameValuePair;
    import org.apache.http.ssl.SSLContextBuilder;
    import org.apache.http.util.EntityUtils;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    
    import javax.net.ssl.HostnameVerifier;
    import javax.net.ssl.SSLContext;
    import javax.net.ssl.SSLSession;
    import java.io.IOException;
    import java.io.UnsupportedEncodingException;
    import java.security.GeneralSecurityException;
    import java.security.cert.CertificateException;
    import java.security.cert.X509Certificate;
    import java.util.ArrayList;
    import java.util.HashMap;
    import java.util.List;
    import java.util.Map;
    
    /**
     * @author code4crafter@gmail.com
     * Date: 17/3/27
     */
    public abstract class HttpClientUtils {
    
        public static Map<String, List<String>> convertHeaders(Header[] headers) {
            Map<String, List<String>> results = new HashMap<String, List<String>>();
            for (Header header : headers) {
                List<String> list = results.get(header.getName());
                if (list == null) {
                    list = new ArrayList<String>();
                    results.put(header.getName(), list);
                }
                list.add(header.getValue());
            }
            return results;
        }
    
        /**
         * http的get请求
         * @param url
         */
        public static String get(String url) {
            return get(url, "UTF-8");
        }
    
        public static Logger logger = LoggerFactory.getLogger(HttpClientUtils.class);
    
        /**
         * http的get请求
         * @param url
         */
        public static String get(String url, String charset) {
            HttpGet httpGet = new HttpGet(url);
            return executeRequest(httpGet, charset);
        }
    
        /**
         * http的get请求，增加异步请求头参数
         * @param url
         */
        public static String ajaxGet(String url) {
            return ajaxGet(url, "UTF-8");
        }
    
        /**
         * http的get请求，增加异步请求头参数
         *
         * @param url
         */
        public static String ajaxGet(String url, String charset) {
            HttpGet httpGet = new HttpGet(url);
            httpGet.setHeader("X-Requested-With", "XMLHttpRequest");
            return executeRequest(httpGet, charset);
        }
    
        /**
         * @param url
         * @return
         */
        public static String ajaxGet(CloseableHttpClient httpclient, String url) {
            HttpGet httpGet = new HttpGet(url);
            httpGet.setHeader("X-Requested-With", "XMLHttpRequest");
            return executeRequest(httpclient, httpGet, "UTF-8");
        }
    
        /**
         * http的post请求，传递map格式参数
         */
        public static String post(String url, Map<String, String> dataMap) {
            return post(url, dataMap, "UTF-8");
        }
    
        /**
         * http的post请求，传递map格式参数
         */
        public static String post(String url, Map<String, String> dataMap, String charset) {
            HttpPost httpPost = new HttpPost(url);
            try {
                if (dataMap != null) {
                    List<NameValuePair> nvps = new ArrayList<NameValuePair>();
                    for (Map.Entry<String, String> entry : dataMap.entrySet()) {
                        nvps.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
                    }
                    UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(nvps, charset);
                    formEntity.setContentEncoding(charset);
                    httpPost.setEntity(formEntity);
                }
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            return executeRequest(httpPost, charset);
        }
    
        /**
         * http的post请求，增加异步请求头参数，传递map格式参数
         */
        public static String ajaxPost(String url, Map<String, String> dataMap) {
            return ajaxPost(url, dataMap, "UTF-8");
        }
    
        /**
         * http的post请求，增加异步请求头参数，传递map格式参数
         */
        public static String ajaxPost(String url, Map<String, String> dataMap, String charset) {
            HttpPost httpPost = new HttpPost(url);
            httpPost.setHeader("X-Requested-With", "XMLHttpRequest");
            try {
                if (dataMap != null) {
                    List<NameValuePair> nvps = new ArrayList<NameValuePair>();
                    for (Map.Entry<String, String> entry : dataMap.entrySet()) {
                        nvps.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
                    }
                    UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(nvps, charset);
                    formEntity.setContentEncoding(charset);
                    httpPost.setEntity(formEntity);
                }
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            return executeRequest(httpPost, charset);
        }
    
        /**
         * http的post请求，增加异步请求头参数，传递json格式参数
         */
        public static String ajaxPostJson(String url, String jsonString) {
            return ajaxPostJson(url, jsonString, "UTF-8");
        }
    
        /**
         * http的post请求，增加异步请求头参数，传递json格式参数
         */
        public static String ajaxPostJson(String url, String jsonString, String charset) {
            HttpPost httpPost = new HttpPost(url);
            httpPost.setHeader("X-Requested-With", "XMLHttpRequest");
            
            StringEntity stringEntity = new StringEntity(jsonString, charset);// 解决中文乱码问题
            stringEntity.setContentEncoding(charset);
            stringEntity.setContentType("application/json");
            httpPost.setEntity(stringEntity);
            return executeRequest(httpPost, charset);
        }
    
        /**
         * 执行一个http请求，传递HttpGet或HttpPost参数
         */
        public static String executeRequest(HttpUriRequest httpRequest) {
            return executeRequest(httpRequest, "UTF-8");
        }
    
        /**
         * 执行一个http请求，传递HttpGet或HttpPost参数
         */
        public static String executeRequest(HttpUriRequest httpRequest, String charset) {
            CloseableHttpClient httpclient;
            if ("https".equals(httpRequest.getURI().getScheme())) {
                httpclient = createSSLInsecureClient();
            } else {
                httpclient = HttpClients.createDefault();
            }
            String result = "";
            try {
                try {
                    CloseableHttpResponse response = httpclient.execute(httpRequest);
                    HttpEntity entity = null;
                    try {
                        entity = response.getEntity();
                        result = EntityUtils.toString(entity, charset);
                    } finally {
                        EntityUtils.consume(entity);
                        response.close();
                    }
                } finally {
                    httpclient.close();
                }
            } catch (IOException ex) {
                ex.printStackTrace();
            }
            return result;
        }
    
        public static String executeRequest(CloseableHttpClient httpclient, HttpUriRequest httpRequest, String charset) {
            String result = "";
            try {
                try {
                    CloseableHttpResponse response = httpclient.execute(httpRequest);
                    HttpEntity entity = null;
                    try {
                        entity = response.getEntity();
                        result = EntityUtils.toString(entity, charset);
                    } finally {
                        EntityUtils.consume(entity);
                        response.close();
                    }
                } finally {
                    httpclient.close();
                }
            } catch (IOException ex) {
                ex.printStackTrace();
            }
            return result;
        }
    
        /**
         * 创建 SSL连接
         */
        public static CloseableHttpClient createSSLInsecureClient() {
            try {
                SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(new TrustStrategy() {
                    @Override
                    public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                        return true;
                    }
                }).build();
                SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new HostnameVerifier() {
                    @Override
                    public boolean verify(String hostname, SSLSession session) {
                        return true;
                    }
                });
                return HttpClients.custom().setSSLSocketFactory(sslsf).build();
            } catch (GeneralSecurityException ex) {
                throw new RuntimeException(ex);
            }
        }
    }

### 运行 ###

由于网络等原因，我们发现并不能全部下载成功，不过可以多次运行尝试，可以实现较高的下载成功率。

![1eb4cecf0aa98b1d95c9b16f027a10f2.png][]

![51f131aafd67e7a75cd57948b9b2a711.png][]

![f4e2324a2e7bdd72dc695c7d626cd5e8.png][]

666，厉害了。。

####  ####

**往期推荐**

[牛逼！SpringBoot+Vue企业级支付系统！附源码！][SpringBoot_Vue]

[为什么建议大家使用 Linux 开发！][Linux]

[IDEA的5大调试技巧，你学废了吗？][IDEA_5]

[让SpringBoot不需要Controller、Service、DAO、Mapper，卧槽！这款工具][SpringBoot_Controller_Service_DAO_Mapper]

[一张900w的数据表，16s执行的SQL优化到300ms？][900w_16s_SQL_300ms]

![03c6866f1f3111e0f85ef8cf771485a2.png][]

[bf3b3a2836580ef35bd8eddc051b5dca.png]: /images/20220828/40ef4360661249f7895370345a521439.png
[0ba4caa620543c6be8cf936943a191ea.png]: /images/20220828/0f0463ee112344b9b691ccb7caf4ee91.png
[3401e2512f48068fae727ecbc6e4d44d.png]: /images/20220828/fc0ac9dbfb174260859f28d85cd98254.png
[dd13ac6c47d24ddd5f387cd7be7967fc.png]: /images/20220828/f59bb40a9d324b82be0147087fdeccee.png
[1eb4cecf0aa98b1d95c9b16f027a10f2.png]: /images/20220828/a4746b36b58641d4af4ac168d7008a05.png
[51f131aafd67e7a75cd57948b9b2a711.png]: /images/20220828/2b8e36c34917424bbaf009f5bbab1472.png
[f4e2324a2e7bdd72dc695c7d626cd5e8.png]: /images/20220828/5a1045002bc74355985d45d578d67d5a.png
[SpringBoot_Vue]: http://mp.weixin.qq.com/s?__biz=MzU3NDE0NjMwNw%3D%3D&chksm=fd346b33ca43e225b33e1fe9deb152660b57c494f6cced29f7041ba599ba925ef7d32125d846&idx=1&mid=2247498079&scene=21&sn=a0f3c1fed1e33d33cf9d54905466926f#wechat_redirect
[Linux]: http://mp.weixin.qq.com/s?__biz=MzU3NDE0NjMwNw%3D%3D&chksm=fd346b08ca43e21ee67c8abd13bb06e21eaf3892d46b9d0454438d0d9f5a42b1bd7b81bf93d5&idx=1&mid=2247498084&scene=21&sn=2589cd885586c47741341796aaad2d7b#wechat_redirect
[IDEA_5]: http://mp.weixin.qq.com/s?__biz=MzU3NDE0NjMwNw%3D%3D&chksm=fd346b06ca43e2106f7d727f3143a88fc7cb94a42047675b873e4c580e9c929165ab8cba7f84&idx=1&mid=2247498090&scene=21&sn=f9ab3d8904aca1b30daa46691fda369a#wechat_redirect
[SpringBoot_Controller_Service_DAO_Mapper]: http://mp.weixin.qq.com/s?__biz=MzU3NDE0NjMwNw%3D%3D&chksm=fd346b1eca43e2084a534556652a1b3bbfee54603b12674aa02288fa9cadb7638d7342ac4afc&idx=1&mid=2247498098&scene=21&sn=54581eb93979b9382a1503477fc58284#wechat_redirect
[900w_16s_SQL_300ms]: http://mp.weixin.qq.com/s?__biz=MzU3NDE0NjMwNw%3D%3D&chksm=fd346be6ca43e2f062e79ee5d18afef2d6de588af36423b2d6959731356630d9e33cb9a709e3&idx=1&mid=2247498122&scene=21&sn=c5370a753aa1f4f5a9edb3fecfba3048#wechat_redirect
[03c6866f1f3111e0f85ef8cf771485a2.png]: /images/20220828/09856bf406374f40b5bd118fb4f46192.png