【JAVA-爬虫】使用 Jsoup+HttpClient 爬取网页信息

待我称王封你为后i 2021-12-12 05:43 286阅读 0赞

![format_png][]

### 思路 ###

> 1.添加相依依赖
> 
> 2.编写工具类，根据url（需要爬取的页面）参数，使用HttpClient连接到网页获取网页源码
> 
> 3.根据获取到的html格式的网页源码，使用 Jsoup获取所需的 Element 元素及各属性值

###  代码 ###

一、依赖

<dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpclient</artifactId>
        <version>4.5.6</version>
    </dependency>
    
    <dependency>
        <groupId>org.jsoup</groupId>
        <artifactId>jsoup</artifactId>
        <version>1.8.3</version>
    </dependency>

二、工具类：

package com.gourd.base.utils;
    
    import lombok.extern.slf4j.Slf4j;
    import org.apache.http.HttpEntity;
    import org.apache.http.client.config.RequestConfig;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.config.Registry;
    import org.apache.http.config.RegistryBuilder;
    import org.apache.http.conn.socket.ConnectionSocketFactory;
    import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
    import org.apache.http.conn.socket.PlainConnectionSocketFactory;
    import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
    import org.apache.http.conn.ssl.SSLContexts;
    import org.apache.http.conn.ssl.TrustStrategy;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClientBuilder;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
    import org.apache.http.util.EntityUtils;
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.select.Elements;
    
    import javax.net.ssl.SSLContext;
    import java.io.IOException;
    import java.security.KeyManagementException;
    import java.security.KeyStore;
    import java.security.KeyStoreException;
    import java.security.NoSuchAlgorithmException;
    
    /**
     * JsoupHttpClient工具类
     *
     * @author gourd
     */
    @Slf4j
    public class JsoupHttpClientUtils {
    
        private final static String DEFAULT_CHARSET = "utf8";
    
        /**
         * 根据url获取网页源码
         *
         * @param url 需要爬取的网页URL
         * @return
         */
        public static String getHtmlByUrl(String url) {
            // 获取到的网页源码
            String html = null;
            // 建立一个新的请求客户端
            CloseableHttpClient httpClient = null;
            if (url.startsWith("https://")) {
                httpClient = getHttpsClient();
            } else {
                httpClient = HttpClients.createDefault();
            }
    
            // 使用HttpGet的方式请求网址
            HttpGet httpGet = new HttpGet(url);
            // 模拟浏览器
            httpGet.setHeader("User-Agent",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
    
            // 使用代理 IP
    //        HttpHost proxy = new HttpHost("118.114.77.47", 8080);
            RequestConfig config = RequestConfig.custom()
                    .setConnectionRequestTimeout(10000)
                    // 设置连接超时时间 10秒钟
                    .setConnectTimeout(10000)
                    // 设置读取超时时间10秒钟
                    .setSocketTimeout(10000)
                    .build();
            httpGet.setConfig(config);
            // 获取网址的返回结果
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpGet);
                if (response != null && response.getStatusLine().getStatusCode() == 200) {
                    //获取返回结果中的实体
                    HttpEntity entity = response.getEntity();
                    //将返回的实体输出
                    html = EntityUtils.toString(entity, DEFAULT_CHARSET);
                } else {
                    log.error("获取网页源码失败");
                }
            } catch (IOException e) {
                log.error("获取网页源码异常：", e);
            } finally {
                // 关闭资源
                try {
                    if (response != null) {
                        response.close();
                    }
                } catch (IOException e) {
                    log.error("response关闭错误，请检查原因");
                }
                try {
                    if(httpClient != null){
                        httpClient.close();
                    }
                } catch (IOException e) {
                    log.error("httpClient关闭错误，请检查原因");
                }
            }
            return html;
        }
    
        /**
         * 根据html获取元素
         *
         * @param html    网页源码
         * @param tagName 标签名
         * @return
         */
        public static Elements getElements(String html, String tagName) {
            // 解析网页 得到文档对象
            Document doc = Jsoup.parse(html);
            // 获取tag是tagName的所有DOM元素，数组
            Elements elements = doc.getElementsByTag(tagName);
            return elements;
        }
    
    
        /**
         * 获取https连接（不验证证书）
         *
         * @return
         */
        private static CloseableHttpClient getHttpsClient() {
            RegistryBuilder<ConnectionSocketFactory> registryBuilder = RegistryBuilder.<ConnectionSocketFactory>create();
            ConnectionSocketFactory plainSF = new PlainConnectionSocketFactory();
            registryBuilder.register("http", plainSF);
            // 指定信任密钥存储对象和连接套接字工厂
            try {
                KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
                // 信任任何链接
                TrustStrategy anyTrustStrategy = new TrustStrategy() {
    
                    @Override
                    public boolean isTrusted(java.security.cert.X509Certificate[] arg0, String arg1) throws java.security.cert.CertificateException {
                        return true;
                    }
                };
                SSLContext sslContext = SSLContexts.custom().useTLS().loadTrustMaterial(trustStore, anyTrustStrategy).build();
                LayeredConnectionSocketFactory sslSF = new SSLConnectionSocketFactory(sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
                registryBuilder.register("https", sslSF);
            } catch (KeyStoreException e) {
                throw new RuntimeException(e);
            } catch (KeyManagementException e) {
                throw new RuntimeException(e);
            } catch (NoSuchAlgorithmException e) {
                throw new RuntimeException(e);
            }
            Registry<ConnectionSocketFactory> registry = registryBuilder.build();
            // 设置连接管理器
            PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(registry);
            // 构建客户端
            return HttpClientBuilder.create().setConnectionManager(connManager).build();
        }
    
    
    }

三、调用测试：

import com.gourd.base.utils.JsoupHttpClientUtils;
    import io.swagger.annotations.Api;
    import io.swagger.annotations.ApiOperation;
    import org.jsoup.select.Elements;
    import org.springframework.web.bind.annotation.GetMapping;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.RequestParam;
    import org.springframework.web.bind.annotation.RestController;
    
    /**
     * @author gourd
     * createAt: 2018/9/17
     */
    
    @RestController
    @Api(description = "爬虫控制器")
    @RequestMapping("/spider")
    public class SpiderController {
    
        @GetMapping(value = "/spider" )
        @ApiOperation(value = "获取网页爬虫数据", notes = "获取网页爬虫数据")
        public void logout(@RequestParam String url){
            String htmlByUrl = JsoupHttpClientUtils.getHtmlByUrl(url);
            Elements elements = JsoupHttpClientUtils.getElements(htmlByUrl, "a");
    
            System.out.println("success");
        }
    
    
    }

[format_png]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly90aW1nc2EuYmFpZHUuY29tL3RpbWc_aW1hZ2UmcXVhbGl0eT04MCZzaXplPWI5OTk5XzEwMDAwJnNlYz0xNTczNzA2MTk5NjYxJmRpPWUyM2MzNDNkNzEyZTZlMDllZDJmMWFjNGEyNGU4OTY3JmltZ3R5cGU9MCZzcmM9aHR0cCUzQSUyRiUyRnBob3RvY2RuLnNvaHUuY29tJTJGMjAxNTA4MDYlMkZtcDI2MDI0Nzc4XzE0Mzg4MjgzMzMxMTRfMi5qcGVn?x-oss-process=image/format,png