JAVA抓取JS渲染完后的网页数据

逃离我推掉我的手 2022-02-20 05:43 670阅读 0赞

# JAVA抓取JS渲染完后的网页数据 #

*  **maven 依赖**

<dependency>
                <groupId>net.sourceforge.htmlunit</groupId>
                <artifactId>htmlunit</artifactId>
                <version>2.34.1</version>
            </dependency>
            
            <dependency>
                <groupId>commons-io</groupId>
                <artifactId>commons-io</artifactId>
                <version>2.6</version>
            </dependency>
    
            <dependency>
                <groupId>org.jsoup</groupId>
                <artifactId>jsoup</artifactId>
                <version>1.8.3</version>
            </dependency>

*  **编写工具类**

import com.gargoylesoftware.htmlunit.BrowserVersion;
    import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
    import com.gargoylesoftware.htmlunit.WebClient;
    import com.gargoylesoftware.htmlunit.html.HtmlPage;
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    
    import java.util.*;
    
    public class ReptilePicUtil {
    
        /**
         * 获取页面文档字串(等待异步JS执行)
         *
         * @param url 页面URL
         * @return
         * @throws Exception
         */
        public static String getHtmlPageResponse(String url) throws Exception {
            String result = "";
    
            final WebClient webClient = new WebClient(BrowserVersion.CHROME);
    
            webClient.getOptions().setThrowExceptionOnScriptError(false);//当JS执行出错的时候是否抛出异常
            webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);//当HTTP的状态非200时是否抛出异常
            webClient.getOptions().setActiveXNative(false);
            webClient.getOptions().setCssEnabled(false);//是否启用CSS
            webClient.getOptions().setJavaScriptEnabled(true); //很重要，启用JS
            webClient.setAjaxController(new NicelyResynchronizingAjaxController());//很重要，设置支持AJAX
    
            webClient.getOptions().setTimeout(30000);//设置“浏览器”的请求超时时间
            webClient.setJavaScriptTimeout(30000);//设置JS执行的超时时间
    
            HtmlPage page;
            try {
                page = webClient.getPage(url);
            } catch (Exception e) {
                webClient.close();
                throw e;
            }
            webClient.waitForBackgroundJavaScript(30000);//该方法阻塞线程
    
            result = page.asXml();
            webClient.close();
    
            return result;
        }
    
    
        //解析xml获取ImageUrl地址
        public static List<String> getImageUrl(String html,String className){
            List<String> result = new ArrayList<>();
            Document document = Jsoup.parse(html);//获取html文档
            List<Element> infoListEle = document.getElementsByClass(className);//获取元素节点等
            infoListEle.forEach(element -> {
                result.add(element.attr("src"));
            });
            return result;
        }
    
    public static void main(String[] args) throws Exception {
            String xml = ReptilePicUtil.getHtmlPageResponse("http://www.baidu.com");
            System.out.println(JSON.toJSONString(ReptilePicUtil.getImageUrl(xml,"index-logo-src")));
        }
    
    }

*  **控制台打印结果**  
    ![在这里插入图片描述][20190412172747848.png]
 *  **访问图片**

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3h1bHZlamllMDE0Mw_size_16_color_FFFFFF_t_70]

[20190412172747848.png]: /images/20220220/343b6e4cc1204bd8b7e216b372ac57ef.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3h1bHZlamllMDE0Mw_size_16_color_FFFFFF_t_70]: /images/20220220/673fb0d3e935433d9e7d39e98fa78ce3.png