网络爬虫：爬取动态网页（例如javascript）

ゝ一世哀愁。 2022-08-18 11:39 270阅读 0赞

经过一段时间的摸索，终于有了一套爬取动态网页的方法，此方法适合大多数的动态网页爬取，至于另外少数的动态网页爬取，还必须利用其它的办法。在此分享给大家。

举例：例如在百度中搜索成语词典，显示如下，需要爬取所有的成语词汇。我们可以点击下一页查看，有经验的同学一眼就可以看出这里是使用javascript异步加载的。在网页源码上是找不到的。下面就介绍爬取所有词典的方法  
![这里写图片描述][20151214200244616]

**1.点击鼠标右键–>审查元素**

![这里写图片描述][20151214200753766]

**2.选择Network按钮**

![这里写图片描述][20151214200937117]

**3.选择下一页按钮** *得到的效果是如下图，我们可以发现出现两个网址，*

![这里写图片描述][20151214201217475]  
![这里写图片描述][20151214204358464]

为了清楚显示我们想要的东西，我们选择js按钮  
![这里写图片描述][20151214204242077]

**4.将鼠标放在网址那，点击右键选择open link in new tab**

![这里写图片描述][20151214201833921]

**5.以上四部便可以得到大致是一个json格式的内容，做到这一步，算是完成了一大部分的工作，由于为了方便解析内容，需要将网站稍作修改，这一步大家需要去尝试在此我将这部分（&cb=jQuery1102011321965302340686\_1450094493974）去掉后，内容变成纯的json格式文档。接下来的和爬取静态网页的方法一样了**

**6. 第五步可以得到这样的网址：=1450094493985”>https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource\_id=28204&from\_mid=1&&format=json&ie=utf-8&oe=utf-8&query=%E6%88%90%E8%AF%AD%E8%AF%8D%E5%85%B8%E4%B8%8B%E8%BD%BD&sort\_key=&sort\_type=1&stat0=&stat1=&stat2=&stat3=&pn=30&rn=30&=1450094493985，pn=30，表示每页显示30个，我们需要改变pn的值来爬取所有的成语，具体操作是pn+=30。**

**所有的步骤说完了，接下来上代码：**

import java.io.BufferedReader;
    import java.io.ByteArrayOutputStream;
    import java.io.File;
    import java.io.FileNotFoundException;
    import java.io.IOException;
    import java.io.InputStream;
    import java.io.InputStreamReader;
    import java.io.PrintStream;
    import java.io.UnsupportedEncodingException;
    import java.net.HttpURLConnection;
    import java.net.MalformedURLException;
    import java.net.URL;
    import java.net.URLConnection;
    
    import org.json.JSONArray;
    import org.json.JSONObject;
    
    
    public class BaiduIdiomCrawer {
        
        public static void main(String[] args) {
            //BaiduIdiomCrawer test=new BaiduIdiomCrawer();
            new BaiduIdiomCrawer().test01();
        }
    
        public void test01() {
            //String myurl = "https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=6848&from_mid=1&&format=json&ie=utf-8&oe=utf-8&query=%E6%88%90%E8%AF%AD&sort_key=&sort_type=1&stat0=&stat1=&stat2=&stat3=&pn=30&rn=30&cb=jQuery110206992364453617483_1449371069677&_=1449371069681";
            //String myurl="https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=6848&from_mid=1&&format=json&ie=utf-8&oe=utf-8&query=%E6%88%90%E8%AF%AD&sort_key=&sort_type=1&stat0=&stat1=&stat2=&stat3=&pn=30&rn=30&_=1449371069681";
            String myurl;
            String content = new String();
            StringBuffer sb = new StringBuffer();
            try {
                String url2="https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id="
                        + "28204&from_mid=1&&format=json&ie=utf-8&oe=utf-8&"
                        + "query=%E6%88%90%E8%AF%AD%E8%AF%8D%E5%85%B8%E4%B8%8B%E8%BD%BD&sort_key=&"
                        + "sort_type=1&stat0=&stat1=&stat2=&stat3=&pn=";
                String url1="&rn=30&_=1449376371822";
                //String url2="https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=6848&from_mid=1&&format=json&ie=utf-8&oe=utf-8&query=%E6%88%90%E8%AF%AD&sort_key=&sort_type=1&stat0=&stat1=&stat2=&stat3=&pn=";
                //String url1="&rn=30&_=1449371069681";
                for (int j = 0; j < 1098; j++) {
                    myurl = url2 + j * 30 + url1;
                    System.out.println(j);
                    //content=this.downloadToString(myurl, "utf-8");
                    content=this.downloadToString(myurl);
                    JSONObject jo = new JSONObject(content.toString());
                    JSONArray jsonarray = jo.getJSONArray("data");
                    if(j==690){
                        System.out.println(j);
                    }
                    JSONObject jo1 = jsonarray.getJSONObject(0);
                    JSONArray jsonarray1 = jo1.getJSONArray("result");
                    for (int i = 0; i < jsonarray1.length(); i++) {
                        JSONObject user = (JSONObject) jsonarray1.get(i);
                        String userName = (String) user.get("ename");
                        //System.out.println(userName);
                        sb.append(userName + "\n");
                    }
                }
            } catch (Exception e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            try {
                PrintStream ps=new PrintStream(new File("result.txt"),"utf-8");
                ps.print(sb.toString());
                ps.close();
            } catch (FileNotFoundException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (UnsupportedEncodingException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            System.out.println(sb.toString());
        }
    
    
        public String downloadToString(String url, String charset) throws Exception {
            ByteArrayOutputStream output = new ByteArrayOutputStream();
            try {
                HttpURLConnection http = (HttpURLConnection) (new URL(url)).openConnection();
                http.setReadTimeout(5 * 60 * 1000);
                //http.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)");
                http.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.1.141.64 Safari/537.31");
                http.setRequestMethod("GET");
                http.setUseCaches(false);
                InputStream input = http.getInputStream();
                int ch = -1;
                while((ch = input.read()) != -1)
                    output.write(ch);
                input.close();
            } catch(Exception ex) {
                ex.printStackTrace(System.out);
            }
            return output.toString(charset);
        }
        public String downloadToString(String u){
            StringBuffer sb = new StringBuffer();
            try {
                URL url = new URL(u);
                URLConnection connection = url.openConnection();
                InputStream is = connection.getInputStream();
    
                InputStreamReader isr = new InputStreamReader(is,"UTF-8");
                BufferedReader br = new BufferedReader(isr);
                String line;
    
                while((line=br.readLine())!=null){
                    sb.append(line);
                }
                br.close();
                isr.close();
                is.close();
            } catch (MalformedURLException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            return sb.toString();
    
        }
    }

大功告成，大家有问题请留言！！！

[20151214200244616]: /images/20220731/bdfd488b6c7343f1abd177938e68448c.png
[20151214200753766]: /images/20220731/72d87dcf732340468f9b72cb77e9dda7.png
[20151214200937117]: /images/20220731/b7cf2c9f45c3415892caf6f590be8ef6.png
[20151214201217475]: /images/20220731/dccca456090a4444ad4cc3663b68c949.png
[20151214204358464]: /images/20220731/e8c4fb8ce9ab4ca7ad42a77951342917.png
[20151214204242077]: /images/20220731/be5fbf5e2f5a48aaaf112fd35f0dc78c.png
[20151214201833921]: /images/20220731/0a1546b440a14a3cb80436507cb50350.png