Java 简单爬虫

淩亂°似流年 2021-09-19 15:00 272阅读 0赞

# demo1 #

package com.javabase;
    
    import java.io.BufferedReader;
    import java.io.IOException;
    import java.io.InputStreamReader;
    import java.net.HttpURLConnection;
    import java.net.URL;
    import java.util.List;
    
    public class WabpageDemo {
        public static void main(String[] args) throws IOException {
            // TODO Auto-generated method stub
            URL url;
            HttpURLConnection urlCon;
            //实例化
            url=new URL("https://www.taobao.com ");
            //执行url的openConnection的方法，方法返回值为Connection连接
            urlCon=(HttpURLConnection) url.openConnection();
            //执行了getResponseCode方法，返回int状态值
            int rscode=urlCon.getResponseCode();
            if(rscode==200){   //rscode 返回两百表示成功
                BufferedReader reader=new BufferedReader
                        (new InputStreamReader(urlCon.getInputStream(),"UTF-8"));
                String line;
                List<String> alist;
                while((line=reader.readLine())!=null){
                	System.out.println(line);
                	
    //                if((line.startsWith("<a"))&&(line.contains("data-cid")))
    //                    System.out.println(line.substring(line.indexOf("\">")+2, line.indexOf("</a>")));
                }
            }
        }
    }

# demo2 #

需要添加工具包：

<dependency>
    	    <groupId>org.jsoup</groupId>
    	    <artifactId>jsoup</artifactId>
    	    <version>1.11.3</version>
    	</dependency>

package com.javabase;
    
    import java.io.IOException;
    
    import org.jsoup.Connection;
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    import org.jsoup.select.Elements;
    
    public class WabpageDemo2 {
        public static void main(String[] args) {
            // TODO Auto-generated method stub
            try {
                Document doc=Jsoup.connect("https://www.taobao.com").get();
    //            Connection con=Jsoup.connect("http://www1.sxcredit.gov.cn/public/infocomquery.do?method=publicIndexQuery");
    //            con.data("query.enterprisename", "兴");
    //            Document doc2=con.timeout(100000).post();
                Elements es=doc.getElementsByTag("a");
                for(Element e:es){
                    String text=e.text();
                    System.out.println(text);    
                    String url=e.attr("href");  
                    System.out.println(url); 
                }
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    
    }

# demo3 #

package com.javabase;
    
    import java.io.IOException;
    
    import org.jsoup.Connection;
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    import org.jsoup.select.Elements;
    
    public class WabpageDemo3 {
        public static void main(String[] args) {
            // TODO Auto-generated method stub
            try {
                Document doc=Jsoup.connect("https://blog.csdn.net/h356363/article/details/90579050").get();
                Elements es=doc.getElementsByTag("li");
                for(Element e:es){
                	Element e1=e.child(0);
                	System.out.println(e1.text());
                	System.out.println(e1.attr("href"));
                	
                }
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    
    }