java爬虫入门--用jsoup爬取汽车之家的新闻

客官°小女子只卖身不卖艺 2022-06-07 08:07 184阅读 0赞

public static boolean isContainChinese(String str) {  
            Pattern p = Pattern.compile("[\u4e00-\u9fa5]");  
            Matcher m = p.matcher(str);  
            if (m.find()) {  
                return true;  
            }  
            return false;  
        }  
          
     
        /** 
         * 从汽车之家抓新闻 
         * @param size 
         * @param baseUrl 
         * @param domainName 
         * @param newsListId 
         * @param newsContentClass 
         * @param titleTagOrClass 
         * @param limitHref 
         * @param dateTag 
         * @param needDeleteAlt 
         * @return 
         */  
        public static ArrayList<News> getNewsFromCarHome(int size,String baseUrl,String domainName,String newsListId,  
                String newsContentClass,String titleTag,String dateTag,String needDeleteAlt){  
            ArrayList<News> newsList = new ArrayList<News>();  
            Document doc;  
            Elements elements =null;  
            Element title =null;  
            News news = null;  
            try {  
                doc = Jsoup.connect(baseUrl).get();  
                elements = (Elements) doc.getElementById(newsListId).children();  
                if(elements!=null&&elements.size()>0){  
                    for(Element ele:elements){  
                        news = new News();  
                        title = ele.select("a").first();  
                        if(title==null){  
                            continue;  
                        }  
                        news.setTitle(title.getElementsByTag(titleTag).text());  
                        if(news.getTitle()==null||news.getTitle().equals("")){  
                            continue;  
                        }  
                        news.setHref(domainName+title.attr("href"));  
                        if(dateTag!=null){  
                            String date=ele.select("i").text();  
                              news.setDate(date);  
                        }  
                        String newsUrl =news.getHref();  
                        if (isContainChinese(news.getHref())) {  
                            newsUrl = URLEncoder.encode(news.getHref(), "utf-8").toLowerCase().replace("%3a", ":").replace("%2f", "/");  
                        }  
                        Document newsDoc = Jsoup.connect(newsUrl).get();  
                        String text=newsDoc.getElementsByClass(newsContentClass).html();  
                        if(text.indexOf("未经许可")>0||text.indexOf("禁止转载")>0||text.indexOf("公众号")>0||text.indexOf("公众账号")>0){  
                            continue;  
                        }  
                         text=replaceImgSrcFromDataSrc(text,true,needDeleteAlt);  
                         int index=text.lastIndexOf("（");  
                         if(index>0){  
                             text=text.substring(0,index);  
                         }  
                         StringBuffer textBuffer=new StringBuffer(5);  
                         textBuffer.append("<!DOCTYPE html><html><head><meta name=\"content-type\" content=\"text/html; charset=UTF-8\">");  
                         textBuffer.append("</head><body>");  
                         textBuffer.append(deleteSource(text));  
                         textBuffer.append("</body></html>");  
                         news.setContent(textBuffer.toString());  
                         news.setContent(textBuffer.toString());  
                         System.out.println("标题====="+news.getTitle());  
                         System.out.println("href====="+news.getHref());  
                         System.out.println("content====="+news.getContent());  
                         newsList.add(news);  
                        if(newsList.size()==size){  
                            break;  
                        }  
                    }  
                }  
            } catch (Exception e) {  
                e.printStackTrace();  
            }  
            return newsList;  
        }  
          
        private static String deleteImg(String text) {  
            return text.replaceAll("<img [^>]*>", "");  
        }  
        private static String deleteSource(String text) {  
             return text.replaceAll("\\(.*?\\)|\\[.*?]", "");  
         }  
        /** 
         * 删除a标签中的href 
         * @param content 
         * @return 
         */  
        public static String removeHref(String content){    
            Document document = Jsoup.parse(content);    
            Elements elements = document.select("a[href]");    
            for(Element el:elements){    
                el.removeAttr("href");    
            }    
            return document.html();    
        }    
          
          
        /** 
         * 将htmlBody中所有img标签中的src内容替换为原data-src的内容, <br/> 
         * 如果不报含data-src,则src的内容不会被替换 <br/> 
         * @param htmlBody html内容 
         * @param needDeleteAlt 需要剔除的图片的alt信息 
         * @param imgUrlNeedAddProtocolPrefix 图片的url是否需要添加http协议前缀 
         * @return 返回替换后的内容 
         */  
        public static String replaceImgSrcFromDataSrc(String htmlBody,boolean imgUrlNeedAddProtocolPrefix,String needDeleteAlt) {  
            Document document = Jsoup.parseBodyFragment(htmlBody);  
            List<Element> nodes = document.select("img");  
            int nodeLenth = nodes.size();  
            if(nodeLenth==0){  
                return htmlBody;  
            }  
            for (int i = 0; i < nodeLenth; i++) {  
                Element e = nodes.get(i);  
                String dataSrc = e.attr("data-src");  
                if (StringUtils.isNotBlank(dataSrc)) {  
                    e.attr("src", dataSrc);  
                    e.removeAttr("data-src");  
                }  
            }  
            if (htmlBody.contains("<html>")) {  
                if(needDeleteAlt==null&&!imgUrlNeedAddProtocolPrefix){  
                    return document.toString();  
                }else if(needDeleteAlt==null&&imgUrlNeedAddProtocolPrefix){  
                    return document.toString().replace("src=\"//", "src=\"http://");  
                }else if(needDeleteAlt!=null&&imgUrlNeedAddProtocolPrefix){  
                    return document.toString().replace("src=\"//", "src=\"http://").replace("alt="+needDeleteAlt, "");  
                }  
                return document.toString().replace("alt="+needDeleteAlt, "");  
            } else {  
                if(needDeleteAlt==null&&!imgUrlNeedAddProtocolPrefix){  
                    return document.select("body>*").toString();  
                }else if(needDeleteAlt==null&&imgUrlNeedAddProtocolPrefix){  
                    return document.select("body>*").toString().replace("src=\"//", "src=\"http://");  
                }else if(needDeleteAlt!=null&&imgUrlNeedAddProtocolPrefix){  
                    return document.select("body>*").toString().replace("src=\"//", "src=\"http://").replace("alt="+needDeleteAlt, "");  
                }  
                return document.select("body>*").toString().replace("alt="+needDeleteAlt, "");  
            }  
          
        }  
        public static void main(String[] args) throws Exception{  
            getNewsFromCarHome(1,"http://m.autohome.com.cn/channel","http://m.autohome.com.cn","list","details","h4","time","汽车之家"); 
        }

![Center][]

[Center]: /images/20220607/50b0a451be48403398aa3e48e9e7b41c.png