java 一个简单的爬虫项目详解

╰半橙微兮° 2022-04-06 02:10 340阅读 0赞

#### 我们在不久的将来就会相遇的，在那之前，我要成为一名优秀的程序员 ####

爬虫对我们来说是一个既爱又恨的家伙。我记得我大学时期，有个朋友学会了python的爬虫之后，整天就去爬各种网站，不亦乐乎。在工程中，经常需要做爬虫相关的业务。爬虫一般是和多线程挂钩的，今天先详细介绍一个简单版，后续增加一个线程池版本的。

实现的功能：从文件中读取url列表，然后把每个页面的结果都存储在一个文件中，同时去掉html中的标签和javascript代码。

首先，创建一个maven工程，然后引入依赖

<dependency>
                <groupId>net.sourceforge.htmlunit</groupId>
                <artifactId>htmlunit</artifactId>
                <version>2.32</version>
    </dependency>

这个包是专门用于爬取页面的。

//web页面获取的实体类
    public class WebEntity { 
        private WebClient       webclient;      //模拟浏览器对象
        private String          url;            //请求url
    
        public WebEntity(String url) { 
            WebClient webclient = new WebClient(BrowserVersion.CHROME);
            webclient.getOptions().setJavaScriptEnabled(false);
            webclient.getOptions().setCssEnabled(false);
            webclient.getOptions().setUseInsecureSSL(false);
            webclient.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0");
            this.webclient = webclient;
            this.url = url;
        }
    
        //获取页面
        public HtmlPage executeReq() { 
            int time = 1;
            while (time <= 5) { 
                try { 
                    return webclient.getPage(url);
                } catch (IOException e) { 
                }
                time++;
            }
            return null;
        }
    
    
        //打印信息
        @Override
        public String toString() { 
            return String.format("url = %s", url);
        }
    
        //get、set方法
        public WebClient getWebclient() { 
            return webclient;
        }
    
        public void setWebclient(WebClient webclient) { 
            this.webclient = webclient;
        }
    
        public String getUrl() { 
            return url;
        }
    
        public void setUrl(String url) { 
            this.url = url;
        }
    }

//爬取页面
    public class Craw { 
        private static volatile Craw singleton;
        private Craw(){ 
    
        }
        public static Craw getInstance() { 
            if (singleton == null) { 
                synchronized (Craw.class) { 
                    if (singleton == null) { 
                        singleton = new Craw();
                        return singleton;
                    }
                }
            }
            return singleton;
        }
    
        public HtmlPage parsePage(WebEntity webEntity){ 
            HtmlPage page = null;
            try{ 
                page = webEntity.executeReq();
            }catch (Exception e){ 
                System.err.println("获取页面失败");
            }
            return page;
        }
    
    }

//主类，包含了从文件中读取和写入东西
    public class Pachong { 
    
    
        static void outfile(File file, HtmlPage page){ 
            FileOutputStream fileOutputStream = null;
            BufferedOutputStream bufferedOutputStream = null;
            try { 
                if (file != null){ 
                    fileOutputStream = new FileOutputStream(file);
                    bufferedOutputStream = new BufferedOutputStream(fileOutputStream);
                    if (page != null){ 
                        String ans = page.asXml();
                        String anstrue = LabelUtil.handleHtmlLabel(ans);
                        bufferedOutputStream.write(anstrue.getBytes());
                    }
                    bufferedOutputStream.flush();
                    bufferedOutputStream.close();
                    fileOutputStream.close();
    
                }
            }catch (IOException e){ 
                e.printStackTrace();
            }
        }
    
        public static void main(String args[]){ 
            String filename = "d://test/in.txt";
            File file = new File(filename);
            InputStream inputStream = null;
            try { 
                inputStream = new FileInputStream(file);
                BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
                String line = null;
                int cnt = 0;
                while ((line = bufferedReader.readLine()) != null){ 
                    WebEntity webEntity = new WebEntity(line);
                    Craw craw = Craw.getInstance();
                    HtmlPage page = craw.parsePage(webEntity);
    
                    if (page != null){ 
                        String filenametmp = "d://test/"+cnt + ".txt";
                        File file1 = new File(filenametmp);
                            if (!file1.exists()){ 
                                file1.createNewFile();
                                outfile(file1,page);
                            }else { 
                                outfile(file1,page);
                            }
                    }
                    cnt ++ ;
                }
                inputStream.close();
                bufferedReader.close();
            }catch (IOException e){ 
                e.printStackTrace();
    
            }
    
        }
    }

//去掉页面中的html标签和js代码
    public class LabelUtil { 
        public static String handleHtmlLabel(String html){ 
            String noHTMLString = "";
            html = html.replaceAll("&", "&");
            Matcher m = Pattern
                    .compile("&#(\\d+);", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL | Pattern.CANON_EQ)
                    .matcher(html);
            boolean b = false;
            int i = 0;
            while (m.find()) { 
                if (i > 500) { 
                    System.out.println(i);
                }
                i++;
                html = html.replace("&#" + m.group(1) + ";", (char) Integer.parseInt(m.group(1)) + "");
                b = true;
            }
            if (!b) { 
                m = Pattern
                        .compile("&#x([\\da-f]+);",
                                Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL | Pattern.CANON_EQ)
                        .matcher(html);
                int j = 0;
                while (m.find()) { 
                    if (j > 500) { 
                        System.out.println(j);
                    }
                    j++;
                    html = html.replaceAll("&#[x|X]" + m.group(1) + ";", (char) Integer.parseInt(m.group(1), 16) + "");
                }
            }
            String scl = "<script";//8
            String scr = "</script>";//9
            int indexl = -1;
            indexl = html.indexOf(scl);
            long mm = html.length();
            while (indexl != -1){ 
                int indexr = -1;
                indexr = html.indexOf(scr);
                if (indexl != 0){ 
                    String x = html.substring(0,indexl);
                    int n = html.length();
                    if (indexr != n-9 && indexr != -1) {  ;
                        String y = html.substring(indexr+9,n-1);
                        html = x+y;
    
                    }else if (indexr == n-9 || indexr == -1){ 
                        html= x;
    
                    }
                }else { 
                    int n = html.length();
                    if (indexr != n-9 && indexr != -1){ 
                        String y = html.substring(indexr+9,n-1);
                        html = y;
    
                    }else if(indexr == n-9){ 
                        html = "";
                    }else if(indexr == -1){ 
                        html = "";
                    }
                }
                indexl = -1;
                indexl = html.indexOf(scl);
            }
            noHTMLString = html.replaceAll("<\\s*(?:br|Br|BR|bR|div|DIV|Div|p|P|td|TD|Td)\\s*(?:[^>])*\\s*>", "")
                    .replaceAll("", "").replaceAll(" ", "").replaceAll("\\<.*?\\>", "")
                    .replaceAll("&(?:g|l)t", "");
            String x = "";
            Pattern pattern = Pattern.compile("\\s*|\t|\r|\n");
            Matcher matcher = pattern.matcher(noHTMLString);
            x = matcher.replaceAll("");
            return x.trim();
        }
    }