使用java+WebMagic实现小说爬取

淡淡的烟草味﹌ 2022-02-12 11:43 274阅读 0赞

1、首先百度搜索webmagic网站，网址如下：[https://github.com/code4craft/webmagic/releases/tag/WebMagic-0.7.3][https_github.com_code4craft_webmagic_releases_tag_WebMagic-0.7.3]

2、接下来下载[webmagic-0.7.3-all.tar.gz 压缩包解压][webmagic-0.7.3-all.tar.gz]

3、创建一个项目，在项目中创建一个lib文件夹，将WebMagic中的jar包复制到lib文件夹里面，选取所有的jar包（按下shift可以全选），右键->build path,然后创建一个class 文件，具体执行代码如下：

package test20190428;
    
    import java.util.Iterator;
    import java.util.List;
    import java.util.Set;
    import java.util.TreeSet;
    
    import us.codecraft.webmagic.Page;
    import us.codecraft.webmagic.Site;
    import us.codecraft.webmagic.Spider;
    import us.codecraft.webmagic.processor.PageProcessor;
    
    public class BookProcessor implements PageProcessor{
    	private static int count=1;
    	@Override
    	public Site getSite() {
    		// TODO Auto-generated method stub
    		Site site=Site.me();
    		site.setTimeOut(3000);
    		site.setRetryTimes(3);
    		site.setSleepTime(3000);
    		return site;
    	}
    
    	@Override
    	public void process(Page page) {
    		// TODO Auto-generated method stub
    		if(count==1)
    		{
    			//获取每一页的网址链接
    			List <String> all = page.getHtml().regex("/1_1999/\\d+.html").all();
    			//获取每一页的网址链接，字符串拼接
    			Set set = new TreeSet();
    			set.addAll(all);
    			Iterator<String> iterator = all.iterator();
    			while(iterator.hasNext())
    			{
    				String url2="https://www.biquyun.com/"+iterator.next();
    				 page.addTargetRequest(url2);
    			}
    			count++;
    		}
    		
    		if(count==2)
    		{
    			//获取每一页的页面文字
    			List <String> all2 = page.getHtml().xpath("//div[@id='content']/text()").all();
    			Iterator<String> iterator = all2.iterator();
    			while(iterator.hasNext())
    			{
    				System.out.println(iterator.next());
    			}
    		}
    		
    	}
    	
    	public static void main(String[] args) {
    		// TODO Auto-generated method stub
    		BookProcessor bookprocessor = new BookProcessor();
    		Spider create = Spider.create(bookprocessor);
    		create.addUrl("https://www.biquyun.com/1_1999/");
    		create.thread(5);
    		create.run();
    	}
    
    }

实现结果：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpamlhMTExMTEx_size_16_color_FFFFFF_t_70][]

[https_github.com_code4craft_webmagic_releases_tag_WebMagic-0.7.3]: https://github.com/code4craft/webmagic/releases/tag/WebMagic-0.7.3
[webmagic-0.7.3-all.tar.gz]: https://github.com/code4craft/webmagic/releases/download/WebMagic-0.7.3/webmagic-0.7.3-all.tar.gz
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpamlhMTExMTEx_size_16_color_FFFFFF_t_70]: /images/20220212/f22645ec3aca4f0d97ee3bce381b2b4e.png