WebMagic爬取应用市场应用信息

叁歲伎倆 2022-05-19 01:49 167阅读 0赞

### WebMagic资料 ###

官方教程 [http://webmagic.io/docs/zh/][http_webmagic.io_docs_zh]

官方网址 [http://webmagic.io/][http_webmagic.io]

### 代码实现 ###

爬取华为应用市场应用信息，统计华为应用市场应用数量，启动20个线程，自定义MyPiple来保存数据。

AppStoreProcessor.java主类

import us.codecraft.webmagic.Page;
    import us.codecraft.webmagic.Site;
    import us.codecraft.webmagic.Spider;
    import us.codecraft.webmagic.processor.PageProcessor;
    import us.codecraft.webmagic.selector.Selectable;
    
    /**
     * @author wzj
     * @create 2018-07-17 22:06
     **/
    public class AppStoreProcessor implements PageProcessor
    {
        // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
        private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
    
        public void process(Page page)
        {
            //获取名称
            String name = page.getHtml().xpath("//p/span[@class='title']/text()").toString();
    
            page.putField("appName",name );
    
            if (page.getResultItems().get("appName") == null)
            {
                //skip this page
                page.setSkip(true);
            }
    
            //获取页面其他链接
            Selectable links = page.getHtml().links();
            page.addTargetRequests(links.regex("(http://app.hicloud.com/app/C\\d+)").all());
        }
    
    
        public Site getSite()
        {
            return site;
        }
    
        public static void main(String[] args)
        {
            Spider.create(new AppStoreProcessor())
    
                    .addUrl("http://app.hicloud.com")
                    .addPipeline(new MyPipeline())
                    .thread(20)
                    .run();
        }
    }

自定义Piple，将结果保存到D:\\\\cache\\\\appstore.txt中

import us.codecraft.webmagic.ResultItems;
    import us.codecraft.webmagic.Task;
    import us.codecraft.webmagic.pipeline.Pipeline;
    
    import java.io.FileWriter;
    import java.io.IOException;
    
    /**
     * @author wzj
     * @create 2018-07-17 22:16
     **/
    public class MyPipeline implements Pipeline
    {
        /**
         * 写入文件
         */
        private FileWriter fileWriter;
    
        /*
         * 统计数目
         */
        private int count = 1;
    
    
        public MyPipeline()
        {
            try
            {
                this.fileWriter = new FileWriter("D:\\cache\\appstore.txt");
            }
            catch (IOException e)
            {
                e.printStackTrace();
            }
        }
    
        /**
         * Process extracted results.
         *
         * @param resultItems resultItems
         * @param task        task
         */
        public void process(ResultItems resultItems, Task task)
        {
            String appName = resultItems.get("appName");
            String content = String.valueOf(count) + " " + appName;
    
            try
            {
                fileWriter.write(content);
                fileWriter.write("\n");
                fileWriter.flush();
            }
            catch (IOException e)
            {
                e.printStackTrace();
            }
    
            System.out.println(content);
    
            count ++;
        }
    }

[http_webmagic.io_docs_zh]: http://webmagic.io/docs/zh/
[http_webmagic.io]: http://webmagic.io/