Java项目之seek-poetry（爬虫）

Myth丶恋晨 2024-04-17 15:06 11阅读 0赞

### 1.爬虫模块 ###

public class Crawler { 
    	//打印日志
        private final Logger logger = LoggerFactory.getLogger(Crawler.class);
    
        //放置文档页面（超链接）
        private final Queue<Page> docQueue = new LinkedBlockingQueue<>();
        //放置详情页面(处理完成，数据在dataSet)
        private final Queue<Page> detailQueue = new LinkedBlockingQueue<>();
        //采集器
        private final WebClient webClient;
        //解析器
        private final List<Parse> parseList = new LinkedList<>();
        //清洗器——管道
        private final List<Pipeline> pipelineList = new LinkedList<>();
        //线程调度器
        private final ExecutorService executorService;
    
        public Crawler(){ 
            this.webClient = new WebClient(BrowserVersion.FIREFOX_38);
            this.executorService = Executors.newFixedThreadPool(8,
                    new ThreadFactory() { 
                private final AtomicInteger id = new AtomicInteger(0);
                @Override
                public Thread newThread(Runnable r) { 
                    Thread thread = new Thread(r);
                    thread.setName("Crawler-Thread"+id.getAndIncrement());
                    return thread;
                }
            });
        }
        //启动爬虫
        public void start(){ 
            //1.爬取+解析
            //2.清洗
            this.executorService.submit(this::parse);
            this.executorService.submit(this::pipeline);
        }
        private void parse(){ 
            while (true){ 
                try { 
                    Thread.sleep(1000);
                } catch (InterruptedException e) { 
                    logger.error("Parse occur Exception{}.",e.getMessage());
                }
                final Page page = this.docQueue.poll();
                if (page == null){ 
                    continue;
                }
                this.executorService.submit(new Runnable() { 
                    @Override
                    public void run() { 
                        try { 
                            //采集
                            HtmlPage htmlPage = Crawler.this.webClient.getPage(page.getUrl());
                            page.setHtmlPage(htmlPage);
                            for (Parse parse:Crawler.this.parseList){ 
                                parse.parse(page);
                            }
                            if (page.isDetail()){ 
                                Crawler.this.detailQueue.add(page);
                            }else{ 
                                Iterator<Page> iterator = page.getSubPage().iterator();
                                while (iterator.hasNext()){ 
                                    Page subPage = iterator.next();
                                    Crawler.this.docQueue.add(subPage);
                                    iterator.remove();
                                }
                            }
                        } catch (IOException e) { 
                            logger.error("Parse occur Exception{}.",e.getMessage());
                        }
                    }
                });
            }
        }
    
        private void pipeline(){ 
            while (true){ 
                try { 
                    Thread.sleep(1000);
                } catch (InterruptedException e) { 
                    logger.error("Pipeline occur Exception{}.",e.getMessage());
                }
                final Page page = this.detailQueue.poll();
                if (page == null){ 
                    continue;
                }
                this.executorService.submit(new Runnable() { 
                    @Override
                    public void run() { 
                        for (Pipeline pipeline : Crawler.this.pipelineList){ 
                            pipeline.pipeline(page);
                        }
                    }
                });
    
            }
        }
    
        //爬虫停止
        public void stop(){ 
            if (this.executorService!=null && !this.executorService.isShutdown()){ 
                this.executorService.shutdown();
            }
            logger.info("Crawler stopped...");
        }
        public void addParse(Parse parse){ 
            this.parseList.add(parse);
        }
        public void addPipeline(Pipeline pipeline){ 
            this.pipelineList.add(pipeline);
        }
        public void addPage(Page page){ 
            this.docQueue.add(page);
        }
    
    }
    //---------------------------------Common-----------------------------------
    @Data
    public class Page { 
        //数据网站的根地址
        // 比如：https://so.gushiwen.org
        private final String base;
        //具体网页的路径
        private final String path;
        //网页DOM对象
        private HtmlPage htmlPage;
        //标识网页是否是详情页
        private final boolean detail;
        //子页面对象集合
        private Set<Page> subPage = new HashSet<>();
        //数据对象(封装后的hashMap）
        private DataSet dataSet = new DataSet();
        public String getUrl(){ 
            return this.base+this.path;
        }
    }
    
    
    public class DataSet { 
        //data把DOM解析，清洗之后存储的数据
        //比如：
        //标题：送孟浩然之广陵
        //作者：李白
        //正文：XXX
        private Map<String,Object> data = new HashMap<>();
    
        public void putData(String key,Object value){ 
    
            this.data.put(key,value);
        }
        public Object getData(String key){ 
    
            return this.data.get(key);
        }
        public Map<String,Object> getData(){ 
    
            return new HashMap<>(this.data);
        }
    }
    //---------------------------------解析--------------------------------------
    
    public interface Parse { 
        //解析页面
        void parse(Page page);
    }
    
    //链接解析
    public class DocumentParse implements Parse { 
        @Override
        public void parse(final Page page) { 
            if (page.isDetail()){ 
                return;
            }
            HtmlPage htmlPage = page.getHtmlPage();
            //分析网页源码
            htmlPage.getBody().getElementsByAttribute("div",
                    "class","typecont").
                    forEach(div -> { 
                        DomNodeList<HtmlElement> anodeList = div.
                                getElementsByTagName("a");
                        anodeList.forEach(
                                aNode -> { 
                                    String path = aNode.getAttribute("href");
                                    Page subpage = new Page(page.getBase(), path, true);
                                    page.getSubPage().add(subpage);
                                }
                        );
                    });
        }
    }
    
    //详情页面解析
    public class DataPageParse implements Parse { 
        @Override
        public void parse(final Page page) { 
            if(!page.isDetail()){ 
                return;
            }
    
            HtmlPage htmlPage = page.getHtmlPage();
            HtmlElement body = htmlPage.getBody();
            //标题htmlunit
            String titlePath = "//div[@class='cont']/h1/text()";
            DomText titleDom =(DomText)body.getByXPath(titlePath).get(0);
            String title = titleDom.asText();
    
            //作者
            String authorPath = "//div[@class='cont']/p/a[2]";
            HtmlAnchor authorDom = (HtmlAnchor) body.getByXPath(authorPath).get(0);
            String author = authorDom.asText();
    
            //朝代
            String dynastyPath = "//div[@class='cont']/p/a[1]";
            HtmlAnchor dynastyDom = (HtmlAnchor) body.getByXPath(dynastyPath).get(0);
            String dynasty = authorDom.asText();
    
            //正文
            String contextPath = "//div[@class='cont']/div[@class='contson']";
            HtmlDivision contentDom = (HtmlDivision) body.getByXPath(contextPath).get(0);
            String content = contentDom.asText();
    
            page.getDataSet().putData("title",title);
            page.getDataSet().putData("dynasty",dynasty);
            page.getDataSet().putData("author",author);
            page.getDataSet().putData("content",content);
    
        }
    }
    
    
    
    
    //---------------------------------管道------------------------------------
    public interface Pipeline { 
    
        //管道处理page中的数据
        void pipeline(final Page page);
    }
    
    
    //将数据插入数据库中
    public class DatabasePopeline implements Pipeline{ 
        private final Logger logger = LoggerFactory.getLogger(DatabasePopeline.class);
    
        private final DataSource dataSource;
    
        public DatabasePopeline(DataSource dataSource) { 
            this.dataSource = dataSource;
        }
    
        @Override
        public void pipeline(final Page page) { 
    
            String title = ((String) page.getDataSet().getData("title"));
            String dynasty = (String) page.getDataSet().getData("dynasty");
            String author = ((String) page.getDataSet().getData("author"));
            String content = ((String) page.getDataSet().getData("content"));
    
            String sql = "insert into poetry_info values (?,?,?,?);";
            try (Connection connection = dataSource.getConnection();
                 PreparedStatement statement =connection.prepareStatement(sql)){ 
                statement.setString(1,title);
                statement.setString(2,dynasty);
                statement.setString(3,author);
                statement.setString(4,content);
                statement.executeUpdate();
            }catch (SQLException e){ 
                logger.error("Database insert occur Exception{}.",e.getMessage());
            }
        }
    }
    
    //打印处理后的数据
    public class ConsolePipeline implements Pipeline { 
        @Override
        public void pipeline(final Page page) { 
            Map<String,Object> data = page.getDataSet().getData();
            //存储
            System.out.println(data);
        }
    }

### 2.配置模块 ###

@Data
    public class ConfigProperties { 
        private String crawlerBase;
        private String crawlerPath;
        private boolean crawlerDetail;
    
        private String dbUsername;
        private String dbPassword;
        private String dbUrl;
        private String dbDriverClass;
        private boolean enableConsole ;
    
        public ConfigProperties(){ 
            //从外部文件加载
            InputStream inputStream = ConfigProperties.class.getClassLoader().
                    getResourceAsStream("Config.properties");
            Properties p = new Properties();
            try { 
                p.load(inputStream);
    
            } catch (IOException e) { 
                e.printStackTrace();
            }
            this.crawlerBase = String.valueOf(p.get("crawler.base"));
            this.crawlerPath = String.valueOf(p.get("crawler.path"));
            this.crawlerDetail = Boolean.parseBoolean(String.valueOf(p.get("crawler.detail")));
            this.dbUsername = String.valueOf(p.get("db.username"));
            this.dbPassword = String.valueOf(p.get("db.password"));
            this.dbUrl = String.valueOf(p.get("db.url"));
            this.dbDriverClass = String.valueOf(p.get("db.driver_class"));
            this.enableConsole = Boolean.valueOf(String.valueOf(p.getProperty("config.enable_console","false")));
    
        }
    
        public static void main(String[] args) { 
            new ConfigProperties();
        }
    }
    
    
    
    
    
    
    public class ObjectFactory { 
        private static final ObjectFactory instance = new ObjectFactory();//单例模式
    
        //存放所有对象
        private  final Map<Class,Object> objectHashMap = new HashMap<>();
        private ObjectFactory(){ 
            //1.初始化配置对象
            initConfigProperties();
            //2.数据源对象
            initDataSource();
            //3爬虫对象
            initCrawler();
            //4.web对象
            initWebController();
        }
    
        private void initWebController() { 
            DataSource dataSource = getObject(DataSource.class);
            AnalyzeDao analyzeDao = new AnalyzeDaoImpl(dataSource);
            AnalyzeService analyzeService = new AnalyzeServiceImpl(analyzeDao);
    
            WebController webController = new WebController(analyzeService);
            objectHashMap.put(WebController.class,webController);
        }
    
        private void initCrawler() { 
            ConfigProperties configProperties = new ConfigProperties();
            DruidDataSource dataSource = getObject(DataSource.class);
            final Page page = new Page(configProperties.getCrawlerBase(),
                    configProperties.getCrawlerPath(),
                    configProperties.isCrawlerDetail());
            Crawler crawler = new Crawler();
            crawler.addParse(new DocumentParse());
            crawler.addParse(new DataPageParse());
            if (configProperties.isEnableConsole()){ 
                crawler.addPipeline(new ConsolePipeline());
            }
            crawler.addPipeline(new DatabasePopeline(dataSource));
            crawler.addPage(page);
            objectHashMap.put(Crawler.class,crawler);
        }
    
        private void initConfigProperties(){ 
            ConfigProperties configProperties = new ConfigProperties();
            objectHashMap.put(ConfigProperties.class,configProperties);
    
    
        }
    
        private void initDataSource(){ 
            ConfigProperties configProperties = getObject(ConfigProperties.class);
            DruidDataSource dataSource = new DruidDataSource();
            dataSource.setUsername(configProperties.getDbUsername());
            dataSource.setPassword(configProperties.getDbPassword());
            dataSource.setDriverClassName(configProperties.getDbDriverClass());
            dataSource.setUrl(configProperties.getDbUrl());
    
            objectHashMap.put(DataSource.class,dataSource);
        }
    
        public  <T> T getObject(Class classz){ 
            if (!objectHashMap.containsKey(classz)){ 
                throw new IllegalArgumentException("Class"+classz
                .getName()+"not found Object");
            }
            return (T)objectHashMap.get(classz);
        }
        public static ObjectFactory getInstance(){ 
            return instance;
        }
    /* private void printObjectList(){ System.out.println("----------------ObjectFactoryList--------------"); for (Map.Entry<Class,Object> entry : objectHashMap.entrySet()){ System.out.println(String.format("\t[%s]==>[%s]", entry.getKey().getCanonicalName(), entry.getValue().getClass().getCanonicalName())); } System.out.println("------------------------------------------------"); }*/
    }

### 3.分析模块 ###

//------------------------------存放诗的信息------------------------------------
    @Data
    public class PoetryInfo { 
        private String title;
        private String dynasty;
        private String author;
        private String content;
    }
    
    
    
    public interface AnalyzeDao { 
        //分析唐诗中作者的创作数量
        List<AuthorCount> analyzeAuthorCount();
        //查询所有诗文，提供给业务层进行分析
        List<PoetryInfo> queryAllPoetryInfo();
    }
    
    public class AnalyzeDaoImpl implements AnalyzeDao { 
        private final DataSource dataSource;
        public AnalyzeDaoImpl(DataSource dataSource) { 
            this.dataSource = dataSource;
        }
    
        @Override
        public List<AuthorCount> analyzeAuthorCount() { 
            List<AuthorCount> datas = new ArrayList<>();
            //try()自动关闭
            String sql = "select count(*) as count,author from" +
                    " poetry_info group by author;";
            try (Connection connection = dataSource.getConnection();
                 PreparedStatement statement = connection.prepareStatement(sql);
                 ResultSet rs = statement.executeQuery();
                 ){ 
                while (rs.next()){ 
                    AuthorCount authorCount = new AuthorCount();
                    authorCount.setAuthor(rs.getString("author"));
                    authorCount.setCount(rs.getInt("count"));
                    datas.add(authorCount);
                }
    
            } catch (SQLException e) { 
                e.printStackTrace();
            }
            return datas;
        }
    
        @Override
        public List<PoetryInfo> queryAllPoetryInfo() { 
            List<PoetryInfo> datas = new ArrayList<>();
            String sql = "select title,dynasty,author,content from poetry_info;";
            try(Connection connection = dataSource.getConnection();
                PreparedStatement statement = connection.prepareStatement(sql);
                ResultSet rs = statement.executeQuery();
                    ){ 
                while (rs.next()){ 
                    PoetryInfo poetryInfo = new PoetryInfo();
                    poetryInfo.setTitle(rs.getString("title"));
                    poetryInfo.setDynasty(rs.getString("dynasty"));
                    poetryInfo.setAuthor(rs.getString("author"));
                    poetryInfo.setContent(rs.getString("content"));
                    datas.add(poetryInfo);
                }
    
            } catch (SQLException e) { 
                e.printStackTrace();
            }
            return datas;
        }
    }
    
    
    public interface AnalyzeService { 
        //分析唐诗中作者的创作数量
        List<AuthorCount> analyzeAuthorCount();
        //词云分析
        List<WordCount> analyzeWordCloud();
    }
    
    public class AnalyzeServiceImpl implements  AnalyzeService{ 
        private final AnalyzeDao analyzeDao;
    
        public AnalyzeServiceImpl(AnalyzeDao analyzeDao) { 
            this.analyzeDao = analyzeDao;
        }
    
        @Override
        public List<AuthorCount> analyzeAuthorCount() { 
    
            //排序方式：
            //1.DAO层SQL排序
            //2.业务层Service进行数据排序
            List<AuthorCount> authorCounts = analyzeDao.analyzeAuthorCount();
            //降序排序
            authorCounts.sort((o1, o2) -> o2.getCount() - o1.getCount());
            return authorCounts;
        }
    
        @Override
        public List<WordCount> analyzeWordCloud() { 
            //1.查询出所有数据
            //2.取出title content
            //3.分词-过滤w、null、空-length<2
            //4.统计
            //<词：词频>
            Map<String,Integer> map = new HashMap<>();
            List<PoetryInfo> poetryInfos = analyzeDao.queryAllPoetryInfo();
            for (PoetryInfo poetryInfo :poetryInfos){ 
                List<Term> terms = new ArrayList<>();
                String title = poetryInfo.getTitle();
                String content = poetryInfo.getContent();
    
                terms.addAll(NlpAnalysis.parse(title).getTerms());
                terms.addAll(NlpAnalysis.parse(content).getTerms());
                Iterator<Term>iterator = terms.iterator();
    
                while (iterator.hasNext()){ 
                    Term term = iterator.next();
                    //词性的过滤
                    if (term.getNatureStr() == null || term.getNatureStr().equals("w")){ 
                        iterator.remove();
                        continue;
                    }
                    //词的过滤
                    if(term.getRealName().length()<2){ 
                        iterator.remove();
                        continue;
                    }
                    //统计
                    String realName = term.getRealName();
                    Integer count = 0;
                    if (map.containsKey(realName)){ 
                        count =  map.get(realName)+1;
                    }else { 
                        count = 1;
                    }
                    map.put(realName,count);
                }
            }
            List<WordCount> wordCounts = new ArrayList<>();
            for (Map.Entry<String,Integer> entry: map.entrySet()){ 
                WordCount wordCount = new WordCount();
                wordCount.setCount(entry.getValue());
                wordCount.setWord(entry.getKey());
                wordCounts.add(wordCount);
            }
            return wordCounts;
        }
    
    /* 测试 public static void main(String[] args) { Result result = NlpAnalysis.parse("松下问童子，言师采药去。" + "只在此山中，云深不知处。 "); List<Term> terms = result.getTerms(); for (Term term:terms) { System.out.println(term); } }*/
    }
    
    
    @Data
    public class AuthorCount { 
        private String author;
        private Integer count;
    }
    
    
    public class WordCount { 
        private String word;
        private Integer count;
    }

### 4.web模块 ###

public class WebController { 
        private final AnalyzeService analyzeService;
    
        public WebController(AnalyzeService analyzeService) { 
            this.analyzeService = analyzeService;
        }
        //-> http://127.0.0.1:4567/
        //->/analyze/author_count
        private List<AuthorCount> analyzeAythorCount(){ 
            return analyzeService.analyzeAuthorCount();
        }
        //-> http://127.0.0.1:4567/
        //->/analyze/word_cloud
        private List<WordCount> analyzeWordCloud(){ 
            return analyzeService.analyzeWordCloud();
        }
    
        public void launch(){ 
            ResponseTransformer transformer = new JSONResponseTransformer();
    
            Spark.staticFileLocation("/static");
            Spark.get("/analyze/author_count", ((request, response) ->
                analyzeAythorCount()),transformer);
            Spark.get("/analyze/word_cloud",((request, response) ->
                    analyzeWordCloud()),transformer);
        }
    
        public static class JSONResponseTransformer implements ResponseTransformer{ 
            //Object——>Gson
           private Gson gson = new Gson();
            @Override
            public String render(Object o) throws Exception { 
                return gson.toJson(o);
            }
        }

### 5.项目主类 ###

public class TangshiAnalyzeApplication { 
        private static final Logger LOGGER = LoggerFactory.
                getLogger(TangshiAnalyzeApplication.class);
    
        public static void main(String[] args) { 
    
            WebController webController = ObjectFactory.getInstance().getObject(WebController.class);
    
            //运行了web服务，提供接口
            LOGGER.info("Web Server Launch...");
            webController.launch();
    
          /* if (args.length==1&&args[0].equals("run-crawler")){ Crawler crawler = ObjectFactory.getInstance().getObject(Crawler.class); LOGGER.info("Crawler started..."); crawler.start(); }*/
        }
    }

### 6.前端模块 ###

<!DOCTYPE html>
    <html lang="en">
    <head>
        <meta charset="UTF-8">
        <title>唐诗分析报告</title>
        <link type="text/css" href="assets/css/base.css">
        <script type="text/javascript" src="assets/js/echarts.min.js"></script>
        <script type="text/javascript" src="assets/js/echarts-wordcloud.min.js"></script>
        <script type="text/javascript" src="assets/js/jquery-3.3.1.min.js"></script>
        <script type="text/javascript" src="assets/js/api.js"></script>
        <script> </script>
    </head>
    <body id="box">
    <h1>唐诗分析报告</h1>
    <div>
        <div>
            <input type="button" value="创作数量排行榜" onclick="creationRanking('content')">
            <input type="button" value="诗词用词云图" onclick="cloudWorld('content')">
        </div>
        <div id="content" style="width: 800px ; height: 500px;">
    
        </div>
    </div>
    </body>
    </html>

/* api.js 1. 主要用于前端页面与后端的交互 前端：HTML+CSS+JavaScript开发的浏览器识别的Web程序 后端：Java开发服务程序，通过HTTP协议提供Web API接口 前端和后端交互：通信协议用的是HTTP协议 */
    function creationRanking(id) { 
        //HTTP Method : GET
        //HTTP URL : 请求地址（服务端提供的API接口）
        $.get({ 
            url: "/analyze/author_count",
            dataType: "json",
            method: "get",
            success: function (data, status, xhr) { 
                //echarts图表对象
                var myChart = echarts.init(document.getElementById(id));
                var options = { 
                    //图标的标题
                    title: { 
                        text: '唐诗创作排行榜'
                    },
    
                    tooltip: { },
                    //柱状图的提示信息
                    legend: { 
                        data: ['数量(首)']
                    },
                    //X轴的数据：作者
                    xAxis: { 
                        data: []
                    },
                    //Y轴的数据：创作的数量
                    yAxis: { },
                    series: [{ 
                        name: '创作数量',
                        type: 'bar',
                        data: []
                    }]
                };
    
                //List<AuthorCount>
                for (var i=0; i< data.length; i++) { 
                    var authorCount  = data[i];
                    options.xAxis.data.push(authorCount.author);
                    options.series[0].data.push(authorCount.count);
                }
                myChart.setOption(options, true);
            },
            error: function (xhr, status, error) { 
    
            }
        });
    }
    
    function cloudWorld(id) { 
        $.get({ 
            url: "/analyze/word_cloud",
            dataType: "json",
            method: "get",
            success: function (data, status, xhr) { 
                var myChart = echarts.init(document.getElementById(id));
                var options = { 
                    series: [{ 
                        type: 'wordCloud',
                        shape: 'pentagon',
                        left: 'center',
                        top: 'center',
                        width: '80%',
                        height: '80%',
                        right: null,
                        bottom: null,
                        sizeRange: [12, 60],
                        rotationRange: [-90, 90],
                        rotationStep: 45,
                        gridSize: 8,
                        drawOutOfBound: false,
                        textStyle: { 
                            normal: { 
                                fontFamily: 'sans-serif',
                                fontWeight: 'bold',
                                color: function () { 
                                    //rgb(r,g,b)
                                    return 'rgb(' + [
                                        Math.round(Math.random() * 160),
                                        Math.round(Math.random() * 160),
                                        Math.round(Math.random() * 160)
                                    ].join(',') + ')';
                                }
                            },
                            emphasis: { 
                                shadowBlur: 10,
                                shadowColor: '#333'
                            }
                        },
                        // Data is an array. Each array item must have name and value property.
                        data: []
                    }]
                };
                for (var i=0 ;i<data.length; i++) { 
                    var wordCount = data[i];
                    //wordCount => 词 ： 词频
                    options.series[0].data.push({ 
                        name: wordCount.word,
                        value: wordCount.count,
                        textStyle: { 
                            normal: { },
                            emphasis: { }
                        }
                    });
                }
                myChart.setOption(options, true);
            },
            error: function (xhr, status, error) { 
    
            }
        });
    }

#### 效果展示 ####

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3BlcmZlY3RtYXRjaF9H_size_16_color_FFFFFF_t_70]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3BlcmZlY3RtYXRjaF9H_size_16_color_FFFFFF_t_70 1]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3BlcmZlY3RtYXRjaF9H_size_16_color_FFFFFF_t_70]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/32f42b213d4441db90661261cf99e3a5.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3BlcmZlY3RtYXRjaF9H_size_16_color_FFFFFF_t_70 1]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/0f8375fda9b541a2812f2486d27b11c0.png