springboot2.1入门系列十二 springboot使用jsoup爬取全国的省市数据

末蓝、 2022-03-01 06:44 692阅读 0赞

国家统计局的统计用区划和城乡划分代码的网址为[http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/][http_www.stats.gov.cn_tjsj_tjbz_tjyqhdmhcxhfdm] 通过爬取该网址可以获取最新的全国省市区乡镇村的数据。

本文通过springboot结合jsoup技术实现模拟爬虫爬取全国的省市数据，其他层级的数据如县乡镇村的数据可以由同学们自己练习。

一、数据库表

create table province
    (
       id                   int not null auto_increment,
       name                 varchar(64),
       code                 varchar(32),
       primary key (id)
    )
    ENGINE = InnoDB
    DEFAULT CHARSET = utf8;
    
    create table city
    (
       id                   int not null auto_increment,
       province_id          int,
       name                 varchar(64),
       code                 varchar(32),
       primary key (id)
    )
    ENGINE = InnoDB
    DEFAULT CHARSET = utf8;

二、创建工程demo012

pom.xml的内容为

<project xmlns="http://maven.apache.org/POM/4.0.0"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
        <parent>
            <groupId>com.yinww</groupId>
            <artifactId>demo-springboot2</artifactId>
            <version>0.0.1-SNAPSHOT</version>
        </parent>
        <artifactId>demo012</artifactId>
    
        <dependencies>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
            <dependency>
                <groupId>org.mybatis.spring.boot</groupId>
                <artifactId>mybatis-spring-boot-starter</artifactId>
                <version>1.3.2</version>
            </dependency>
            <dependency>
                <groupId>mysql</groupId>
                <artifactId>mysql-connector-java</artifactId>
                <scope>runtime</scope>
            </dependency>
            <dependency>
                <groupId>org.jsoup</groupId>
                <artifactId>jsoup</artifactId>
                <version>1.11.3</version>
            </dependency>
        </dependencies>
    </project>

三、Java类

主类

package com.yinww.demo.springboot2.demo012;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    
    @SpringBootApplication
    public class Demo012Application {
    
        public static void main(String[] args) {
            SpringApplication.run(Demo012Application.class, args);
        }
    
    }

工具类：

public class HttpUtil {
    
        public static Document get(String url, String charset) throws IOException {
            String userAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36";
            URL url2 = new URL(url);
            HttpURLConnection connection = (HttpURLConnection)url2.openConnection();
            connection.setRequestMethod("GET");
            //是否允许缓存，默认true。
            connection.setUseCaches(Boolean.FALSE);
            //设置请求头信息
            connection.addRequestProperty("Connection", "close");
            connection.addRequestProperty("user-agent", userAgent);
            //设置连接主机超时（单位：毫秒）
            connection.setConnectTimeout(80000);
             //设置从主机读取数据超时（单位：毫秒）  
            connection.setReadTimeout(80000);
            //开始请求
            try {
                Document doc = Jsoup.parse(connection.getInputStream(), charset, url);
                return doc;
            } catch (Exception e) {
                System.out.println("parse error: " + url);
                e.printStackTrace();
            }
            return null;
        }
        
    }

控制器

@RestController
    public class SpiderController {
        
        @Autowired
        private SpiderService spiderService;
        
        @GetMapping({"/", ""})
        public Object spider() throws Exception {
            String url = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/";
            String charset = "gb2312";
            Document rootDoc = HttpUtil.get(url, charset);
            if(rootDoc == null) {
                return 0;
            }
            Element firstElement = rootDoc.getElementsByClass("center_list_contlist").get(0);
            String yearHref = firstElement.select("a").get(0).attr("href"); // 最近一个年份的省份链接
            Document doc = HttpUtil.get(yearHref, charset);
            
            // 遍历所有的省
            Elements provinceElements = doc.getElementsByClass("provincetr");
            for (Element element : provinceElements) {
                Elements aEles = element.select("a");
                for (Element aEle : aEles) {
                    String name = aEle.text();
                    String provincesHref = aEle.attr("href");
                    String code = provincesHref.substring(0, provincesHref.indexOf("."));
                    int index = yearHref.lastIndexOf("/") + 1;
                    provincesHref = yearHref.substring(0, index) + provincesHref;
                    Province province = new Province(name, code);
                    spiderService.saveProvince(province);
                    getCites(provincesHref, charset, province.getId());
                }
            }
            
            return "spider crawl end.";
        }
        
        private void getCites(String url, String charset, int provinceId) throws Exception {
            Document rootDoc = HttpUtil.get(url, charset);
            if(rootDoc != null) {
                Elements cityElements = rootDoc.getElementsByClass("citytr");
                for (Element cityElement : cityElements) {
                    Element aEle = cityElement.select("a").get(1); // 第二个是市的名字
                    String name = aEle.text();
                    String cityHref = aEle.attr("href");
                    City city = new City();
                    city.setName(name);
                    city.setProvinceId(provinceId);
                    int start = cityHref.lastIndexOf("/") + 1;
                    String code = cityHref.substring(start, cityHref.indexOf("."));
                    city.setCode(code);
                    spiderService.saveCity(city);
                }
            }
        }
    
    }

四、运行程序

启动程序后，访问 [http://localhost:8080/][http_localhost_8080]  等待程序运行结束，再查看数据库可获取到31条省的数据，343条市的数据。

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Jvd2VpMDI2_size_16_color_FFFFFF_t_70][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Jvd2VpMDI2_size_16_color_FFFFFF_t_70 1][]

整体思路还是比较清晰，重点在于掌握jsoup的几个api的用法，其他的都是常规的springboot代码。

本文内容到此结束，更多内容可关注公众号和个人微信号：

![2018111718160012.jpg][]![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Jvd2VpMDI2_size_16_color_FFFFFF_t_70 2][]

[http_www.stats.gov.cn_tjsj_tjbz_tjyqhdmhcxhfdm]: http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/
[http_localhost_8080]: http://localhost:8080/
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Jvd2VpMDI2_size_16_color_FFFFFF_t_70]: /images/20220301/ced56a128de5408d821f9f1a7fde5632.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Jvd2VpMDI2_size_16_color_FFFFFF_t_70 1]: /images/20220301/b376729ca1b448e29906d7aa41d7bc86.png
[2018111718160012.jpg]: https://img-blog.csdnimg.cn/2018111718160012.jpg
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Jvd2VpMDI2_size_16_color_FFFFFF_t_70 2]: /images/20220301/4b876f4f501b4bdbb82bb85c91174db8.png