爬虫模式-JAVA获取省市区编码

末蓝、 2024-04-08 08:16 43阅读 0赞

#### 起因 ####

准备做一个省级联动的功能，从网页上查找了很多资源符合要求的很少（缺失港澳台数据，不好实现直辖市、特别行政区判断）。复制代码，能直接运行。

数据来源：中国统计局标准 http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/  
运行环境：jdk8；

#### 代码 ####

> jar 包

<dependency>
        <groupId>org.jsoup</groupId>
        <artifactId>jsoup</artifactId>
        <version>1.11.3</version>
        </dependency>
        
        <dependency>
        <groupId>cn.hutool</groupId>
        <artifactId>hutool-all</artifactId>
        <version>4.5.11</version>
        </dependency>
        
        <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        
        <dependency>
        <groupId>org.projectlombok</groupId>
        <artifactId>lombok</artifactId>
        <optional>true</optional>
        </dependency>
        <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-test</artifactId>
        </dependency>
        
        <dependency>
        <groupId>com.baomidou</groupId>
        <artifactId>mybatis-plus-boot-starter</artifactId>
        <version>3.3.0</version>
        </dependency>
        
        <dependency>
        <groupId>mysql</groupId>
        <artifactId>mysql-connector-java</artifactId>
        <version>5.1.35</version>
        </dependency>
        
        <dependency>
        <groupId>com.baomidou</groupId>
        <artifactId>mybatis-plus-generator</artifactId>
        <version>3.3.0</version>
        </dependency>
        <dependency>
        <groupId>org.apache.velocity</groupId>
        <artifactId>velocity-engine-core</artifactId>
        <version>2.1</version>
        </dependency>

> \_服务实现类 \_ChinasServiceImpl

package com.hn.yuan.city.service.impl;
    
    import cn.hutool.core.date.DateUtil;
    import cn.hutool.core.util.IdUtil;
    import cn.hutool.http.HttpRequest;
    import cn.hutool.http.HttpUtil;
    import com.hn.yuan.city.entity.Chinas;
    import com.hn.yuan.city.mapper.ChinasMapper;
    import com.hn.yuan.city.service.ChinasService;
    import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
    import com.hn.yuan.reptileCity.demoTest.Test;
    import lombok.var;
    import org.apache.commons.lang3.StringUtils;
    import org.jsoup.Jsoup;
    import org.jsoup.select.Elements;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.stereotype.Service;
    import java.util.Date;
    import java.util.concurrent.atomic.AtomicInteger;
    
    /**
     * <p>
     * 服务实现类
     * </p>
     *
     * @author XIAOCAO
     * 
     */
    @Service
    public class ChinasServiceImpl extends ServiceImpl<ChinasMapper, Chinas> implements ChinasService {
        
    
    
        @Autowired
        private ChinasMapper chinasMapper;
    
        /*什么情况下使用AtomicInteger
        1、作为多个线程同时使用的原子计数器。
        2、在比较和交换操作中实现非阻塞算法。
        https://www.jianshu.com/p/073096a729f6
         */
        public String addChinas(Chinas chinas) {
        
            chinasMapper.insert(chinas);
            return "成功";
        }
    
        private static final Logger log = LoggerFactory.getLogger(Test.class);
    
        //AtomicInteger类是系统底层保护的int类型，通过提供执行方法的控制进行值的原子操作
        private static AtomicInteger atomicInteger = new AtomicInteger(0);
        private static AtomicInteger atomicIntegerErrorNum = new AtomicInteger(0);
    
    
        public void test(String appendProvide, String appendCity) {
        
    
            System.out.println("===============开始抓取数据=================");
            var url = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/index.html";
            String html = HttpUtil.get(url);
            var htmlDoc = Jsoup.parse(html);
            var selectClasses = htmlDoc.getElementsByClass("provincetr");
            var startDateStr = DateUtil.formatDateTime(new Date());
            long startDate = System.currentTimeMillis();
    //        ExecutorService executorService = new ThreadPoolExecutor(4, 5, 1L, TimeUnit.SECONDS, new ArrayBlockingQueue(4), Executors.defaultThreadFactory());
            for (int i = 0; i < selectClasses.size(); i++) {
        
                int finalI = i;
    //            executorService.execute(() -> {
        
    //                System.out.println(Thread.currentThread().getName() + " " + "--->开始爬数据");
                try {
        
                    //一个线程跑就行
                    startProvide(selectClasses, finalI, baseUrl, appendProvide, appendCity);
                } catch (Exception e) {
        
                    e.printStackTrace();
                }
            }
    
            System.out.println("耗时 =》");
            System.out.println((System.currentTimeMillis() - startDate) / 1000);
    
            System.out.println("开始时间 ==> " + startDateStr);
            System.out.println("结束时间 ==>" + DateUtil.formatDateTime(new Date()));
    
        }
    
    
        private String getHtml(String url) throws Exception {
        
    
            //以原子方式将当前值递增1并在递增后返回新值。它相当于i ++操作。
            atomicInteger.incrementAndGet();
    
            log.info("调用接口次数 ：" + atomicInteger.get());
            log.info("请求开始时间 ==>" + DateUtil.formatDateTime(new Date()));
            log.info(url);
            String html = null;
            try {
        
                html = getHtml2(url);
            } catch (Exception e) {
        
                e.printStackTrace();
                log.info(Thread.currentThread().getName() + " = 超时。。睡5秒再重试");
                Thread.sleep(1000 * 5);
                log.info(Thread.currentThread().getName() + " = 超时重试");
                atomicIntegerErrorNum.incrementAndGet();
                log.info("error num = " + atomicIntegerErrorNum.get());
    
                try {
        
                    //重试1次
                    html = getHtml2(url);
                } catch (Exception e2) {
        
                    e2.printStackTrace();
    
                    log.info("error num = " + atomicIntegerErrorNum.get());
                    log.info(Thread.currentThread().getName() + " 第二次超时重试");
                    log.info(Thread.currentThread().getName() + " = 第二次超时。。睡12分钟再重试");
                    log.info("error num = " + atomicIntegerErrorNum.get());
    
                    Thread.sleep(1000 * 60 * 12);
    
                    //重试2次
                    html = getHtml2(url);
    
                }
    
            }
            log.info("请求结束时间 ==>" + DateUtil.formatDateTime(new Date()));
    
            System.out.println("");
            System.out.println("");
            return html;
    
        }
    
        private String getHtml2(String url) throws Exception {
        
    
            //链式构建请求
            return HttpRequest.get(url)
                    .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36")//头信息，多个头信息多次调用此方法即可
                    .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")//头信息，多个头信息多次调用此方法即可
                    .header("Cookie", "_trs_uv=" + IdUtil.simpleUUID() + "; SF_cookie_1=" + IdUtil.fastUUID())
                    .header("If-None-Match", "f32-5d4bccaa05a80-gzip")
    //                    .header("If-Modified-Since",new Date())
    //                    .form(paramMap)//表单内容
                    .timeout(15000)//超时，毫秒
                    .execute().body();
    
        }
    
        //国家统计局链接
        public static String baseUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/";
    
    
        /**
         * @param selectClasses
         * @param i
         * @param baseUrl
         * @param appendProvide 需要查询的省  ,null表示查询所有
         * @throws Exception
         */
        private void startProvide(Elements selectClasses, Integer i, String baseUrl, String appendProvide, String appendCity) throws Exception {
        
    
            var provideCodes = selectClasses.get(i).children();
            //1.省份 provincetr
            for (int provideCodeIndex = 0; provideCodeIndex < provideCodes.size(); provideCodeIndex++) {
        
                var provideCodeUrl = provideCodes.get(provideCodeIndex).select("a").attr("href");
                var provideName = provideCodes.get(provideCodeIndex).select("a").text();
    //            System.out.println("省份 = " + provideName);
                if (!StringUtils.isBlank(provideCodeUrl)) {
        
                    if (StringUtils.isBlank(appendProvide)) {
        
                        provide(provideCodeUrl, provideName, null);
                    } else {
        
                        if (appendProvide.equals(provideName)) {
        
                            System.out.println("要的省份 = " + provideName);
                            provide(provideCodeUrl, provideName, appendCity);
                        }
                    }
                }
            }
        }
    
    
        /**
         * 通过省，市获取下面的数据
         *
         * @param provideCodeUrl
         */
        public void provide(String provideCodeUrl, String provideName, String appendCity) throws Exception {
        
            var provideCode = provideCodeUrl.split("\\.")[0];
    
    //                    String gotoCityHtml = HttpUtil.get(baseUrl + provideCodeUrl);
            String gotoCityHtml = getHtml(baseUrl + provideCodeUrl);
            var cityHtmlDoc = Jsoup.parse(gotoCityHtml);
            Elements selectCityClass = cityHtmlDoc.select(".citytr");
    
            //拼接省code码插入数据库
            var len = provideCode.length();
            var provideCode2 = provideCode;
            if (len < 6) {
        
                len = 6 - len;
                for (int l = 0; l < len; l++) {
        
                    provideCode2 += "0";
                }
            }
    
            if (StringUtils.isBlank(appendCity)) {
        
                //todo 保存到数据库  保存省
                System.out.println("保存数据库的id：" + provideCode2 + "省名称：" + provideName + "code码：" + provideCode2 + "level：" + 1);
                Chinas chinas = new Chinas().setCode(Long.valueOf(provideCode2)).setName(provideName).setPid(Long.valueOf(0)).setLevel("1");
                this.addChinas(chinas);
                city(selectCityClass, provideName, provideCode, null);
            } else {
        
                city(selectCityClass, provideName, provideCode, appendCity);
            }
    
        }
    
    
        /**
         * 城市
         *
         * @param selectCityClass
         * @param provideName
         * @param provideCode
         * @throws Exception
         */
        private void city(Elements selectCityClass, String provideName, String provideCode, String appendCity) throws Exception {
        
            //2.城市  citytr
            for (int cityIndex = 0; cityIndex < selectCityClass.size(); cityIndex++) {
        
                var gotoCountyUrl = selectCityClass.get(cityIndex).select("td").get(1).select("a").attr("href");
                var cityName = selectCityClass.get(cityIndex).select("td").get(1).select("a").text();
                if (StringUtils.isBlank(gotoCountyUrl)) {
        
                    continue;
                }
    
                if (StringUtils.isBlank(appendCity)) {
        
                    this.appendCity(gotoCountyUrl, provideName, provideCode, cityName);
                } else {
        
                    if (appendCity.equals(cityName)) {
        
                        this.appendCity(gotoCountyUrl, provideName, provideCode, cityName);
                    }
                }
    
    
            }
        }
    
    
        private void appendCity(String gotoCountyUrl, String provideName, String provideCode, String cityName) throws Exception {
        
            //String countytr = HttpUtil.get(baseUrl + gotoCountyUrl);
            boolean flag = true;
            String countytr = getHtml(baseUrl + gotoCountyUrl);
            var countytrDoc = Jsoup.parse(countytr);
            Elements countyClass = countytrDoc.select(".countytr");
            if (countyClass.size() == 0) {
        
                flag = false;
                countyClass = countytrDoc.select(".towntr");
            }
    
            var strCityUrl = gotoCountyUrl.split("/");
            var cityCode = strCityUrl[strCityUrl.length - 1].split("\\.")[0];
            var cityCodeLen = cityCode.length();
            var cityCode2 = cityCode;
            if (cityCodeLen < 6) {
        
                cityCodeLen = 6 - cityCodeLen;
                for (int l = 0; l < cityCodeLen; l++) {
        
                    cityCode2 += "0";
                }
            }
            if (cityName.equals("市辖区")) {
        
                cityName = provideName;
            }
            cityCode = cityCode2;
    
            //拼接省code码插入数据库
            var len = provideCode.length();
            var provideCode2 = provideCode;
            if (len < 6) {
        
                len = 6 - len;
                for (int l = 0; l < len; l++) {
        
                    provideCode2 += "0";
                }
            }
            if (!flag) {
        
                System.out.println("直辖市，进入此方法");
            } else {
        
                //todo 保存到数据库   保存城市
                System.out.println("保存数据库的市id：" + cityCode + " 市名称：" + cityName + " code码：" + provideCode2 + "level：" + 2);
                Chinas chinas = new Chinas().setCode(Long.valueOf(cityCode)).setName(cityName).setPid(Long.valueOf(provideCode2)).setLevel("2");
                this.addChinas(chinas);
                county(countyClass,
                        provideName,
                        provideCode, cityCode);
            }
    
        }
    
    
        /**
         * 街道
         *
         * @param countyClass
         * @param provideCode
         */
        private void county(Elements countyClass, String provideName, String provideCode, String cityCode) throws Exception {
        
            //3.县区  countytr
            for (int county = 0; county < countyClass.size(); county++) {
        
                var gotoTownUrl = countyClass.get(county).select("td").get(1).select("a").attr("href");
                var countyName = countyClass.get(county).select("td").get(1).select("a").text();
                if (StringUtils.isBlank(gotoTownUrl)) {
        
                    continue;
                }
    
                String gotoTownUrls = StringUtils.substringBetween(gotoTownUrl, "/", ".");
    
                //todo 保存到数据库
                System.out.println("保存数据库的县 区数据id：" + gotoTownUrls + " 县区名称：" + countyName + " code码：" + cityCode + "level：" + 3);
                Chinas chinas = new Chinas().setCode(Long.valueOf(gotoTownUrls)).setName(countyName).setPid(Long.valueOf(cityCode)).setLevel("3");
                this.addChinas(chinas);
            }
        }
    
    }

> \_服务类 \_ChinasService

package com.hn.yuan.city.service;
    
    import com.hn.yuan.city.entity.Chinas;
    import com.baomidou.mybatisplus.extension.service.IService;
    
    /**
     * <p>
     *  服务类
     * </p>
     * @author XIAOCAO
     * 
     */
    public interface ChinasService extends IService<Chinas> {
        
        public void test(String appendProvide, String appendCity);
    }

> 测试类 testDemo

package com.hn.yuan.city;
    
    import com.hn.yuan.city.service.ChinasService;
    import org.junit.Test;
    import org.junit.runner.RunWith;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.boot.test.context.SpringBootTest;
    import org.springframework.test.context.junit4.SpringRunner;
    
    
    @RunWith(SpringRunner.class)
    @SpringBootTest
    public class testDemo {
        
    
        @Autowired
        private ChinasService chinasService;
    
        @Test
        public void tests() {
        
            //获取所有的数据
            chinasService.test(null, null);
            //获取指定的城市
            //chinasService.test("河南省", "郑州市");
    
        }
    
    }

> 注意：controller、service、mapper、entity层通过mybatis-plus自动生成的；此方法不涉及controller,mapper,entity；  
> 全面省市区县居委5级详情看转载：[https://blog.csdn.net/qq\_15421685/article/details/124754314][https_blog.csdn.net_qq_15421685_article_details_124754314]

#### Mysql数据表数据包含港澳台 ####

> **https://pan.baidu.com/s/1CwnmUGSPFzhZdwIWNTwd\_Q?pwd=1102**  
> **提取码：1102**

![在这里插入图片描述][5483b471975f4b77a27fac75bb488965.jpeg_pic_center]

> 各位看官》创作不易，点个赞！！！  
> 诸君共勉：万事开头难，只愿肯放弃。

免责声明：本文章仅用于学习参考

--------------------

[实战模式-Vue+Java后台实现省市区三级联动][-Vue_Java]  
[实战模式-微信小程序java后台+mysql实现省市区三级联动][-_java_mysql]  
[微信小程序组件简易实现省市区三级联动][Link 1]  
免责声明：本文章仅用于学习参考

[https_blog.csdn.net_qq_15421685_article_details_124754314]: https://blog.csdn.net/qq_15421685/article/details/124754314
[5483b471975f4b77a27fac75bb488965.jpeg_pic_center]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/08/8d1f2477c8594805812b76676a4dd425.jpeg
[-Vue_Java]: https://blog.csdn.net/SoulNone/article/details/126317092
[-_java_mysql]: https://blog.csdn.net/SoulNone/article/details/126317291
[Link 1]: https://blog.csdn.net/SoulNone/article/details/126317657