java爬取携程酒店数据存到本地数据库

我不是女神ヾ 2023-08-17 16:57 109阅读 0赞

话不多说，直接上代码

1：配置依赖

</dependency>
        
        <dependency>
          <groupId>org.jsoup</groupId>
          <artifactId>jsoup</artifactId>
          <version>1.8.3</version>
        </dependency>
    
        <dependency>
          <groupId>org.json</groupId>
          <artifactId>json</artifactId>
          <version>20160810</version>
        </dependency>
    
    
        <dependency>
          <groupId>com.alibaba</groupId>
          <artifactId>fastjson</artifactId>
          <version>1.2.47</version>
        </dependency>
    
        <dependency>
          <groupId>mysql</groupId>
          <artifactId>mysql-connector-java</artifactId>
          <version>5.1.47</version>
        </dependency>
    
        <dependency>
          <groupId>org.mybatis</groupId>
          <artifactId>mybatis</artifactId>
          <version>3.5.1</version>
        </dependency>

2：数据库表bean对象

public class Hotel {
        /**
         * 酒店id，酒店价格，酒店所在城市，酒店名称，酒店图片，酒店经纬度（地图），酒店地址,星级，评分
         */
        private String hid;
        private String price;
        private  String city;
        private String name;
        private String pic;
        private String xpath;
        private String ypath;
        private String addr;
        private String start;
        private String score;
    
        public String getHid() {
            return hid;
        }
    
        public void setHid(String hid) {
            this.hid = hid;
        }
    
        public String getPrice() {
            return price;
        }
    
        public void setPrice(String price) {
            this.price = price;
        }
    
        public String getCity() {
            return city;
        }
    
        public void setCity(String city) {
            this.city = city;
        }
    
        public String getName() {
            return name;
        }
    
        public void setName(String name) {
            this.name = name;
        }
    
        public String getPic() {
            return pic;
        }
    
        public void setPic(String pic) {
            this.pic = pic;
        }
    
        public String getXpath() {
            return xpath;
        }
    
        public void setXpath(String xpath) {
            this.xpath = xpath;
        }
    
        public String getYpath() {
            return ypath;
        }
    
        public void setYpath(String ypath) {
            this.ypath = ypath;
        }
    
        public String getAddr() {
            return addr;
        }
    
        public void setAddr(String addr) {
            this.addr = addr;
        }
    
        public String getStart() {
            return start;
        }
    
        public void setStart(String start) {
            this.start = start;
        }
    
        public String getScore() {
            return score;
        }
    
        public void setScore(String score) {
            this.score = score;
        }
    
        @Override
        public String toString() {
            return "Hotel{" +
                    "hid='" + hid + '\'' +
                    ", price='" + price + '\'' +
                    ", city='" + city + '\'' +
                    ", name='" + name + '\'' +
                    ", pic='" + pic + '\'' +
                    ", xpath='" + xpath + '\'' +
                    ", ypath='" + ypath + '\'' +
                    ", addr='" + addr + '\'' +
                    ", start='" + start + '\'' +
                    ", score='" + score + '\'' +
                    '}';
        }
    }

3：数据库配置

driver=com.mysql.jdbc.Driver
    url=jdbc:mysql://localhost:3306/数据库表名
    username=***
    password=***

4：mybatis配置

<?xml version="1.0" encoding="UTF-8" ?>
    <!DOCTYPE configuration
            PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
            "http://mybatis.org/dtd/mybatis-3-config.dtd">
    <configuration>
        <properties resource="jdbc.properties"></properties>
        <environments default="development">
            <environment id="development">
                <transactionManager type="JDBC"/>
                <dataSource type="POOLED">
                    <property name="driver" value="${driver}"/>
                    <property name="url" value="${url}"/>
                    <property name="username" value="${username}"/>
                    <property name="password" value="${password}"/>
                </dataSource>
            </environment>
        </environments>
        <mappers>
            <mapper resource="HotelMapper.xml"/>
        </mappers>
    </configuration>

5：dao接口，为插入数据做准备

public interface HotelMapper {
        /**
         * 插入数据
         * @param hotel
         * @return
         */
        public int insert (Hotel hotel);
    }

6：做dao类的映射文件

<!DOCTYPE mapper
            PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
            "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
    <mapper namespace="com.yc.dao.HotelMapper">
        <insert id="insert" parameterType="com.yc.model.Hotel">
            insert into hotel(id,hid,name,price,city,xpath,ypath,pic,addr,start,score)
            values (null,#{hid},#{name},#{price},#{city},#{xpath},#{ypath},#{pic},#{addr},#{start},#{score})
        </insert>
    </mapper>

7：终极奥义，爬虫代码

public class Test02 {
    
    
        public static void main(String[] args) throws IOException, InterruptedException {
            Hotel hotel = new Hotel();
            String resource = "mybatis-config.xml";
            InputStream inputStream = null;
            try {
                //读取配置文件
                inputStream = Resources.getResourceAsStream(resource);
            } catch (IOException e) {
                e.printStackTrace();
            }
            SqlSessionFactory sqlSessionFactory = new SqlSessionFactoryBuilder().build(inputStream);
            //注册mybatis 工厂
    
            //得到连接对象
            SqlSession sqlSession = sqlSessionFactory.openSession(true);
            //从mybatis中得到dao对象
            HotelMapper mapper = sqlSession.getMapper(HotelMapper.class);
            System.out.println(getHotelInfoByCity("guangzhou32"));
            String str = "guangzhou32/p";
            String str2;
            int x=0;
            for (int i = 1; i < 100; i+=2) {
                str2 = str + i;
                Thread.sleep(5000);
                List<List<String>> lists = getHotelInfoByCity(str2);
                for (List<String> list : lists) {
                    hotel.setHid(list.get(0));
                    hotel.setCity(list.get(1));
                    hotel.setPrice(list.get(2));
                    hotel.setName(list.get(3));
                    hotel.setXpath(list.get(4));
                    hotel.setYpath(list.get(5));
                    hotel.setPic(list.get(6));
                    hotel.setAddr(list.get(7));
                    hotel.setScore(list.get(8));
                    if (list.get(9)=="" ||list.get(9).isEmpty()){
                        list.set(9,"0");
                    }
                    hotel.setStart(list.get(9).charAt(list.get(9).length()-1)+"");
                    System.out.println(mapper.insert(hotel));
                    sqlSession.commit();
                }
                x+=25;
                System.out.println("已爬取"+x+"条数据");
            }
    
        }
    
        public static List<List<String>> getHotelInfoByCity(String city) throws IOException {
    
            //首先输入要爬的网页
            URL url = new URL("https://hotels.ctrip.com/hotel/" + city);
    
            //建立连接
            URLConnection urlConnection = url.openConnection();
            HttpURLConnection connection = (HttpURLConnection) urlConnection;
    
            // 建立一个读取流从连接中读取
            BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(),
                    "utf-8"));
    
            String current; // 爬虫爬下来的每一行数据
            // 是否找到关键行1
            boolean hadfindhtllst = false;
            // 是否找到关键行2
            boolean hadfindhotelPositionJSON = false;
            // 内含hotelid,酒店剩余房间数
            char[] htllist = new char[10005];
            // 存放目标char[]->String
            String roomnum = null;
            // 内涵hotelid,酒店全称，酒店地址，酒店精确经纬度
            char[] hotelPositionJSON = new char[50005];
            // 存放目标char[]->String
            String hotelinfo = null;
    
            while ((current = in.readLine()) != null) {
                // 如果还没找到关键行
                if (hadfindhtllst == false) {
                    if (current.length() < 19) {
                        // 如果该行长度小于19 直接跳过
                        continue;
                    }
                    current.getChars(12, 19, htllist, 0);
                    if (charsequals(htllist, "htllist", 7)) {
                        // 找到了关建行
                        hadfindhtllst = true;
                        current.getChars(12, current.length(), htllist, 0);
                        roomnum = new String(htllist, 0, current.length() - 12);
                    }
                }
                // 如果还没找到关键行
                if (hadfindhotelPositionJSON == false) {
                    if (current.length() < 25) {
                        // 如果改行长度小于25 直接跳过
                        continue;
                    }
                    current.getChars(8, 25, hotelPositionJSON, 0);
                    if (charsequals(hotelPositionJSON, "hotelPositionJSON", 17)) {
                        // 找到了关建行
                        hadfindhotelPositionJSON = true;
                        current.getChars(8, current.length(), hotelPositionJSON, 0);
                        hotelinfo = new String(hotelPositionJSON, 0, current.length() - 8);
                    }
                }
            }
    
            // 经过爬虫后，有用的信息只有 String roomnum(内含酒店id，酒店剩余房间数)
            // String hotelinfo(内含酒店id，酒店全名，酒店位置，酒店经纬度，酒店评分)
            //System.err.println(roomnum+"........");
            //System.err.println(hotelinfo+"--------");
    
            if (roomnum == null || hotelinfo == null) {
                return null;
            }
    
            List<List<String>> lists = new ArrayList<List<String>>();
    
            // 添加酒店id，酒店城市，酒店余房数量
            // cs为"出现的次数
            for (int i = 0, cs = 0, now = 0; i < roomnum.length(); i++) {
                if (roomnum.charAt(i) == '\"') {
                    cs++;
                    if (cs % 8 == 3) {
                        List<String> list = new ArrayList<String>();
                        String temp = "";
                        while (roomnum.charAt(++i) != '\"') {
                            temp += roomnum.charAt(i);
    
                        }
                        i--;
                        list.add(temp);
                        if (city.equals("chengdu28")) {
                            list.add("成都");
                        } else if (city.equals("guangzhou32")) {
                            list.add("广州");
                        } else if (city.equals("beijing1")) {
                            list.add("北京");
                        } else {
                            list.add("未知城市");
                        }
                        lists.add(list);
                    }
                    if (cs % 8 == 7) {
                        String temp = "";
                        while (roomnum.charAt(++i) != '\"') {
                            temp += roomnum.charAt(i);
    
                        }
                        i--;
                        lists.get(now++).add(temp);
                    }
                }
            }
    
            // 添加酒店名，纬度lat，经度lon，酒店照片，酒店地址，酒店评分，酒店接待能力（星级）
            for (int i = 0, cs = 0, now = 0; i < hotelinfo.length(); i++) {
                if (hotelinfo.charAt(i) == '\"') {
                    cs++;
                    if (cs % 56 == 7 || cs % 56 == 11 || cs % 56 == 15
                            || cs % 56 == 23 || cs % 56 == 27
                            || cs % 56 == 31 || cs % 56 == 43) {
                        String temp = "";
                        while (hotelinfo.charAt(++i) != '\"') {
                            temp += hotelinfo.charAt(i);
                        }
                        i--;
                        lists.get(now).add(temp);
                    }
                    if (cs % 56 == 43) {
                        now++;
                    }
                }
            }
            return lists;
        }
    
        // 查询char[]型 与 String型是否相等
        public static boolean charsequals(char[] left, String right, int length) {
            boolean result = true;
            for (int i = length - 1; i >= 0; i--) {
                result &= left[i] == right.charAt(i);
            }
            return result;
        }
    }