爬取链家任意城市二手房数据(天津)

桃扇骨 2023-06-03 04:00 6阅读 0赞

1 #!/usr/bin/env python
      2 # -*- coding: utf-8 -*-
      3 # @Time    : 2019-08-16 12:40
      4 # @Author  : Anthony
      5 # @Email   : ianghont7@163.com
      6 # @File    : 爬取链家任意城市二手房数据.py
      7 
      8 
      9 import requests
     10 from lxml import etree
     11 import time
     12 import xlrd
     13 import os
     14 import xlwt
     15 from xlutils.copy import copy
     16 
     17 # 伪装请求
     18 headers = {
     19     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.7 Safari/537.36'
     20 }
     21 
     22 xlsInfo = {}
     23 
     24 def catchHouseDetail(url):
     25     # 通过requests模块模拟get请求
     26     page_text = requests.get(url, headers=headers, stream=True)
     27 
     28     # 将互联网上获取的页面数据加载到etree对象中
     29     tree = etree.HTML(page_text.text)
     30 
     31     # 定位页面标签位置装入一个list中
     32     li_list = tree.xpath('//div[@class="leftContent"]/ul/li')
     33     all_house_list = []
     34     # 遍历列表中每一个字段
     35     for li in li_list:
     36         info = []
     37         # info = {}
     38         # info["房屋标题"] = li.xpath('.//div[@class="info clear"]/div[@class="title"]/a/text()')[0]
     39         # info["小区名称"] = li.xpath('.//div[@class="address"]/div[@class="houseInfo"]/text()')[0].split('|')[1]
     40         # info['建筑面积'] = li.xpath('.//div[@class="address"]/div[@class="houseInfo"]/text()')[0].split('|')[2]
     41         # info['房屋朝向'] = li.xpath('.//div[@class="address"]/div[@class="houseInfo"]/text()')[0].split('|')[3]
     42         # info['装修情况'] = li.xpath('.//div[@class="address"]/div[@class="houseInfo"]/text()')[0].split('|')[4]
     43         # info['所在楼层'] = li.xpath('.//div[@class="flood"]/div[@class="positionInfo"]/text()')[0].split(' ')[0]
     44         # info['所在区域'] = li.xpath('.//div[@class="flood"]/div[@class="positionInfo"]/a/text()')[0]
     45         # info['总价'] = li.xpath('.//div[@class="priceInfo"]/div[@class="totalPrice"]/span/text()')[0] + '万'
     46         # info['每平米售价'] = li.xpath('.//div[@class="priceInfo"]/div[@class="unitPrice"]/span/text()')[0]
     47         # info['房屋关注人数'] = li.xpath('.//div[@class="followInfo"]/text()')[0].split('/')[0]
     48         # info['房屋发布时间'] = li.xpath('.//div[@class="followInfo"]/text()')[0].split('/')[1]
     49 
     50         #房屋标题
     51         houseTitle = li.xpath('.//div[@class="info clear"]/div[@class="title"]/a/text()')[0]
     52         #小区名称
     53         houseName = li.xpath('.//div[@class="address"]/div[@class="houseInfo"]/text()')[0].split('|')[1]
     54         #建筑面积
     55         houseArea = li.xpath('.//div[@class="address"]/div[@class="houseInfo"]/text()')[0].split('|')[2]
     56         #房屋朝向
     57         houseTowards = li.xpath('.//div[@class="address"]/div[@class="houseInfo"]/text()')[0].split('|')[3]
     58         #装修情况
     59         houseFinish = li.xpath('.//div[@class="address"]/div[@class="houseInfo"]/text()')[0].split('|')[4]
     60         #所在楼层
     61         houseFloor = li.xpath('.//div[@class="flood"]/div[@class="positionInfo"]/text()')[0].split(' ')[0]
     62         #所在区域
     63         houseSite = li.xpath('.//div[@class="flood"]/div[@class="positionInfo"]/a/text()')[0]
     64         #总价
     65         housePrices = li.xpath('.//div[@class="priceInfo"]/div[@class="totalPrice"]/span/text()')[0] + '万'
     66         #每平米售价
     67         houseSquarePrices = li.xpath('.//div[@class="priceInfo"]/div[@class="unitPrice"]/span/text()')[0]
     68         #房屋关注人数
     69         houseFollowers = li.xpath('.//div[@class="followInfo"]/text()')[0].split('/')[0]
     70         #房屋发布时间
     71         houseTime = li.xpath('.//div[@class="followInfo"]/text()')[0].split('/')[1]
     72         info.append(houseTitle)
     73         info.append(houseName)
     74         info.append(houseArea)
     75         info.append(houseTowards)
     76         info.append(houseFinish)
     77         info.append(houseFloor)
     78         info.append(houseSite)
     79         info.append(housePrices)
     80         info.append(houseSquarePrices)
     81         info.append(houseFollowers)
     82         info.append(houseTime)
     83         all_house_list.append(info)
     84     if if_xls_exits() == True:
     85         write_excel_xls_append(xlsInfo["xlsName"],all_house_list)
     86 
     87 
     88 #获取数据写入xls表格中
     89 def write_excel_xls(path, sheet_name, value):
     90     index = len(value)  # 获取需要写入数据的行数
     91     workbook = xlwt.Workbook()  # 新建一个工作簿
     92     sheet = workbook.add_sheet(sheet_name)  # 在工作簿中新建一个表格
     93     for i in range(0, index):
     94         for j in range(0, len(value[i])):
     95             sheet.write(i, j, value[i][j])  # 像表格中写入数据（对应的行和列）
     96     workbook.save(path)  # 保存工作簿
     97     print("xls格式表格写入数据成功！")
     98 
     99 
    100 
    101 def write_excel_xls_append(path, value):
    102     index = len(value)  # 获取需要写入数据的行数
    103     workbook = xlrd.open_workbook(path)  # 打开工作簿
    104     sheets = workbook.sheet_names()  # 获取工作簿中的所有表格
    105     worksheet = workbook.sheet_by_name(sheets[0])  # 获取工作簿中所有表格中的的第一个表格
    106     rows_old = worksheet.nrows  # 获取表格中已存在的数据的行数
    107     new_workbook = copy(workbook)  # 将xlrd对象拷贝转化为xlwt对象
    108     new_worksheet = new_workbook.get_sheet(0)  # 获取转化后工作簿中的第一个表格
    109     for i in range(0, index):
    110         for j in range(0, len(value[i])):
    111             new_worksheet.write(i + rows_old, j, value[i][j])  # 追加写入数据，注意是从i+rows_old行开始写入
    112     new_workbook.save(path)  # 保存工作簿
    113     print("xls格式表格【追加】写入数据成功！")
    114 
    115 
    116 
    117 
    118 def if_xls_exits():
    119     while True:
    120         book_name_xls = '天津链家二手房信息表.xls'
    121         sheet_name_xls = '房屋信息'
    122         value_title = [["房屋标题", "房屋户型", "建筑面积", "房屋朝向", "装修情况", "所在楼层", "所在区域", "总价", "每平米售价", "房屋关注人数", "房屋发布时间"], ]
    123         if os.path.exists('./%s'%book_name_xls):
    124             xlsInfo["xlsName"] = book_name_xls
    125             return True
    126         else:
    127             write_excel_xls(book_name_xls, sheet_name_xls, value_title)
    128             continue
    129 
    130 
    131 
    132 def catch():
    133     pages = ['https://tj.lianjia.com/ershoufang/pg{}/'.format(x) for x in range(1, 1000)]
    134     for page in pages:
    135         try:
    136             info = catchHouseDetail(page)
    137         except:
    138             pass
    139         time.sleep(3)
    140 
    141 
    142 if __name__ == '__main__':
    143     catch()

## **效果图：** ##

![1394549-20190816193237550-1251091806.png][]

转载于:https://www.cnblogs.com/ipyanthony/p/11365962.html

[1394549-20190816193237550-1251091806.png]: /images/20230601/58b1e1a0041843689b26a693bbbd619d.png