python爬虫——爬取房天下

柔情只为你懂 2021-07-27 00:39 870阅读 0赞

## python爬虫——爬取房天下 ##

**话不多说，直接上代码！**

import requests as req
    import time
    import pandas as pd
    from bs4 import BeautifulSoup
    from sqlalchemy import create_engine
     
    global info
    
    def getHouseInfo(url):
    	info = { }
    	soup = BeautifulSoup(req.get(url).text,"html.parser")
    	resinfo = soup.select(".tab-cont-right .trl-item1")
    	# 获取户型、建筑面积、单价、朝向、楼层、装修情况
    	for re in resinfo:
    		tmp = re.text.strip().split("\n")
    		name = tmp[1].strip()
    		if("朝向" in name):
    			name = name.strip("进门")
    		if("楼层" in name):
    			name = name[0:2]
    		if("地上层数" in name):
    			name = "楼层"
    		if("装修程度" in name):
    			name = "装修"
    		info[name] = tmp[0].strip()
    	xiaoqu = soup.select(".rcont .blue")[0].text.strip()
    	info["小区名字"] = xiaoqu
    	zongjia = soup.select(".tab-cont-right .trl-item")
    	info["总价"] = zongjia[0].text
    	return info
     
    domain = "http://esf.anyang.fang.com/"
    city = "house/"
    
    #获取总页数
    def getTotalPage():
    	res = req.get(domain+city+"i31")
    	soup = BeautifulSoup(res.text, "html.parser")
    	endPage = soup.select(".page_al a").pop()['href']
    	pageNum = endPage.strip("/").split("/")[1].strip("i3")
    	print("loading.....总共 "+pageNum+" 页数据.....")
    	return pageNum
     
    # 分页爬取数据
    def pageFun(i):
    	pageUrl = domain + city + "i3" +i
    	print(pageUrl+" loading...第 "+i+" 页数据.....")
    	res = req.get(pageUrl)
    	soup = BeautifulSoup(res.text,"html.parser")
    	houses = soup.select(".shop_list dl")
    	pageInfoList = []
    	for house in houses:
    		try:
    			# print(domain + house.select("a")[0]['href'])
    			info = getHouseInfo(domain + house.select("a")[0]['href'])
    			pageInfoList.append(info)
    			print(info)
    		except Exception as e:
    			print("---->出现异常，跳过 继续执行",e)
    	df =  pd.DataFrame(pageInfoList)
    	return df
     
    connect = create_engine("mysql+pymysql://root:root@localhost:3306/houseinfo?charset=utf8")
     
    for i in range(1,int(getTotalPage())+1):
    	try:
    		df_onePage = pageFun(str(i))
    	except Exception as e:
    		print("Exception",e)
    	pd.io.sql.to_sql(df_onePage, "city_house_price", connect, schema="houseinfo", if_exists="append")

**运行结果：**  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2d1b3h1eWluZw_size_16_color_FFFFFF_t_70_pic_center]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2d1b3h1eWluZw_size_16_color_FFFFFF_t_70_pic_center 1]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2d1b3h1eWluZw_size_16_color_FFFFFF_t_70_pic_center]: /images/20210726/07a71e0e54ad4af5a5fee6b69d126c37.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2d1b3h1eWluZw_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20210726/ce74c49a908d4b8ba3034ece092062d9.png