python：爬取天气预报+美女网站数据信息

我不是女神ヾ 2022-02-03 09:13 456阅读 0赞

一、爬取湛江天气预报信息

url：[http://www.weather.com.cn/weather/10128100101A.shtml][http_www.weather.com.cn_weather_10128100101A.shtml]

1.思路分析：

①先爬取网页所有数据，div/ul/li，获得7天天气预报的所有信息；

②li下的所有数据进行提取数据；

③保存文件。 --文件操作、json模块。

import requests
    import lxml.html
    import json
    
    def parse_url(url,header):
        """"解析url地址，获得网页的所有数据内容"""
        response = requests.get(url,headers=header)
        # return response.text  会出现乱码
    
        #改为content二进制数据，再转为字符串decode即可
        return response.content.decode("utf-8")  
    
    
    def get_weather_datas(html_content):
    
        """"从所有html数据信息中获取ul>li标签中的所有天气信息"""
    
        metree = lxml.html.etree
    
        # 1 获得解析对象(构造一个 XPath 解析对象并对 HTML 文本进行自动修正)
        parser = metree.HTML(html_content,metree.HTMLParser())
    
        # 2 使用Xpath语法获得li所有标签
        li_list = parser.xpath("//div[@class='c7d']/ul[@class='t clearfix']/li")
        # print(li_list)
        # print(len(li_list))  7
    
        # 3 获得li标签下的所有信息
            # 先创建一个空列表
        data = []
            #  循环遍历
        for ele in li_list:
            # 创建一个空字典
            item = {}
            # 继续使用XPath语法
            item["date"] = ele.xpath("./h1/text()")[0]
            item["天气"] = ele.xpath("./p[@class='wea']/text()")[0]
            # item["wea"] = ele.xpath("./p[@class='wea']/@title")[0]  通过title属性获取
            item["最低温度"] = ele.xpath("./p[@class='tem']/i/text()")[0]
            item["最高温度"] = ele.xpath("./p[@class='tem']/span/text()")[0]
            # print(item["最高温度"])
            data.append(item)
            # print(data)
        return data
    
    def save_weather_file(datas):
        """"保存文件"""
    
        # 列表转json字符串,并保存到文件中
        json_strs = json.dumps(datas,ensure_ascii=False,indent=2)
        # print(json_strs)
        # print(type(json_strs))
        with open("./file/weather.json","w",encoding="utf-8") as files:
            files.write(json_strs)
        print("数据保存成功！")
    
    def main():
        # ①div/ul/li，获得7天天气预报的所有信息；  --lxml、requests、XPath   --list类型
        # 网址、请求头
        # print("HelloWorld...")
        http_url = "http://www.weather.com.cn/weather/10128100101A.shtml"
        headers = \
            {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"}
        # 解析url
        html_data = parse_url(http_url,headers)
        # print(html_data)
    
        # ②li下的所有数据进行提取数据；  --对上述的7条数据进行处理，提取数据  XPath   --列表
        weather_datas = get_weather_datas(html_data)
        # print(weather_datas)
    
    
        # ③保存文件。  --文件操作、json模块
        save_weather_file(weather_datas)
    
    
    """"python程序入口，通常写在程序末尾处"""
    if __name__ == '__main__':
        main()

2.爬取的数据：

![20190509093532428.png][]  ![20190509093638918.png][]

--------------------

二 、爬取美女网站信息

url：http://www.xiaohuar.com/list-1-1.html

1.思路分析

①获得所有美女图片的div列表；

②在当前美女的div中获得所有信息；

③保存数据。

import requests
    import lxml.html
    import json
    
    def parse_url(url,header):
        """"解析url地址，获取所有网页数据信息"""
    
        response = requests.get(url,headers=header)
        return response.content.decode("gbk")
    
    def get_xiaohua_datas(html_content):
        """"获取数据内容"""
    
        metree = lxml.html.etree
        # 解析对象
        parser = metree.HTML(html_content,metree.HTMLParser())
        # 解析获得②在当前美女的div中获得所有信息
        div_list = parser.xpath("//div[@class='item_list infinite_scroll']/div")
            # 创建空列表
        data = []
        # print(div_list)
        # print(len(div_list))  25
        for ele in div_list:
            item = {}  #创建空字典
            # 继续使用xpath 语法
            item["title"] = ele.xpath("./div[@class='item_t']/div[@class='img']/a/img/@alt")[0]
            item["name"] = ele.xpath("./div[@class='item_t']/div[@class='img']/span/text()")[0]
            item["school"] = ele.xpath("./div[@class='item_t']/div[@class='img']/div[@class='btns']/a/text()")[0]
            item["like_count"] = ele.xpath("./div[@class='item_b clearfix']/div[@class='items_likes fl']/em/text()")[0]
            # print(item["like_count"])
            data.append(item)
        # print(data)
        return data
    
    def save_xiaohua_file(datas):
        """"保存文件"""
    
        json_strs = json.dumps(datas,ensure_ascii=False,indent=2)
        with open("./file/xiaohua.json","w",encoding="utf-8") as files:
            files.write(json_strs)
        print("数据保存成功！")
    
    def main():
        xiaohua_url = "http://www.xiaohuar.com/list-1-1.html"
        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"}
        html_data = parse_url(xiaohua_url,headers)
        # print(html_data)
    
        # ①获得所有美女图片的div列表
        xiaohua_datas = get_xiaohua_datas(html_data)
    
        # ③保存数据
        save_xiaohua_file(xiaohua_datas)
    
    
    if __name__ == '__main__':
        main()

2.爬取的数据

[
      {
        "title": "大连国际舞蹈学校校花王钰萌",
        "name": "王钰萌",
        "school": "大连国际舞蹈学校",
        "like_count": "159"
      },
      {
        "title": "南昌大学校花曾阳",
        "name": "曾阳",
        "school": "南昌大学",
        "like_count": "220"
      },
      {
        "title": "中国民航大学校花张金玉",
        "name": "张金玉",
        "school": "中国民航大学",
        "like_count": "109"
      },
      {
        "title": "天津财经大学校花卓娅祺",
        "name": "卓娅祺",
        "school": "天津财经大学",
        "like_count": "361"
      },
      {
        "title": "新疆农业大学校花麦合丽娅",
        "name": "麦合丽娅",
        "school": "新疆农业大学",
        "like_count": "53"
      },
      {
        "title": "成都职业技术学院校花杨萍",
        "name": "杨萍",
        "school": "成都职业技术学院",
        "like_count": "108"
      },
      {
        "title": "东北师范大学校花尹思凝",
        "name": "尹思凝",
        "school": "东北师范大学",
        "like_count": "109"
      },
      {
        "title": "北京理工大学珠海学院校花韦若愚",
        "name": "韦若愚",
        "school": "北京理工大学珠海学院",
        "like_count": "122"
      },
      {
        "title": "厦门理工学院校花袁慧",
        "name": "袁慧",
        "school": "厦门理工学院",
        "like_count": "78"
      },
      {
        "title": "湖北艺术学院校花王媛茜",
        "name": "王媛茜",
        "school": "湖北艺术学院",
        "like_count": "96"
      },
      {
        "title": "文光中心校花陈里佳",
        "name": "陈里佳",
        "school": "文光中心",
        "like_count": "48"
      },
      {
        "title": "大连外国语大学校花高梦馨",
        "name": "高梦馨",
        "school": "大连外国语大学",
        "like_count": "115"
      },
      {
        "title": "舟山技师学院校花宋世杰",
        "name": "宋世杰",
        "school": "舟山技师学院",
        "like_count": "99"
      },
      {
        "title": "上海财经大学校花徐逸岑",
        "name": "徐逸岑",
        "school": "上海财经大学",
        "like_count": "123"
      },
      {
        "title": "武汉大学校花丁婷婷",
        "name": "丁婷婷",
        "school": "武汉大学",
        "like_count": "121"
      },
      {
        "title": "行健学院校花徐艳琛",
        "name": "徐艳琛",
        "school": "行健学院",
        "like_count": "149"
      },
      {
        "title": "上海交通大学校花唐雨乔",
        "name": "唐雨乔",
        "school": "上海交通大学",
        "like_count": "105"
      },
      {
        "title": "温州大学校花汤以斯贴",
        "name": "汤以斯贴",
        "school": "温州大学",
        "like_count": "289"
      },
      {
        "title": "华东大学校花赵梦洁",
        "name": "赵梦洁",
        "school": "华东大学",
        "like_count": "604"
      },
      {
        "title": "鄞州职业高级中学校花翁川美",
        "name": "翁川美",
        "school": "鄞州职业高级中学",
        "like_count": "109"
      },
      {
        "title": "中央戏剧学院校花刘垚昕",
        "name": "刘垚昕",
        "school": "中央戏剧学院",
        "like_count": "585"
      },
      {
        "title": "星源初中校花廖炯炅",
        "name": "廖炯炅",
        "school": "星源初中",
        "like_count": "99"
      },
      {
        "title": "广州华夏职业学院校花邓杏琳",
        "name": "邓杏琳",
        "school": "广州华夏职业学院",
        "like_count": "97"
      },
      {
        "title": "芷江师范校花滕之雅",
        "name": "滕之雅",
        "school": "芷江师范",
        "like_count": "208"
      },
      {
        "title": "铁岭师范校花施玉",
        "name": "施玉",
        "school": "铁岭师范",
        "like_count": "186"
      }
    ]

--------------------

三、总结

爬取数据的基本步骤（以案例二为例）

1.写入对应的url

2.请求头，解析url地址，获取所有网页数据信息：

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"} 
    
    html_data = parse_url(xiaohua_url,headers)

3.获取数据后可以获得所有美女图片的div列表

xiaohua_datas = get_xiaohua_datas(html_data)

4.根据需要的内容来保存数据

save_xiaohua_file(xiaohua_datas)

[http_www.weather.com.cn_weather_10128100101A.shtml]: http://www.weather.com.cn/weather/10128100101A.shtml
[20190509093532428.png]: https://img-blog.csdnimg.cn/20190509093532428.png
[20190509093638918.png]: /images/20220203/67de69cc19a1489ca4a0ec4c71283edb.png