爬虫之requests+lxml+json获取糗百某分类数据（返回数据是html）

雨点打透心脏的1/2处 2021-12-21 05:11 73阅读 0赞

import requests, json
    from lxml import etree
    
    
    class QiubaiSpider(object):
    
        # 初始化url
        def __init__(self):
            self.origin_url = "https://www.qiushibaike.com/hot/page/{}/"
            self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"}
    
        # 因为这里发现url规律比较好找，可以事先构造url列表
        def get_url_list(self):
            url_list = [self.origin_url.format(page_num) for page_num in range(1, 13)]
            return url_list
    
        # 发送请求获取数据
        def parse_url(self, url):
            response = requests.get(url, headers=self.headers)
            return response.text
    
        # 提取数据
        def get_content_list(self, html_str):
            html = etree.HTML(html_str)
            # 经过分析发现，每一条数据都是存在一个div标签中的，所以这里按照div分组
            group_div_list = html.xpath("//div[@id='content-left']/div")
            content_list = []
            for div in group_div_list:
                content_dict = {}
                content_dict["user_name"] = div.xpath(".//h2/text()")[0].strip() if len(div.xpath("//h2/text()")) != 0 else None
                content_dict["user_age"] = div.xpath(".//div[@class='articleGender womenIcon']/text()")[0] if len(div.xpath(".//div[@class='articleGender womenIcon']/text()")) != 0 else None
                content_dict["user_gender"] = div.xpath(".//div[@class='author clearfix']/div/@class")[0] if len(div.xpath(".//div[@class='author clearfix']/div/@class")) != 0 else None
                content_dict["user_image"] = div.xpath(".//div[@class='author clearfix']/a/img/@src")
                content_dict["user_image"] = "https" + content_dict["user_image"][0] if len(div.xpath(".//div[@class='author clearfix']/a/img/@src")) != 0 else None
                content_dict["comments"] = div.xpath(".//div[@class='content']/span/text()") if len(div.xpath(".//div[@class='content']/span/text()")) != 0 else None
                content_dict["comments"] = [i.strip() for i in content_dict["comments"]]
                content_dict["stats-vote"] = div.xpath(".//div[@class='stats']/span[@class='stats-vote']/i/text()")[0] if len(div.xpath(".//div[@class='stats']/span[@class='stats-vote']/i/text()")) != 0 else None
                content_dict["stats-comments"] = div.xpath(".//div[@class='stats']/span[@class='stats-comments']//i/text()")[0] if len(div.xpath(".//div[@class='stats']/span[@class='stats-comments']//i/text()")) != 0 else None
                content_list.append(content_dict)
            return content_list
    
        # 保存数据
        def save_data(self,content_list):
            with open("qiubai.txt", "a", encoding="utf-8") as f:
                for content_dict in content_list:
                    print(content_dict)
                    f.write(json.dumps(content_dict, ensure_ascii=False))
                    f.write("\n")
            print("保存成功")
    
        def run(self):
            # 1-获取url
            url_list = self.get_url_list()
            # 2-发送请求,获取数据
            for url in url_list:
                # 3-提取数据
                html_str = self.parse_url(url)
                content_list = self.get_content_list(html_str)
                # 4-保存
                self.save_data(content_list)
    
    
    if __name__ == '__main__':
        qiubai = QiubaiSpider()
        qiubai.run()