爬虫:爬取糗事百科数据

青旅半醒 2022-01-26 04:45 367阅读 0赞

在这里插入图片描述

  1. import requests
  2. from lxml import etree
  3. from fake_useragent import UserAgent
  6. class Qiu:
  8.     def __init__(self):
  9.         self.count = 1
  11.     def __call__(self, *args, **kwargs):
  12.         self.get_max_page()
  14.     def get_html(self, base_url):
  15.         # 随机产生一个浏览器信息
  16.         headers = {"User-Agent": UserAgent().random}
  17.         response = requests.get(base_url, headers=headers)
  18.         html = response.text
  19.         # print(html)
  20.         html_xml = etree.HTML(html)
  21.         return html_xml
  24.     def get_max_page(self):
  25.         base_url = "https://www.qiushibaike.com/8hr/page/2/"
  26.         html_xml = self.get_html(base_url)
  27.         # 获取最大页码
  28.         max_page = int(html_xml.xpath("//a/span[@class='page-numbers']/text()")[-1].strip())
  29.         # print(max_page)
  30.         self.get_data(max_page)
  32.     def get_data(self, max_page):
  34.         for page in range(1, max_page + 1):
  35.             print("===================第{}页开始下载=========================".format(page))
  36.             page_url = "https://www.qiushibaike.com/8hr/page/{}/".format(page)
  37.             # print(page_url)
  38.             html_xml = self.get_html(page_url)
  39.             # 缩小范围
  40.             li_list = html_xml.xpath("//li[contains(@id, 'qiushi_tag_')]")
  41.             # print(len(li_list))
  43.             for li in li_list:
  44.                 # 获取图片
  45.                 pic = li.xpath(".//a[contains(@class, 'recmd-left')]/img/@src")[0]
  46.                 # if "/w/150/h/112" in pic:
  47.                 #     pic = "https:" + pic[:-12]
  48.                 # else:
  49.                 #     pic = ""
  51.                 # 三元表达式 实现上面的代码
  52.                 pic = "https:" + pic[:-12] if "/w/150/h/112" in pic else ""
  53.                 # print(pic)
  55.                 # 获取昵称
  56.                 nike_name = li.xpath(".//span[@class='recmd-name']/text()")[0]
  57.                 # print(nike_name)
  59.                 # 获取内容
  60.                 content = li.xpath(".//a[@class='recmd-content']/text()")
  61.                 content = content[0] if content else ""
  62.                 # print(content)
  64.                 # 获取好笑数量
  65.                 laught_num = li.xpath(".//div[@class='recmd-num']/span[1]/text()")[0]
  66.                 # if "万" in laught_num:
  67.                 #     laught_num = int(float(laught_num[:-1]) * 10000)
  68.                 # else:
  69.                 #     laught_num = int(laught_num)
  70.                 laught_num = int(float(laught_num[:-1]) * 10000) if "万" in laught_num else int(laught_num)
  71.                 # print(laught_num)
  73.                 # 评论数量
  74.                 comment_num = li.xpath(".//div[@class='recmd-num']/span[4]/text()")
  75.                 comment_num = int(comment_num[0]) if comment_num else 0
  76.                 # print(comment_num)
  78.                 qiu_dict = {
  79.                     "pic": pic,
  80.                     "nike_name": nike_name,
  81.                     "content": content,
  82.                     "laught_num": laught_num,
  83.                     "comment_num": comment_num,
  85.                 }
  86.                 print(self.count, qiu_dict)
  87.                 self.count += 1
  90. if __name__ == '__main__':
  91.     qiu = Qiu()
  92.     qiu()

发表评论

表情：
评论列表 （有 0 条评论，367人围观）

还没有评论，来说两句吧...

相关阅读

    相关 糗事百科小爬虫

    这几天看了不少phtyon 的基础，试着做了一个daemo 但不是很成功 不知道家里网络不太好还正则匹配的不好，re.findall 的数据不是特别的稳定，有时候要加载很长

    r囧r小猫r囧r小猫/ 0 赞/ 268 阅读