python 爬虫爬取小说信息

桃扇骨 2022-04-02 11:42 670阅读 0赞

1.进入小说主页（以下示例是我在网上随便找的一片小说），获取该小说的名称、作者以及相关描述信息

2.获取该小说的所有章节列表信息（最重要的是每个章节的链接地址href）

3.根据每个章节的地址信息下载每个章节的内容并解析

4.将解析后的内容打印并写入文件或者数据库

示例代码v1版如下（仅供学习交流）：

  1. # ！/usr/bin/env python
  2. # -*-coding:utf-8-*-
  3. """
  4. @Author  : xiaofeng
  5. @Time    : 2018/12/26 11:41
  6. @Desc : Less interests,More interest.
  7. @Project : python_appliction
  8. @FileName: dianxs.py
  9. @Software: PyCharm
  10. @Blog    ：https://blog.csdn.net/zwx19921215
  11. """
  13. import requests
  14. import time
  15. from lxml import html
  16. import os
  18. """
  19. 简单爬取小说文章内容（'殿行说小说网'）
  20. """
  23. class Dianxs():
  24.     # 构造函数初始化
  25.     def __init__(self, host, url, headers, path):
  26.         self.host = host
  27.         self.url = url
  28.         self.headers = headers
  29.         self.path = path
  31.     """
  32.     下载并解析小说主页信息，获取小说列表
  33.     """
  35.     def download_page(self):
  36.         response = requests.get(url=self.url, headers=self.headers)
  37.         text = html.fromstring(response.text)
  38.         novel = text.xpath('//div[@class="info"]/h1/text()')
  39.         author = text.xpath('//div[@class="info"]/p[1]/text()')
  40.         act = text.xpath('//div[@class="info"]/p[2]/text()')
  41.         last_update = text.xpath('//div[@class="info"]/p[3]/text()')
  42.         last_chapter_href = text.xpath('//div[@class="info"]/p[4]/a/@href')
  43.         last_chapter_title = text.xpath('//div[@class="info"]/p[4]/a/text()')
  44.         introduce = text.xpath('//div[@class="info"]/p[@class="introduce"]/text()')
  45.         print('---------------------------description--------------------------------')
  46.         print(novel)
  47.         # author[0].replace('\xa0', '')
  48.         print(author)
  49.         print(act)
  50.         print(last_update)
  51.         print(last_chapter_title, ' , ', last_chapter_href)
  52.         print('简介：', introduce)
  53.         print('-----------------------------------------------------------------------')
  54.         print('\n')
  55.         chapters = text.xpath('//div[@class="section-panel section-list"]/dl/dd/a/text()')
  56.         hrefs = text.xpath('//div[@class="section-panel section-list"]/dl/dd/a/@href')
  57.         print(chapters)
  58.         print(hrefs)
  59.         print('\n')
  61.         for href in hrefs:
  62.             time.sleep(1)
  63.             address = self.host + href
  64.             self.parse_html(address)
  66.     """
  67.     解析文章内容
  68.     @:param address 章节页地址
  69.     """
  71.     def parse_html(self, address):
  72.         response = requests.get(url=address, headers=self.headers, timeout=10)
  73.         if response.status_code != 200:
  74.             self.parse_html(address)
  75.         text = html.fromstring(response.text)
  76.         title = text.xpath('//div[@class="read-title"]/h2/text()')
  77.         content = text.xpath('//div[@class="read-content"]/p/text()')
  78.         print('-------- ', title, '-----------')
  79.         print(content)
  80.         print('\n')
  81.         # ''.join(content)：list集合转字符串；list = list(string)：字符串转list集合
  82.         title_str = ''.join(title)
  83.         content_str = ''.join(content)
  84.         self.write_to_file(title_str, content_str)
  86.     """
  87.     章节内容写入文件
  88.     """
  90.     def write_to_file(self, title, content):
  91.         flag = os.path.exists(self.path)
  92.         if not flag:
  93.             # 'w' 表示写模式，没有文件则会创建一个
  94.             f = open(self.path, 'w')
  95.             f.close()
  97.         # with 可以不用显示调用close方法
  98.         # 'a' 表示追加写入
  99.         with open(self.path, mode='a', encoding='utf-8') as file:
  100.             file.write(title + '\n')
  101.             file.writelines(content)
  102.             file.write('\n\n')
  105. if __name__ == '__main__':
  106.     host = 'https://www.dianxs.com'
  107.     url = 'https://www.dianxs.com/book/64554/'
  108.     headers = {
  109.         'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
  110.     }
  111.     path = 'G:/test/novel.txt'
  112.     app = Dianxs(host, url, headers, path)
  113.     app.download_page()

控制台输出：

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3eDE5OTIxMjE1_size_16_color_FFFFFF_t_70

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3eDE5OTIxMjE1_size_16_color_FFFFFF_t_70 1

文件写入内容：

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3eDE5OTIxMjE1_size_16_color_FFFFFF_t_70 2

v2版：从网站主页开始深度优先爬取所有分类栏目以及栏目下所有小说信息

改进v2版示例如下

  1. # ！/usr/bin/env python
  2. # -*-coding:utf-8-*-
  3. """
  4. @Author  : xiaofeng
  5. @Time    : 2018/12/26 11:41
  6. @Desc : Less interests,More interest.
  7. @Project : python_appliction
  8. @FileName: dianxs2.py
  9. @Software: PyCharm
  10. @Blog    ：https://blog.csdn.net/zwx19921215
  11. """
  13. import os
  15. import requests
  16. from lxml import html
  18. """
  19. 简单爬取小说文章内容（'殿行说小说网'）
  20. """
  23. class Dianxs():
  24.     # 构造函数初始化
  25.     def __init__(self, host, url, headers, path):
  26.         self.host = host
  27.         self.url = url
  28.         self.headers = headers
  29.         self.path = path
  30.         self.novel_name = ''
  32.     """
  33.     主站导航栏栏目列表爬取
  34.     """
  36.     def nav_page(self):
  37.         print('------------------殿行说----------------------------')
  38.         response = requests.get(url=self.host, headers=self.headers)
  39.         text = html.fromstring(response.text)
  40.         nav_list = text.xpath('//ul[@class="nav"]/li/a/text()')
  41.         nav_href_list = text.xpath('//ul[@class="nav"]/li/a/@href')
  42.         nav_list.pop(0)
  43.         nav_href_list.pop(0)
  44.         print(nav_list)
  45.         print(nav_href_list)
  46.         i = 0
  47.         for nav_item in nav_href_list:
  48.             address = self.host + nav_item
  49.             nav_title = nav_list[i]
  50.             self.nav_item(address, nav_title)
  51.             i += 1
  53.     """
  54.     小说栏目下所有章节爬取
  55.     """
  57.     def nav_item(self, url, nav_title):
  58.         response = requests.get(url=url, headers=self.headers)
  59.         text = html.fromstring(response.text)
  60.         novel_list = text.xpath(
  61.             '//div[@class="panel new-xs-list w300 w265 fr simple"]/ul/li//span[@class="xs-name"]/a/text()')
  62.         novel_list_href = text.xpath(
  63.             '//div[@class="panel new-xs-list w300 w265 fr simple"]/ul/li//span[@class="xs-name"]/a/@href')
  64.         print('--------------------', nav_title, '-----------------')
  65.         print(novel_list)
  66.         print(novel_list_href)
  67.         print('\n')
  69.         for nov_item in novel_list_href:
  70.             self.url = self.host + nov_item
  71.             self.download_page()
  73.     """
  74.     每个章节下载，并解析小说主页信息，获取小说列表
  75.     """
  77.     def download_page(self):
  78.         response = requests.get(url=self.url, headers=self.headers)
  79.         text = html.fromstring(response.text)
  80.         novel = text.xpath('//div[@class="info"]/h1/text()')
  81.         author = text.xpath('//div[@class="info"]/p[1]/text()')
  82.         act = text.xpath('//div[@class="info"]/p[2]/text()')
  83.         last_update = text.xpath('//div[@class="info"]/p[3]/text()')
  84.         last_chapter_href = text.xpath('//div[@class="info"]/p[4]/a/@href')
  85.         last_chapter_title = text.xpath('//div[@class="info"]/p[4]/a/text()')
  86.         introduce = text.xpath('//div[@class="info"]/p[@class="introduce"]/text()')
  87.         print('---------------------------description--------------------------------')
  88.         print(novel)
  89.         # author[0].replace('\xa0', '')
  90.         print(author)
  91.         print(act)
  92.         print(last_update)
  93.         print(last_chapter_title, ' , ', last_chapter_href)
  94.         print('简介：', introduce)
  95.         print('-----------------------------------------------------------------------')
  96.         print('\n')
  97.         chapters = text.xpath('//div[@class="section-panel section-list"]/dl/dd/a/text()')
  98.         hrefs = text.xpath('//div[@class="section-panel section-list"]/dl/dd/a/@href')
  99.         print(chapters)
  100.         print(hrefs)
  101.         print('\n')
  103.         for href in hrefs:
  104.             # time.sleep(1)
  105.             address = self.host + href
  106.             self.novel_name = novel
  107.             self.parse_html(address)
  109.     """
  110.     解析文章内容
  111.     @:param address 章节页地址
  112.     """
  114.     def parse_html(self, address):
  115.         response = requests.get(url=address, headers=self.headers, timeout=10)
  116.         if response.status_code != 200:
  117.             self.parse_html(address)
  118.         text = html.fromstring(response.text)
  119.         title = text.xpath('//div[@class="read-title"]/h2/text()')
  120.         content = text.xpath('//div[@class="read-content"]/p/text()')
  121.         print('-------- ', title, '-----------')
  122.         print(content)
  123.         print('\n')
  124.         # ''.join(content)：list集合转字符串；list = list(string)：字符串转list集合
  125.         title_str = ''.join(title)
  126.         content_str = ''.join(content)
  127.         self.write_to_file(title_str, content_str)
  129.     """
  130.     章节内容写入文件
  131.     """
  133.     def write_to_file(self, title, content):
  134.         file_path = self.path + ''.join(self.novel_name)
  135.         if not os.path.exists(file_path):
  136.             os.makedirs(file_path)
  138.         file_name = file_path + '/' + title + '.txt'
  139.         flag = os.path.exists(file_name)
  140.         if not flag:
  141.             # 'w' 表示写模式，没有文件则会创建一个
  142.             try:
  143.                 f = open(file_name, 'w')
  144.                 f.close()
  145.             except Exception as e:
  146.                 print(e)
  147.                 # 可将发生异常信息的章节收集，进行后续处理
  148.                 # todo
  150.         # with 可以不用显示调用close方法
  151.         # 'a' 表示追加写入
  152.         with open(file_name, mode='a', encoding='utf-8') as file:
  153.             file.write(title + '\n')
  154.             file.writelines(content)
  155.             file.write('\n\n')
  158. if __name__ == '__main__':
  159.     host = 'https://www.dianxs.com'
  160.     url = 'https://www.dianxs.com/book/64554/'
  161.     headers = {
  162.         'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
  163.     }
  164.     path = 'G:/殿兴说/'
  165.     app = Dianxs(host, url, headers, path)
  166.     app.nav_page()
  167.     # app.download_page()

20181227140428747.png

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3eDE5OTIxMjE1_size_16_color_FFFFFF_t_70 3

发表评论

表情：
评论列表 （有 0 条评论，670人围观）

还没有评论，来说两句吧...

相关阅读

    相关 python 爬虫爬取小说信息

    1.进入小说主页（以下示例是我在网上随便找的一片小说），获取该小说的名称、作者以及相关描述信息 2.获取该小说的所有章节列表信息（最重要的是每个章节的链接地址href） 3

    桃扇骨桃扇骨/ 0 赞/ 671 阅读