Python爬虫爬取小说网站

迷南。 2022-02-03 22:47 390阅读 0赞

\#!/usr/bin/env python  
\# -\*- coding: utf-8 -\*-  
\# @Author : Woolei  
\# @File : book136\_singleprocess.py  
  
import requests  
import time  
import os  
from bs4 import BeautifulSoup  
  
  
headers = \{  
'User-Agent':  
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'  
\}  
  
  
\# 获取小说章节内容，并写入文本  
def getChapterContent(each\_chapter\_dict):  
content\_html = requests.get(each\_chapter\_dict\['chapter\_url'\], headers=headers).text  
soup = BeautifulSoup(content\_html, 'lxml')  
content\_tag = soup.find('p', \{'id': 'content'\})  
p\_tag = content\_tag.find\_all('p')  
print('正在保存的章节 --> ' + each\_chapter\_dict\['name'\])  
for each in p\_tag:  
paragraph = each.get\_text().strip()  
with open(each\_chapter\_dict\['name'\] + r'.txt', 'a', encoding='utf8') as f:  
f.write(' ' + paragraph + '\\n\\n')  
f.close()  
  
  
\# 获取小说各个章节的名字和url  
def getChapterInfo(novel\_url):  
chapter\_html = requests.get(novel\_url, headers=headers).text  
soup = BeautifulSoup(chapter\_html, 'lxml')  
chapter\_list = soup.find\_all('li')  
chapter\_all\_dict = \{\}  
for each in chapter\_list:  
import re  
chapter\_each = \{\}  
chapter\_each\['name'\] = each.find('a').get\_text() \# 获取章节名字  
chapter\_each\['chapter\_url'\] = each.find('a')\['href'\] \# 获取章节url  
chapter\_num = int(re.findall('\\d+', each.get\_text())\[0\]) \# 提取章节序号  
chapter\_all\_dict\[chapter\_num\] = chapter\_each \# 记录到所有的章节的字典中保存  
return chapter\_all\_dict  
  
  
if \_\_name\_\_ == '\_\_main\_\_':  
start = time.clock() \# 记录程序运行起始时间  
novel\_url = 'https://www.136book.com/sanshengsanshimenglitaohua/' \# 这里以三生三世十里桃花为例  
novel\_info = getChapterInfo(novel\_url) \# 获取小说章节记录信息  
dir\_name = '保存的小说路径'  
if not os.path.exists(dir\_name):  
os.mkdir(dir\_name)  
os.chdir(dir\_name) \# 切换到保存小说的目录  
for each in novel\_info:  
getChapterContent(novel\_info\[each\])  
\# time.sleep(1)  
end = time.clock() \# 记录程序结束的时间  
print('保存小说结束，共保存了 %d 章，消耗时间：%f s' % (len(novel\_info), (end - start)))

![1042758-20180424102052765-527177669.png][]

![1042758-20180424102137928-1219874271.png][]

![1042758-20180424102205129-1070953855.png][]

[1042758-20180424102052765-527177669.png]: /images/20220204/0dfe48b216ac40daa39b8ac718dac342.png
[1042758-20180424102137928-1219874271.png]: /images/20220204/7239da2847d242dc81debba401432076.png
[1042758-20180424102205129-1070953855.png]: /images/20220204/ce5d7f9d34da4e24b14134390da3db0e.png