抓取猫眼电影排行前一百

淡淡的烟草味﹌ 2022-05-18 07:37 167阅读 0赞

代码主要是大神写的：[https://cuiqingcai.com/5534.html][https_cuiqingcai.com_5534.html]

需要看具体细节的去该网址：

import json
 import requests
 from requests.exceptions import RequestException
 import re
 import time
 
 def get_one_page(url): 
 try: ####一定要加Header，大神的没加，跑不起来
 header = {
 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 UBrowser/6.1.2107.204 Safari/537.36'}
 response = requests.get(url,headers=header)
 if response.status_code == 200:
 return response.text
 return None
 except RequestException:
 return None
 
 def parse_one_page(html): #解析网页，获取想要的信息
 #print(html)
 pattern = re.compile('<dd>.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name"><a'
 + '.*?>(.*?)</a>.*?star">(.*?).*?releasetime">(.*?)'
 + '.*?integer">(.*?).*?fraction">(.*?).*?</dd>', re.S)
 #print(pattern)
 items = re.findall(pattern, html)
 #print(items)
 #exit()
 for item in items:
 yield {
 'index': item[0],
 'image': item[1],
 'title': item[2],
 'actor': item[3].strip()[3:],
 'time': item[4].strip()[5:],
 'score': item[5] + item[6]
 }
 
 def write_to_file(content): #写入TXT文件
 with open('result.txt', 'a', encoding='utf-8') as f:
 f.write(json.dumps(content, ensure_ascii=False) + '\n')
 
 def main(offset):
 url = 'http://maoyan.com/board/4?offset=' + str(offset) #offset值得是偏移量，每个
 html = get_one_page(url) #网页上面是十部电影，offset
 for item in parse_one_page(html): #下一页就是+10
 print(item) #比较有规律，F12打开源码看看
 write_to_file(item)
 
 if __name__ == '__main__':
 for i in range(10): #这里就是遍历前十页
 main(offset=i * 10)
 time.sleep(1)

[https_cuiqingcai.com_5534.html]: https://cuiqingcai.com/5534.html