猫眼电影爬取

谁借莪１个温暖的怀抱￠ 2021-11-26 20:22 446阅读 0赞

'''爬取猫眼电影TOP100，并将其保存'''
    from urllib import request
    import re
    import csv
    import time
    import random
    import os
    
    
    class MaoyanSpider:
        def __init__(self):
            self.url = 'https://maoyan.com/board/4?offset={}'
            self.ua_list = [
                'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
                'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
                'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
                'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB7.0)',
                'Mozilla/5.0 (Windows; U; Windows NT 6.1; ) AppleWebKit/534.12 (KHTML, like Gecko) Maxthon/3.0 Safari/534.12',
                'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.33 Safari/534.3 SE 2.X MetaSr 1.0',
                'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201'
            ]
    
        # 获取页面
        def get_page(self, url):
            # 每次使用随机的User-Agent
            headers = {
        'User-Agent': random.choice(self.ua_list)}
            req = request.Request(url=url, headers=headers)
            res = request.urlopen(req)
            html = res.read().decode('utf-8')
            # 调用解析函数
            self.parse_page(html)
    
        # 解析页面
        def parse_page(self, html):
            pattern = re.compile(
                r'<div class="movie-item-info">.*?title="(.*?)".*?class="star">(.*?)</p>.*?class="releasetime">(.*?)</p>',
                re.S)
            r_list = pattern.findall(html)
            self.save_info(r_list)
    
        # 打印页面
        # def save_info(self, r_list):
        #     one_film_dict = {}
        #     for rt in r_list:
        #         one_film_dict['name'] = rt[0].strip()
        #         one_film_dict['stars'] = rt[1].strip()
        #         one_film_dict['time'] = rt[2].strip()
        #         print(one_film_dict)
    
        # 保存页面
        def save_info(self, r_list):
            film_list = []
            with open('./maoyan.csv', 'a', encoding='utf-8', newline='') as f:
                # 方法一：一条一条写入csv
                # for rt in r_list:
                #     writer = csv.writer(f)
                #     writer.writerow([rt[0].strip(), rt[1].strip(), rt[2].strip()])
                # 方法二：一次性写入csv,减少IO
                writer = csv.writer(f)
                for rt in r_list:
                    # 把处理过的数据定义成元组
                    t = (rt[0].strip(), rt[1].strip(), rt[2].strip())
                    film_list.append(t)
                writer.writerows(film_list)
    
        # 主函数
        def main(self):
            if os.path.exists('./maoyan.csv'):
                os.remove('./maoyan.csv')
            with open('./maoyan.csv', 'a', encoding='utf-8', newline='') as f:
                writer = csv.writer(f)
                writer.writerow(['电影名称', '主演', '上映时间'])
            i = 1
            for offset in range(0, 91, 10):
                url = self.url.format(offset)
                self.get_page(url)
                print('第{}页成功下载'.format(i))
                i += 1
                # time.sleep(random.randint(1, 4))
    
    
    if __name__ == '__main__':
        start = time.time()
        spider = MaoyanSpider()
        spider.main()
        end = time.time()
        print('程序执行时间为: %.2f' % (end - start))

转载于:https://www.cnblogs.com/yuxiangyang/p/11212544.html