python3分别用多进程，多线程，协程爬取豆瓣top250数据（python经典编程案例）

小咪咪 2023-01-23 11:54 208阅读 0赞

#### 一. 多进程爬取 ####

import requests
    import time
    import multiprocessing
    import random
    from lxml import etree
    from multiprocessing import Queue
    
    
    class DouBanSpider:
        """爬虫类"""
            self.headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
            self.baseURL = "https://movie.douban.com/top250"
            # dataQueue：队列存储数据
            self.dataQueue = Queue()
            self.num = 1
    
        def loadPage(self, url):
            """向url发送请求,获取响应内容"""
            # 随机休眠0-1秒，避免爬虫过快，会导致爬虫被封禁
            time.sleep(random.random())
            return requests.get(url, headers=self.headers).content
    
        def parsePage(self, url):
            """根据起始url提取所有的url"""
            content = self.loadPage(url)
            html = etree.HTML(content)
            node_list = html.xpath("//div[@class='info']")
            for node in node_list:
                # 每部电影的标题
                title = node.xpath(".//span[@class='title']/text()")[0]
                # 每部电影的评分
                score = node.xpath(".//span[@class='rating_num']/text()")[0]
                # 将数据存储到队列里
                self.dataQueue.put(score + "\t" + title)
    
            # 只有在第一页的时候才获取所有url组成的列表，其它翻页就不再获取
            if url == self.baseURL:
                return [self.baseURL + link for link in html.xpath("//div[@class='paginator']/a/@href")]
    
        def startWork(self):
            link_list = self.parsePage(self.baseURL)
            process_list = []
            for link in link_list:
                process = multiprocessing.Process(target=self.parsePage, args=[link])
                process.start()
                process_list.append(process)
    
            # 父进程等待所有子进程结束，自己再结束
            for process in process_list:
                process.join()
    
            # 循环get队列的数据，直到队列为空则退出
            while not self.dataQueue.empty():
                print(self.num)
                print(self.dataQueue.get())
                self.num += 1
    
    
    if __name__ == "__main__":
        spider = DouBanSpider()
        start = time.time()
        spider.startWork()
        stop = time.time()
        print(f"\n[LOG]: { stop} - { start} seconds...")

#### 二. 多进程爬取 ####

import requests
    import random
    import time
    import threading
    from lxml import etree
    from queue import Queue
    
    
    class DouBanSpider:
        """爬虫类"""
            self.headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
            self.baseURL = "https://movie.douban.com/top250"
            # dataQueue：队列存储数据
            self.dataQueue = Queue()
            self.num = 1
    
        def loadPage(self, url):
            """向url发送请求,获取响应内容"""
            time.sleep(random.random())
            return requests.get(url, headers=self.headers).content
    
        def parsePage(self, url):
            """根据起始url提取所有的url"""
            content = self.loadPage(url)
            html = etree.HTML(content)
            node_list = html.xpath("//div[@class='info']")
            for node in node_list:
                title = node.xpath(".//span[@class='title']/text()")[0]
                score = node.xpath(".//span[@class='rating_num']/text()")[0]
                # 将数据存储到队列里
                self.dataQueue.put(score + "\t" + title)
    
            # 只有在第一页的时候才获取所有url组成的列表，其它翻页就不再获取
            if url == self.baseURL:
                return [self.baseURL + link for link in html.xpath("//div[@class='paginator']/a/@href")]
    
        def startWork(self):
            """开始"""
            link_list = self.parsePage(self.baseURL)
            thread_list = []
            for link in link_list:
                thread = threading.Thread(target=self.parsePage, args=[link])
                thread.start()
                thread_list.append(thread)
    
            # 父线程等待所有子线程结束，自己再结束
            for thread in thread_list:
                thread.join()
    
            # 循环get队列的数据，直到队列为空则退出
            while not self.dataQueue.empty():
                print(self.num)
                print(self.dataQueue.get())
                self.num += 1
    
    
    if __name__ == "__main__":
        spider = DouBanSpider()
        start = time.time()
        spider.startWork()
        stop = time.time()
        print(f"\n[LOG]: { stop - start} seconds...")

#### 三. 协程爬取 ####

"""协程爬虫"""
    
    import time
    import gevent  
    import requests
    import random
    from lxml import etree
    from queue import Queue
    from gevent import monkey
    
    # gevent 让我们可以按同步的方式来写异步程序
    # monkey.patch_all() 会在Python程序执行时动态的将网络库（socket,select,thread)
    # 替换掉，变成异步的库，让我们的程序可以异步的方式处理网络相关的任务
    monkey.patch_all()
    
    
    class DouBanSpider:
        """爬虫类"""
            self.headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
            self.baseURL = "https://movie.douban.com/top250"
            # dataQueue：队列存储数据
            self.dataQueue = Queue()
            self.num = 1
    
        def loadPage(self, url):
            """向url发送请求,获取响应内容"""
            time.sleep(random.random())
            return requests.get(url, headers=self.headers).content
    
        def parsePage(self, url):
            """根据起始url提取所有的url"""
            content = self.loadPage(url)
            html = etree.HTML(content)
            node_list = html.xpath("//div[@class='info']")
    
            for node in node_list:
                title = node.xpath(".//span[@class='title']/text()")[0]
                score = node.xpath(".//span[@class='rating_num']/text()")[0]
                self.dataQueue.put(score + "\t" + title)
    
            if url == self.baseURL:
                return [self.baseURL + link for link in html.xpath("//div[@class='paginator']/a/@href")]
    
        def startWork(self):
            """开始"""
            link_list = self.parsePage(self.baseURL)
            print(link_list)
            jobs = [gevent.spawn(self.parsePage, link) for link in link_list]
            # 父线程阻塞，等待所有任务结束后继续执行
            gevent.joinall(jobs)
            # 循环get队列的数据，直到队列为空则退出
            while not self.dataQueue.empty():
                print(self.num)
                print(self.dataQueue.get())
                self.num += 1
    
    
    if __name__ == "__main__":
        spider = DouBanSpider()
        start = time.time()
        spider.startWork()
        stop = time.time()
        print(f"\n[LOG]: { stop} - { start} seconds...")