多进程爬虫

- 日理万妓 2022-05-16 22:24 156阅读 0赞

\# !/usr/bin/env python  
\# \_\*\_ coding:utf-8 \_\*\_

import requests  
from lxml import etree

import time

from multiprocessing import Process  
from multiprocessing import JoinableQueue as Queue

\# 反爬 限制频率

class QiushiSpider(object):  
    def \_\_init\_\_(self):  
        self.base\_url = 'https://www.qiushibaike.com/8hr/page/\{\}'  
        self.headers = \{"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"\}

\# 记录个数  
        self.count = 0

\# 1. url队列  
        self.url\_queue = Queue()

\# 2. 响应对列  
        self.response\_queue = Queue()

\# 3. 数据队列  
        self.data\_queue = Queue()

\# 1.组合列表 url  
    def get\_url\_list(self):  
        \# return \[self.base\_url.format(i) for i in range(1, 14)\]  
        \# 入对垒 url  
        for i in range(1, 14):  
            self.url\_queue.put(self.base\_url.format(i))

\# 2. 发请求  
    def send\_request(self):  
        while True:  
            \# 取出 url 对=列 中url  
            url = self.url\_queue.get()

response = requests.get(url, headers=self.headers)  
            print(response.url)

if response.status\_code == 200:

\# data = response.content.decode()

\# 入 response 队列  
                self.response\_queue.put(response)  
            else:  
                self.url\_queue.put(url)

\# 计数器减一  
            self.url\_queue.task\_done()

\# 3. 解析数据  
    def analysis\_data(self):  
        while True:  
            \# 取出 response队列 中的数据  
            data = self.response\_queue.get().content.decode()

\# 1.转换类型  
            html\_data = etree.HTML(data)

\# 2.解析 所有的 段子  
            div\_list = html\_data.xpath('//div\[@id="content-left"\]/div')

print("解析个数:", len(div\_list))  
            for div in div\_list:  
                nick\_name = div.xpath('.//h2/text()')\[0\]

\#  入数据 队列  
                self.data\_queue.put(nick\_name)

\# 计数器减一  
            self.response\_queue.task\_done()

\# 4.保存数据  
    def save\_data(self):  
        while True:  
            \# 从 数据队列 取数据  
            data = self.data\_queue.get()

print(data.strip())

self.count += 1  
            print(self.count)

\# 减一  
            self.data\_queue.task\_done()

\# 调度任务  
    def start\_work(self):

t\_list = \[\]

\# 1. urllsit 线程  
        t\_url = Process(target=self.get\_url\_list)  
        t\_list.append(t\_url)