scrapy进阶

一时失言乱红尘 2021-09-17 21:48 355阅读 0赞

# scrapy进阶 #

### item ###

*  Item是保存爬取数据的容器,它的使用方法和字典类似。不过,相比字典,Item提供了额外的保护机制,可以避免拼写错误或者定义字段错误

import scrapy
    
    class Minimp4SpiderItem(scrapy.Item):
        # define the fields for your item here like:
        # 保存的键
        title = scrapy.Field()
        introduce = scrapy.Field()
        link = scrapy.Field()
        passwd = scrapy.Field()

##### 爬虫文件的运用 #####

from ..items import Minimp4SpiderItem
    
    # 在回调函数中实例item对象,采用字典赋值的方式,最后yield item实例化对象
    item = Minimp4SpiderItem()
    item["title"] = title
    item["introduce"] = response.meta
    item["link"] = link
    item["passwd"] = passwd
    yield item

### Pipeline保存数据 ###

*  Scrapy 提供了 pipeline 模块来执行保存数据的操作。在创建的 Scrapy 项目中自动创建了一个 [pipeline.py][] 文件，同时创建了一个默认的 Pipeline 类

process_item:
        该方法用来处理数据 筛选 过滤 保存等
        item 是爬虫文件返回的数据
        spider 获取该数据的爬虫 
    
    注意：
        1. return item 不要删除
            作用：为了以后的管道文件处理用的
        2. 写好的管道文件 要激活 在配置文件中
    
    激活：
        ITEM_PIPELINES = {
            'tencent.pipelines.TencentPipeline': 300,
            'tencent.pipelines.TencentPipeline2': 299,
        }
    
    
        tencent.pipelines.TencentPipeline ： 管道的路径
    
        300： 代表激活的优先级 越小越优先
    
        数字大小：【1-1000】

##### settings设置 #####

> 如果要使用管道必须激活settings的ITEM\_PIPELINES

# 激活管道
    ITEM_PIPELINES = { 
       'minimp4spider.pipelines.Minimp4SpiderPipeline': 300,
    }
    
    # mysql数据库连接信息
    MYSQL_HOST = '127.0.0.1'
    MYSQL_KEY = '1234'
    MYSQL_USER = 'root'
    MYSQL_DB = 'scrapy'
    
    # 图片保存的路径
    IMAGES_STORE = r'F:\python0828\2019-01-19-Scrapy进阶\Code\WuZhuLiangYuan\Pictures'

##### 保存到mysql数据库 #####

import pymysql
    
    
    class Minimp4SpiderPipeline(object):
        # 保存到mysql
        def __init__(self, host, user, passwd, db):
            self.host = host
            self.user = user
            self.passwd = passwd
            self.db = db
    
        @classmethod
        def from_crawler(cls, crawler):
            return cls(
                host=crawler.settings.get('MYSQL_HOST'),
                user=crawler.settings.get('MYSQL_USER'),
                passwd=crawler.settings.get('MYSQL_KEY'),
                db=crawler.settings.get('MYSQL_DB'),
            )
    
        # 保存到mysql
        def open_spider(self, spider):
            # 打开数据库连接
            self.mysql = pymysql.connect(host=self.host, user=self.user, passwd=self.passwd, port=3306, db=self.db,
                                         charset="utf8")
    
        def process_item(self, item, spider):
            data = dict(item)
            title = data['title']
            introduce = data['introduce']
            link = data['link']
            passwd = data['passwd']
            cursor = self.mysql.cursor()
            sql = "insert into minimp4(title,introduce,link,passwd) values ('{}','{}','{}','{}') ".format(title, introduce, link, passwd)
            cursor.execute(sql)
            self.mysql.commit()
            return item
    
        def close_spider(self, spider):
            self.mysql.close()

##### 保存到redis #####

import redis
    class FirstspiderPipeline2(object):
        # 保存到redis
        def __init__(self):
            self.myredis = redis.StrictRedis(host='127.0.0.1', port=6379, db=2, password="密码")
    
        def process_item(self, item, spider):
            data = dict(item)
            self.myredis.lpush("liangyuan", data)
            return item

##### 图片的下载与保存 #####

*  【图片文件】

1.  先继承ImagesPipeline
2.  发请求
3.  激活 配置文件激活
4.  指定图片路径 配置文件中指定

from scrapy.pipelines.images import ImagesPipeline
    
    class ImagePipeline(ImagesPipeline):
        # 下载图片
        def get_media_requests(self, item, info):
            data = dict(item)
            # 获取图片链接
            avatar = data['avatar']
            yield scrapy.Request(avatar)
    
        def file_path(self, request, response=None, info=None):
            # 获取图片名
            imgname = request.url.split('/')[-1]
            if ".jpg" or ".png" in imgname:
                name = imgname
            else:
                name = imgname + '.jpg'
            # 图片保存路径
            return './%s.jpg' % (name)

### 下载器中间件**Downloader Middlewares** ###

'''
    下载器中间件：
    【每一个请求都会经过下载器中间件】
    【每一个响应都会经过下载器中间件】
    请求：
        1. 引擎-->请求-->[下载器中间件]-->请求-->下载器
        设置请求：
            1. 随机替换请求头
            2. IP替换
            3. Cookie池
        在下载器的时候请求
    响应：
        2. 下载器-->响应-->[下载器中间件]-->响应-->引擎
     
    【process_request】
    每一个请求都会经过这个方法
    作用：
    用来处理请求
    1. None --> 下载器
    2. Response --> process_response --> 引擎 --> 爬虫文件
    3. Request --> 交给下载器
    4. 异常 process_exception 处理异常
    【process_response】
    每个响应都会经过这个方法
    '''
    
    '''
    自定义中间件：
        1. 定义好的中间件要激活
        2. 激活的中间件的顺序不要随便设置
        3. 设置的顺序最好在上个中间件和下个中间件之间。
        4. 最好将原来的中间件改为None
    DOWNLOADER_MIDDLEWARES_BASE = {
        # Engine side
        'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
        'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
        'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
        'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
        'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
        'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
        'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
        'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
        'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
        'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
        'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
        'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
        'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
        'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
        # Downloader side
    }
    '''
    
    '''
    【断点续爬】
        当Scrapy停掉以后 下次开启的时候接着上次的爬
    
        scrapy crawl spiderName -s JOBDIR=DIR
    '''

# middelwares文件中配置
    from fake_useragent import UserAgent
    import requests
    # 随机替换请求头
    class UserAgentMiddleware(object):
        def __init__(self):
            self.UA = UserAgent()
    
        def process_request(self, request, spider):
            request.headers["User-Agent"] = self.UA.random
    
    # 随机替换ip 
    # 这里使用了ip池
    class ProxyMiddleware(object):
        def __init__(self):
            self.IP = requests.get("http://127.0.0.1:8080/get/")
    
        def process_request(self, request, spider):
            request.headers["Proxy"] = "https://{}".format(self.IP)

##### 激活settings的DOWNLOADER\_MIDDLEWARES #####

DOWNLOADER_MIDDLEWARES = { 
        # 'minimp4spider.middlewares.Minimp4SpiderDownloaderMiddleware': 543,
        'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
        'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': None,
        'minimp4spider.middlewares.UserAgentMiddleware': 500,   # 值不要乱写,根据源码的配置写
        'minimp4spider.middlewares.ProxyMiddleware': 750,
    }
    
    ''' DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550, 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560, 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580, 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590, 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, # Downloader side } '''

[pipeline.py]: http://pipeline.py