scrapy 之 4K美食图片下载

ゝ一纸荒年。 2022-04-12 12:53 263阅读 0赞

这里不说废话了直接上代码

# -*- coding: utf-8 -*-
    import scrapy
    from .. items import ImagenetItem
    
    class ImageSpider(scrapy.Spider):
        name = 'image'
        allowed_domains = ['pic.netbian.com']
        # 请求最开始的url
        start_urls = ['http://pic.netbian.com/4kmeishi/']
        def parse(self, response):
            # 根据响应(response)来找到指定的内容   现在找到的是img的src属性
            img_list = response.xpath('//ul[@class="clearfix"]/li/a/img/@src')
            print(img_list)
            # 找到了很多src属性值   现在进行遍历   分别使用每一个
            for img in img_list:
                # 使用在items.py中创建的数据模型item
                item = ImagenetItem()
                # 拼接url   得到完整的下载地址
                src = 'http://pic.netbian.com' + img.extract()
                # 将得到的下载地址   放入到数据模型中
                # item['src'] = src
                # 下载地址要包在列表当中
                item['src'] = [src]
                # 将数据模型传输给管道
                yield item
    
            next_url = response.xpath('//div[@class="page"]/a[text()="下一页"]/@href').extract()
            # print('---------------------------------------------')
            # print(next_url)
            if len(next_url) != 0:
                print('----------------')
                url = 'http://pic.netbian.com' + next_url[0]
                # 将url传给scrapy.Request   得到的结果继续用self.parse进行处理
                yield scrapy.Request(url=url,callback=self.parse)

items.py

# -*- coding: utf-8 -*-
    
    # Define here the models for your scraped items
    #
    # See documentation in:
    # https://doc.scrapy.org/en/latest/topics/items.html
    
    import scrapy
    
    
    class ImagenetItem(scrapy.Item):
        # define the fields for your item here like:
        # name = scrapy.Field()
        src = scrapy.Field()
        pass

settings.py

# -*- coding: utf-8 -*-
    
    # Scrapy settings for imageNet project
    #
    # For simplicity, this file contains only settings considered important or
    # commonly used. You can find more settings consulting the documentation:
    #
    #     https://doc.scrapy.org/en/latest/topics/settings.html
    #     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
    #     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
    
    BOT_NAME = 'imageNet'
    
    SPIDER_MODULES = ['imageNet.spiders']
    NEWSPIDER_MODULE = 'imageNet.spiders'
    
    
    # Crawl responsibly by identifying yourself (and your website) on the user-agent
    #USER_AGENT = 'imageNet (+http://www.yourdomain.com)'
    
    # Obey robots.txt rules
    # ROBOTSTXT_OBEY = True
    
    # Configure maximum concurrent requests performed by Scrapy (default: 16)
    #CONCURRENT_REQUESTS = 32
    
    # Configure a delay for requests for the same website (default: 0)
    # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
    # See also autothrottle settings and docs
    #DOWNLOAD_DELAY = 3
    # The download delay setting will honor only one of:
    #CONCURRENT_REQUESTS_PER_DOMAIN = 16
    #CONCURRENT_REQUESTS_PER_IP = 16
    
    # Disable cookies (enabled by default)
    #COOKIES_ENABLED = False
    
    # Disable Telnet Console (enabled by default)
    #TELNETCONSOLE_ENABLED = False
    
    # Override the default request headers:
    #DEFAULT_REQUEST_HEADERS = {
    #   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    #   'Accept-Language': 'en',
    #}
    
    # Enable or disable spider middlewares
    # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
    #SPIDER_MIDDLEWARES = {
    #    'imageNet.middlewares.ImagenetSpiderMiddleware': 543,
    #}
    
    # Enable or disable downloader middlewares
    # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
    #DOWNLOADER_MIDDLEWARES = {
    #    'imageNet.middlewares.ImagenetDownloaderMiddleware': 543,
    #}
    
    # Enable or disable extensions
    # See https://doc.scrapy.org/en/latest/topics/extensions.html
    #EXTENSIONS = {
    #    'scrapy.extensions.telnet.TelnetConsole': None,
    #}
    
    # Configure item pipelines
    # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
    ITEM_PIPELINES = {
       # 'imageNet.pipelines.ImagenetPipeline': 300,
        #scrapy中  专门负责图片下载的管道
        'scrapy.pipelines.images.ImagesPipeline':1
    }
    # 图片的存储路径
    IMAGES_STORE = 'imageDownLoad'
    # 图片的下载地址  根据item中的字段来设置哪一个内容需要被下载
    IMAGES_URL_FIELD = 'src'
    # Enable and configure the AutoThrottle extension (disabled by default)
    # See https://doc.scrapy.org/en/latest/topics/autothrottle.html
    #AUTOTHROTTLE_ENABLED = True
    # The initial download delay
    #AUTOTHROTTLE_START_DELAY = 5
    # The maximum download delay to be set in case of high latencies
    #AUTOTHROTTLE_MAX_DELAY = 60
    # The average number of requests Scrapy should be sending in parallel to
    # each remote server
    #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
    # Enable showing throttling stats for every response received:
    #AUTOTHROTTLE_DEBUG = False
    
    # Enable and configure HTTP caching (disabled by default)
    # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
    #HTTPCACHE_ENABLED = True
    #HTTPCACHE_EXPIRATION_SECS = 0
    #HTTPCACHE_DIR = 'httpcache'
    #HTTPCACHE_IGNORE_HTTP_CODES = []
    #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'