python框架scrapy实现增量爬虫（含redis）

爱被打了一巴掌 2022-11-09 14:18 201阅读 0赞

# 本案例以信用中国武清开放数据为例 #

redis相关知识详见：  
https://blog.csdn.net/weixin\_44675384/article/details/90723634?spm=1001.2014.3001.5501  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDY3NTM4NA_size_16_color_FFFFFF_t_70]

# spider部分： #

import scrapy
    import json
    from wq_hhb_heimd.items import WqHhbHeimdItem
    import redis
    import hashlib
    
    class HhbHeimdSpider(scrapy.Spider):
        name = 'hhb_heimd'
        allowed_domains = ['credit.tjwq.gov.cn/protal/getBlackListByPage']
        start_urls = ['http://credit.tjwq.gov.cn/protal/getBlackListByPage']
        #post请求需要增加参数，这里对start_urls进行重写
        def start_requests(self):
            # for i in range(1,6900):
            for i in range(1,20):
                data={ 
                    'pageNum':f"{i}",
                    'pageSize':"10",
    
                }
                for url in self.start_urls:
                    yield  scrapy.FormRequest(url=url,formdata=data,callback=self.parse)
    
        def parse(self, response):
            conn = redis.Redis(host='127.0.0.1', port=6379, db=1)
            # print(response.text)
            data_dict=json.loads(response.text)
            # print(data_dict)
    
    
            for data in data_dict['rows']:
                # print(data)
                item = WqHhbHeimdItem()
                item['ztmc']=data['corpName']
                item['zzjgdm']=data['creditCode']
                item['gszch']=data['orgCode']
                item['zcjdjg']=data['publishOrgan']
                item['lrrq']=data['confirmTime']
    
                # print(item)
                #数据插入redis
                # conn.rpush("item",item)
    
                # 将解析到的数据值生成一个唯一的标识进行redis存储
                source = item['ztmc'] + item['zzjgdm']
                source_id = hashlib.sha256(source.encode()).hexdigest()
                # 将解析内容的唯一表示存储到redis的data_id中
                ex = conn.sadd('data_id', source_id)
    
                if ex == 1:
                    print('该条数据没有爬取过，可以爬取......')
                    print(item)
                    yield item
                else:
                    print('该条数据已经爬取过了，不需要再次爬取了!!!')
    
    
                # yield item

# items #

import scrapy
    
    class WqHhbHeimdItem(scrapy.Item):
        # define the fields for your item here like:
        ztmc = scrapy.Field()
        zzjgdm = scrapy.Field()
        gszch = scrapy.Field()
        zcjdjg = scrapy.Field()
        lrrq = scrapy.Field()

# setting和pipelines 无较大改变，常规处理，如不理解可以私信 #

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDY3NTM4NA_size_16_color_FFFFFF_t_70]: /images/20221023/900c97ca23f64504b7f721682396f1b3.png