python3 scrapy爬取微信公众号及历史信息V2.0

╰半橙微兮° 2021-06-24 15:59 417阅读 0赞

代码部分，日后补充：

# -*- coding: utf-8 -*-
    # @Time : 2018/2/25 14:24
    # @Author : 蛇崽
    # @Email : 643435675@QQ.com
    # @File : WeChatSogou.py（微信公众号爬虫：不包含浏览量跟评论）
    import scrapy
    import re
    import time
    import json
    import requests
    from fake_useragent import UserAgent
    from scrapy import Request
    from scrapy_redis.spiders import RedisSpider
    from News_scrapy.items import NewsItem
    from News_scrapy.constants.WeChatSource import WXSource
    
    
    class WeChatSogouSpider(scrapy.Spider):
        name = "WeChatAiWeiBang"
        allowed_domains = ["aiweibang.com", 'mp.weixin.qq.com']
        start_urls = ['http://top.aiweibang.com/user/search_advanced']
    
        def parse(self, response):
            wx_source = WXSource.get_dict()
            for v_wx_source in wx_source:
                try:
                    type = v_wx_source.split('-')[0]
                    channel = v_wx_source.split('-')[1]
                    print("正在抓取:", type, channel)
                    url = 'http://top.aiweibang.com/user/search?kw={}'.format(channel)
                    yield scrapy.Request(url=url,callback=self.get_target_gzh,meta={
       'type':type,'channel':channel})
                except:
                    continue
                    print('wx_source error ===', v_wx_source)
    
        def get_target_gzh(self,response):
            print('get_target_gzh == ',response)
    
            type = response.meta['type']
            channel = response.meta["channel"]
    
            gzh_url = 'http://top.aiweibang.com/user/getsearch'
            formdata = {
       'PageIndex': '1', 'PageSize': '10', 'Kw': channel}
            yield scrapy.FormRequest(url=gzh_url,callback=self.get_v_gzh,formdata=formdata,meta={
       'type':type,'channel':channel})
    
    
        def get_v_gzh(self,response):
            print('get_v_gzh jsondata === ',response.body)
    
            type = response.meta['type']
            channel = response.meta["channel"]
    
            jsondatas = json.loads(str(response.body.decode('utf-8')))
            first_gzh_Id = jsondatas['data']['data'][0]['Id']
            print('first_gzh_ID ==== ',first_gzh_Id)
    
            if first_gzh_Id:
                gzh_url = 'http://top.aiweibang.com/article/{}'.format(first_gzh_Id)
                yield scrapy.Request(url=gzh_url,callback=self.get_article_gzh,meta={
       'type':type,'channel':channel,'gzh_ID':first_gzh_Id})
    
        def get_article_gzh(self,response):
            print('get_article_gzh === ',response.body)
    
            type = response.meta['type']
            channel = response.meta["channel"]
            gzh_ID = response.meta["gzh_ID"]
    
            # 获取全部文章列表的url
            all_url = 'http://top.aiweibang.com/article/getarticles'
            formadata = {
       'PageIndex': '1', 'PageSize': '20', 'Type': '0', 'Wechat': gzh_ID}
    
            yield scrapy.FormRequest(url=all_url,formdata=formadata,callback=self.get_all_article,meta={
       'type':type,'channel':channel,'gzh_ID':gzh_ID})
    
    
        def get_all_article(self,response):
            print('get_all_article ***** the last *** ',response)
    
            type = response.meta['type']
            channel = response.meta["channel"]
    
            jsondatas = json.loads(str(response.body.decode('utf-8')))
            # 获取拼接aid
            aids = jsondatas['data']['data']
            for aid in aids:
                print('aid -- ',aid)
                v_aid = aid['Id']
                print('v_aid ',v_aid)
                if v_aid:
                    detail_url = 'http://top.aiweibang.com/article/url?aid={}'.format(v_aid)
                    yield scrapy.Request(url=detail_url,callback=self.get_wx_url,meta={
       'type':type,'channel':channel})
    
    
        def get_wx_url(self,response):
            print('get_wx_url ----- ',response)
            print('wx_url 00000000000 ',response.url)