批量全站抓取，结构化抓取, 智能解析

小鱼儿 2023-10-05 08:52 11阅读 0赞

### 批量全站抓取，结构化抓取, 智能解析。 ###

###### 思路: ######

###### 分析匹配有url链接，加入队列，等待循环抓取。 ######

###### 通过xpath匹配，判断正文格式不对不进行抓取。 ######

###### 通过xml直接转dict, 直接转化成json结构化 。 ######

###### 出于隐私考虑，网址及结果已打码。 ######

# -*- coding: utf-8 -*-
    # @author Mengtao
    import html
    import json
    import re
    import requests
    import xmltodict
    
    from lxml import etree
    from lxml.html import tostring
    from fake_useragent import UserAgent
    
    
    class ConstructionSpider:
        headers = {
        
            'User-Agent': UserAgent().random,
        }
    
        def start_requests(self, url):
            response = requests.get(url, headers=self.headers)
            response.encoding='utf-8'
            return response.text
    
        def parse(self, response):
            # 有正文内容
            parse_html = etree.HTML(response)
            result = parse_html.xpath('//div[@class="main-r"]')
            if result:
                # content_html_str = etree.tostring(result[0], encoding='utf-8', pretty_print=True, method='html').decode('utf-8')  # 转为字符串
    
                content_html_str = etree.tostring(result[0]).decode("utf-8")
    
                data = html.unescape(content_html_str)
                with open('content_html_str.html', 'w',encoding='utf-8') as w:
                    w.write(content_html_str)
    
                # data = content_html_str.replace('<br>', '').replace(' ', '')
                convertedDict = xmltodict.parse(data)
                a = json.dumps(convertedDict, ensure_ascii=False)
                a = json.loads(a)
                self.deal(a)
                print(a)
                with open('result.json', 'w',encoding='utf-8') as w:
                    w.write(json.dumps(a, ensure_ascii=False))
            else:
                pass
    
        def parse_url(self, response):
            parse_html = etree.HTML(response)
            result = parse_html.xpath('//div[@class="list-left-nav"]//ul/li/a/@href')
            for i in result:
                base = re.findall(r'.*\.\./(.*)/', i, re.S)
                # print(i)
                if base:
                    if '/' in base[0]:
                        url = "https://www.xxxxx.com/lvxingxinxi/" + base[0]
                    else:
                        url = "https://www.xxxxx.com/lvxingxinxi/zj/" + base[0]
                # yield url
    
        def deal(self, data):
            if isinstance(data, dict):
                for i in list(data.keys()):
                    if i in ['@class', '@style', '@alt', '@oldsrc', "@complete", "@align", '@color', '@colspan', "@rowspan",
                             '@width', '@cellspacing', '@cellpadding', '@border', '@target'] or not data[i]:
                        data.pop(i)
                    else:
                        self.deal(data[i])
            elif isinstance(data, list):
                for j in range(len(data)):
                    if data[j]:
                        self.deal(data[j])
    
        def main(self):
            url = 'https://www.xxxxx.com/lvxingxinxi/zj/jczzzj/'
            response = self.start_requests(url)
            self.parse(response)
            self.parse_url(response)
    
    
    if __name__ == '__main__':
        ConstructionSpider().main()

抓取效果可以转成json格式，键基本上都是html标签，值代表数据内容。

{
        
      "div": {
        
        "h3": {
        
          "#text": "XXXX"
        },
        "div": {
        
          "div": [
            {
        
              "img": {
        
                "@src": "./images/P02020072137XXXX.jpg"
              }
            },
            {
        
              "div": [
                {
        
                  "h4": {
        
                    "i": {
        },
                    "#text": "设备配备及功能介绍"
                  }
                },
                {
        
                  "p": [
                    {
        
                      "strong": "设备配备介绍"
                    },
                    {
        
                      "strong": "功能介绍："
                    },
                    "打印（疫情期间暂不开放）"
                  ]
                }
              ]
            },
            {
        
              "div": [
                {
        
                  "h4": {
        
                    "i": {
        },
                    "#text": "自助设备操作流程"
                  }
                },
                {
        
                  "p": [
                    "点击首页“进入”按钮。"
                  ]
                }
              ]
            }
          ]
        }
      }
    }