「数据分析师的网络爬虫」动态页面和Ajax渲染页面抓取

超、凢脫俗 2022-10-16 10:26 303阅读 0赞

文章目录

  • 内容介绍
  • Ajax抓取示例
  • JS动态加载示例

内容介绍

开发环境为 Python3.6 ，爬虫项目全部内容索引目录

看懂Python爬虫框架，所见即所得一切皆有可能

本文介绍动态页面和Ajax渲染页面数据抓取的示例，以及相应的页面分析过程，你会发现本来想想复杂的网页爬虫居然比那些非动态网页的抓取要更简单。

虽说不会敲代码的 Python数据分析师 不是好的数据分析师，但你不是正儿八经的开发人员，代码敲的那么溜有什么用？学点数据爬虫基础能让繁琐的数据CV工作（Ctrl+C，Ctrl+V）成为自动化就足够了。

Ajax抓取示例

现在越来越多的网页的原始HTML文档不包括任何数据，而是采用Ajax统一加载。发送Ajax请求道网页更新的过程：

  • 发送请求。
  • 解析内容。
  • 渲染网页。

打开浏览器的开发者工具，到Networkk选项卡，使用XHR过滤工具。需要按照对应all_config_file.py文件建立对应相应文件夹修改该配置并且开启相关服务。

在这里插入图片描述

all_config_file.py

  1. #coding=utf-8
  2. __author__ = 'Mr数据杨'
  3. __explain__ = '各目标网站爬虫脚本配置文件'
  5. #加载引用模块
  6. import time
  7. import pymongo
  8. import pandas as pd
  10. def news_page_num():
  11.     page_num=input("输入每个网站页面爬取的页面数:")
  12.     return int(page_num)
  14. def title_error_num():
  15.     title_error_num=input("输入错误标题爬取最大数:")
  16.     return int(title_error_num)
  18. def body_error_num():
  19.     body_error_num=input("输入错误页面爬取最大数:")
  20.     return int(body_error_num)
  23. def mongodb_client():
  24.     # 获取mongoClient对象
  25.     client = pymongo.MongoClient("localhost", 27017)
  26.     # 获取使用的database对象
  27.     db = client.news
  28.     print("加载MongoDB数据库完毕......")
  29.     return db
  31. db=mongodb_client()
  33. def time_today():
  34.     # 全局函数
  35.     time_today = time.strftime('%Y-%m-%d', time.localtime(time.time()))
  36.     print("加载全局日期函数完毕......")
  37.     return time_today
  40. # 错误日志信息
  41. def error_text_title(text,time_today):
  42.     print("加载错误信息日志完毕......")
  43.     with open("logs/" + time_today + " news_title_error.txt", "a") as f:
  44.         f.write(text + '\n')
  46. # 错误日志信息
  47. def error_text_body(text,time_today):
  48.     with open("logs/" + time_today + " news_body_error.txt", "a") as f:
  49.         f.write(text + '\n')
  51. # 找到每个爬取网页的链接
  52. def get_title_links_from_MongoDB(label, type):
  53.     result = []
  54.     for item in db.news_tmp.find({ 'label': label, 'type': type}, { 'url': 1, '_id': 1}):
  55.         result.append(item)
  56.     result = pd.DataFrame(result, columns=['url', '_id'])
  57.     return result

主程序

  1. #加载引用模块
  2. import urllib
  3. import urllib.request
  4. import requests
  5. import datetime
  6. from bs4 import BeautifulSoup
  7. import all_config_file
  8. from all_config_file import error_text_title
  9. from all_config_file import error_text_body
  10. from all_config_file import get_title_links_from_MongoDB
  12. cqcoal = "http://news.cqcoal.com/manage/newsaction.do?method:webListPageNewsArchivesByTypeid"
  13. print("加载目标网址完毕......")
  14. db = all_config_file.mongodb_client()
  15. time_today = all_config_file.time_today()
  16. def cqcoal_title_start(num):
  17.     def start_type(url, label, typeid, pagenum, type):
  18.         try:
  19.             page_num = 1
  20.             while page_num <= pagenum:
  21.                 print("开始爬取：" + url)
  22.                 page_num += 1
  23.                 user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
  24.                 headers = { 'User-Agent': user_agent}
  25.                 req = urllib.request.Request(url, headers=headers)
  26.                 response = requests.get(url, headers=headers, timeout=10)
  27.                 post_param = { 'pageNum': pagenum, 'pageSize': '20', 'jsonStr': typeid}
  28.                 # post_param = post_param.format(typeid)
  29.                 return_data = requests.post(url, data=post_param, verify=False)
  30.                 content = return_data.text
  31.                 # print(content)
  32.                 if label == 'news.cqcoal.com':
  33.                     one_page = get_cqcoal_page_news(content, type)
  34.                     print('新闻抓取完毕')
  35.         except:
  36.             error = str(url+ " label:" + label + " gd:" + str(typeid) + " pagenum:" + str(pagenum) + " type:" + type + ' 未抓取到')
  37.             error_text_title(error, time_today)
  38.             print (error)
  39.     def get_cqcoal_page_news(content, type):
  40.         l = content.split("},{")
  41.         for i in [*range(len(l))]:
  42.             url = "http://news.cqcoal.com/blank/nc.jsp?mid=" + l[i][l[i].find("id") + 4:l[i].find("typeid") - 2]
  43.             title = l[i][l[i].find("title") + 8:l[i].find("shorttitle") - 3]
  44.             typename = l[i][l[i].find("typename") + 11:l[i].find("typeid2") - 3]
  45.             timeStamp = l[i][(l[i].find("pubdate") + 10):(l[i].find("senddate") - 3)]
  46.             description = l[i][l[i].find("description") + 14:l[i].find("filename") - 3]
  47.             timeStamp = int(timeStamp)
  48.             dateArray = datetime.datetime.utcfromtimestamp(timeStamp)
  49.             pubdate = dateArray.strftime("%Y-%m-%d")
  50.             one_page = { 'title': title, 'url': url, 'date': pubdate, 'type': type, 'label': 'news.cqcoal.com'}
  51.             db.news_tmp.insert_one(one_page)
  52.         return one_page
  53.     ###news.cqcoal.com
  54.     def start_Cqcoal_supply_and_demand():
  55.         start_type(cqcoal, 'news.cqcoal.com', '{"typeid":"238"}', num, 'supply_and_demand')
  56.     def start_Cqcoal_price():
  57.         start_type(cqcoal, 'news.cqcoal.com', '{"typeid":"234"}', num, 'price')
  58.     def start_Cqcoal_dynamic():
  59.         start_type(cqcoal, 'news.cqcoal.com', '{"typeid":"235"}', num, 'dynamic')
  60.     def start_Cqcoal_international():
  61.         start_type(cqcoal, 'news.cqcoal.com', '{"typeid":"236"}', num, 'international')
  62.     def start_Cqcoal_comment():
  63.         start_type(cqcoal, 'news.cqcoal.com', '{"typeid":"14"}', num, 'comment')
  64.     def start_Cqcoal_transportation():
  65.         start_type(cqcoal, 'news.cqcoal.com', '{"typeid":"239"}', num, 'transportation')
  66.     def start_Cqcoal_economics():
  67.         start_type(cqcoal, 'news.cqcoal.com', 'road_price}', num, 'economics')
  68.     def start_Cqcoal_policy():
  69.         start_type(cqcoal, 'news.cqcoal.com', '{"typeid":"230"}', num, 'policy')
  70.     def start_Cqcoal_correlation():
  71.         start_type(cqcoal, 'news.cqcoal.com', '{"typeid":"237"}', num, 'correlation')
  72.     def start_Cqcoal_expert():
  73.         start_type(cqcoal, 'news.cqcoal.com', '{"typeid":"232"}', num, 'expert')
  74.     start_Cqcoal_transportation()
  75.     start_Cqcoal_supply_and_demand()
  76.     start_Cqcoal_price()
  77.     start_Cqcoal_policy()
  78.     start_Cqcoal_international()
  79.     start_Cqcoal_expert()
  80.     start_Cqcoal_economics()
  81.     start_Cqcoal_dynamic()
  82.     start_Cqcoal_correlation()
  83.     start_Cqcoal_comment()
  84. def cqcoal_body_start():
  85.     def get_new_body(label, type):
  86.         link_list = get_title_links_from_MongoDB(label, type)
  87.         if label == 'news.cqcoal.com':
  88.             try:
  89.                 for url in link_list['url']:
  90.                     news_body, news_body_1 = get_news_Cqcoal_text(url,label, type)
  91.                     if news_body is not None:
  92.                         db.news_tmp.update({ 'url': url}, { "$set": { 'newsbody': news_body}})
  93.                         db.news_tmp.update({ 'url': url}, { "$set": { 'newsbody_1': news_body_1}})
  94.                 print("网站：" + label + " 类型：" + type + "内容爬取完毕!")
  95.             except:
  96.                 error = str(url + " error:" + ' label:' + label + " type：" + type)
  97.                 #error_text(error)
  98.                 print(error)
  100.     def get_news_Cqcoal_text(url,label, type):
  101.         # html = urllib.request.urlopen(url,timeout=5)
  102.         id = url.split('=')[1]
  103.         url = 'http://news.cqcoal.com/manage/newsaction.do?method:getNewsAddonarticle'
  104.         post_param = { 'id': id}
  105.         # return_data = requests.post(url,data =post_param, verify = False)
  106.         try:
  107.             return_data = requests.post(url, data=post_param, verify=False, timeout=120)
  108.         except:
  109.             print("error label:", url, " Time out!")
  110.             error = str(url + " error:" + ' label:' + label + " type：" + type)
  111.             error_text_body(error, time_today)
  112.             return None, None
  113.         return_data = return_data.text
  114.         try:
  115.             newsBody = return_data[return_data.find("body") + 7:return_data.find("xh") - 3]
  116.             newsBody_1 = return_data[return_data.find("body") + 7:return_data.find("xh") - 3]
  117.             print(url + " 记录爬取完毕")
  118.             return newsBody, newsBody_1
  119.         except:
  120.             print("error label:", url, " type:", type)
  121.             error = str(url + " error:" + ' label:' + label + " type：" + type)
  122.             error_text_body(error, time_today)
  123.             return None, None
  125.     ###news.cqcoal.com
  126.     def start_body_Cqcoal_transportation():
  127.         get_new_body('news.cqcoal.com', 'transportation')
  128.     def start_body_Cqcoal_supply_and_demand():
  129.         get_new_body('news.cqcoal.com', 'supply_and_demand')
  130.     def start_body_Cqcoal_price():
  131.         get_new_body('news.cqcoal.com', 'price')
  132.     def start_body_Cqcoal_policy():
  133.         get_new_body('news.cqcoal.com', 'policy')
  134.     def start_body_Cqcoal_international():
  135.         get_new_body('news.cqcoal.com', 'international')
  136.     def start_body_Cqcoal_expert():
  137.         get_new_body('news.cqcoal.com', 'expert')
  138.     def start_body_Cqcoal_dynamic():
  139.         get_new_body('news.cqcoal.com', 'dynamic')
  140.     def start_body_Cqcoal_economics():
  141.         get_new_body('news.cqcoal.com', 'economics')
  142.     def start_body_Cqcoal_correlation():
  143.         get_new_body('news.cqcoal.com', 'correlation')
  144.     def start_body_Cqcoal_comment():
  145.         get_new_body('news.cqcoal.com', 'comment')
  146.     start_body_Cqcoal_transportation()
  147.     start_body_Cqcoal_supply_and_demand()
  148.     start_body_Cqcoal_price()
  149.     start_body_Cqcoal_policy()
  150.     start_body_Cqcoal_international()
  151.     start_body_Cqcoal_expert()
  152.     start_body_Cqcoal_economics()
  153.     start_body_Cqcoal_dynamic()
  154.     start_body_Cqcoal_correlation()
  155.     start_body_Cqcoal_comment()

JS动态加载示例

浏览器渲染引擎：渲染引擎的职责就是渲染，即在浏览器窗口中显示所请求的内容。浏览器向服务器发送请求，得到服务器返回的资源文件后，经过需要渲染引擎的处理，将资源文件显示在浏览器窗口中。

目前使用较为广泛的渲染引擎有两种：

  • webkit——使用者有Chrome, Safari
  • Geoko——使用者有Firefox

渲染主流程：渲染引擎首先通过网络获得所请求文档的内容，通常以8K分块的方式完成。下面是渲染引擎在取得内容之后的基本流程：

  • 解析html以构建dom树 -> 构建render树 -> 布局render树 -> 绘制render树渲染引擎开始解析html，并将标签转化为内容树中的dom节点。如果遇到JS，那么此时会启用单独连接进行下载，并且在下载后进行解析。
  • 接着，它解析外部CSS文件及style标签中的样式信息。这些样式信息以及html中的可见性指令将被用来构建另一棵树——render树。
  • Render树由一些包含有颜色和大小等属性的矩形组成，它们将被按照正确的顺序显示到屏幕上。
  • Render树构建好了之后，将会执行布局过程，它将确定每个节点在屏幕上的确切坐标。
  • 再下一步就是绘制，即遍历render树，并使用UI后端层绘制每个节点。

渲染动态网页，有两种选择：

  • 自己从头实现一个浏览器渲染引擎，在合适的时机返回构建的dom树或render树。
  • 这需要进行大量的工作，需要考虑html、js、css等不同格式文件的解析方式以及解析顺序等。

以36氪主页抓取为实例。
在这里插入图片描述

  1. import warnings
  2. warnings.filterwarnings("ignore")
  3. import time
  4. import requests
  5. import pymongo
  6. import pandas as pd
  7. import re
  8. from lxml import etree
  10. # 获取mongoClient对象
  11. client = pymongo.MongoClient("localhost", 27017)
  12. # 获取使用的database对象
  13. db = client.news
  14. today=time.strftime('%Y.%m.%d',time.localtime(time.time()))
  16. def main(page_num):
  17.     #开始爬取数据设定抓取页面数字
  18.     n=int(page_num)
  20.     def start_crawler(pro,col,adress):
  21.         i=1
  22.         while i<= n:
  23.             t = time.time()
  24.             url = "https://36kr.com/api/search-column/{}?per_page=20&page={}&_=".format(pro,i)+str(int(t))
  25.             i+=1
  26.             post_param = { 'per_page':'20',\
  27.                           'page':str(i),\
  28.                           '_':int(t)}
  29.             time.sleep(2)
  30.             user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
  31.             headers = {  'User-Agent' : user_agent }
  32.             return_data = requests.get(url,data =post_param, verify = False)
  33.             one_page = get_news_link_to_mongodb(return_data,col,url,adress)
  34.         print (adress+" "+col+' 新闻标题抓取完毕')
  36.     def get_news_link_to_mongodb(return_data,col,url,adress):
  37.         content = return_data.json().get('data').get('content')
  38.         for i in return_data.json().get('data').get('items'):
  39.             one_page = { 'title': i["title"], \
  40.                         'url_html': "https://36kr.com/p/"+str(i["id"])+".html",\
  41.                         'url_json': "https://36kr.com/p/"+str(i["id"])+".json",\
  42.                         'summary':i["summary"],\
  43.                         'tags': re.sub(r'[0-9"\[\]]','',i["extraction_tags"]).replace(",,"," ").replace(",",""),\
  44.                         'label':col,\
  45.                         'adress':adress,\
  46.                         'work_date':time.strftime('%Y.%m.%d',time.localtime(time.time()))}
  47.             db.kr36.insert_one(one_page)
  48.         print("已经完成抓取 "+adress+" "+col+" 板块页面："+url)
  50.     def news_body_start(label,adress,today):
  51.         url_list = []
  52.         for item in db.kr36.find({ 'label': label,'adress':adress,'work_date':today}, { 'url_json': 1}):
  53.             url_list.append(item)
  54.         url_list = pd.DataFrame(url_list, columns=['url_json'])
  55.         for i in url_list["url_json"]:
  56.             html = requests.get(i)
  57.             a=html.json().get('props').get('detailArticle|post').get('content')
  58.             sel = etree.HTML(a)
  59.             clear_content = sel.xpath('string(//*)')
  60.             db.kr36.update({ 'url_json': i}, { "$set": { 'newsbody': clear_content}})
  61.             print(i+" 抓取完毕")
  62.         print (adress+" "+label+" "+"新闻主体抓取完毕")
  64.     start_crawler('23','大公司',"36kr")
  65.     start_crawler('221','消费',"36kr")
  66.     start_crawler('225','娱乐',"36kr")
  67.     start_crawler('218','前沿技术',"36kr")
  68.     start_crawler('219','汽车交通',"36kr")
  69.     start_crawler('208','区块链',"36kr")
  70.     start_crawler('103','技能get',"36kr")
  72.     news_body_start("大公司","36kr",today)
  73.     news_body_start("消费","36kr",today)
  74.     news_body_start("娱乐","36kr",today)
  75.     news_body_start("前沿技术","36kr",today)
  76.     news_body_start("汽车交通","36kr",today)
  77.     news_body_start("区块链","36kr",today)
  78.     news_body_start("技能get","36kr",today)
  80.     #导出模块
  81.     name = { 'adress':'36kr'}
  82.     search_res = db.kr36.find(name)
  83.     list_=[]
  84.     for i in search_res:
  85.         list_.append(i)
  86.     ddf=pd.DataFrame(list_,columns=["title","url_html","tags","labels","adress","summary","newsbody"])
  87.     ddf.to_csv("36氪首页news.csv",encoding="utf_8_sig")
  89. if __name__ == '__main__':
  90.     page_num=input("输入需要抓取的页数")
  91.     main(page_num)

发表评论

表情：
评论列表 （有 0 条评论，303人围观）

还没有评论，来说两句吧...

相关阅读