python爬虫爬取知网

谁践踏了优雅 2021-07-26 13:49 1381阅读 0赞

python爬虫爬取知网

话不多说，直接上代码！

  1. import requests 
  2. import re
  3. import time
  4. import xlrd
  5. from xlrd import open_workbook
  6. from xlutils.copy import copy
  8. class BeautifulPicture():
  9.     def get_pic(self):
  10.         data = xlrd.open_workbook(r'C:\\flim\library_my\new.xls')  # 打开xls文件，自己新建这个文件在运行路径，不然不行的哦
  11.         table = data.sheets()[0]  # 打开第一张表
  12.         table2 = data.sheets()[1]  # 打开第一张表
  13.         i = table.nrows #上一次爬到的表1的行数
  14.         i1 = 0
  15.         i2 = table2.nrows #上一次爬到的表2的行数
  16.         told = 0
  17.         rb = open_workbook(r'C:\\flim\library_my\new.xls','utf-8')
  18.         wb = copy(rb) # 将上一次爬到的复制到新表里，并在新表里继续添加纪录
  19.         # 通过get_sheet()获取的sheet有write()方法
  20.         ws = wb.get_sheet(0)
  21.         ws1 = wb.get_sheet(1)
  22.         p = 1  # 这里是页数
  23.         for num in range(p, p + 100):
  24.             # 这里的num是页码
  25.             web_url ='http://kns.cnki.net/kns/brief/brief.aspx?curpage=%s&Reco' \
  26.                      'rdsPerPage=50&QueryID=8&ID=&turnpage=1&tpagemode=L&dbPref' \
  27.                      'ix=SCDB&Fields=&DisplayMode=listmode&PageName=ASP.brief_' \
  28.                      'default_result_aspx#J_ORDER&' % num #这里的URL实现了二次加载
  29.             print('搜素页的URL=',web_url)
  30.             # 这里开始是时间控制
  31.             t = int(time.clock())
  32.             print(t / 60, '分钟')
  33.             useTime = t - told
  34.             # 如果一个周期的时间使用太短，则等待一段时间
  35.             # 主要用于防止被禁
  36.             if (useTime < 120 and useTime > 10):
  37.                 print("useTime=%s" % useTime)
  38.                 whiteTime = 120 - useTime
  39.                 print("等待%s秒" % whiteTime)
  40.                 time.sleep(whiteTime)
  41.             told = int(time.clock())
  42.             print(t)
  43.             print('开始网页get请求')
  44.             r = self.request(web_url)
  45.             # 这里是报错的解释，能知道到底是因为什么不能继续爬了
  46.             # 一开始会看爬到的源代码，但是之后正式开始爬的时候，打印页面源代码会拉低爬虫效率
  47.             yan = re.search(r'参数错误', r.text)
  48.             if yan != None:
  49.                 print("参数")
  50.                 break
  51.             yan = re.search(r'验证码', r.text)
  52.             if yan != None:
  53.                 print("验证")
  54.                 break
  55.             #这里开始抓列表里每一个文献的url
  56.             soup = re.findall(r'<TR([.$\s\S]*?)</TR>', r.text)
  57.             #print(soup)#测试打印
  58.             for a in soup:
  59.                 print("-", i1)
  60.                 #print(a)#测试代码
  61.                 i1 += 1
  62.                 name = re.search(r'_blank.*<', a)
  63.                 #print('初次取的=',name)#测试代码
  64.                 name = name.group()[8:-1]
  65.                 #print('第二次=',name)#测试代码
  66.                 name = re.sub(r'<font class=Mark>', '', name)
  67.                 #print('第三次=', name) # 测试代码
  68.                 name = re.sub(r'</font>', '', name)
  69.                 #print('第四次=', name) # 测试代码
  70.                 url = re.search(r'href=.*? ', a)#将’‘看做一个子表达式，惰性匹配一次就可以了
  71.                 url = url.group()
  72.                 #print('爬取的详情页的URL=',url)#测试代码
  73.                 # 将爬来的相对地址，补充为绝对地址
  74.                 url = "http://kns.cnki.net/KCMS/" + url[11:-2]#数字是自己数的。。。
  75.                 #print("url:%s" % url) # 这里是写代码时测试留下的print记录
  76.                 #下面是参考文献详情的URL
  77.                 FN = re.search(r'FileName.*?&', url)#.group()#出现错误 没有匹配！！！
  78.                 if FN !=None:#测试代码
  79.                     FN = re.search(r'FileName.*?&', url).group()
  80.                 #print(FN)#测试代码
  81.                 DN = re.search(r'DbName.*?&', url)#.group()
  82.                 if DN !=None:#测试代码
  83.                     DN=re.search(r'DbName.*?&', url).group()
  84.                 #print(DN) #测试代码
  85.                 DC = re.search(r'DbCode.*?&', url).group()
  86.                 DUrl = "http://kns.cnki.net/KCMS/detail/frame/list.aspx?%s%s%sRefType=1" % (FN, DN, DC)
  87.                 #print('DUrl=',DUrl)#测试代码
  88.                 # 这里打开文献详情页
  89.                 R = self.request(DUrl)
  90.                 #如果没有参考文献，则认为是劣质文献，不爬，转爬下一篇
  91.                 isR = re.search(r'参考文献', R.text)
  92.                 if i1 == 1:
  93.                     print("没有参考文献的文章:%s" % name)
  94.                 if isR == None:
  95.                     continue
  96.                 # 详情页
  97.                 print(i)
  98.                 print("文章名字:%s" % name)
  99.                 d = self.request(url).text
  100.                 #print('d=',d)#测试代码
  101.                 # 这里是文献摘要，
  102.                 summary = re.search(r'(?<=name="ChDivSummary">).+?(?=</span>)', d)
  103.                 summary=summary.group()
  104.                 #print('摘要=',summary)
  105.                 type = re.search(r'"\).html\(".*?"', d)
  106.                 type = type.group()[9:-1]
  107.                 ins = re.search(r'TurnPageToKnet\(\'in\',\'.*?\'', d)
  108.                 if ins == None:
  109.                     continue
  110.                 ins = ins.group()[21:-1]
  111.                 wt = re.findall(r'TurnPageToKnet\(\'au\',\'.*?\'', d)
  112.                 writer = ""
  113.                 for w in wt:
  114.                     writer = writer + "," + w[21:-1]
  115.                 writer = writer[1:]
  116.                 ws.write(i, 0, name)    #文献名
  117.                 ws.write(i, 1, writer)  #作者名
  118.                 ws.write(i, 2, type)    #文献类别
  119.                 ws.write(i, 15, num)    #列表的页码
  120.                 ws.write(i, 3, summary)  # 摘要
  121.                 ws.write(i, 16, time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) #记录的时间
  122.                 # 这里是文献的关键词，最多可以记录8个关键词
  123.                 kw = re.findall(r'TurnPageToKnet\(\'kw\',\'.*?\'', d)
  124.                 tnum = 0
  125.                 for tkw in kw:
  126.                     tnum += 1
  127.                     tkw = tkw[21:-1]
  128.                     if tnum > 8:
  129.                         break
  130.                     ws.write(i, 3 + tnum, tkw)
  131.                  # 这里是文献的来源基金
  132.                 fund = re.search(r'TurnPageToKnet\(\'fu\',\'.*?\'', d)
  133.                 if fund != None:
  134.                     fund = fund.group()[21:-1]
  135.                     ws.write(i, 11, fund)
  136.                 # 分类号
  137.                 cn = re.search(r'ZTCLS.*?</p', d)
  138.                 if cn != None:
  139.                     cn = cn.group()[19:-3]
  140.                     ws.write(i, 12, cn)
  141.                 # 这是机构
  142.                 jg = re.search(r'TurnPageToKnet\(\'in\',\'.*?\'', d)
  143.                 if jg != None:
  144.                     jg = jg.group()[21:-1]
  145.                     #print(jg)
  146.                     ws.write(i, 17, jg)
  147.                 # 是否为核心期刊
  148.                 sourinfo = re.search(r'sourinfo([.$\s\S]*?)</div', d)
  149.                 if sourinfo != None:
  150.                     sourinfo = sourinfo.group()
  151.                     # print(sourinfo)
  152.                     from_ = re.search(r'title.*</a', sourinfo).group()
  153.                     from_ = re.sub(r'title">.*?>', '', from_)
  154.                     from_ = re.sub(r'</a', '', from_)
  155.                     ws.write(i, 13, from_)
  156.                     core = re.search(r'中文核心期刊', sourinfo)
  157.                     if core != None:
  158.                         print(core.group())
  159.                         ws.write(i, 14, "中文核心期刊")
  160.                  # ws.write(i, 0, a)
  161.                 i += 1 # 增加页码的计数
  162.         wb.save('new.xls') # 保存表格*******有改变 有问题
  164.     def request(self, url):  # 返回网页的response
  165.         # print(url)
  166.         # 这里是伪造浏览器信息，和伪造来源
  167.         user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Ch' \
  168.                      'rome/58.0.3029.81 Safari/537.36'
  169.         referer = "http://kns.cnki.net/kns/brief/result.aspx"
  170.         # 这里是伪造cookie，你要从浏览器里复制出来，粘贴在这里
  171.         # 可以把里面的时间不断更新，这样能爬久一点
  172.         cookie = 'cnkiUserKey=e0b40271-cdc0-fd83-36fa-9d4c5f8f2a86; RsPerPage=5' \
  173.                  '0; KNS_DisplayModel=custommode@CJFQ; UM_distinctid=161275ba757a' \
  174.                  '16-0ed232d4ab2da2-6010107f-15f900-161275ba75895e; Ecp_Cli' \
  175.                  'entId=3170915092701569567; ASP.NET_SessionId=3dnqvlsxovsla4' \
  176.                  'ywvr2ykp4c; SID_kns=123106; SID_crrs=125131; SID_klogin=12514' \
  177.                  '3; SID_krsnew=125132; SID_kcms=124113; SID_knsdelivery=125124;' \
  178.                  ' CNZZDATA3258975=cnzz_eid%3D995934993-1516843818-http%253A%252' \
  179.                  'F%252Fkns.cnki.net%252F%26ntime%3D1517383256; Ecp_session=1; ' \
  180.                  'c_m_LinID=LinID=WEEvREcwSlJHSldRa1FhdkJkVWI2VUJNODFzUHlUaC9LN' \
  181.                  'XZrUDkrcXhYbz0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4ggI8Fm4gTkoU' \
  182.                  'KaID8j8gFw!!&ot=02/01/2018 18:13:09; LID=WEEvREcwSlJHSldRa1F' \
  183.                  'hdkJkVWI2VUJNODFzUHlUaC9LNXZrUDkrcXhYbz0=$9A4hF_YAuvQ5obgVAqN' \
  184.                  'KPCYcEjKensW4ggI8Fm4gTkoUKaID8j8gFw!!; c_m_expire=2018-02-01 ' \
  185.                  '18:13:09; Ecp_LoginStuts=%7B%22IsAutoLogin%22%3Afalse%2C%22U' \
  186.                  'serName%22%3A%22KT1008%22%2C%22ShowName%22%3A%22%25E8%25A5%2' \
  187.                  '5BF%25E5%258D%2597%25E7%259F%25B3%25E6%25B2%25B9%25E5%25A4%' \
  188.                  '25A7%25E5%25AD%25A6%22%2C%22UserType%22%3A%22bk%22%2C%22r%2' \
  189.                  '2%3A%22RrMW7A%22%7D'
  190.         headers = { 'User-Agent': user_agent,
  191.                    "Referer": referer,
  192.                    "cookie": cookie}
  193.         r = requests.get(url, headers=headers, timeout=30)
  194.         return r
  196. print(time.clock())
  197. beauty = BeautifulPicture()  # 创建类的实例
  198. beauty.get_pic()  # 执行类中的方法

发表评论

表情：
评论列表 （有 0 条评论，1381人围观）

还没有评论，来说两句吧...

相关阅读