python爬虫3——爬取腾讯招聘全部招聘信息

柔光的暖阳◎ 2022-05-09 15:46 460阅读 0赞

python爬虫2中，已经有了初步的代码，之后做了优化

增加了工作职责、工作要求：

![70][]

获取的数据有：

![70 1][]

代码如下：

#!/usr/bin/env python
    # -*- coding:utf-8 -*-
    
    from bs4 import BeautifulSoup
    import urllib2
    import urllib
    import json    # 使用了json格式存储
    
    
    def tengxun(detail,num):
        url = 'https://hr.tencent.com/'
        # detail = 'position.php?&start=0#a'
        request = urllib2.Request(url + detail)
        response =urllib2.urlopen(request)
        resHtml = response.read()
        soup = BeautifulSoup(resHtml,'html.parser', from_encoding='utf-8')
        result = soup.select(".even")
        result += soup.select(".odd")
        # print len(result)
        # 处理页面
        items = []
        for node in result:
            item = {}
            # 职位名
            zname = node.select('td')[0].get_text()
            # 职位类别
            ztype = node.select('td')[1].get_text()
            # 人数
            znum = node.select('td')[2].get_text()
            # 地点
            zlocal = node.select('td')[3].get_text()
            # 发布时间
            ztime = node.select('td')[4].get_text()
            # 链接
            detailLink = node.select('td a')[0].attrs['href']
            # 获取工作职责、工作要求
            request1 = urllib2.Request(url + detailLink)
            response1 = urllib2.urlopen(request1)
            jobHtml = response1.read()
            soup1 = BeautifulSoup(jobHtml, 'html.parser', from_encoding='utf-8')
            # print len(soup1.select('ul.squareli'))
            # 工作职责
            jobRes = ''
            for li in soup1.select('ul.squareli')[0].select('li') :
                jobRes += li.get_text() + '\n'
            # 工作要求
            jobReq = ''
            for li in soup1.select('ul.squareli')[1].select('li') :
                jobReq += li.get_text() + '\n'
            # print jobReq
            # 将数据存入item中
            item['zname']=zname;
            item['detailLink'] = detailLink;
            item['ztype']=ztype
            item['znum'] = znum
            item['zlocal'] = zlocal
            item['ztime'] = ztime
            item['jobRes'] = jobRes
            item['jobReq'] = jobReq
            # 处理工作职责和工作要求
            items.append(item)
    
        origin = []
        print(len(items))
    
        # 以json格式输出到文件中
        # 禁用ascii编码，按utf-8编码
        output = open('tencent.json'+ str(num), 'w')
        for i in origin:
            items.append(i)
        line = json.dumps(items, ensure_ascii=False);
        # print line
        output.write(line.encode('utf-8'))
        output.close()
    
    # print resHtml
    for i in range(303):
        print("进行到第" + str(i) + "页")
        url = 'position.php?&start='+ str(i * 10) +'#a'
        tengxun(url, i)

取出来的json数据：

![70 2][]

[70]: /images/20220506/4e73877106144f57a787a8944c0e88ba.png
[70 1]: /images/20220506/83f9f64eedff4f558c1bcee3685a3e05.png
[70 2]: /images/20220506/edfb80da9fd14f0f870f5f54c2b4d46f.png