selenium爬拉勾网职位信息并保存到mongoDB数据库

梦里梦外; 2022-05-15 01:09 256阅读 0赞

import pymongo
    
    from selenium import webdriver
    from lxml import etree
    import re
    import time
    
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    
    
    class LagouSpider(object):
        def __init__(self):
            self.driver = webdriver.Chrome()
            self.url = "https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput="
            self.positions = []
    
        def connectDB(self, host, port):
            """
            连接数据库
            :param host: IP号
            :param port: 端口号
            :return:
            """
            client = pymongo.MongoClient(host=host, port=port)
            return client
    
        def run(self):
            self.driver.get(self.url)
            while True:
                # 获取page_source
                source = self.driver.page_source
                # 直到页面加载出来"//div[@class='pager_container']/span[last()]" 才执行下面的活动
                WebDriverWait(driver=self.driver, timeout=10).until(
                    EC.presence_of_element_located((By.XPATH, "//div[@class='pager_container']/span[last()]"))
                )
                # 调用parse_list_page获取所有职位的超链接方法
                self.parse_list_page(source)
                time.sleep(2)
                # 下一页
                next_btn = self.driver.find_element_by_xpath("//div[@class='pager_container']/span[last()]")
                # 判断是否是最后一页，如果是break,否则点击“下一页”
                if "pager_next_disabled" in next_btn.get_attribute("class"):
                    break
                else:
                    next_btn.click()
                time.sleep(1)
    
        def parse_list_page(self, source):
            """
            获取所有职位的超链接
            :param source:
            :return:
            """
            html = etree.HTML(source)
            # 获取职位详情的超链接links列表
            links = html.xpath("//a[@class='position_link']/@href")
            # 遍历links列表
            for link in links:
                # 调用request_detail_page请求职位详情页方法
                self.request_detail_page(link)
                time.sleep(1)
    
        def request_detail_page(self, url):
            """
            请求职位详情页
            :param url:请求职位详情页的url
            :return:
            """
            # self.driver.get(url)
            self.driver.execute_script("window.open('%s')" % url)
            self.driver.switch_to.window(self.driver.window_handles[1])
            WebDriverWait(driver=self.driver, timeout=10).until(
                EC.presence_of_element_located((By.XPATH, "//div[@class='job-name']/span[@class='name']"))
            )
            source = self.driver.page_source
            # 调用parse_detail_page获取职位“字段”的方法
            self.parse_detail_page(source)
            # 关闭当前这个详情页
            self.driver.close()
            # 继续切换回职位列表页
            self.driver.switch_to.window(self.driver.window_handles[0])
    
        def parse_detail_page(self, source):
            """
            获取职位详情页的“字段”
            :param source:
            :return:
            """
            html = etree.HTML(source)
            position_name = html.xpath("//span[@class='name']/text()")[0]
            job_request = html.xpath("//dd[@class='job_request']//span")
            salary = job_request[0].xpath("./text()")[0].strip()
            city = job_request[1].xpath("./text()")[0].strip()
            # 去掉无用的字符
            city = re.sub(r"[\s/]", "", city)
            work_year = job_request[2].xpath("./text()")[0].strip()
            work_year = re.sub(r"[\s/]", "", work_year)
            education = job_request[3].xpath("./text()")[0].strip()
            education = re.sub(r"[\s/]", "", education)
            desc = "".join(html.xpath("//dd[@class='job_bt']//text()")).strip()
            company_name = html.xpath("//h2[@class='fl']/text()")[0].strip()
    
            position = {
                'name': position_name,
                'company_name': company_name,
                'salary': salary,
                'city': city,
                'work_year': work_year,
                'education': education,
                'desc': desc
    
            }
            self.positions.append(position)
            # print(position)
            # 链接数据库
            client = self.connectDB('127.0.0.1', 27017)
            db = client['test']
            p = db['lagou']
            # 保存至数据库
            p.insert(position)
    
            print("*" * 40)
    
    
    if __name__ == '__main__':
        spider = LagouSpider()
        spider.run()