Python，网络爬虫selenium与pyautogui抓取新浪微博用户数据

你的名字 2022-12-31 13:20 169阅读 0赞

# Python，网络爬虫selenium与pyautogui抓取新浪微博用户数据 #

不需要登陆新浪微博账户，直接运行就可以通过python爬虫爬取新浪微博用户数据。本例selenium与pyautogui结合，爬取十大城市的地铁（轨道交通）官方微博的粉丝数量。最终把数据存入excel和csv文件。

python代码：

import re
    import time
    import pandas as pd
    from tqdm import tqdm
    import pyautogui
    from selenium import webdriver
    
    
    def get_city_data(city):
        url = f'https://weibo.com/{city[1]}'
        chromepath = r'D:\program\chromedriver_win32\chromedriver.exe'
        driver = webdriver.Chrome(executable_path=chromepath)
    
        driver.get(url)
        time.sleep(15)
    
        file_path = r'd:\web\{city}_{file_time}.html'.format(city=city[1],
                                                             file_time=time.strftime('%Y%m%d%H%M', time.localtime()))
    
        pyautogui.hotkey('ctrl', 's')
        time.sleep(1)
    
        pyautogui.hotkey('delete')
        time.sleep(1)
    
        pyautogui.typewrite(file_path)
        time.sleep(1)
    
        pyautogui.hotkey('enter')
        time.sleep(1)
    
        pyautogui.hotkey('ctrl', 'enter')
        time.sleep(15)
    
        f = open(file=file_path, mode='r', encoding='utf-8')
        content = str(f.read())
    
        fan_num_pattern = re.compile(r'粉丝\((.*?)\)', re.S)
        fan_num = re.search(fan_num_pattern, content)
        num = fan_num.groups()[0]
    
        driver.close()
        driver.quit()
    
        return city[0], int(num), time.strftime('%Y-%m-%d %H:%M', time.localtime())
    
    
    def main():
        city = [('北京', 'bjsubway'),
                ('上海', 'shmetro'),
                ('广州', 'gzmtr'),
                ('深圳', 'szmcservice'),
                ('成都', '2384889627'),
                ('西安', 'xianditie'),
                ('武汉', '3186945861'),
                ('重庆', '2152519810'),
                ('杭州', 'hangzhouditie'),
                ('南京', '2638276292'),
                ('天津', 'tjdtjt'),
                ]
    
        city_data = []
        pbar = tqdm(total=len(city), leave=True)
        for c in city:
            result = get_city_data(c)
            print(result)
            city_data.append(list(result))
    
            pbar.update(1)
    
        col = ['城市', '粉丝数量', '统计时间']
        df = pd.DataFrame(data=city_data, columns=col)
        df = df.sort_values(by=col[1], axis=0, ascending=False)  # 降序
    
        # 排序后重置index，
        # 否则索引是混乱的
        df = df.reset_index(drop=True)
    
        # 因为默认的pandas起始索引从0开始，
        # 为了使数据行的初始索引（起始索引index）从1开始
        df.index = df.index + 1
    
        print(df.head(10))
    
        df.to_excel('city.xls', encoding='utf-8')
        df.to_csv('city.csv', encoding='utf-8')
    
    
    if __name__ == '__main__':
        main()

生成的excel表数据输出：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3poYW5ncGhpbA_size_16_color_FFFFFF_t_70][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3poYW5ncGhpbA_size_16_color_FFFFFF_t_70]: /images/20221120/e01b8adfc606416aaee9cde212fab4d0.png