Python，网络爬虫爬取百度贴吧元数据存入excel

谁借莪１个温暖的怀抱￠ 2022-12-29 09:47 212阅读 0赞

import requests
    import re
    import time
    import pandas as pd
    from tqdm import trange
    
    
    def get_city_data(city):
        url = f'https://tieba.baidu.com/f?ie=utf-8&kw={city}'
        r = requests.get(url=url, timeout=30)
    
        card_menNum_pattern = re.compile(r'<span class="card_menNum">(.*?)</span>', re.S)
        card_menNum_result = re.search(card_menNum_pattern, r.text)
        card_menNum = card_menNum_result.groups()[0]
        card_menNum = card_menNum.replace(',', '')
    
        card_infoNum_pattern = re.compile(r'<span class="card_infoNum">(.*?)</span>', re.S)
        card_infoNum_result = re.search(card_infoNum_pattern, r.text)
        card_infoNum = card_infoNum_result.groups()[0]
        card_infoNum = card_infoNum.replace(',', '')
    
        return city, card_menNum, card_infoNum, time.strftime('%Y-%m-%d', time.localtime())
    
    
    def main():
        city = ['北京', '上海', '广州', '深圳', '成都',
                '天津', '南京', '杭州', '重庆', '武汉',
                '长沙', '厦门', '郑州', '沈阳', '西安',
                '大连', '济南', '青岛', '苏州', '昆明',
                '合肥', '福州', '哈尔滨', '太原', '宁波',
                '贵阳', '南宁', '长春', '南昌', '石家庄', '东莞',
                ]
    
        city_data = []
        for c, _ in zip(city, trange(len(city))):
            result = get_city_data(c)
            city_data.append(list(result))
            time.sleep(3)
    
        col = ['城市', '关注', '发帖', '时间']
        df = pd.DataFrame(data=city_data, columns=col)
        df = df.sort_values(by=col[1], axis=0, ascending=False)  # 降序
    
        # 排序后重置index，
        # 否则索引是混乱的
        df = df.reset_index(drop=True)
    
        # 因为默认的pandas起始索引从0开始，
        # 为了使数据行的初始索引（起始索引index）从1开始
        df.index = df.index + 1
    
        df.to_excel('city.xls', encoding='utf-8')
    
        print(df.head(10))
    
    
    if __name__ == '__main__':
        main()

输出：

城市       关注        发帖          时间
    1   成都  3306318  93013066  2020-12-22
    2   深圳  2993915  67340313  2020-12-22
    3   北京  2665298  43814643  2020-12-22
    4   广州  2657354  58574023  2020-12-22
    5   郑州  2576916  93291514  2020-12-22
    6   重庆  2383480  63210904  2020-12-22
    7   上海  2147023  15954088  2020-12-22
    8   西安  2009421  81562527  2020-12-22
    9   杭州  2001683  24184211  2020-12-22
    10  武汉  2000696  40914598  2020-12-22