python爬虫爬取腾讯新闻

红太狼 2021-07-27 01:32 804阅读 0赞

## python爬虫爬取腾讯新闻 ##

**话不多说，直接上代码！**

import requests
    from bs4 import BeautifulSoup
    
    def getHTMLText(url):
        try:
            r = requests.get(url, timeout = 30)
            r.raise_for_status()
            #r.encoding = 'utf-8'
            return r.text
        except:
            return ""
    
    def getContent(url):
        html = getHTMLText(url)
        # print(html)
        soup = BeautifulSoup(html, "html.parser")
        title = soup.select("div.hd > h1")
        print(title[0].get_text())
        time = soup.select("div.a_Info > span.a_time")
        print(time[0].string)
        author = soup.select("div.qq_articleFt > div.qq_toolWrap > div.qq_editor")
        print(author[0].get_text())
        paras = soup.select("div.Cnt-Main-Article-QQ > p.text")
        for para in paras:
            if len(para) > 0:
                print(para.get_text())
                print()
        fo = open("text.txt", "w+")
        fo.writelines(title[0].get_text() + "\n")
        fo.writelines(time[0].get_text() + "\n")
        for para in paras:
            if len(para) > 0:
                fo.writelines(para.get_text() + "\n\n")
        fo.writelines(author[0].get_text() + '\n')
        fo.close()
        article = { 
            'Title' : title[0].get_text(),
            'Time' : time[0].get_text(),
            'Paragraph' : paras,
            'Author' : author[0].get_text()
        }
        print(article)
        
    def main():
        url = "http://news.qq.com/a/20170504/012032.htm"
        getContent(url);
    main()

**运行结果：**  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2d1b3h1eWluZw_size_16_color_FFFFFF_t_70_pic_center]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2d1b3h1eWluZw_size_16_color_FFFFFF_t_70_pic_center]: /images/20210726/928d3de00b574714b0103ce287395b87.png