python利用requests和lxml库爬取豆瓣top250的电影信息并保存到csv文件中

旧城等待， 2022-01-21 23:45 187阅读 0赞

### 准备 ###

开发环境：windows 7

开发工具：pycharm

python版本：python 3.7

用到的库：requests，lxml，csv

requests，和lxml库是第三方库需要自行安装，安装方法：pip install requests，pip install lxml

csv库为python自带库不需安装

### 关键步骤 ###

1.  确定url，设置请求头user-agent，利用requests库请求并得到网页源码
2.  分析返回的源码，利用xpath解析源码得到想要获取的内容并以字典的方式存到列表中
3.  遍历列表将数据写入csv文件中

### 实现代码 ###

# coding = utf-8
    # Author:'Alvin'
    # Date: '2018/8/10'
     
    import requests
    from lxml import html
    import csv
     
    #step1 确定url，设置请求头user-agent，利用requests库请求并得到网页源码
    def get_content():
        url = 'https://movie.douban.com/top250'
        user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
        headers = {
            "User_Agent": user_agent
        }
        response = requests.get(url, headers=headers)
        get_movie_list(response.content)
     
    #setp2 分析返回的源码，利用xpath解析源码得到想要获取的内容并以字典的方式存到列表中
    def get_movie_list(source):
        selector = html.document_fromstring(source)
        movieList = selector.xpath('//div[@class="info"]')
        movies = []
        for movie in movieList:
            title = movie.xpath('div[@class="hd"]/a/span[@class="title"]/text()')
            other = movie.xpath('div[@class="hd"]/a/span[@class="other"]/text()')
            url = movie.xpath('div[@class="hd"]/a/@href')
            rate_num = movie.xpath('div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()')
            quote = movie.xpath('div[@class="bd"]/p[@class="quote"]/span/text()')
     
            movieDic = {}
            movieDic["title"] = ''.join(title[0] + other[0])
            movieDic["url"] = url[0]
            movieDic["star"] = rate_num[0]
            movieDic["quote"] = quote[0]
            movies.append(movieDic)
     
        write_data(movies)
     
    #step3 遍历列表将数据写入csv文件中
    def write_data(movies):
        with open("douban.csv", 'w', encoding="UTF-8") as f:
            writer = csv.DictWriter(f, fieldnames=['title', 'star', 'quote', 'url'])
            writer.writeheader()
            for mov in movies:
                print("正在写入：" + mov["title"])
                writer.writerow(mov)
     
     
    if __name__ == "__main__":
        get_content()