Python爬虫+数据可视化教学：分析猫咪交易数据

待我称王封你为后i 2022-09-03 10:21 202阅读 0赞

# 前言 #

各位，七夕快到了，想好要送什么礼物了吗？

昨天有朋友私信我，问我能用Python分析下网上小猫咪的数据，是想要送一只给女朋友，当做礼物。

## [Python从零基础入门到实战系统教程、源码、视频][Python] ##

网上的数据太多、太杂，而且我也不知道哪个网站的数据比较好。所以，只能找到一个猫咪交易网站的数据来分析了

地址：

http://www.maomijiaoyi.com/

# ![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L20wXzQ4NDA1Nzgx_size_16_color_FFFFFF_t_70][] #

# 爬虫部分 #

请求数据

import requests
    
    url = f'http://www.maomijiaoyi.com/index.php?/chanpinliebiao_c_2_1--24.html'
    headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'
    }
    response = requests.get(url=url, headers=headers)
    print(response.text)

解析数据

# 把获取到的 html 字符串数据转换成 selector 对象 这样调用
    selector = parsel.Selector(response.text)
    # css 选择器只要是根据标签属性内容提取数据 编程永远不看过程 只要结果
    href = selector.css('.content:nth-child(1) a::attr(href)').getall()
    areas = selector.css('.content:nth-child(1) .area .color_333::text').getall()
    areas = [i.strip() for i in areas] # 列表推导式

提取标签数据

for index in zip(href, areas):
        # http://www.maomijiaoyi.com/index.php?/chanpinxiangqing_224383.html
        index_url = 'http://www.maomijiaoyi.com' + index[0]
        response_1 = requests.get(url=index_url, headers=headers)
        selector_1 = parsel.Selector(response_1.text)
        area = index[1]
        # getall 取所有 get 取一个
        title = selector_1.css('.detail_text .title::text').get().strip()
        shop = selector_1.css('.dinming::text').get().strip()  # 店名
        price = selector_1.css('.info1 div:nth-child(1) span.red.size_24::text').get()  # 价格
        views = selector_1.css('.info1 div:nth-child(1) span:nth-child(4)::text').get()  # 浏览次数
        # replace() 替换
        promise = selector_1.css('.info1 div:nth-child(2) span::text').get().replace('卖家承诺: ', '')  # 浏览次数
        num = selector_1.css('.info2 div:nth-child(1) div.red::text').get()  # 在售只数
        age = selector_1.css('.info2 div:nth-child(2) div.red::text').get()  # 年龄
        kind = selector_1.css('.info2 div:nth-child(3) div.red::text').get()  # 品种
        prevention = selector_1.css('.info2 div:nth-child(4) div.red::text').get()  # 预防
        person = selector_1.css('div.detail_text .user_info div:nth-child(1) .c333::text').get()  # 联系人
        phone = selector_1.css('div.detail_text .user_info div:nth-child(2) .c333::text').get()  # 联系方式
        postage = selector_1.css('div.detail_text .user_info div:nth-child(3) .c333::text').get().strip()  # 包邮
        purebred = selector_1.css(
            '.xinxi_neirong div:nth-child(1) .item_neirong div:nth-child(1) .c333::text').get().strip()  # 是否纯种
        sex = selector_1.css(
            '.xinxi_neirong div:nth-child(1) .item_neirong div:nth-child(4) .c333::text').get().strip()  # 猫咪性别
        video = selector_1.css(
            '.xinxi_neirong div:nth-child(2) .item_neirong div:nth-child(4) .c333::text').get().strip()  # 能否视频
        worming = selector_1.css(
            '.xinxi_neirong div:nth-child(2) .item_neirong div:nth-child(2) .c333::text').get().strip()  # 是否驱虫
        dit = {
            '地区': area,
            '店名': shop,
            '标题': title,
            '价格': price,
            '浏览次数': views,
            '卖家承诺': promise,
            '在售只数': num,
            '年龄': age,
            '品种': kind,
            '预防': prevention,
            '联系人': person,
            '联系方式': phone,
            '异地运费': postage,
            '是否纯种': purebred,
            '猫咪性别': sex,
            '驱虫情况': worming,
            '能否视频': video,
            '详情页': index_url,
        }

保存数据

import csv # 内置模块
    
    f = open('猫咪1.csv', mode='a', encoding='utf-8', newline='')
    csv_writer = csv.DictWriter(f, fieldnames=['地区', '店名', '标题', '价格', '浏览次数', '卖家承诺', '在售只数',
                                               '年龄', '品种', '预防', '联系人', '联系方式', '异地运费', '是否纯种',
                                               '猫咪性别', '驱虫情况', '能否视频', '详情页'])
    csv_writer.writeheader() # 写入表头
    csv_writer.writerow(dit)
    print(title, area, shop, price, views, promise, num, age,
          kind, prevention, person, phone, postage, purebred, sex, video, worming, index_url, sep=' | ')

得到数据

![3bcc73c53c17b33d3bc20f5fae643714.png][]

# 数据可视化部分 #

词云图

from pyecharts import options as opts
    from pyecharts.charts import WordCloud
    from pyecharts.globals import SymbolType
    from pyecharts.globals import ThemeType
    
    
    words = [(i,1) for i in cat_info['品种'].unique()]
    c = (
        WordCloud(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
        .add("", words,shape=SymbolType.DIAMOND)
        .set_global_opts(title_opts=opts.TitleOpts(title=""))
    )
    c.render_notebook()

![4abe1c8565bae0368263dce224bdfc0c.png][]

交易品种占比图

from pyecharts import options as opts
    from pyecharts.charts import TreeMap
    
    pingzhong = cat_info['品种'].value_counts().reset_index()
    data = [{'value':i[1],'name':i[0]} for i in zip(list(pingzhong['index']),list(pingzhong['品种']))]
    
    c = (
        TreeMap(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
        .add("", data)
        .set_global_opts(title_opts=opts.TitleOpts(title=""))
        .set_series_opts(label_opts=opts.LabelOpts(position="inside"))
    )
    
    c.render_notebook()

![ee466065247076d76e5578a3b297a9c3.png][]

均价占比图

from pyecharts import options as opts
    from pyecharts.charts import PictorialBar
    from pyecharts.globals import SymbolType
    
    location = list(price['品种'])
    values = list(price['价格'])
    
    c = (
        PictorialBar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
        .add_xaxis(location)
        .add_yaxis(
            "",
            values,
            label_opts=opts.LabelOpts(is_show=False),
            symbol_size=18,
            symbol_repeat="fixed",
            symbol_offset=[0, 0],
            is_symbol_clip=True,
            symbol=SymbolType.ROUND_RECT,
        )
        .reversal_axis()
        .set_global_opts(
            title_opts=opts.TitleOpts(title="均价排名"),
            xaxis_opts=opts.AxisOpts(is_show=False),
            yaxis_opts=opts.AxisOpts(
                axistick_opts=opts.AxisTickOpts(is_show=False),
                axisline_opts=opts.AxisLineOpts(
                    linestyle_opts=opts.LineStyleOpts(opacity=0),
                
                ),
            ),
        )
        .set_series_opts(
            label_opts=opts.LabelOpts(position='insideRight')
        )
    )
    
    c.render_notebook()

![c95711b2ca502417718a99f72734b34e.png][]

猫龄柱状图

from pyecharts import options as opts
    from pyecharts.charts import Bar
    from pyecharts.faker import Faker
    
    x = ['1-3个月','3-6个月','6-9个月','9-12个月','1年以上']
    y = [69343,115288,18239,4139,5]
    
    c = (
        Bar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
        .add_xaxis(x)
        .add_yaxis('', y)
        .set_global_opts(title_opts=opts.TitleOpts(title="猫龄分布"))
    )
    
    c.render_notebook()

![05baf72bc5f0a28bd7bd015b652085b7.png][]

[Python]: https://links.jianshu.com/go?to=https%3A%2F%2Fnote.youdao.com%2Fs%2FPTxuKwE0
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L20wXzQ4NDA1Nzgx_size_16_color_FFFFFF_t_70]: /images/20220829/471f28cb1c0f417bae44fb56f37a22c6.png
[3bcc73c53c17b33d3bc20f5fae643714.png]: /images/20220829/8a7f71d112d54b34843fda752d2c1cff.png
[4abe1c8565bae0368263dce224bdfc0c.png]: /images/20220829/7fb1e63b931744b990040a37be151ae6.png
[ee466065247076d76e5578a3b297a9c3.png]: /images/20220829/dbae32465bd846fda598d8df4ef6cab9.png
[c95711b2ca502417718a99f72734b34e.png]: /images/20220829/1236ed69d9f84b1dbaa1211abbcb76a7.png
[05baf72bc5f0a28bd7bd015b652085b7.png]: /images/20220829/5469bcac5b07400fa33ab61c7ac4097f.png