汽车之家爬虫实例

我就是我 2022-05-19 11:40 205阅读 0赞

import json
    import time
    import urllib
    import urllib2
    import re
    
    
    
    #获取城市列表
    url='https://dealer.autohome.com.cn/DealerList/GetAreasAjax?provinceId=340000&cityId=0&brandid=0&manufactoryid=0&seriesid=0&isSales=0'
    html=urllib2.urlopen(url)
    html=html.read()
    value=json.loads(html.decode('gbk'))
    lis=[]
    r=len(value['AreaInfoGroups'])
    for i in range(0,r):
        len1=len(value['AreaInfoGroups'][i]['Values'])
        for j in range(0,len1):
            len2=len(value['AreaInfoGroups'][i]['Values'][j]['Cities'])
            for z in range(0,len2):
                lis.append( str(value['AreaInfoGroups'][i]['Values'][j]['Cities'][z]['Pinyin'])) 
    
    print lis

\-----------------------------------------------------------------------------------------------------------------------------------------------------------------------

#coding:utf-8
 import json
 import time
 import urllib
 import urllib2
 import re
 
 h=open('qichezhijia01.txt','a')
 def run(city,num):
 url='https://dealer.autohome.com.cn/'+str(city)+'/0/0/0/0/'+str(num)+'/1/0/0.html'
 html=urllib2.urlopen(url)
 html=html.read()
 
 ss=get(city,html)
 if ss=='q':
 return 'q'
 
 def get(city,html):
 #print html
 reg4='(.*?).*?' #地址
 reg1='<a class="link" href="//dealer.autohome.com.cn/.*?/#pvareaid=.*?" target=" _blank">.*?(.*?).*?' #名称
 reg3='(.*?).*?'#电话
 reg2='(.*?).*?' #主营品牌
 reg=reg1+reg2+reg3+reg4
 reg=reg.decode('utf-8')
 reg = re.compile(ur'%s' % reg,re.S)
 items=re.findall(reg,html)
 if not items:
 return 'q'
 for i in items:
 ss=city+'///'
 for v in i:
 ss+=v
 ss=ss+'///'
 h.write( ss+'\n' )
 h.flush()
 time.sleep(0.1)
 
 
 
 
 lis=['beijing', 'tianjin', 'shanghai', 'chongqing', 'hefei', 'wuhu', 'bangbu', 'huainan', 'maanshan', 'huaibei',
 'tongling', 'anqing', 'huangshan', 'chuzhou', 'fu_yang', 'su_zhou', 'liuan', 'bozhou', 'chizhou', 'xuancheng',
 'aomen', 'fuzhou', 'xiamen', 'putian', 'sanming', 'quanzhou', 'zhangzhou', 'nanping', 'longyan', 'ningde',
 'guangzhou', 'shaoguan', 'shenzhen', 'zhuhai', 'shantou', 'foshan', 'jiangmen', 'zhanjiang', 'maoming',
 'zhaoqing', 'huizhou', 'meizhou', 'shanwei', 'heyuan', 'yangjiang', 'qingyuan', 'dongguan', 'zhongshan',
 'chaozhou', 'jieyang', 'yunfu', 'nanning', 'liuzhou', 'guilin', 'wuzhou', 'beihai', 'fangchenggang', 'qinzhou',
 'guigang', 'yu_lin', 'baise', 'hezhou', 'hechi', 'laibin', 'chongzuo', 'guiyang', 'liupanshui', 'zunyi',
 'anshun', 'bijie', 'tongren', 'xingyishi', 'kaili', 'duyunshi', 'lanzhou', 'jiayuguan', 'jinchang', 'baiyin',
 'tianshui', 'wuwei', 'zhangye', 'pingliang', 'jiuquan', 'qingyang', 'dingxi', 'longnan', 'linxia', 'gannan',
 'shijiazhuang', 'tangshan', 'qinhuangdao', 'handan', 'xingtai', 'baoding', 'zhangjiakou', 'chengde', 'cangzhou',
 'langfang', 'hengshui', 'haerbin', 'qiqihaer', 'jixi', 'hegang', 'shuangyashan', 'daqing', 'yichun', 'jiamusi',
 'qitaihe', 'mudanjiang', 'heihe', 'suihua', 'daxinganling', 'zhengzhou', 'kaifeng', 'luoyang', 'pingdingshan',
 'anyang', 'hebi', 'xinxiang', 'jiaozuo', 'puyang', 'xuchang', 'luohe', 'sanmenxia', 'nanyang', 'shangqiu',
 'xinyang', 'zhoukou', 'zhumadian', 'jiyuan', 'wuhan', 'huangshi', 'shiyan', 'yichang', 'xiangyang', 'ezhou',
 'jingmen', 'xiaogan', 'jingzhou', 'huanggang', 'xianning', 'suizhou', 'enshi', 'xiantao', 'qianjiang', 'tianmen',
 'shennongjia', 'changsha', 'zhuzhou', 'xiangtan', 'hengyang', 'shaoyang', 'yueyang', 'changde', 'zhangjiajie', 'yiyang',
 'chenzhou', 'yongzhou', 'huaihua', 'loudi', 'xiangxi', 'haikou', 'sanya', 'sanshashi', 'danzhou', 'wuzhishan', 'qionghai',
 'wenchang', 'wanning', 'dongfang', 'dingan', 'tunchang', 'chengmai', 'lingao', 'baisha', 'changjiang', 'ledong',
 'lingshui', 'baoting', 'qiongzhong', 'changchun', 'jilinshi', 'siping', 'liaoyuan', 'tonghua', 'baishan', 'songyuan',
 'baicheng', 'yanbian', 'nanjing', 'wuxi', 'xuzhou', 'changzhou', 'suzhou', 'nantong', 'lianyungang', 'huaian', 'yancheng',
 'yangzhou', 'zhenjiang', 'tai_zhou', 'suqian', 'nanchang', 'jingdezhen', 'ping_xiang', 'jiujiang', 'xinyu', 'yingtan',
 'ganzhou', 'jian', 'yi_chun', 'fu_zhou', 'shangrao', 'shenyang', 'dalian', 'anshan', 'fushun', 'benxi', 'dandong',
 'jinzhou', 'yingkou', 'fuxin', 'liaoyang', 'panjin', 'tieling', 'chaoyang', 'huludao', 'huhehaote', 'baotou', 'wuhai',
 'chifeng', 'tongliao', 'eerduosi', 'hulunbeier', 'bayannaoer', 'wulanchabu', 'xinganmeng', 'xilinguolemeng', 'alashanmeng',
 'yinchuan', 'shizuishan', 'wuzhong', 'guyuan', 'zhongwei', 'xining', 'haidong', 'haibei', 'huangnan', 'hai_nan',
 'guoluo', 'yushu', 'haixi', 'taiyuan', 'datong', 'yangquan', 'changzhi', 'jincheng', 'shuozhou', 'jinzhong', 'yuncheng',
 'xinzhou', 'linfen', 'lvliang', 'jinan', 'qingdao', 'zibo', 'zaozhuang', 'dongying', 'yantai', 'weifang', 'jining', 'taian',
 'weihai', 'rizhao', 'laiwu', 'linyi', 'dezhou', 'liaocheng', 'binzhou', 'heze', 'chengdu', 'zigong', 'panzhihua', 'luzhou',
 'deyang', 'mianyang', 'guangyuan', 'suining', 'neijiang', 'leshan', 'nanchong', 'meishan', 'yibin', 'guangan', 'dazhou',
 'yaan', 'bazhong', 'ziyang', 'aba', 'ganzi', 'liangshan', 'xian', 'tongchuan', 'baoji', 'xianyang', 'weinan', 'yanan',
 'hanzhong', 'yulin', 'ankang', 'shangluo', 'taiwan', 'lasa', 'rikaze', 'changdou', 'linzhi', 'shannan', 'naqu', 'ali',
 'wulumuqi', 'kelamayi', 'tulufan', 'hami', 'changji', 'boertala', 'bayinguoleng', 'akesu', 'kezilesu', 'kashen', 'hetian',
 'yili', 'tacheng', 'aletai', 'shihezi', 'alaer', 'tumushuke', 'wujiaqu', 'beitunshi', 'tiemenguanshi', 'shuangheshi',
 'kekedalashi', 'kunyu', 'xianggang', 'kunming', 'qujing', 'yuxi', 'baoshan', 'zhaotong', 'lijiang', 'puer', 'lincang',
 'chuxiong', 'honghe', 'wenshan', 'xishuangbanna', 'dali', 'dehong', 'nujiang', 'diqing', 'hangzhou', 'ningbo', 'wenzhou',
 'jiaxing', 'huzhou', 'shaoxing', 'jinhua', 'quzhou', 'zhoushan', 'taizhou', 'lishui']
 
 for i in lis:
 print i
 j=1
 while 1: 
 try:
 ss=run(i,j)
 if ss=='q':
 break
 j+=1
 print j
 except:
 break