Python爬取图片-蒲公英云

Python爬取图片

灰太狼 2021-04-08 04:26 873阅读 0赞

import requests                   # 模块导入的俩种方法
from multiprocessing import Pool
import re
def get(url):
    ret=requests.get(url)
    if ret.status_code==200:
        return  ret.content.decode('gbk')
def call_back(arg):
    ret = com.finditer(arg)
    dict_lst=[]
    for i in ret:
        dic = {
            'png': i.group('png'),
            'name': i.group('name'),
            'place': i.group('place')
        }
        dict_lst.append(dic)
    for i in  dict_lst:
        res=subget(i['png'])
        write_func(i['name'],i['place'],res)
    return dict_lst
def subget(url):
    if 'https' in url:
        ret = requests.get(url)
        if ret.status_code == 200:
            return ret.content
        else:
            pass
    else:
        n_url = 'http://www.xiaohuar.com' + url
        ret = requests.get(n_url)
        if ret.status_code == 200:
            return ret.content
        else:
            pass
def write_func(path,place,picture):
    with open(r'E:\text1\爬虫\text_png\%s-%s.png' %(path,place),'wb') as f:
         f.write(picture)
'''我要爬取的网页的特征'''
'''http://www.xiaohuar.com/list-1-0.html'''
'''http://www.xiaohuar.com/list-1-43.html'''
if __name__ =='__main__':
    com = re.compile(
        '<div class="item_t">(?:.*?)src="(?P<png>.*?)"(?:.*?)<span class="price">(?P<name>.*?)</span>(?:.*?)'
        '<a href="http://www.xiaohuar.com/" class="img_album_btn">(?P<place>.*?)</a>', re.S)
    pool=Pool(3)
    res_lst=[]
    for i in range(40):
        pool.apply_async(get,args=('http://www.xiaohuar.com/list-1-%s.html' %i,),callback=call_back)
    pool.close()
    pool.join()