mysql百度图片爬虫_百度图片爬虫-python版-如何爬取百度图片?

本是古典何须时尚 2022-10-31 13:36 321阅读 0赞

这一篇我想写写如何爬取百度图片的爬虫，这个爬虫也是：搜搜gif(在线制作功能点我) 的爬虫代码，其实爬虫整体框架还是差不多的，但就是会涉及到图片的的一些处理，还是花费了我不少时间的，所以我请阅读的本爬虫的孩子还是认真一些，毕竟程序猿都不容易啊。

附代码：

PS:不会python的孩子赶快去补补吧，先把基础搞清楚再说

\#coding:utf-8

"""

Createdon2015-9-17

@author:huangxie

"""

importtime,math,os,re,urllib,urllib2,cookielib

frombs4importBeautifulSoup

importtime

importre

importuuid

importjson

fromthreadingimportThread

fromQueueimportQueue

importMySQLdbasmdb

importsys

importthreading

importutils

importimitate\_browser

fromMySQLdb.constants.REFRESHimportSTATUS

reload(sys)

sys.setdefaultencoding('utf-8')

DB\_HOST='127.0.0.1'

DB\_USER='root'

DB\_PASS='root'

proxy=\{u'http':u'222.39.64.13:8118'\}

TOP\_URL="http://image.baidu.com/i?tn=resultjsonavatarnew&ie=utf-8&word=\{word\}&pn=\{pn\}&rn=\{rn\}"

KEYWORD\_URL="https://www.baidu.com/s?ie=utf-8&f=8&tn=baidu&wd=\{wd\}"

"""

i\_headers=\{'User-Agent':'Mozilla/5.0(WindowsNT6.1)AppleWebKit/537.11(KHTML,likeGecko)Chrome/23.0.1271.64Safari/537.11',

'Accept':'json;q=0.9,\*/\*;q=0.8',

'Accept-Charset':'utf-8;q=0.7,\*;q=0.3',

'Accept-Encoding':'gzip',

'Connection':'close',

'Referer':None\#注意如果依然不能抓取的话，这里可以设置抓取网站的host

"""

i\_headers=\{'User-Agent':'Mozilla/5.0(WindowsNT6.1)AppleWebKit/537.36(KHTML,likeGecko)Chrome/31.0.1650.48'\}

defGetDateString():

x=time.localtime(time.time())

foldername=str(x.\_\_getattribute\_\_("tm\_year"))+"-"+str(x.\_\_getattribute\_\_("tm\_mon"))+"-"+str(x.\_\_getattribute\_\_("tm\_mday"))

returnfoldername

classBaiduImage(threading.Thread):

def\_\_init\_\_(self):

Thread.\_\_init\_\_(self)

self.browser=imitate\_browser.BrowserBase()

self.chance=0

self.chance1=0

self.request\_queue=Queue()

self.wait\_ana\_queue=Queue()

\#self.key\_word\_queue.put((("动态图",0,24)))

self.count=0

self.mutex=threading.RLock()\#可重入锁，使单线程可以再次获得已经获得的锁

self.commit\_count=0

self.ID=500

self.next\_proxy\_set=set()

self.dbconn=mdb.connect(DB\_HOST,DB\_USER,DB\_PASS,'sosogif',charset='utf8')

self.dbconn.autocommit(False)

self.dbcurr=self.dbconn.cursor()

self.dbcurr.execute('SETNAMESutf8')

"""

defrun(self):

whileTrue:

self.get\_pic()

"""

defwork(self,item):

print"startthread",item

whileTrue:\#MAX\_REQUEST条以上则等待

self.get\_pic()

self.prepare\_request()

defformat\_keyword\_url(self,keyword):

returnKEYWORD\_URL.format(wd=keyword).encode('utf-8')

defgenerateSeed(self,url):

html=self.browser.openurl(url).read()

ifhtml:

try:

soup=BeautifulSoup(html)

trs=soup.find('p',id='rs').find('table').find\_all('tr')\#获得所有行

fortrintrs:

ths=tr.find\_all('th')

forthinths:

a=th.find\_all('a')\[0\]

keyword=a.text.strip()

if"动态图"inkeywordor"gif"inkeyword:

print"keyword",keyword

self.dbcurr.execute('selectidfrominfowhereword=%s',(keyword))

y=self.dbcurr.fetchone()

ifnoty:

self.dbcurr.execute('INSERTINTOinfo(word,status,page\_num,left\_num,how\_many)VALUES(%s,0,0,0,0)',(keyword))

self.dbconn.commit()

except:

pass

defprepare\_request(self):

self.lock()

self.dbcurr.execute('select\*frominfowherestatus=0')

result=self.dbcurr.fetchone()

ifresult:

id,word,status,page\_num,left\_num,how\_many=result

self.request\_queue.put((id,word,page\_num))

ifpage\_num==0andleft\_num==0andhow\_many==0:

url=self.format\_keyword\_url(word)

self.generateSeed(url)

html=""

try:

url=self.format\_top\_url(word,page\_num,24)

html=self.browser.openurl(url).read()

exceptExceptionaserr:

print"err",err

\#pass

ifhtml!="":

how\_many=self.how\_many(html)

print"how\_many",how\_many

ifhow\_many==None:

how\_many=0

t=math.ceil(how\_many/24\*100)\#只要前1/100即可

num=int(t)

foriinxrange(0,num-1):

self.dbcurr.execute('INSERTINTOinfo(word,status,page\_num,left\_num,how\_many)VALUES(%s,%s,%s,%s,%s)',(word,0,i\*24,num-i,how\_many))

self.dbcurr.execute('updateinfoSETstatus=1WHEREid=%s',(id))\#置为已经访问

self.dbconn.commit()

self.unlock()

defstart\_work(self,req\_max):

foriteminxrange(req\_max):

t=threading.Thread(target=self.work,args=(item,))

t.setDaemon(True)

t.start()

deflock(self):\#加锁

self.mutex.acquire()

defunlock(self):\#解锁

self.mutex.release()

defget\_para(self,url,key):

values=url.split('?')\[-1\]

forkey\_valueinvalues.split('&'):

value=key\_value.split('=')

ifvalue\[0\]==key:

returnvalue\[1\]

returnNone

defmakeDateFolder(self,par,child):

\#self.lock()

ifos.path.isdir(par):

path=par+'//'+GetDateString()

newFolderName=path+'//'+child

ifnotos.path.isdir(path):

os.mkdir(path)

ifnotos.path.isdir(newFolderName):

os.mkdir(newFolderName)

returnnewFolderName

else:

returnpar

\#self.unlock()

defparse\_json(self,data):

ipdata=json.loads(data)

try:

ifipdata\['imgs'\]:

forninipdata\['imgs'\]:\#data子项

ifn\['objURL'\]:

try:

proxy\_support=urllib2.ProxyHandler(proxy)

opener=urllib2.build\_opener(proxy\_support)

urllib2.install\_opener(opener)

\#print"proxy",proxy

self.lock()

self.dbcurr.execute('selectIDfrompic\_infowhereobjURL=%s',(n\['objURL'\]))

y=self.dbcurr.fetchone()

\#print"y=",y

ify:

print"databaseexist"

self.unlock()\#continue前解锁

continue

else:

real\_extension=utils.get\_extension(n\['objURL'\])

req=urllib2.Request(n\['objURL'\],headers=i\_headers)

resp=urllib2.urlopen(req,None,5)

dataimg=resp.read()

name=str(uuid.uuid1())

filename=""

iflen(real\_extension)>4:

real\_extension=".gif"

real\_extension=real\_extension.lower()

ifreal\_extension==".gif":

filename=self.makeDateFolder("E://sosogif","d"+str(self.count%60))+"//"+name+"-www.sosogif.com-搜搜gif贡献"+real\_extension

self.count+=1

else:

filename=self.makeDateFolder("E://sosogif","o"+str(self.count%20))+"//"+name+"-www.sosogif.com-搜搜gif贡献"+real\_extension

self.count+=1

"""

name=str(uuid.uuid1())

filename=""

iflen(real\_extension)>4:

real\_extension=".gif"

filename=self.makeDateFolder("E://sosogif","d"+str(self.count%60))+"//"+name+"-www.sosogif.com-搜搜gif贡献"+real\_extension

self.count+=1

"""

try:

ifnotos.path.exists(filename):

file\_object=open(filename,'w+b')

file\_object.write(dataimg)

file\_object.close()

self.anaylis\_info(n,filename,real\_extension)\#入库操作

else:

print"fileexist"

exceptIOError,e1:

print"e1=",e1

pass

self.unlock()

exceptIOError,e2:

\#print"e2=",e2

pass

self.chance1+=1

exceptExceptionasparse\_error:

print"parse\_error",parse\_error

pass

deftitle\_dealwith(self,title):

\#print"title",title

a=title.find("")

temp1=title\[0:a\]

b=title.find("")

temp2=title\[a+8:b\]

temp3=title\[b+9:len(title)\]

return(temp1+temp2+temp3).strip()

defanaylis\_info(self,n,filename,real\_extension):

print"success."

\#ifself.wait\_ana\_queue.qsize()!=0:

\#n,filename,real\_extension=self.wait.ana\_queue.get()

\#self.lock()

objURL=n\['objURL'\]\#图片地址

fromURLHost=n\['fromURLHost'\]\#来源网站

width=n\['width'\]\#宽度

height=n\['height'\]\#高度

di=n\['di'\]\#用来唯一标识

type=n\['type'\]\#格式

fromPageTitle=n\['fromPageTitle'\]\#来自网站

keyword=self.title\_dealwith(fromPageTitle)

cs=n\['cs'\]\#未知

os=n\['os'\]\#未知

temp=time.time()

x=time.localtime(float(temp))

acTime=time.strftime("%Y-%m-%d%H:%M:%S",x)\#爬取时间

self.dbcurr.execute('selectIDfrompic\_infowherecs=%s',(cs))

y=self.dbcurr.fetchone()

ifnoty:

print'addpic',filename

self.commit\_count+=1

self.dbcurr.execute('INSERTINTOpic\_info(objURL,fromURLHost,width,height,di,type,keyword,cs,os,acTime,filename,real\_extension)VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',(objURL,fromURLHost,width,height,di,type,keyword,cs,os,acTime,filename,real\_extension))

ifself.commit\_count==10:

self.dbconn.commit()

self.commit\_count=0

\#self.unlock()

defformat\_top\_url(self,word,pn,rn):

url=TOP\_URL.format(word=word,pn=pn,rn=rn).encode('utf-8')

returnurl

defhow\_many(self,data):

try:

ipdata=json.loads(data)

ifipdata\['displayNum'\]>0:

how\_many=ipdata\['displayNum'\]

returnint(how\_many)

else:

return0

exceptExceptionase:

pass

defget\_pic(self):

"""

word="gif"

pn=0

rn=24

ifself.key\_word\_queue.qsize()!=0:

word,pn,rn=self.key\_word\_queue.get()

url=self.format\_top\_url(word,pn,rn)

globalproxy

ifurl:

try:

html=""

try:

req=urllib2.Request(url,headers=i\_headers)

response=urllib2.urlopen(req,None,5)

\#print"url",url

html=self.browser.openurl(url).read()

exceptExceptionaserr:

print"err",err

\#pass

ifhtml:

how\_many=self.how\_many(html)

\#how\_many=10000

print"how\_many",how\_many

word=self.get\_para(url,"word")

rn=int(self.get\_para(url,"rn"))

t=math.ceil(how\_many/rn)

num=int(t)

foriteminxrange(0,num-1):

"""

try:

globalproxy

print"sizeofqueue",self.request\_queue.qsize()

ifself.request\_queue.qsize()!=0:

id,word,page\_num=self.request\_queue.get()

u=self.format\_top\_url(word,page\_num,24)

self.lock()

self.dbcurr.execute('updateinfoSETstatus=1WHEREid=%s',(id))

self.dbconn.commit()

ifself.chance>0orself.chance1>1:\#任何一个出问题都给换代理

ifself.ID%100==0:

self.dbcurr.execute("selectcount(\*)fromproxy")

forrinself.dbcurr:

count=r\[0\]

ifself.ID>count:

self.ID=50

self.dbcurr.execute("select\*fromproxywhereID=%s",(self.ID))

results=self.dbcurr.fetchall()

forrinresults:

protocol=r\[1\]

ip=r\[2\]

port=r\[3\]

pro=(protocol,ip+":"+port)

ifpronotinself.next\_proxy\_set:

self.next\_proxy\_set.add(pro)

self.chance=0

self.chance1=0

self.ID+=1

self.unlock()

proxy\_support=urllib2.ProxyHandler(proxy)

opener=urllib2.build\_opener(proxy\_support)

urllib2.install\_opener(opener)

html=""

try:

req=urllib2.Request(u,headers=i\_headers)

\#print"u=",u

response=urllib2.urlopen(req,None,5)

html=response.read()

ifhtml:

\#print"html",type(html)

self.parse\_json(html)

exceptExceptionasex1:

\#print"error=",ex1

pass

self.chance+=1

ifself.chance>0orself.chance1>1:

iflen(self.next\_proxy\_set)>0:

protocol,socket=self.next\_proxy\_set.pop()

proxy=\{protocol:socket\}

print"changeproxyfinished<

exceptExceptionase:

print"error1",e

pass

if\_\_name\_\_=='\_\_main\_\_':

app=BaiduImage()

app.start\_work(80)

\#app.generateSeed()

while1:

pass

mysql百度图片爬虫_百度图片爬虫-python版-如何爬取百度图片?

发表评论取消回复

还没有评论，来说两句吧...

相关阅读

相关 python爬取百度图片

相关爬虫实战：通过百度关键词爬取大量图片

相关 Ajax爬取百度图片

相关 php爬去百度图片,Python爬虫：通过关键字爬取百度图片

相关 mysql百度图片爬虫_百度图片爬虫-python版-如何爬取百度图片?

相关 Python 爬取百度图片

相关爬虫：爬取百度图片

相关 python爬取百度图片（动态页面）

相关用python爬虫爬取百度图片

相关 python爬虫爬取百度文档