html股票数据代码,股票数据的网站抓取（4.2）代码优化

深藏阁楼爱情的钟 2022-10-06 11:48 125阅读 0赞

\#coding=utf-8

from selenium import webdriver

import time

import os

import re

import sys

import threading

import Queue

import Tkinter as tk

from selenium.common.exceptions import NoSuchElementException

def myinit():

reload(sys)

sys.setdefaultencoding('utf8')

\#获取屏幕分辨率

def getscreeninfo():

\#这个没考虑多显示器的问题

info=\{\}

root = tk.Tk()

info\['width\_px'\] = root.winfo\_screenwidth()

info\['height\_px'\] = root.winfo\_screenheight()

info\['width\_mm'\] = root.winfo\_screenmmwidth()

info\['height\_mm'\] = root.winfo\_screenmmheight()

\# 2.54 cm = in

info\['width\_in'\] = info\['width\_mm'\] / 25.4

info\['height\_in'\] = info\['height\_mm'\] / 25.4

info\['width\_dpi'\] = info\['width\_px'\]/info\['width\_in'\]

info\['height\_dpi'\] = info\['height\_px'\]/info\['height\_in'\]

return info

\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#

\#生成多个浏览器实例

def makebr(number):

brs=\[\]

for i in range(number):

br=webdriver.Firefox()

brs.append(br)

return brs

\#设置所有浏览器实例大小

def setbrsize(brs,x,y):

for i in brs:

i.set\_window\_size(200,200)

\#排列打开的浏览器

def layoutbrs(brs,x=0,y=0,dx=10,dy=10,mode=0,hang=0):

\#mode 0,横向, 1，纵向 2，矩形 3，对角线

\#hang 0为尽量按正方形排列

screeninfo=getscreeninfo()

if(mode==0):

dy=0

elif(mode==1):

dx=0

if(mode!=2):

for i in brs:

i.set\_window\_position(x,y)

x=x+dx

y=y+dy

else:

\#这个以后有空再弄

pass

\#hang=math.ceil(math.sqrt(len(dbrs)))

\#关闭所有浏览器

def closebr(brs):

for i in brs:

i.quit()

\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#

\#获得指定股票相关数据

def getinfo(mydriver,gourl):

title='//\*\[@id="name"\]'

code='//\*\[@id="code"\]'

hexinshujuxpath="/html/body/div\[14\]/div\[1\]/div\[4\]/div\[1\]"

restr="：".decode('utf8')

myre=re.compile(restr,re.I|re.M|re.S)

linetext=""

errorinfo=""

mydriver.get(gourl)

try:

gupiaoming=mydriver.find\_element\_by\_xpath(title).text

gupiaocode=mydriver.find\_element\_by\_xpath(code).text

hexinshuju=mydriver.find\_element\_by\_class\_name('pad5')

shujuhang=hexinshuju.find\_elements\_by\_tag\_name('tr')

for i in range(len(shujuhang)-2):

shujulie=shujuhang\[i\].find\_elements\_by\_tag\_name('td')

tmpshuju=myre.split(shujulie\[0\].text)

linetext=linetext+"~"+tmpshuju\[1\]

shuju=myre.split(shujuhang\[8\].text)

linetext=linetext+"~"+shuju\[1\]

tmpshuju=myre.split(shujuhang\[9\].text)

linetext=linetext+"~"+tmpshuju\[1\]

linetext="%s~%s%s"%(gupiaoming,gupiaocode,linetext)

\#print "数据：",linetext

except NoSuchElementException,e:

\#print "不是股票"

pass

except Exception:

errorinfo= "非预期错误"+gourl

print errorinfo

finally:

return linetext,errorinfo

\#获得所有股票链接

def geturls(br):

\#通过link对象获得链接地址的text文本

def getlinkurl(linklist):

my=\[\]

for x in linklist:

my.append(x.get\_attribute('href'))

return my

sz=\[\]

sh=\[\]

br.get("http://quote.eastmoney.com/stocklist.html")

shxpath="/html/body/div\[9\]/div\[2\]/div/ul\[1\]"

szxpath="/html/body/div\[9\]/div\[2\]/div/ul\[2\]"

shgupiao=br.find\_element\_by\_xpath(shxpath)

szgupiao=br.find\_element\_by\_xpath(szxpath)

shgupiaolist=shgupiao.find\_elements\_by\_tag\_name('a')

szgupiaolist=szgupiao.find\_elements\_by\_tag\_name('a')

sh=getlinkurl(shgupiaolist)

sz=getlinkurl(szgupiaolist)

return sh,sz

\#多线程执行用的函数

def thread\_getinfo(br,jobslink\_queue,jieguo\_queue,errorinfo\_queue):

while True:

try:

\#获得队列里的地址

url=jobslink\_queue.get(False) \#False =Don't wait

except Queue.Empty:

print "完成退出"

\#br.quit()

return

\#print url

if(url!=None):

(linetext,errorinfo)=getinfo(br,url)

if(linetext!=""):

jieguo\_queue.put(linetext)

if(errorinfo!=""):

errorinfo\_queue.put(errorinfo)

\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#

\#多线程控制函数

def saveinfoabc(info\_filename,error\_filename,urllist):

jobslink=Queue.Queue(0)

jieguo=Queue.Queue(0)

errorsinfo=Queue.Queue(0)

for x in urllist\[200:250\]:\#为测试方便这里只取了50，如果要全下载，取消\[\]就好

jobslink.put(x)

\#启动线程

for x in range(THREAD\_NUM):

t=threading.Thread(target=thread\_getinfo,args=(brs\[x\],jobslink,jieguo,errorsinfo))

t.start()

f=open(info\_filename,'w')

e=open(error\_filename,'w')

mycount=0

while (threading.activeCount()>1) or (not jobslink.empty()):

while jieguo.qsize()>0 or errorsinfo.qsize()>0:

if(jieguo.qsize()>0):

jieguotxt=jieguo.get()

f.write(jieguotxt+"\\n")

if(errorsinfo.qsize()>0):

error=errorsinfo.get()

e.write(error+"\\n")

mycount=mycount+1

if(mycount%100)==0:

print "%d: 活动线程：%d,剩余连接数：%d,结果剩余条数：%d,错误剩余条数:%d"%(mycount,threading.activeCount(),jobslink.qsize(),jieguo.qsize(),errorsinfo.qsize())

time.sleep(0.01)

e.close()

f.close()

print "数据下载完成"

\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#

myinit()

br=webdriver.Firefox()

print "获得所有链接地址"

(sh,sz)=geturls(br)

br.quit()

\#info\_filename='shinfo.txt'

\#error\_filename='sherror.txt'

THREAD\_NUM=10

brs=makebr(THREAD\_NUM)

setbrsize(brs,200,300)

layoutbrs(brs ,0,0,80,80,0)

saveinfoabc('shinfo.txt','sherror.txt',sh)

saveinfoabc('szinfo.txt','szerror.txt',sz)

closebr(brs)

html股票数据代码,股票数据的网站抓取（4.2）代码优化

发表评论取消回复

还没有评论，来说两句吧...

相关阅读

相关 python股票数据分析的修改、清洗数据

相关股票基础知识 - 通过股票前缀代码判断股票是什么类型，属于哪个板块

相关 java获取股票创业指数_java抓取东方财富股票数据（附源码）

相关 html股票数据代码,股票数据的网站抓取（4.2）代码优化

相关股票实时数据接口

相关股票数据爬虫

相关 Python获取股票历史数据和收盘数据的代码实现

相关股票实时数据接口

相关使用scrapy抓取股票代码

相关项目：股票数据分析（ARMA）