实现用火狐进行动态网页抓取

我就是我 2022-06-03 09:16 240阅读 0赞

# -*- coding: utf-8 -*-
    import sys,re
    from lxml import etree
    import  time
    from selenium import webdriver
    import os
    import pymysql
    
    host = '服务器IP'
    dbName = '数据库名'
    user = '数据库登陆用户名'
    password = '数据库登陆密码'
    db = pymysql.connect(host, user, password, dbName, charset='utf8')
    cursor = db.cursor()
    cursor.execute("DROP TABLE IF EXISTS EMPLOYEE")  
    sql = """CREATE TABLE EMPLOYEE (SYMBOL CHAR(6) NOT NULL,NAME CHAR(10),PRICE CHAR(10))"""
    cursor.execute(sql)
    
    driver = webdriver.Firefox()  #打开火狐浏览器
    driver.set_window_position(x=70,y=30)   #设置火狐浏览器位置
    driver.set_window_size(width=1064, height=524) #设置火狐浏览器窗口大小
    f = open("symbol.txt", 'r')   # 以只读方式打开当前路径下的symbol.txt文件
    for line in f.readlines():
       line = line.replace("\n", "")  # 提取每行数据，去掉换行符
       url = "url1"+line+"url2"  # url拼接
       driver.get(url)  #输入需要访问的URL
       time.sleep(3)    #等待3秒
       content = driver.page_source.encode('utf-8') #网页内容转码utf8
       html = etree.HTML(content)
       price = html.xpath('.//*[@class="ico_data col_open"]/text()') #提取价格
       name = html.xpath('.//*[@class="ico_name label_open"]/text()') #提取名称
       symbolname = ''.join(name)
       price = ''.join(price)
       print(line, symbolname, price)
       cursor = db.cursor()
       sql1 = ("INSERT INTO EMPLOYEE(SYMBOL, NAME, PRICE) VALUES ('%s', '%s', '%s')" % (str(line), symbolname, str(price)))
       cursor.execute(sql1)
       db.commit()  #提交数据
    db.close() #关闭数据库
    f.close()  #关闭symbol文件
    driver.quit() #退出浏览器