scrapy mysql的同步插入与异步插入

刺骨的言语ヽ痛彻心扉 2021-06-24 16:09 406阅读 0赞

主要代码是在Pipeline中进行编写，上完整代码：

同步插入代码：

# 同步插入
    class MysqlPipeline2(object):
        """ 同步操作 """
    
        def __init__(self):
            # 建立连接
            self.conn = pymysql.connect('localhost', 'root', 'Abcd1234', 'test')  # 有中文要存入数据库的话要加charset='utf8'
            # 创建游标
            self.cursor = self.conn.cursor()
    
        def process_item(self, item, spider):
            # sql语句
            insert_sql = """ insert into test_zxf(quote,author,tags,born_date,born_location) VALUES(%s,%s,%s,%s,%s) """
            # 执行插入数据到数据库操作
            self.cursor.execute(insert_sql, (item['quote'], item['author'], item['tags'], item['born_date'],
                                             item['born_location']))
            # 提交，不进行提交无法保存到数据库
            self.conn.commit()
    
        def close_spider(self, spider):
            # 关闭游标和连接
            self.cursor.close()
            self.conn.close()

mysql异步插入代码：

class MysqlPipeline(object):
    
        def __init__(self):
            self.host = settings.DB_HOST
            self.port = settings.DB_PORT
            self.user = settings.DB_USER
            self.pwd = settings.DB_PWD
            self.name = settings.DB_NAME
            self.charset = settings.DB_CHARSET
    
            self.connect()
    
        def connect(self):
            self.conn = pymysql.connect(host=self.host,
                                        port=self.port,
                                        user=self.user,
                                        password=self.pwd,
                                        db=self.name,
                                        charset=self.charset)
            self.cursor = self.conn.cursor()
    
        def close_spider(self, spider):
            self.conn.close()
            self.cursor.close()
    
        def process_item(self, item, spider):
            try:
                print('在插入 *************************************** ')
                # sql = 'insert into zb_contest_data(craw_date, cd_filename, cd_content, create_time) values("%s", "%s", "%s", "%s")' % (
                # item['craw_date'], item['file_name'], str(item['file_content']), item['create_time'])
                file_count = pymysql.escape_string(item['file_content'])
                sql = 'insert into zb_contest_data(craw_date, cd_filename, cd_content, create_time) values("%s", "%s", "%s", "%s")' % (
                    item['craw_date'], item['file_name'], file_count, item['create_time'])
    
                # sql = 'insert into zb_contest_data(craw_date, cd_filename, cd_content, create_time) values("%s", "%s", "%s", "%s")' % (
                # item['craw_date'], item['file_name'], item['file_content'], item['create_time'])
    
                # 执行sql语句
                self.cursor.execute(sql)
                self.conn.commit()
                pass
            except Exception as e:
                print(e)
                pass
            return item

以上就是scrapy中同步与异步插入的代码片段