puppeteer爬虫

Dear 丶 2023-02-16 12:00 7阅读 0赞

## 一、爬虫 ##

//生成一个npm的仓库
    npm init -y

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3_size_16_color_FFFFFF_t_70]

//安装nodemon
    npm i nodemon -S

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3_size_16_color_FFFFFF_t_70 1]

npm i puppeteer -S

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3_size_16_color_FFFFFF_t_70 2]

### 全局安装yarn ###

npm i yarn -g 
    //yarn是facebook出的一个node.js的包原理方案,相对会快一些
    yarn config set registry https://registry.npm.taobao.org

## 二、爬虫的代码 ##

npm i koa koa-router koa2-cors -S

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3_size_16_color_FFFFFF_t_70 3]

//package.json
    {
      "name": "server",
      "version": "1.0.0",
      "description": "",
      "main": "index.js",
      "scripts": {
        "start": "nodemon index.js",
        "test": "echo \"Error: no test specified\" && exit 1"
      },
      "keywords": [],
      "author": "",
      "license": "ISC",
      "dependencies": {
        "nodemon": "^2.0.4",
        "puppeteer": "^3.3.0"
      }
    }

//配置index.js
    const run = require('./reptile/db');
    const koa = require("koa");
    const Router = require('koa-router');
    const router = new Router();
    var cors = require('koa2-cors');
    const app =  new koa();
    router.get("/",async ctx=>{ 
        var data = await run();
    
        ctx.body = data;
    })
    app.use(cors());
    app.use(router.routes())
    app.listen(8080)

//爬取堆糖
    const puppeteer = require("puppeteer");
    const url = "https://www.duitang.com/"
    async function run() { 
        const browser = await puppeteer.launch()
        const page = await browser.newPage();
        await page.goto(url, { 
            waitUntil: 'networkidle2'
        })
        await page.waitFor(2000)
        await page.hover(".dynamic-feed-item")
        let res = await page.evaluate(() => { 
            var $ = window.jQuery
            var items = $(".dynamic-feed-item")
            // .user-info .avatar --src属性 $(".user-info .avatar").attr("src")
            // .user-info .username $(".user-info .username").html()
            var links = [];
            if(items.length>0){ 
                items.each((index,item)=>{ 
                    let avatar = $(item).find(".user-info .avatar").attr("src");
                    let username = $(item).find(".user-info .username").html();
                    links.push({ 
                        avatar:avatar,
                        username:username
                    })
                })
            }
            return links;
           
        })
        console.log(res);
        browser.close();
        return res;
    }
    run();
    // module.exports = run;

npm start

访问：http://localhost:8080/  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3_size_16_color_FFFFFF_t_70 4]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200609180618294.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3_size_16_color_FFFFFF_t_70 1]: https://img-blog.csdnimg.cn/20200609174734455.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3_size_16_color_FFFFFF_t_70 2]: https://img-blog.csdnimg.cn/20200609174809445.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3_size_16_color_FFFFFF_t_70 3]: https://img-blog.csdnimg.cn/20200609175212591.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3_size_16_color_FFFFFF_t_70 4]: https://img-blog.csdnimg.cn/20200609181140165.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTU0OTk3,size_16,color_FFFFFF,t_70