guimaizi’s blog

从前


title: puppeteer异步并发方案 url: 535.html id: 535 comments: false categories:

  • 安全/代码 date: 2019-10-04 23:16:18 tags:

博主是一名优秀的爬虫师,欢迎联系博主交流爬虫技术。

关于puppeteer的并发方案: node代码:

const puppeteer = require('puppeteer');
async function process_page(page){
    //page.evaluate(() => alert(document.domain));
    await page.on('dialog', async dialog => {
        await dialog.dismiss();})
    console.log(page.url());
    await page.close();
}
async function process_queue(browser,page,list_url){
    await page.evaluate(list_url => {
        for(i=0;i<list_url.length;i++){
        window.open(list_url[i]);
        }
        return 1;
    },list_url);
    while(allPages=await browser.pages(),allPages.length-2<list_url.length){
        await page.waitFor(3000);
}
    for(j=0;j<allPages.length;j++){
        if (allPages[j]!=page){
        process_page(allPages[j]);
    }
    }
}
(async () => {
  try {
        const list_url=['https://www.w3school.com.cn/js/js_loop_for.asp','http://www.voidcn.com/article/p-stvclize-bte.html',
        'https://xiday.com/2019/09/21/puppeteer-run-js/','https://im.qq.com/','http://192.168.0.225/vul/frame221.html','https://en.mail.qq.com/','https://www.1688.com/',
    'https://github.com/','https://www.baidu.com/','http://www.guimaizi.com/','https://fanyi.baidu.com/?aldtype=16047#auto/zh','http://es6.ruanyifeng.com/#docs/promise',
'https://security.alibaba.com/','https://security.alipay.com/home.htm','https://security.pingan.com/','https://bugbounty.huawei.com/hbp/#/home'];
        const browser = await puppeteer.launch({
            headless: false,
            slowMo:150,
            timeout:5000});
        const page = await browser.newPage();
        set_list=[]
        for(i=0;i<list_url.length;i++){
            if(set_list.length>=5){
            console.log(set_list);
            await process_queue(browser,page,set_list)
            set_list=[];
        }
        set_list.push(list_url[i]);
    }
        console.log(set_list)
        await process_queue(browser,page,set_list)
        //await browser.close();
} catch (err) {
  console.error(err);
  //await browser.close();
}
})();

如果对您有帮助,请打赏杯奶茶吧 微信支付: