im using nodejs / puppeteer to login my users to a remote website ... here is how it works

clients connects to nodejs server trough socket.io , client sends start_tunnel to nodejs server to start puppeteer and node calls run(socket , data.token ); which runs the puppeteer

    io.on('connection' , function(socket){

        socket.on('start_tunnel' , function (data) {
            fullfillCaptcha[socket.id] = null ;
            set_stat(socket.id , 1 );
            run(socket , data.token );
        })

        socket.on('get_captcha_from_client' , function (data) {

            fullfillCaptcha[socket.id](data);

        })

    });

    var fullfillCaptcha = {};
    var pay_stats = {} ;

    function captchaPromise(id){
        return  new Promise(resolve => fullfillCaptcha[id] = resolve);
    }

and here is the run function which lunches the puppeteer .... i've mented the code so its easy to read ... basically it opens a webpage containing a form with captcha , takes screenshot from captcha image and send it to client , receives typed captcha from client , put it in the input and submits the form

async function run(socket , token ) {

   /// OPENING THE WEB PAGE 
    const browser = await puppeteer.launch({headless: true , args:['--no-sandbox']});
    const page = await browser.newPage();
    await page.goto('/' + token );

    await Promise.race([
        page.waitForNavigation(),
        page.waitForSelector(".Error")
    ]);

   /// CHECKING FOR ERROR

    if (await page.$(".Error"))
    {
        socket.emit('payment_connection_error' );
        set_stat(socket.id , 4 );
        browser.close();
        return ;
    }



   /// TAKING SCREENSHOT FROM CAPTCHA IMAGE

    console.log( ' current url  ' + page.url() );
    let element =  await page.$('#security');

    await element.screenshot({path: 'public_html/captcha/'+  socket.id+'.png'  });
    set_stat(socket.id , 2 );

    /// SENDING CAPTCHA IMAGE FOR CLIENT 
    socket.emit('send_captcha_to_client' , {text : socket.id+'.png'  });

    /// WAITING FOR CLIENT TO TYPE THE CAPTCHA AND SEND IT BACK TO SERVER 

    var captcha = await captchaPromise(socket.id);

    if( typeof(captcha.W_CAPTCHA) == 'undefined' || captcha.W_CAPTCHA == 'cancel' )
    {
        console.log('canceling ... ');
        set_stat(socket.id , 3 );
        browser.close();
        return ;
    }
    console.log( ' captcha confirm -> ' + captcha.W_CAPTCHA );
    set_stat(socket.id , 3 );

    /// TYPING CAPTCHA IN THE INPUT AND SUBMITTING THE FORM 

    await page.$eval('#CAPTCHA', (el , _captcha) => el.value = _captcha.W_CAPTCHA , captcha );
    await page.click('#doPay');


    /// CHECKING FOR SUCCESS OR ERROR OF FORM AFTER 

    await Promise.race([
        page.waitForNavigation(),
        page.waitForSelector(".Error")
    ]);


    if (await page.$(".Error"))
    {
        const pay_error = await page.$eval('.Error', (element) => {
        return element.innerHTML
    });

        console.log(" error : " + pay_error )
        socket.emit('payment_error' , {text : pay_error });
    }
else
    {

        console.log('all ok') ;
        await page.click('#doSubmitTop');
        await page.waitForSelector('#payment-result');
        console.log( ' current url  ' + page.url() );
        socket.emit('payment_result');



    }

    browser.close();
}

here is the problem , we're having lots of server crush due to lack of RAM and its all puppeteer using the ram ... we have a decent vps (4G RAM , 2 CPU ) ... and when we're testing there is not much server load can be seen

so i was wondering if anything im doing wrong here or is there anyway to see a log or something to see what went wrong ?

im using nodejs / puppeteer to login my users to a remote website ... here is how it works

clients connects to nodejs server trough socket.io , client sends start_tunnel to nodejs server to start puppeteer and node calls run(socket , data.token ); which runs the puppeteer

    io.on('connection' , function(socket){

        socket.on('start_tunnel' , function (data) {
            fullfillCaptcha[socket.id] = null ;
            set_stat(socket.id , 1 );
            run(socket , data.token );
        })

        socket.on('get_captcha_from_client' , function (data) {

            fullfillCaptcha[socket.id](data);

        })

    });

    var fullfillCaptcha = {};
    var pay_stats = {} ;

    function captchaPromise(id){
        return  new Promise(resolve => fullfillCaptcha[id] = resolve);
    }

and here is the run function which lunches the puppeteer .... i've mented the code so its easy to read ... basically it opens a webpage containing a form with captcha , takes screenshot from captcha image and send it to client , receives typed captcha from client , put it in the input and submits the form

async function run(socket , token ) {

   /// OPENING THE WEB PAGE 
    const browser = await puppeteer.launch({headless: true , args:['--no-sandbox']});
    const page = await browser.newPage();
    await page.goto('http://example./init/' + token );

    await Promise.race([
        page.waitForNavigation(),
        page.waitForSelector(".Error")
    ]);

   /// CHECKING FOR ERROR

    if (await page.$(".Error"))
    {
        socket.emit('payment_connection_error' );
        set_stat(socket.id , 4 );
        browser.close();
        return ;
    }



   /// TAKING SCREENSHOT FROM CAPTCHA IMAGE

    console.log( ' current url  ' + page.url() );
    let element =  await page.$('#security');

    await element.screenshot({path: 'public_html/captcha/'+  socket.id+'.png'  });
    set_stat(socket.id , 2 );

    /// SENDING CAPTCHA IMAGE FOR CLIENT 
    socket.emit('send_captcha_to_client' , {text : socket.id+'.png'  });

    /// WAITING FOR CLIENT TO TYPE THE CAPTCHA AND SEND IT BACK TO SERVER 

    var captcha = await captchaPromise(socket.id);

    if( typeof(captcha.W_CAPTCHA) == 'undefined' || captcha.W_CAPTCHA == 'cancel' )
    {
        console.log('canceling ... ');
        set_stat(socket.id , 3 );
        browser.close();
        return ;
    }
    console.log( ' captcha confirm -> ' + captcha.W_CAPTCHA );
    set_stat(socket.id , 3 );

    /// TYPING CAPTCHA IN THE INPUT AND SUBMITTING THE FORM 

    await page.$eval('#CAPTCHA', (el , _captcha) => el.value = _captcha.W_CAPTCHA , captcha );
    await page.click('#doPay');


    /// CHECKING FOR SUCCESS OR ERROR OF FORM AFTER 

    await Promise.race([
        page.waitForNavigation(),
        page.waitForSelector(".Error")
    ]);


    if (await page.$(".Error"))
    {
        const pay_error = await page.$eval('.Error', (element) => {
        return element.innerHTML
    });

        console.log(" error : " + pay_error )
        socket.emit('payment_error' , {text : pay_error });
    }
else
    {

        console.log('all ok') ;
        await page.click('#doSubmitTop');
        await page.waitForSelector('#payment-result');
        console.log( ' current url  ' + page.url() );
        socket.emit('payment_result');



    }

    browser.close();
}

here is the problem , we're having lots of server crush due to lack of RAM and its all puppeteer using the ram ... we have a decent vps (4G RAM , 2 CPU ) ... and when we're testing there is not much server load can be seen

so i was wondering if anything im doing wrong here or is there anyway to see a log or something to see what went wrong ?

• javascript
• node.js
• puppeteer

1 Answer 1

Memory crash with Puppeteer happens quite often. Some pages may consume even a GB of memory so it's hard to predict how many instances you can run in parallel.

If you are running multiple Puppeteer Browser instances which I expect then it's easy to run out of memory with few tabs opened.

Some methods to make your Puppeteer usage more memory efficient:

• scale with tabs (pages) not browser instances (less stable solution but opening multiple pages from one browser is more efficient)
• abort requests for assets that are not needed (JS, CSS, fonts, images, etc.) https://github./GoogleChrome/puppeteer/blob/master/docs/api.md#pagesetrequestinterceptionvalue
• queue and run tasks in a pool to limit the concurency

To ensure that Puppeteer won't kill other processes you could run it inside the docker container with limited resources.

We use this class https://github./apifytech/apify-js/blob/master/src/autoscaled_pool.js to autoscale Puppeteer tasks based on available memory in a container (running as close to 100% CPU and 100% memory as possible).