CODEWITHSUNDEEP
javascriptnode.jswindowsasync-awaitpuppeteer
naqushab
naqushab

Reputation: 804

Puppeteer opening chrome instance for each file at once

I am trying to automate a workflow in which I have a list of files from a directory and I put them in an array. Then for each file in array I call a function for Chrome automation.

const path = require('path');
const chalk = require('chalk');
const puppeteer = require('puppeteer');

module.exports = {
    generateOutput : async(fileName, url = "https://example.com/") => {
        const filePath = path.join(process.cwd(), fileName);
        const outputFilePath = path.join(process.cwd(), "OutputFiles");
        try{
            const browser = await puppeteer.launch({headless: false});
            process.setMaxListeners(0);
            const page = await browser.newPage();
            await page._client.send('Page.setDownloadBehavior', {behavior: 'allow', downloadPath: outputFilePath});
            page.on('dialog', async dialog => {
                console.log(chalk.magenta("Error Occured: " + dialog.message()));
                await dialog.dismiss();
                await browser.close();
            });
            await page.goto(url, {waitUntil: 'networkidle2'});
            await page.click('#ui-id-9');
            await page.click('#ui-id-18');
            await page
                    .waitForSelector('#ui-id-9')
                    .then(() => console.log(chalk.magenta("Uploader module visible... Uploading the files") ));
            const input = await page.$('#upload-file');
            await input.uploadFile(filePath);
            await page.waitFor(10000);
            await page.click("#up-file");
            await page.waitFor(50000);
            await page
                    .waitForSelector('#ui-id-18')
                    .then(() => console.log(chalk.magenta("Downloader module visible... Downloading the files") ));
            await page.click("#download-td");
            await page.waitFor(100000);
            await browser.close();
        }
        catch(e){
            console.log(chalk.red(fileName + ' has failed in conversion.'));
        }
    }
};

This creates a chrome instance of (say 100 for 100 files) simultaneously. Is there a way to limit the async process. I don't have much exp. in Node so I can't search for right terms.

Upvotes: 3

Views: 2641

Answers (1)

eMad
eMad

Reputation: 1008

One solution is visit urls for each file one by one.

const path = require('path');
const chalk = require('chalk');
const puppeteer = require('puppeteer');

module.exports = {

    start: async() => {
        const browser = await puppeteer.launch({headless: false});

        const page = await browser.newPage();

        // for all the files in array call it one by one
        for (i = 0; i < files.length; i++) {
            await module.exports.generateOutput(page, fileName);
        }

        await browser.close();
    },

    generateOutput : async(page, fileName, url = "https://example.xm/b") => {
        const filePath = path.join(process.cwd(), fileName);
        const outputFilePath = path.join(process.cwd(), "OutputFiles");
        try{
            process.setMaxListeners(0);
            await page._client.send('Page.setDownloadBehavior', {behavior: 'allow', downloadPath: outputFilePath});
            page.on('dialog', async dialog => {
                console.log(chalk.magenta("Error Occured: " + dialog.message()));
                await dialog.dismiss();
                await browser.close();
            });
            await page.goto(coloradoUrl, {waitUntil: 'networkidle2'});
            await page.click('#ui-id-9');
            await page.click('#ui-id-18');
            await page
                    .waitForSelector('#ui-id-9')
                    .then(() => console.log(chalk.magenta("Uploader module visible... Uploading the files") ));
            const input = await page.$('#upload-file');
            await input.uploadFile(filePath);
            await page.waitFor(10000);
            await page.click("#up-file");
            await page.waitFor(50000);
            await page
                    .waitForSelector('#ui-id-18')
                    .then(() => console.log(chalk.magenta("Downloader module visible... Downloading the files") ));
            await page.click("#download-td");
            await page.waitFor(100000);
        }
        catch(e){
            console.log(chalk.red(fileName + ' has failed in conversion.'));
        }
    }
};

Other is to open new tabs for each file close it once its complete. But this could open 100 tabs at once. You can add an upper limit for example open maximum of 10 tabs at once etc. The following code uses delay function to actively wait for number tabs to become less than 10 before opening a new tab

const path = require('path');
const chalk = require('chalk');
const puppeteer = require('puppeteer');

module.exports = {

    delay: async (milisecs) => {
        return new Promise(function(resolve, reject) {
            setTimeout(resolve, milisecs);
        })
    },

    start: async() => {
        const browser = await puppeteer.launch({headless: false});

        // for all the files in array call it one by one
        for (i = 0; i < files.length; i++) {
            pages = await browser.pages();

            /*
            * if number of tabs is less than 10, skips while. Else
            * waits till number of open tabs become less than 10
            */
            while (pages.length == 10) {
                pages = await browser.pages();
                await module.exports.delay(3000);
            }

            // then open a new tab
            const page = await browser.newPage();
            module.exports.generateOutput(page, fileName);
        }

        await browser.close();
    },

    generateOutput : async(page, fileName, url = "https://example.xm/b") => {
        const filePath = path.join(process.cwd(), fileName);
        const outputFilePath = path.join(process.cwd(), "OutputFiles");
        try{
            process.setMaxListeners(0);
            await page._client.send('Page.setDownloadBehavior', {behavior: 'allow', downloadPath: outputFilePath});
            page.on('dialog', async dialog => {
                console.log(chalk.magenta("Error Occured: " + dialog.message()));
                await dialog.dismiss();
                await browser.close();
            });
            await page.goto(url, {waitUntil: 'networkidle2'});
            await page.click('#ui-id-9');
            await page.click('#ui-id-18');
            await page
                    .waitForSelector('#ui-id-9')
                    .then(() => console.log(chalk.magenta("Uploader module visible... Uploading the files") ));
            const input = await page.$('#upload-file');
            await input.uploadFile(filePath);
            await page.waitFor(10000);
            await page.click("#up-file");
            await page.waitFor(50000);
            await page
                    .waitForSelector('#ui-id-18')
                    .then(() => console.log(chalk.magenta("Downloader module visible... Downloading the files") ));
            await page.click("#download-td");
            await page.waitFor(100000);

            await page.close()
        }
        catch(e){
            console.log(chalk.red(fileName + ' has failed in conversion.'));
        }
    }
};

I've modified your code to convey the concept. Of course it won't run until you replace files with your own variable etc

Upvotes: 5

Related Questions