CODEWITHSUNDEEP
node.jspuppeteer
nanquim
nanquim

Reputation: 1924

NodeJS/Puppeteer - Get list of elements by tag name

I need to collect all h1 tags and then pop the first and last ones.

The code works in devtools console, but not in my Node app. I understand page.evaluate does not work directly with DOM, but I follow several examples and did not succeed.

This is the resul I want:

enter image description here

This is my code: 

const puppeteer = require('puppeteer');

//process.on("unhandledRejection");

let liga = async () => {
    const browser = await puppeteer.launch({ headless: true });
    const page = await browser.newPage();
    page.on('console', consoleObj => console.log(consoleObj.text()));
    const URL = 'http://dummy.org.br/grupo';


    //Pegar o dia da semana atual
   ...irrelevant...

    //Vai até a página
    await page.goto(URL, {waitUntil: 'load'});
    console.log('On: ' + page.url());


    //Escolher o estado/região
    const estado = 'RIO DE JANEIRO';
    const selectEstado = await page.$('select[name="busca_grupo_estado"]');
    await selectEstado.type(estado);

    const cidade = 'RIO DE JANEIRO';
    const selectCidade = await page.$('select[name="busca_grupo_cidade"]');
    await selectCidade.type(cidade);

    const btn = await page.$('#frm_busca_uf > div.col-xs-4.no-padding-right > div > div.col-xs-2 > a');
    btn.click();

    //     PROBLEM STARTS HERE

    //Pegar grupos do dia
    //body > div.content-home > div > div.box > div > div:nth-child(2) > div.col-md-12.no-padding > div:nth-child(4) > div:nth-child(2) > div.col-xs-12 > div:nth-child(1) > div:nth-child(1) > h1
    //==xpath: /html/body/div[3]/div/div[2]/div/div[2]/div[2]/div[4]/div[2]/div[3]/div[1]/div[1]/h1

    //........NOT WORKING:
    //this code works fine in chrome console
    /* const grupos = await page.evaluate( () => {
        var data = [];
        var nomes = document.querySelectorAll('h1'); //chrome returns a notelist with 118 elements
        //i don't want the 1s and the last one is undefined
        for(var c = 1; c <= nomes.length-1; c++ ) {
            console.log(nomes[c].textContent);
            data.push(nomes[c].textContent);
            console.log(data[c]);
        }
        return data;
    });

    console.log(grupos[0]);
    console.log(grupos[1]);
    console.log(grupos[2]); */

    //........NOT WORKING:
    //const grupos = await page.$$('h1');

    //........NOT WORKING:
    /* const grupos = await page.evaluate( () => Array.from( document.querySelectorAll( 'h1' ), element => element.textContent) );
    console.log(grupos[0]);
    console.log(grupos[1]); */

    //browser.close();
}

liga().then((value) => {
    console.log(value); 
});

Upvotes: 2

Views: 16332

Answers (2)

Grant Miller
Grant Miller

Reputation: 28999

Use page.waitForNavigation() after clicking the link to filter by region:

const btn = await page.$('#frm_busca_uf a');
await btn.click();

await page.waitForNavigation();

Then, you can scrape the text from the h1 elements and log the results:

const grupos = await page.evaluate(() => Array.from(document.getElementsByTagName('h1'), e => e.textContent));

console.log(grupos[0]); // Encontre uma reunião
console.log(grupos[1]); // Trindade
console.log(grupos[2]); // Paraiso

Note: If you would like to remove the first and last element of the array, you can use grupos.slice(1, -1):

const new_grupos = grupos.slice(1, -1);

... but this should not be necessary, as it appears that the first and last elements are not undefined.

Upvotes: 2

Vaviloff
Vaviloff

Reputation: 16838

You need to actually wait for the page to reload with search results. See the modified currently working code below:

const puppeteer = require('puppeteer');

let liga = async () => {
    const browser = await puppeteer.launch({ headless: true });
    const page = await browser.newPage();
    page.on('console', consoleObj => console.log(consoleObj.text()));
    const URL = 'http://na.org.br/grupo';

    //Vai ate a pagina
    await page.goto(URL, {waitUntil: 'load'});
    console.log('On: ' + page.url());

    //Escolher o estado/regiao
    const estado = 'RIO DE JANEIRO';
    const selectEstado = await page.$('select[name="busca_grupo_estado"]');
    await selectEstado.type(estado);

    const cidade = 'RIO DE JANEIRO';
    const selectCidade = await page.$('select[name="busca_grupo_cidade"]');
    await selectCidade.type(cidade);

    const btn = await page.$('#frm_busca_uf > div.col-xs-4.no-padding-right > div > div.col-xs-2 > a');
    btn.click();

    // Wait for the page to reload with results
    await page.waitForNavigation();

    const grupos = await page.evaluate( () => Array.from( document.querySelectorAll( 'h1' ), element => element.textContent) );

    browser.close();

    return grupos;
}

liga().then((value) => {
    console.log(value); 
});

Upvotes: 3

Related Questions