CODEWITHSUNDEEP
javascriptnode.jspuppeteer
bullettrang
bullettrang

Reputation: 75

Puppeteer: page.evaluate throws TypeError: cannot read innerText of null

I am trying to learn puppeteer and wanted to scrape this MMA web page at https://www.tapology.com/regions .

I wanted to scrape all the region titles such as "US MIDWEST, US NORTHEAST, US SOUTHWEST... etc"

I have tried using await page.waitForSelector to wait for selectors to load in but page just hangs.

I have also tried using innerHTML but its the same result.

Tried using page.$$eval(selector, pageFunction[, ...args]) but returns empty array.

Tried using exact CSS selector from Google dev tools but no success. Tried various combos of CSS selections but I am still not able to scrape the h4's texts.

const puppeteer = require('puppeteer');
const REGIONS_URL = 'https://www.tapology.com/regions';

async function getRegionsNames(url) {
    const browser= await puppeteer.launch();
    const page= await browser.newPage();

  await (async ()=>{
  const MIDWEST_SELECTOR = "#content > div.regionIndex > h4:nth- child(1) > a";
  //await page.waitForSelector(MIDWEST_SELECTOR);  //hangs if used

  const mid = await page.evaluate((sel)=>{         //trying to grab 'MIDWEST'
    return document.querySelector(sel).innerText;
  },MIDWEST_SELECTOR);
 console.log(mid);      //throws error
 await browser.close();
}
getRegionsNames(REGIONS_URL);

(node:23646) UnhandledPromiseRejectionWarning: Error: Evaluation failed: TypeError: Cannot read property 'innerText' of null

Upvotes: 2

Views: 10544

Answers (1)

Saurabh Mistry
Saurabh Mistry

Reputation: 13669

try this : 

app.get('/testing',function(req,res){
    (async () => {

        const browser = await puppeteer.launch({
            headless: true
        });
        const page = await browser.newPage();
        await page.goto('https://www.tapology.com/regions',{waitUntil: 'domcontentloaded'});
        const example  = await page.$('.regionIndex');
        const scrapedData = await page.evaluate(() =>
        Array.from(document.querySelectorAll('h4 a'))
          .map(link => ({
            title: link.innerHTML,
            link: link.getAttribute('href')
          }))
      )
      console.log('scrapedData',scrapedData);
        await page.close();
        await browser.close();
        return res.send(scrapedData);

    })();
});

you will get : 

[
{
title: "US Midwest",
link: "/regions/us-midwest"
},
{
title: "US Northeast",
link: "/regions/us-northeast"
},
{
title: "US Southeast",
link: "/regions/us-southeast"
},
{
title: "US Southwest",
link: "/regions/us-southwest"
},
{
title: "US West",
link: "/regions/us-west"
},
{
title: "Asia Central",
link: "/regions/central-asia"
},
{
title: "Canada",
link: "/regions/canada"
},
{
title: "Europe Balkans",
link: "/regions/europe-balkans"
},
{
title: "Europe Eastern",
link: "/regions/europe-eastern"
},
{
title: "Europe Western",
link: "/regions/western-europe"
},
{
title: "Latin America",
link: "/regions/latin-america"
},
{
title: "Middle East",
link: "/regions/middle-east"
}
]

Upvotes: 4

Related Questions