CODEWITHSUNDEEP
javascriptnode.jsurlpuppeteer
user4487338
user4487338

Reputation: 78

How to use url module in puppeteer page.evaluate

I've already tried everything mentioned in Error: Evaluation Failed: ReferenceError: util is not defined and How to pass required module object to puppeteer page.evaluate. Specifically, I've tried converting url.js using browserify (I've also tried converting url.js and punycode.js together), and I've added the corresponding script (bundle.js) to the page environment.

I'm trying to use the url module inside page.evaluate() in puppeteer. Here's a very simple example to show the error:

const puppeteer = require('puppeteer');

puppeteer.launch({dumpio: true}).then(async browser => {
  const page = await browser.newPage();
  const response = await page.goto('https://www.google.com');
  await page.waitFor(5000);
  const pageUrl = page.url();
  await page.addScriptTag({path: 'bundle.js'});
  await page.evaluate(pageUrl => {
    const anchors = Array.from(document.querySelectorAll('a'));
    for (let anchor of anchors) {
      const href = anchor.getAttribute('href');
      let hrefUrl;
      try {
        hrefUrl = new URL(href);
      } catch (e) {
        hrefUrl = new URL(href, pageUrl);
      }
      console.log(url.format(hrefUrl, {fragment: false}));
    }
  }, pageUrl);
  await page.close();
  await browser.close();
});

This example generates the following error:

(node:23667) UnhandledPromiseRejectionWarning: Error: Evaluation failed: ReferenceError: url is not defined at pageUrl (puppeteer_evaluation_script:11:19) at ExecutionContext.evaluateHandle (/home/webb/node_modules/puppeteer/lib/ExecutionContext.js:97:13) at at process._tickCallback (internal/process/next_tick.js:188:7)

What else do I need to do to get the url module recognized?

Upvotes: 2

Views: 5616

Answers (2)

Azami
Azami

Reputation: 2161

Expose all the functions from the url package using page.exposeFunction.

Iterate over the module's exports and add every function to expose

var url = require('url');

var functionsToExpose = [];
for(let key of Object.keys(url)){
    if(typeof url[key] == 'function'){
        functionsToExpose.push({name: 'url'+key, func: url[key]});
    }
}

Expose them to the page

for(let item of functionsToExpose){
    await page.exposeFunction(item.name, item.func);
}

Every function of the url package will be renamed. url.parse is accessible using urlparse.

Upvotes: 1

vsemozhebuty
vsemozhebuty

Reputation: 13812

Variant with page.exposeFunction():

'use strict';

const url = require('url');
const puppeteer = require('puppeteer');

puppeteer.launch({ dumpio: true }).then(async browser => {
  const page = await browser.newPage();
  await page.exposeFunction('formatURL', formatURL);

  const response = await page.goto('https://www.google.com');
  await page.waitFor(5000);
  const pageUrl = page.url();

  await page.evaluate(async (pageUrl) => {
    const anchors = Array.from(document.querySelectorAll('a'));
    for (const anchor of anchors) {
      const href = anchor.getAttribute('href');
      const hrefUrl = await formatURL(href, pageUrl);
      console.log(hrefUrl);
    }
  }, pageUrl);

  await page.close();
  await browser.close();
});

function formatURL(href, base) {
  try {
    return url.format(new URL(href), { fragment: false });
  } catch (e) {
    return url.format(new URL(href, base), { fragment: false });
  }
}

Upvotes: 4

Related Questions