Accepting cookie policy with CasperJS

Question

I have been trying to log in to Marktplaats.nl (CraigsList-like webpage in Netherlands) using CasperJS. However, I am stuck on accepting the cookie policy.

This is my script so far:

var casper = require('casper').create({   
    verbose: true, 
    logLevel: 'debug',
    pageSettings: {
         loadImages:  false,         
         loadPlugins: false,       
         userAgent: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'
    }
});

var x = require('casper').selectXPath;

casper.start('https://www.marktplaats.nl', function() {
    this.echo(this.getTitle()); //Prints "Marktplaats Cookiewall"
    //POINT1
    //Here I just check and log if the "Agree" button exists.
    if (casper.exists(x("//input[contains(@value, 'Cookies accepteren')]"))) {
        casper.echo("Agree button found");
    }
    else
    {
        casper.echo("Agree button not found");
    }
});

casper.then(function() {
    if(this.getTitle().indexOf("Cookiewall") !== -1)
    { 
     //POINT2           
     //If we are on the cookiewall page, click on agree. 
     casper.echo("Clicking on agree");
     casper.click(x("//input[contains(@value, 'Cookies accepteren')]"));
    }
});

casper.thenOpen('https://www.marktplaats.nl', function() {
//POINT3
//Reloaded page
this.echo('Second Page: ' + this.getTitle());
});

casper.run();

First try to navigate to the homepage (Marked as POINT1 in the code), but get redirected to the Cookiewall page which wants me to accept the cookie policy. Screenshot taken in browser, no connection to CasperJS.

At POINT2 my script clicks on "Cookies accepteren" - this is logged as:

[debug] [phantom] Mouse event 'mousedown' on selector: xpath selector: //input[contains(@value, 'Cookies accepteren')]
[debug] [phantom] Mouse event 'mouseup' on selector: xpath selector: //input[contains(@value, 'Cookies accepteren')]
[debug] [phantom] Mouse event 'click' on selector: xpath selector: //input[contains(@value, 'Cookies accepteren')]

I am new to CasperJS but this looks fine to me.

FInally, at POINT3, I reload the homepage, but again get redirected to the Cookiewall page - casper logs the Cookiewall title and redirections are logged UPDATE after comments: I registered to resource.error, page.error, remote.message and casper.page.onResourceTimeout as per Artjom's comment. 2 ResourceErrors have shown up. I edited this log accordingly:

[info] [phantom] Step anonymous 3/5: done in 2018ms.
[debug] [phantom] opening url: https://www.marktplaats.nl/, HTTP GET
ResourceError: {
    "errorCode": 5,
    "errorString": "Operation canceled",
    "id": 7,
    "status": null,
    "statusText": null,
    "url": "http://s3.amazonaws.com/ki.js/56612/b7M.js"
}
[debug] [phantom] Navigation requested: url=https://www.marktplaats.nl/, type=Other, willNavigate=true, isMainFrame=true
[debug] [phantom] Navigation requested: url=http://www.marktplaats.nl/, type=Other, willNavigate=true, isMainFrame=true
[debug] [phantom] Navigation requested: url=http://www.marktplaats.nl/cookiewall
/?target=http%3A%2F%2Fwww.marktplaats.nl%2F, type=Other, willNavigate=true, isMa
inFrame=true
[debug] [phantom] url changed to "http://www.marktplaats.nl/cookiewall/?target=http%3A%2F%2Fwww.marktplaats.nl%2F"
[debug] [phantom] Successfully injected Casper client-side utilities
[info] [phantom] Step anonymous 5/5 http://www.marktplaats.nl/cookiewall/?target
=http://www.marktplaats.nl/ (HTTP 200)
Second Page: ? Marktplaats - Cookiewall
[info] [phantom] Step anonymous 5/5: done in 2224ms.
[info] [phantom] Done 5 steps in 2243ms
[debug] [phantom] Navigation requested: url=about:blank, type=Other, willNavigate=true, isMainFrame=true
ResourceError: {
    "errorCode": 5,
    "errorString": "Operation canceled",
    "id": 11,
    "status": null,
    "statusText": null,
    "url": "http://s3.amazonaws.com/ki.js/56612/b7M.js"
}
[debug] [phantom] url changed to "about:blank"

I just can't seem to get to the homepage.

Answer 1

You need to do two things:

• Load images, because it seems as the button has no dimensions when images are not loaded.
• Wait a bit after clicking. CasperJS doesn't seem to pick up that there is a page load happening.

Full script:

var casper = require('casper').create({   
    //verbose: true, 
    //logLevel: 'debug',
    pageSettings: {
         loadImages:  true,         
         loadPlugins: false,       
         userAgent: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'
    }
});

var x = require('casper').selectXPath;

var acceptBtn = x("//input[contains(@value, 'Cookies accepteren')]");

casper.start('http://www.marktplaats.nl', function() {
        this.echo(this.getTitle());
    })
    .waitForSelector(acceptBtn)
    .thenClick(acceptBtn)
    .wait(100)
    .then(function(){
        this.echo(this.getTitle());
    })
    .run();