Waiting for AJAX response from a selection of a <select> field in CasperJS

Question

I want to use CasperJS to scrape this website: http://www.agoda.com/hotel-des-arts-saigon-mgallery-collection/hotel/ho-chi-minh-city-vn.html?checkin=2015-11-14&los=2&adults=2&childs=0&rooms=1

I want to make change to the currency options by CasperJS. However, the currency options are wrapped in <select></select> tags which is not embedded in a form. When I then read the table, it shows prices with the old currency. The prices are loaded through AJAX when the currency is changed.

How can I do with CasperJS?

Here is my code:

var casper = require("casper").create({
    verbose: true,
    logLevel: 'error',
    pageSettings: {
        loadImages: false,
    }
});

   var utils = require('utils');

    var url = 'http://www.agoda.com/hotel-des-arts-saigon-mgallery-collection/hotel/ho-chi-minh-city-vn.html?checkin=2015-11-14&los=4&adults=2&childs=0&rooms=1';

    var names = [];
    var prices = [];
    var currency = [];


    function getName() {
        var rows = document.querySelectorAll('table#room-grid-table tbody tr td:first-child .info-container .room-name span');
        return Array.prototype.map.call(rows, function(e) {
            return e.innerHTML;
        });
    }

    function getPrice() {
        var price = document.querySelectorAll('table#room-grid-table tbody tr td:nth-child(3) .price span.sellprice');
        return Array.prototype.map.call(price, function(e) {
            return e.innerHTML;
        });
    }

    casper.start(url, function() {
        this.echo(this.getTitle());
    });

    casper.then(function() {
        this.click('select[id="currency-options"]');
    });

    casper.then(function() {
        this.click('option[value="AED"]');
    });

    casper.then(function() {
        names = this.evaluate(getName);
        prices = this.evaluate(getPrice);
    });

    casper.then(function() {
        utils.dump(names);
        utils.dump(prices);
    })

    casper.run();

Select box:

<select id="currency-options" data-selenium="room-currency">
    <option value="AED">Arab Emirates Dirham (AED)</option>
    <option value="ARS">Argentine Peso (ARS)</option>
    ...
    <option value="USD" selected="">US Dollar (USD)</option>
</select>

Table:

<table id="room-grid-table">
    <tbody data-selenium="room-tbody" data-prebook-url="/NewSite/en-us/Hotel/Prebook/929399">
        <tr>
            <td>...</td>
            <td>...</td>
            <td>
                <div class="price" data-selenium="price">
                    <span class="currency" data-selenium="price-currency">USD</span>
                    <span class="crossout show-cor-tooltip">288.75</span>
                    <span class="sellprice">187.11</span>
                </div>
            </td>
        </tr>
        <tr>...</tr>
        ...
    </tbody>
</table>

Answer 1

Clicking an option field of a select box almost never works, because it is not a clickable UI element in PhantomJS. You have to select the option that you want programmatically through the DOM. For example, this can be done by setting the selectElement.selectedIndex property to the correct index. After setting this, you need to trigger the change event on the select element otherwise no AJAX request is sent to the server.

Here is the code copied from my other answer:

casper.selectOptionByValue = function(selector, valueToMatch){
    this.evaluate(function(selector, valueToMatch){
        var select = document.querySelector(selector),
            found = false;
        Array.prototype.forEach.call(select.children, function(opt, i){
            if (!found && opt.value.indexOf(valueToMatch) !== -1) {
                select.selectedIndex = i;
                found = true;
            }
        });
        // dispatch change event in case there is some kind of validation
        var evt = document.createEvent("UIEvents"); // or "HTMLEvents"
        evt.initUIEvent("change", true, true);
        select.dispatchEvent(evt);
    }, selector, valueToMatch);
};

You need to wait for the AJAX request completion after changing the currency setting. This can be done in different ways:

• Static wait time works, but may wait longer than the request needs:

  casper.wait(5000);
  

• Waiting for a specific selector to appear is more efficient, because it continues with the control flow as soon as the selector is found. You can for example wait for the currency text to change in the table. This can be easily done with XPath:

  var x = require('casper').selectXPath;
  var currency = 'AED';
  ...
  casper.waitForSelector(x("//*[@id='room-grid-table']//*[@class='currency' and contains(text(), '"+currency+"')]"));
  

• You can wait for the text change in a specific selector such as the currency in the table. This won't work if the change is immediate, because CasperJS needs a little time to pick up the change. 20ms should be enough.

  casper.waitForSelectorTextChange("#room-grid-table .currency");
  

• Wait for the AJAX response which works by looking at all received resources:

  casper.waitForResource("Main/GetRoomTypeDetailList").wait(50);
  

---

Full script of version 2:

var casper = require("casper").create({
    verbose: true,
    logLevel: 'error',
    pageSettings: {
        loadImages: false,
    }
});

var utils = require('utils');
var x = require('casper').selectXPath;

var url = 'http://www.agoda.com/hotel-des-arts-saigon-mgallery-collection/hotel/ho-chi-minh-city-vn.html?checkin=2015-11-14&los=4&adults=2&childs=0&rooms=1';

var names = [];
var prices = [];
var currency = [];


function getName() {
    var rows = document.querySelectorAll('table#room-grid-table tbody tr td:first-child .info-container .room-name span');
    return Array.prototype.map.call(rows, function(e) {
        return e.innerHTML;
    });
}

function getPrice() {
    var price = document.querySelectorAll('table#room-grid-table tbody tr td:nth-child(3) .price span.sellprice');
    return Array.prototype.map.call(price, function(e) {
        return e.innerHTML;
    });
}

casper.selectOptionByValue = function(selector, valueToMatch){
    this.evaluate(function(selector, valueToMatch){
        var select = document.querySelector(selector),
            found = false;
        Array.prototype.forEach.call(select.children, function(opt, i){
            if (!found && opt.value.indexOf(valueToMatch) !== -1) {
                select.selectedIndex = i;
                found = true;
            }
        });
        // dispatch change event in case there is some kind of validation
        var evt = document.createEvent("UIEvents"); // or "HTMLEvents"
        evt.initUIEvent("change", true, true);
        select.dispatchEvent(evt);
    }, selector, valueToMatch);
};

casper.start(url, function() {
    this.echo(this.getTitle());
});

var currency = 'AED';
casper.then(function() {
    this.selectOptionByValue('select[id="currency-options"]', currency);
});

casper.waitForSelector(x("//*[@id='room-grid-table']//*[@class='currency' and contains(text(), '"+currency+"')]"));

casper.then(function() {
    names = this.evaluate(getName);
    prices = this.evaluate(getPrice);
});

casper.then(function() {
    utils.dump(names);
    utils.dump(prices);
})

casper.run();