Store multiple scraping data in an Array

Question

First part:

Currently I scrapes a webpage to get links, then I open each scraped link this part work perfectly.

second part:

I check if there is selection of a (select) options field and stored the values(id) in an array this part also works fine.

Third part:

I want to loop through them(selection of options) and fire a click event and wait for AJAX response then extracts infoProduct then stor it in an array.

I have trouble in this part I got an empty array because the return listProducts is called before the first this.eachThen start.

function getInfosProduct(obj,variation) {
    if (!variation) {
        return [{
            name: getName(),
            image: products.image.getElement.link + getImage(),
            url: obj.url
        }];
    } else {
        return {
            name: getName(),
            image: products.image.getElement.link + getImage(),
            url: variation.url,
            idVariation: variation.id,
            descVariation: variation.description
        };
    }
}

function clickVariation(variation) {
    if (variation.ok && variation.id != variation.ignore) {
        chooseVariation(products.selector, variation.id);
        return true;
    }
    return false;
}

casper.getInfosProducts = function(obj) {
    if (obj.level == 0) {
        return this.evaluate(getInfosProduct, obj,false);
    } else {
        listProducts = [];
        this.eachThen(obj.levelVariation, function getInfosProducts(variation) {
            isClick = this.evaluate(clickVariation, variation.data)
            if (isClick) {

                this.waitForSelectorTextChange('.selector', function() {
                    this.echo('The text on .selector has been changed.');
                });

                listProducts.push(this.evaluate(getInfosProduct, obj,variation.data));
                                }
        });
        return listProducts;
    }
};

Function to trigger the change event on the select element

function chooseVariation(selector, valueToMatch) {
    var select = document.querySelectorAll(selector),
        found = false;
    Array.prototype.forEach.call(select, function(opt, i) {
        if (!found && opt.value.indexOf(valueToMatch) !== -1) {
            select.selectedIndex = i;
            found = true;
        }
    });
    // dispatch change event in case there is some kind of validation
    var evt = document.createEvent("UIEvents"); // or "HTMLEvents"
    evt.initUIEvent("change", true, true);
    select[0].dispatchEvent(evt);
}

This is the main fucntion:

    function startSraping(obj) {
        casper.then(function switchAction() {
            switch (obj.action) {

                //......... some code ........

          // iterating through array links and open pages
            case "openLinks":
                this.each(links, function eachOpenLinks(self, link) {
                    if (link.ok) {
                        self.thenOpen(link.url, function thenOpenLinks() {
                            startSraping({
                                url: link.url,
                                action: "getVariations"
                            });
                        });
                    } 
                });
                break;

                    // get all variations for each page opend
                case "getVariations":
                    objVariations = this.getVariations(obj.url);
                    startSraping({
                        url: obj.url,
                        action: "getInfosProducts",
                        objVariations: objVariations
                    });
                    break;

                case "getInfosProducts":
                    this.eachThen(obj.objVariations.list, function(levelVariation) {
                        infosProd = this.getInfosProducts({
                            levelVariation: levelVariation.data,
                            url: obj.url,
                            level: obj.objVariations.level
                        });
                      // Here I got an empty array
                        this.echo(JSON.stringify(infosProd), 'INFO');
                    });
                    break;
            }
        });
    }
    casper.start(url, function start() {
        startSraping({
            variation: variation,
            action: "submitSearch"
        });
    });
    casper.run();

Answer 1

You can't call an asynchronous function (eachThen and waitForSelectorTextChange are both asynchronous) inside of a function that is supposed to return the result from the asynchronous function in a synchronous fashion (general reference). Since CasperJS doesn't support Promises, this get's a little tricky.

I think the following changes should be minimal and get you where you want to go.

casper.getInfosProducts = function(obj, callback) {
    if (obj.level == 0) {
        this.then(function(){
            callback.call(this, arr.push(this.evaluate(getInfosProduct, obj,false));
        });
    } else {
        var listProducts = [];
        this.eachThen(obj.levelVariation, function getInfosProducts(variation) {
            var isClick = this.evaluate(clickVariation, variation.data)
            if (isClick) {
                this.waitForSelectorTextChange('.selector', function() {
                    this.echo('The text on .selector has been changed.');
                    listProducts.push(this.evaluate(getInfosProduct, obj, variation.data));
                });
            }
        });
        this.then(function(){
            callback.call(this, listProducts);
        });
    }
};

In startSraping:

case "getInfosProducts":
    this.eachThen(obj.objVariations.list, function(levelVariation) {
        this.getInfosProducts({
            levelVariation: levelVariation.data,
            url: obj.url,
            level: obj.objVariations.level
        }, function (infosProd){
            // this is the asynchronous callback
            this.echo(JSON.stringify(infosProd), 'INFO');
        });
    });
    break;