Best way to iterate and make an async call during each iteration

Question

If you have to loop and make a bunch of calls to a repository or gateway in my case, how do I do that asynchronously meaning not wrapping my async calls inside a synchronous for loop?

For example, what would be a better approach (restructuring this code) to loop through a set of ids, and make the call to find() below like I'm trying to do?

The goal: I want to take an array of ids, iterate them, and during each iteration, use the id to call find() on my gateway to go get the object for that id, then stuff it into a final array in which I'll return when all said and done.

What I'm using:

• q (for promises)
• co-pg (to hit the database)

someModule.js

var _gateway = require('./database/someGateway');

          var cars = [];
          var car;

             for (var i = 0; i < results.docs.length; i++){

                 var carId = results.docs[i].carId;

                 _gateway.find(carId)
                 .then(function(data){
                         console.log('data[0]: ' + data[0].id);
                         cars.push(data[0]);
                     })
                 .done();
             }

console.log("cars: " + cars.length); // length here is 0 because my asyn calls weren't done yet
             result(cars);

someGateway.js

'use strict';
var Q = require('q');

var _carModel = require('../../models/car');

module.exports = {
    models: {
        car: _carModel
    },
    find: _find
};

function _find(carId)
{
    return _carModel.find(carId);
};

carModel.js

'use strict';

var Q = require('q');
var pg = require('co-pg')(require('pg'));
var config = require('../../models/database-config');

var car = module.exports = {};

car.find = Q.async(function *(id)
{
    var query = 'SELECT id, title, description FROM car WHERE id = ' + id;

    var connectionResults = yield pg.connectPromise(config.connection);

    var client = connectionResults[0];
    var done = connectionResults[1];

    var result = yield client.queryPromise(query);
    done();

    console.log("result.rows[0].id: " + result.rows[0].id);
    return result.rows;
});

so I need help understanding how to refactor my code in someModule.js to get that working properly, so that I make a call to find() for each id, stuff each found car into the array, then return the array. The carModel code is async. It goes out to a physical database to perform the actual query lookup.

UPDATE #1

Ok after a couple more hours of trying all sorts of sh** (q.all(), and a ton of other combinations of callback code, etc.) here's what I have at this point:

someModule.js

var _data;
var Q = require('q');
var _solrClient = require('../models/solr/query');
var _solrEndpoint = "q=_text&indent=true&rows=10";
var _postgreSQLGateway = require('./database/postgreSQLGateway');

module.exports = {
    data: function(data){
        _data = data;
    },
    find: function (text, result){

        if(!searchText){
            result(null);
        };

         _solrClient.query(endpoint, function(results){

             var carIds = [];
             var cars = [];
             var car;

             for (var i = 0; i < results.docs.length; i++){
                 carIds.push(results.docs[i].carId);
             }

             for (var i = 0; i < carIds.length; i++) {

                 var car = _postgreSQLGateway.find(carIds[i], function(o){
                     console.log("i: " + i);
                 });
             };
        });
    }
};

someGateway.js

'use strict';
var Q = require('q');

var _carModel = require('../../models/postgreSQL/car');

module.exports = {
    models: {
        car: _carModel
    },
    find: _find
};

function _find(carId, foundCar)
{
    console.log("CALL MADE");

    _carModel.find(carId)
        .then(function(car){
            console.log("car: " + car[0].id);

            foundCar(car);
        });
};

carModel.js

[same code, has not changed]

Of course I noticed that the for loop fires off all my function calls asyncronously and so when I console.write the i, it's 10 because the for loop is done but then as we know, the rest of the console.logs happen later after the callbacks are done.

So I still can't get this working right...

Also when I was playing around I started down this path but it ended at a brick wall:

var find = Q.async(function(carIds, cars)
{
    var tasks = [];
    var foundCars = [];

    for (var i = 0; i < carIds.length; i++) {
        tasks.push(_postgreSQLGateway.find(carIds[' + i + ']));
    };

    Q.all([tasks.join()]).done(function (values) {
        for (var i = 0; i < values.length; i++) {
            console.log("VALUES: " + values[0]);
            foundCars.push(values[0]);
        }
        cars(foundCars);
    });
});

I ended up with [object promise] every time for values[i] instead of a car for value[i]

Answer 1

This is potentially really easy with the vanilla Promise API from es6 (and replicated by Bluebird and other libs). First map the IDs to an array of promises:

var promises = results.docs.map(function(doc) {
  return _gateway.find(doc.carId);
});

Then create a promise for the aggregate result:

var allDone = Promise.all(promises);

Then inside the done() callback of the aggregate promise, you'll have a final array of results, in the same length and order as the carId array:

allDone.then(function(results) {
  // do something with "results"
});

Answer 2

I don't know the Q promises library, but here's a solution using generic Promises built into node.js. This runs all the requests in parallel and then when all results have been collected, it runs the final .then() handler with all the results:

var _gateway = require('./database/someGateway');

var promises = [];
for (var i = 0; i < results.docs.length; i++) {
    promises.push(_gateway.find(results.docs[i].carId).then(function (data) {
        console.log('data[0]: ' + data[0].id);
        return data[0];
    }));
}
Promise.all(promises).then(function(cars) {
    // cars will be an array of results in order
    console.log("cars: " + cars.length);
    result(cars);
});

Individual promise libraries (like the one I know Bluebird) have features built in that lets you do this kind of activity in even less code, but I've intentionally kept this answer to just using standard promise features.