Getting records from DynamoDB recursively using Q.Promises

Question

I am having trouble implementing Q promises with recursive dynamodb call, new to nodejs and q, considering the limitations of the dynamodb to retrieve results, we need to run recursive query to get the required results.

normally we use the query with Q implementation something like this as

    function getDBResults(){

    var q = Q.defer();

    var params = {
        TableName: mytable,
        IndexName: 'mytable-index',

        KeyConditionExpression: 'id = :id',
        FilterExpression: 'deliveryTime between :startTime and :endTime',

        ExpressionAttributeValues: {
            ':startTime': {
                N: startTime.toString()
            },

            ":endTime": {
                N: endTime.toString()
            },
            ":id": {
                S: id.toString()
            }
        },
        Select: 'ALL_ATTRIBUTES',
        ScanIndexForward: false,
    };


     dynamodb.query(params, function(err, data) {
           if (err) {
               console.log('Dynamo fail ' + err);
               q.reject(err);
           } else {
               console.log('DATA'+ data);
               var results = data.Items;
               q.resolve(results);
           }
       });
      return q.promise;

}



getDBResults.then(
  function(data) {
    // handle data
  },
 function(err) {
        //handle error
  }
);

Using recursive query I can get the results but I need those results to be used in another function, but because of nodejs async nature,the next function calls happens already before the recursive query function finishes its job, now I want that I get all the results from the recursive query function and then get as a promise to a new function and finally handle all the data.

recursive query for dynamodb looks like this.

function getDBResults(){

    //var q = Q.defer();

     params = {
        TableName: mytable,
        IndexName: 'mytable-index',

        KeyConditionExpression: 'id = :id',
        FilterExpression: 'deliveryTime between :startTime and :endTime',

        ExpressionAttributeValues: {
            ':startTime': {
                N: startTime.toString()
            },

            ":endTime": {
                N: endTime.toString()
            },
            ":id": {
                S: id.toString()
            }
        },
        Select: 'ALL_ATTRIBUTES',
        ScanIndexForward: false,
    };



dynamodb.query(params, onQueryCallBack);


}


function onQueryCallBack(err, data) {
if (err) {
    console.log('Dynamo fail ' + err);
    console.error("Could not query db" + err);
} else {


if (typeof data.LastEvaluatedKey != "undefined") {
    console.log("query for more...");
    params.ExclusiveStartKey = data.LastEvaluatedKey;
    dynamodb.query(params, onQueryCallBack);
}
data.Items.forEach(function(item) {
    allResults.push(item);
});
//console.log('NO:OF Results:' + allResults.length);

//q.resolve(tickets);

//});

}

Now I want that I can get the results as promise finally so I can handle them in the next function like this.

getDBResults.then(
  function(data) {
    // handle data
  },
 function(err) {
        //handle error
  }
);

Please help me on this, sorry if its a stupid question but recursive calls with promises have made a hurdle for me.

Thanks

Answer 1

Here is how i solve the problem, Thanks Bergi for your solution as well

function getDBResults() {
    var q = Q.defer();
    var dynamodb = core.getDynamoDB();



    params = {
        TableName: mytable,
        IndexName: 'mytable-index',

        KeyConditionExpression: 'id = :id',
        FilterExpression: 'deliveryTime between :startTime and :endTime',

        ExpressionAttributeValues: {
            ':startTime': {
                N: startTime.toString()
            },

            ":endTime": {
                N: endTime.toString()
            },
            ":id": {
                S: id.toString()
            }
        },
        Select: 'ALL_ATTRIBUTES',
        ScanIndexForward: false,
    };


    var results = [];
    var callback = function(err, data) {
        if (err) {
            console.log('Dynamo fail ' + err);
            q.reject(err);
        } else if (data.LastEvaluatedKey) {
            params.ExclusiveStartKey = data.LastEvaluatedKey;
            dynamodb.query(params, callback);
        } else {
            q.resolve(results);
        }
        data.Items.forEach(function(item) {
            results.push(item);
        });

    }


    dynamodb.query(params, callback);

    return q.promise;
}

Answer 2

First of all, keep the promisified function you already have. Use it as a building block for the recursive solution, instead of trying to alter it!

It might need two small adjustments though:

function getDBResults(startKey){
//                    ^^^^^^^^
    var q = Q.defer();
    var params = {
        ExclusiveStartKey: startKey,
//      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        … // rest as before
    };
    dynamodb.query(params, function(err, data) {
        if (err) {
           q.reject(err);
        } else {
           q.resolve(data);
//                   ^^^^ Not `data.Items`
        }
    });
    return q.promise;
}

Now we can use that to trivially implement the recursive solution:

function getRecursiveDBResults(key) {
    return getDBResults(key).then(function(data) {
        if (typeof data.LastEvaluatedKey != "undefined") {
            return getRecursiveDBResults(data.LastEvaluatedKey).then(items) {
                return data.Items.concat(items);
            });
        } else {
            return data.Items
        }
    });
}