nodejs Q.all promises on function calling itself

Question

I need to make a request to get a list of html, and I need to scan it and loop through it and make more requests for each item in the list found, and those might have lists inside them, and so on till theres none left.

I need a way to keep track of all the requests called and call another function when they're done. The tricky bit is the function calls itself over and over for any list items found in the HTML.

The problem I'm having is using Q promises the only promises it waits for are from first request made, and I cant understand why assuming node works like I think it does, see code:

var _ = require('underscore');
var request = require('request');

var allPromises = [];
var finalArray = [];

var start = function(id) {

  var deferred = Q.defer(); 

  request.get({
    url: 'http://www.example.com/id/' + id
  }, function() {

    _.each(body.items, function(index) {

      var item = this;

      finalArray.push(item);

      if(item.hasMore) {
        start(item.id);
      }
    }

    deferred.resolve();

  });

  allPromises.push(deferred.promise);

}

console.log('Starting');

start(1);

Q.all(allPromises).done(function (values) {
  console.log('All Done');
});

What I thought happens is:

1 - starts() is called for the first time and the first deferred var is created
2 - the first request is made and the first created deferred variable is pushed to the promises array
3 - Q.all is called and waits

4 - The first request's callback is called 5 - if the request contains body.x, start() is called again with a new id
6 - a new promises is created and pushed and a new request is made
7 - the first promise is resolved

assuming this only went one level deep

8 - the second promise is resolved
9 - Q.all calls its callback

but in practice, Q.all calls its callback after the first promise, it doesn't wait for any others even though the second promise is pushed before the first promise is resolved.

Why? And how can I make this work?

Update forgot to add the loop inside the request callback.

Answer 1

Answer to edited question:

var request = require('request');

var finalArray = [];

var start = function(id) {

    var deferred = Q.defer();

    request.get({
        url: 'http://www.example.com/id/' + id
    }, function() {

        var subitems = [];
        _.each(body.items, function(index) {

            var item = this;

            finalArray.push(item);

            if(item.hasMore) {
                subitems.push(start(item.id));
            }
        }
        if (subitems.length) {
            deferred.resolve(Q.all(subitems)); // resolve result of Q.all
        } else {
            deferred.resolve();
        }
    });
    return deferred.promise;
}

start(1).done(function() {
    console.log('All Done');
});

@Bergi's code

var request = require('request');

var start = function(id) {

    var deferred = Q.defer();

    request.get({
        url: 'http://www.example.com/id/' + id
    }, function(err, body) {
        if (err) deferred.reject(err);
        else deferred.resolve(body);
    });
    return deferred.promise.then(function(body) {
        var finalArray = [];
        return Q.all(_.map(body.items, function(index) {
            var item = this;
            finalArray.push(item);
            if(item.hasMore)
                return start(item.id);
            else
                return [];
        })).then(function(moreResults) {
            return finalArray.concat.apply(finalArray, moreResults);
        });
    });
}

start(1).then(function(finalArray) {
    console.log('All Done');
});