CODEWITHSUNDEEP
javascriptnode.jsrecursionstack
kgingeri
kgingeri

Reputation: 320

NodeJS processing for API calls sequentially, but resulting in stack overflow

OK, so I have a situation where I cannot just fire thousands of requests to an API server. I have a Node process (no UI) that I need to have process each API response/update sequentially, waiting for completion before sending the next request. I may be making this more complicated than I think - not sure. I can only figure out how to do this with recursive calls, but this results in a stack overflow as there can be thousands of records. The general process is this:

  1. get rows from SQL table with ID's (result)
  2. formulate and send of an API call to retrieve ID's info
  3. if returned data has image data, write it back to SQL table
  4. wait on this process so not to bombard API server with thousands of requests all at once
  5. repeat until last ID is processed (can be thousands, more than stack space)

Here's sample code (not actual so ignore syntax errors if any)... UPDATED: actual running code with sensitive items removed

var g_con = null;    //...yeah I know, globals are bad

//
//  [ found updating ]
//
function getSetImage(result, row, found) {

  if(row >= result.length) { //...exit on no row or last row processed
    con.end();
    return;
  }

  item = result[row];  //...next SQL row

  if((item !== undefined) && (item.autoid !== undefined)) {

    //...assemble API and send request
    //
    let url =   'https://...API header...'
              + item.autoid
              + '...API params...';

    request(url, (error, response, body) => {

      if(response.statusCode !== 200)
        throw('Server is not responding\n' + response.statusMessage);

      let imageData = JSON.parse(body);
      if((imageData.value[0]        !== undefined) &&
         (imageData.value[0].DETAIL !== undefined) &&
         (imageData.value[0].DETAIL.Value.length)   ) {

        //...post back to SQL
        //
        found++;
        console.log('\n' + item.autoid + '/['+ item.descr + '], ' + 'Found:' + found);

        qry = 'update inventory set image = "'+imageData.value[0].DETAIL.Value+'" where autoid = "'+item.autoid+'";';
        g_con.query(qry, (err) => {
          if (err) {
            console.log('ERROR:',err.message, '\nSQL:['+err.sql+']\n');
            throw err.message;
          }
        });

        row++;
        setTimeout(()=>{getSetImage(result, row, found)}, 0);   //...nested call after SQL

      } else {

        row++;
        process.stdout.write('.');                                   //...show '.' for record, but no image
        setTimeout(()=>{getSetImage(result, row, found)}, 0);   //...nested call after SQL

      }

    }); //...request callback

  }

  // } else {

  //   throw '\nERROR! result['+row+'] undefined? Images found: '+found;
  // }
}


//
//  [ main lines ]
//
(() => {

  let params = null;
  try {

    params = JSON.parse(fs.readFileSync('./config.json'));

    //...load autoids array from SQL inventory table - saving autoids
    //   autoids in INVENTRY join on par_aid's in INVENTRYIMAGES
    //
    g_con = mysql.createConnection(params.SQLConnection);
    g_con.connect((err) => {  if(err) {
                                console.log('ERROR:',err.message);
                                throw err.message;
                              }
                           });

    //...do requested query and return data or an error
    //
    let qry = 'select autoid, descr from inventory order by autoid;';
    g_con.query(qry, (err, results, flds) => {

        if (err || flds === undefined) {
          console.log('ERROR:',err.message, '\nSQL:['+err.sql+']\n');
          throw err.message;
        }

        console.log('Results length:',results.length);
        let row   = 0;
        let found = 0;
        getSetImage(results, row, found);

      });

  }

  catch (err) {
    console.log('Error parsing config parameters!');
    console.log(err);
  }

})();

So here's the answer using Promises (except for MySQL):

//
//  [ found updating ]
//
async function getSetImage(data) {

  for(let item of data) {

    if(item && item.autoid) {

      //...assemble API and send request
      //
      let url   = g_URLHeader + g_URLPartA + item.autoid + g_URLPartB;

      let image = await got(url).json().catch(err => {
                    console.log(err);
                    err.message = 'API server is not responding';
                    throw err;
                  });

      if(image && image.value[0] && image.value[0].DETAIL &&
         image.value[0].DETAIL.Value.length       ) {
           console.log('\nFound: ['+item.autoid+' - '+item.descr
                       + '] a total of ' + g_found + ' in ' + g_count + ' rows');

          g_found++;

          //...post back to SQL
          //
          let qry = 'update inventory set image = "'
                  + image.value[0].DETAIL.Value
                  + '" where autoid = "'
                  + item.autoid+'";';
          await g_con.query(qry, (err) => {
                      if (err) {
                        console.log('ERROR:',err.message, '\nSQL:['+err.sql+']\n');
                        throw err.message;
                      }
                });

      } else {

          process.stdout.write('.');  //...show '.' for record, but no image

      }  //...if/else image.value

      g_count++;

    }  //...if item

  } //...for()

}

Upvotes: 1

Views: 1136

Answers (1)

jfriend00
jfriend00

Reputation: 707456

As I've said in all my comments, this would be a ton simpler using promises and async/await. To do that, you need to switch all your asynchronous operations over to equivalents that use promises.

Here's a general outline based on the original pseudo-code you posted:

// use got() for promise version of request
const got = require('got');

// use require("mysql2/promise" for promise version of mysql

async function getSetImage(data) {

    for (let item of data) {
        if (item && item.id) {
            let url = uriHeader + uriPartA + item.id + uriPartB;
            let image = await got(url).json().catch(err => {
                // log and modify error, then rethrow
                console.log(err);
                err.msg = 'API Server is not responding\n';
                throw err;
            });
            if (image.value && image.value.length) {
                console.log('\nFound image for ' + item.id + '\n');
                let qry = 'update inventory set image = "' + image.value + '" where id = "' + item.id + '";';
                await con.query(qry).catch(err => {
                    console.log('ERROR:', err.message, '\nSQL:[' + err.sql + ']\n');
                    throw err;
                });
            }
        } else {
            // no image data found
            process.stdout.write('.'); //...show '.' for record, but no image
        }
    }
}

//...sql query is done, returning "result" - data rows
getSetImage(result).then(() => {
    console.log("all done");
}).catch(err => {
    console.log(err);
});

Some notes about this code:

  1. The request() library is no longer getting new features and is in maintenance mode and you need to change to a different library to get built-in promise support. You could use request-promise (also in maintenance mode), but I recommend one of the newer libraries such as got() that is more actively being developed. It has some nice features (automatically checks status for you to be 2xx, built-in JSON parsing, etc...) which I've used above to save code.

  2. mysql2/promise has built-in promise support which you get with const mysql = require('mysql2/promise');. I'd recommend you switch to it.

  3. Because of the user of async/await here, you can just loop through your data in a regular for loop. And, no recursion required. And, no stack build-up.

  4. The way promises work by default, any rejected promises will automatically terminate the flow here. The only reason I'm using .catch() in a couple places is just for custom logging and tweaking of the error object. I then rethrow which propagates the error back to the caller for you.

  5. You can tweak the error handling to your desire. The usual convention with promises is to throw an Error object (not a string) and that's often what callers are expecting to see if the promise rejects.

  6. This code can be easily customized to log errors and continue on to subsequent items in the array. Your original code did not appear to do that so I wrote it to abort if it got an error.

Upvotes: 2

Related Questions