Async write file to server in for loop

Question

I am trying to convert my original code to "async" code.

The original code sends a query to the database and retrieves the results. The results also contains an image. I want to asynchronize write the images to my nodejs server. At the moment writing is synchronous, which puts a lot of pressure on the RAM.

Below you could see the original code and my attempt to create asynchronous code, where did I go wrong?

This is the original code:

db.query("SELECT imageID, imageBlob FROM sys.users_image;", function (err, result, fields) {
  console.log("Downloading images from database...")

  if (err) throw err;

    //Make await
    for (let i = 0; i < result.length; i++) {

        let binaryBuffer= result[i].imageBlob;
        let imageID= result[i].imageID;

        let image = binaryBuffer.toString('binary');

        fs.writeFile('public/images/shareImg/'+imageID, image, {encoding: 'base64'},  function (err) {
            if (err) {
                console.log(err)
            } else {
                console.log("file written " + i)
            }
        })
        console.log(i)
    }
});

I looked up multiple tutorials and examples but I can figure it out, this is my attempt.

async function test(result) {

    //Correctly logs result and  result.length
    console.log(result[0])
    console.log(result.length)

    for (let i = 0; i < result.length; i++) {

        //UnhandledPromiseRejectionWarning: ReferenceError: result is not defined
        //Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 2)
        console.log(result)
        
        console.log('Before await for ', i);

        let binaryBuffer= result[i].imageBlob;
        let imageID= result[i].imageID;

        let image = binaryBuffer.toString('binary');

        fs.writeFile('public/images/shareImg/'+imageID, image, {encoding: 'base64'},  function (err) {
            if (err) {
                console.log(err)
            } else {
                console.log("file written " + i)
            }
        })

        let result = await Promise.resolve(i);
        console.log('After await. Value is ', result);
    }
}

//db query
db.query("SELECT imageID, imageBlob FROM sys.users_image;", function (err, result, fields) {
  console.log("Downloading images from database...")

  if (err) throw err;


  //import result from db
    test(result).then(_ => console.log('After test() resolved'));

});

This is the log output:

Downloading images from database...
RowDataPacket {
  imageID: '1609328903457.png',
  imageBlob:
   <Buffer 69 56 42 4f 52 77 30 4b 47 67 6f 41 41 41 41 4e 53 55 68 45 55 67 41 41 42 7a 67 41 41 41 4f 49 43 41 49 41 41 41 41 6c 2f 62 41 69 41 41 41 41 43 58 ... > }
20
(node:24318) UnhandledPromiseRejectionWarning: ReferenceError: result is not defined
(node:24318) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 2)
(node:24318) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.

Answer 1

distilled

import { promises as fs } from 'fs';

async function test() {
  const results = await query('SELECT imageID, imageBlob FROM sys.users_image;');
  for (const result of results) {
    const { imageBlob, imageID } = result;
    const image = imageBlob.toString('binary');
    await fs.writeFile(`public/images/shareImg/${imageID}`, image, {encoding: 'base64'});
  }
}

async function query(q) {
  return new Promise((resolve, reject) => {
    db.query(q, (err, result, fields) => err ? reject(err) : resolve(result));
  });
}

explanation:

The idea is you want to make your code look like it would if everything was blocking line-by-line.

With async and await you can do this. When you mark your function as async it allows you to run await within it, which includes for loops as well. It just so happens node v10+ fs has promises defined within it to allow for easy behavior with async.

Notice how the query function was made. This is the typical way to turn a callback approach into a promise one.

Answer 2

Where did I go wrong : for is synchronous it runs immediately it doesn't wait for writeFile to finish

You should use the for await...of statement to iterating over async iterable.

for instance :

      //Must Node.js v10.23.0
const fs = require("fs");
async function test(result) {
  try {
    let promiseResult = [];
    for await (const data of result) {
      let binaryBuffer = data.imageBlob;
      let imageID = data.imageID;

      let image = binaryBuffer.toString("binary");
      //Write the file
      const writeFile = await fs.await.writeFile(
        "public/images/shareImg/" + imageID,
        image,
        { encoding: "base64" }
      );
      //Save the resultat in a table
      promiseResult.push(writeFile);
    }
    return promiseResult;
  } catch (error) {
    console.log(error);
    throw error;
  }
}

//db query
db.query("SELECT imageID, imageBlob FROM sys.users_image;", function(
  err,
  result,
  fields
) {
  console.log("Downloading images from database...");

  if (err) throw err;

  //import result from db
  test(result).then(_ => console.log("After test() resolved"));
});