CODEWITHSUNDEEP
node.jsasync-awaitsharp
mgbasi
mgbasi

Reputation: 11

Trouble with async/wait file saving and Sharp image processing in NodeJS

I'm trying to make a back end for a NodeJS photography website, but I've hit a snag with processing images with Sharp and then uploading them to AWS S3. NodeJS's asynchronous approach messes up the processing because it tries to upload the files as soon as they've begun to be written, thus throwing an error. I've stripped down the image processing code below, to make troubleshooting easier.

Intended Result

  1. Files are saved from the temporary form upload folders (in this example, I've just stuck random images in an "uploads/" directory) into a processing folder (in this example, a directory called "saved/")
  2. All files are converted to WEBP format and saved as a new file.
  3. Once all WEBP conversions are done, everything should be uploaded to S3 and then the local files deleted.

I've managed to get the initial saving and WEBP conversion steps to work synchronously, but I can't figure out how to get the upload/delete function to wait until after the WEBP files are saved. It waits until after the WEBP conversion has started, but then each file's promise resolves after the upload/delete function fires.

const fs = require('fs')
const path = require('path')
const formidable = require('formidable')
const moment = require('moment')
const sharp = require('sharp')
const dotenv = require("dotenv")
const AWS = require('aws-sdk');

const saveParams = {
    uploadDir: 'upload/',
    saveDir: 'save/'
};

const timeNow = moment().format();

for (const [key, value] of Object.entries(saveParams)) {
    console.log('Checking for ' + value + ' directory')
    if (!fs.existsSync(value)) {
        new Promise((resolve, reject) => {
            console.log(value + ' does not exist—creating it now.')
            fs.mkdirSync(value);
            console.log('Created directory: ' + value)
            resolve();
        });
    }
}

function check(file) {
    console.log('Checking ' + file + ' status.')
    if (fs.existsSync(file)) {
        console.log('File exists at ' + timeNow)
        return (true);
    } else {
        return (false)
    }
}

function checkSave(saveParams, file) {
    if (fs.existsSync(saveParams.saveDir + file)) {
        return ('File already exists')
    } else {
        const startTime = moment().format()
        const data = fs.readFileSync(saveParams.uploadDir + file)
        fs.writeFileSync(saveParams.saveDir + file, data)
        const stopTime = moment().format()
        if (check(saveParams.saveDir + file)) {
            return (file + ' saved at ' + moment().format());
        } else {
            throw new Error(file + "didn't save.")
        }
    }
}

async function sharpAwaitWEBP(saveParams, file, webpFile) {
    const data = await sharp(saveParams.saveDir + file)
        .webp({ lossless: true })
        .toBuffer((err, data, info) => {
            return new Promise((resolve, reject) => {
                fs.writeFileSync(webpFile, data)
                if (err => {
                    console.log(err)
                })
                    console.log('Resolving ', webpFile)
                resolve(webpFile)
            })
        });
};


function webpSave(saveParams, file) {
    var extension = file.substr(file.lastIndexOf('.'));
    var baseName = path.basename(file, extension);
    var webpFile = saveParams.saveDir + baseName + '.webp'
    if (fs.existsSync(webpFile)) {
        return ('File already exists')
    } else {
        new Promise((resolve, reject) => {
            sharpAwaitWEBP(saveParams, file, webpFile);
        })
    }
};

function webpSaveWrapper(files2) {
    return new Promise((resolve, reject) => {
        files2.forEach(file => {
            var extension = file.substr(file.lastIndexOf('.'));
            if (extension !== '.webp' && (extension === '.jpg' || extension === '.png')) {
                console.log('Starting WEBP function.')
                webpSave(saveParams, file)
            }
            resolve();
        });
    })

}

function uploadtoS3(saveParams) {
    console.log('Uploading all files to S3') // Will be a function here to upload all saved and converted files to an S3 bucket
    console.log('Deleting everything. Buh bye now.') // This will delete all local files after they've been uploaded to S3
}


async function processImages(saveParams) {
    console.log('Process started at ' + timeNow);
    const files1 = fs.readdirSync(saveParams.uploadDir);
    files1.forEach(file => {
        console.log('Starting checkSave function.')
        console.log(checkSave(saveParams, file))
    });

    const files2 = fs.readdirSync(saveParams.uploadDir);
    await webpSaveWrapper(files2)
    uploadtoS3(saveParams)
}

processImages(saveParams);

Logged output from the above code

[nodemon] starting `node app.js`
Checking for upload/ directory
Checking for save/ directory
Process started at 2021-03-28T02:42:57-05:00
Starting checkSave function.
File already exists
Starting checkSave function.
Checking save/randomimage-01.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-01.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-02.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-02.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-03.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-03.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-04.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-04.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-05.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-05.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-06.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-06.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-07.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-07.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-08.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-08.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-09.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-09.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-10.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-10.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-11.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-11.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-12.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-12.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-13.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-13.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-14.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-14.jpg saved at 2021-03-28T02:42:57-05:00
Starting checkSave function.
Checking save/randomimage-15.jpg status.
File exists at 2021-03-28T02:42:57-05:00
randomimage-15.jpg saved at 2021-03-28T02:42:57-05:00
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Starting WEBP function.
Uploading all files to S3 //This line and the next one should come after the "Resolve..." lines
Deleting everything. Buh bye now.
Resolving  save/randomimage-02.webp
Resolving  save/randomimage-01.webp
Resolving  save/randomimage-03.webp
Resolving  save/randomimage-04.webp
Resolving  save/randomimage-05.webp
Resolving  save/randomimage-08.webp
Resolving  save/randomimage-06.webp
Resolving  save/randomimage-11.webp
Resolving  save/randomimage-09.webp
Resolving  save/randomimage-07.webp
Resolving  save/randomimage-12.webp
Resolving  save/randomimage-13.webp
Resolving  save/randomimage-14.webp
Resolving  save/randomimage-15.webp
Resolving  save/randomimage-10.webp

Any help is greatly appreciated. This is clearly beyond where I'm comfortable with NodeJS, and is completely overkill when I could just keep my site on Wordpress instead of a custom Node site.

Upvotes: 1

Views: 1106

Answers (1)

mgbasi
mgbasi

Reputation: 11

I figured it out using for (const file of files) {//image processing functions} instead of forEach loops

Upvotes: 0

Related Questions