fs.createWriteStream does not immediately create file?

Question

I have made a simple download from http function as below (error handling is omitted for simplifcation):

function download(url, tempFilepath, filepath, callback) {
    var tempFile = fs.createWriteStream(tempFilepath);
    http.request(url, function(res) {
        res.on('data', function(chunk) {
            tempFile.write(chunk);
        }).on('end', function() {
            tempFile.end();
            fs.renameSync(tempFile.path, filepath);
            return callback(filepath);
        })
    });
}

However, as I call download() tens of times asynchronously, it seldom reports error on fs.renameSync complaining it cannot find file at tempFile.path.

Error: ENOENT, no such file or directory 'xxx'

I used the same list of urls to test it, and it failed about 30% of time. The same list of urls worked when downloaded one by one.

Testing some more, I found out that the following code

fs.createWriteStream('anypath');
console.log(fs.exist('anypath'));
console.log(fs.exist('anypath'));
console.log(fs.exist('anypath'));

does not always print true, but sometimes the first answer prints false.

I am suspecting that too many asynchronous fs.createWriteStream calls cannot guarantee the file creation. Is this true? Are there any methods to guarantee file creation?

Answer 1

I am working on uploading and downloading file (docx, pdf, text, etc) through nodejs request-promise and request libraries.

Problem with request-promise is that they don't promisify pipe method from request package. Hence, we need to do it in the old way.

I was able to come up with the hybrid solution, where I was able to use async/await and Promise() at same time. Here is the example:

    /**
     * Downloads the file.
     * @param {string} fileId : File id to be downloaded.
     * @param {string} downloadFileName : File name to be downloaded.
     * @param {string} downloadLocation : File location where it will be downloaded.
     * @param {number} version : [Optional] version of the file to be downloaded.
     * @returns {string}: Downloaded file's absolute path.
     */
    const getFile = async (fileId, downloadFileName, downloadLocation, version = undefined) => {
        try {
            const url = version ? `http://localhost:3000/files/${fileId}?version=${version}` : 
`${config.dms.url}/files/${fileUuid}`;
            const fileOutputPath = path.join(downloadLocation, fileName);

            const options = {
                method: 'GET',
                url: url,
                headers: {
                    'content-type': 'application/json',
                },
                resolveWithFullResponse: true
            }

            // Download the file and return the full downloaded file path.
            const downloadedFilePath = writeTheFileIntoDirectory(options, fileOutputPath);

            return downloadedFilePath;
        } catch (error) {
           console.log(error);
        }
    };

As you can see in above getFile method, we are using latest ES supported async/await functionality for asynchronous programming. Now, lets look into writeTheFileIntoDirectory method.

/**
 * Makes REST API request and writes the file to the location provided.
 * @param {object} options : Request option to make REST API request.
 * @param {string} fileOutputPath : Downloaded file's absolute path.
 */
const writeTheFileIntoDirectory = (options, fileOutputPath) => {
    return new Promise((resolve, reject) => {
        // Get file downloaded.
        const stream = fs.createWriteStream(fileOutputPath);
        return request
            .get(options.url, options, (err, res, body) => {
                if (res.statusCode < 200 || res.statusCode >= 400) {
                    const bodyObj = JSON.parse(body);
                    const error = bodyObj.error;
                    error.statusCode = res.statusCode;
                    return reject(error);
                }
            })
            .on('error', error => reject(error))
            .pipe(stream)
            .on('close', () => resolve(fileOutputPath));
    });
}

The beauty of nodejs is that it support backward compatibility of different asynchronous implementation. If a method is returning promise, then await will be kicked in and will wait for the method to be completed.

Above writeTheFileIntoDirectory method will download the file and will return positively when the stream is closed successfully, else it will return error.

Answer 2

Here is what I use to get it done:

function download(url, dest) {
    return new Promise((resolve, reject) => {
        http.get(url, (res) => {
            if (res.statusCode !== 200) {
                var err = new Error('File couldn\'t be retrieved');
                err.status = res.statusCode;
                return reject(err);
            }
            var chunks = [];
            res.setEncoding('binary');
            res.on('data', (chunk) => {
                chunks += chunk;
            }).on('end', () => {
                var stream = fs.createWriteStream(dest);
                stream.write(chunks, 'binary');
                stream.on('finish', () => {
                    resolve('File Saved !');
                });
                res.pipe(stream);
            })
        }).on('error', (e) => {
            console.log("Error: " + e);
            reject(e.message);
        });
    })
};

Answer 3

The accepted answer didn't download some of the last bytes for me.
Here's a Q version that works correctly (but without the temp file).

'use strict';

var fs = require('fs'),
    http = require('http'),
    path = require('path'),
    Q = require('q');

function download(url, filepath) {
  var fileStream = fs.createWriteStream(filepath),
      deferred = Q.defer();

  fileStream.on('open', function () {
    http.get(url, function (res) {
      res.on('error', function (err) {
        deferred.reject(err);
      });

      res.pipe(fileStream);
    });
  }).on('error', function (err) {
    deferred.reject(err);
  }).on('finish', function () {
    deferred.resolve(filepath);
  });

  return deferred.promise;
}

module.exports = {
  'download': download
};

Note I'm listening to finish on file stream instead of end on response.

Answer 4

You shouldn't call write on your tempFile write stream until you've received the 'open' event from the stream. The file won't exist until you see that event.

For your function:

function download(url, tempFilepath, filepath, callback) {
    var tempFile = fs.createWriteStream(tempFilepath);
    tempFile.on('open', function(fd) {
        http.request(url, function(res) {
            res.on('data', function(chunk) {
                tempFile.write(chunk);
            }).on('end', function() {
                tempFile.end();
                fs.renameSync(tempFile.path, filepath);
                return callback(filepath);
            });
        });
    });
}

For your test:

var ws = fs.createWriteStream('anypath');
ws.on('open', function(fd) {
    console.log(fs.existsSync('anypath'));
    console.log(fs.existsSync('anypath'));
    console.log(fs.existsSync('anypath'));
});