Sending multiple files to GCS in Node

Question

So I have this function from Google Cloud Storage that I needed to change so it can upload multiple files instead of one. I'm trying to find a good solution to make it always wait until all files are uploaded. Not sure how to make it async - should I do await on stream.on('finish',(...)) or on file.makePublic().then(...) which definitely is a Promiste that I could collect with Promise.all() and then resolve next().

Or if there is already solution for that that Google didn't disclosed in their docs it would be even better.

Function:

function sendUploadsToGCS (req, res, next) {
  if (!req.files) {
    return next()
  }

  let vals = Object.values(req.files)

  for(let f of vals){
    const gcsname = Date.now() + f[0].originalname
    const file = bucket.file(gcsname)

    const stream = file.createWriteStream({
      metadata: {
        contentType: f[0].mimetype
      },
      resumable: false
    })

    stream.on('error', (err) => {
      f[0].cloudStorageError = err
      next(err)
    })

    stream.on('finish', () => {
      f[0].cloudStorageObject = gcsname;
      file.makePublic().then(() => {
        f[0].cloudStoragePublicUrl = getPublicUrl(gcsname)
        console.log('pub url: ', getPublicUrl(gcsname))
        next()
      })
    })

    stream.end(f[0].buffer)
  }
}

Original function (for one file): https://cloud.google.com/nodejs/getting-started/using-cloud-storage#upload_to_cloud_storage

Answer 1

I had similar problem with huge amount of small files. To control uploads I've decided to use p-limit library, which handles concurrent request limit and I am uploading all files only in this one place in code.

This approach allowed me to achieve similar results to this:

Number of files to upload:  186
All files uploaded to GCS:  207.94119999930263  ms

Here is my code for other people dealing with this problem:

const downloadSatelliteFiles = async (files: FileData[]) => {
    const limit = pLimit(5);
    const promises: Promise<void>[] = [];

    files.forEach((file) => {
        promises.push(
            limit(() => {
                uploadFileToGCS(file.fileName, file.buffer, file.contentType);
            })
        );
    });

    await Promise.all(promises);

    return;
};

const uploadFileToGCS = (filename: string, data: any, contentType: string) => {
    return new Promise(async (resolve, reject) => {
        const file = storage.bucket(process.env.GCLOUD_STORAGE_BUCKET).file(filename);

        const stream = file.createWriteStream({
            metadata: {
                contentType,
                cacheControl: "no-cache",
            },
            resumable: false,
        });
        stream.on("error", (err) => {
            console.log("UPLOAD_ERROR");
            console.log(err);
        });
        stream.on("finish", () => {
            resolve("ok");
        });
        stream.end(data);
    });
};

Answer 2

This how I resolved it:

function sendUploadsToGCS (req, res, next) {
  if (!req.files) {
    return next()
  }

  let promises = []
  let vals = Object.values(req.files)

  for(let f of vals){
    const gcsname = Date.now() + f[0].originalname
    const file = bucket.file(gcsname)

    const stream = file.createWriteStream({
      metadata: {
        contentType: f[0].mimetype
      },
      resumable: false
    })

    stream.on('error', (err) => {
      f[0].cloudStorageError = err
      next(err)
    })

    stream.end(f[0].buffer)

    promises.push(
      new Promise ((resolve, reject) => {
        stream.on('finish', () => {
          f[0].cloudStorageObject = gcsname;
          file.makePublic().then(() => {
            f[0].cloudStoragePublicUrl = getPublicUrl(gcsname)
            resolve()
          })
        })
      })
    )
  }
  Promise.all(promises).then(() => next())
}