How to upload files from url in cloud storage node.js client

Question

With Google Cloud Storage version 1 I was successfully using the function below to upload files. It stores the file in a unique location in a temp directory for further processing.

It seems that Cloud Storage version 4 doesn't accept a URL as the source anymore. It will complain that the file doesn't exist.

import { File } from "@google-cloud/storage";
import { join, parse } from "path";
import { generate } from "shortid";
import { URL } from "url";
import { getBucket } from "./helpers";

/**
 * This function takes a HTTP URL and uploads it to a destination in the bucket
 * under the same filename or a generated unique name.
 */
export async function uploadFile(
  url: string,
  destinationDir: string,
  options: { generateUniqueFilename: boolean } = {
    generateUniqueFilename: true
  }
): Promise<File> {
  try {
    const pathname = new URL(url).pathname;
    const { ext, base } = parse(pathname);
    let destination = join(destinationDir, base);

    if (options.generateUniqueFilename) {
      const shortId = generate();

      destination = join(destinationDir, `${shortId}${ext}`);
    }

    const bucket = await getBucket();
    const [file] = await bucket.upload(url, {
      destination,
      public: true
    });

    console.log(`Successfully uploaded ${url} to ${destination}`);

    return file;
  } catch (err) {
    throw new Error(
      `Failed to upload ${url} to ${destinationDir}: ${err.message}`
    );
  }
}

How do I solve this is with the current version? I can't seem to find much info on this. Using gsutil is not an option for me. I need to pass the URL to a Cloud Function and upload from there.

Answer 1

This is what I ended up with:

import { File } from "@google-cloud/storage";
import { join, parse } from "path";
import { generate } from "shortid";
import { URL } from "url";
import { getBucket } from "./helpers";
import * as request from "request";

/**
 * This function takes a http url and uploads it to a destination in the bucket
 * under the same filename or a generated unique name.
 */
export async function uploadFile(
  url: string,
  destinationDir: string,
  options: { generateUniqueFilename: boolean } = {
    generateUniqueFilename: true
  }
) {
  console.log("Upload file from", url);
  const pathname = new URL(url).pathname;
  const { ext, base } = parse(pathname);
  let destination = join(destinationDir, base);

  if (options.generateUniqueFilename) {
    const shortId = generate();

    destination = join(destinationDir, `${shortId}${ext}`);
  }

  const bucket = await getBucket();

  return new Promise<File>((resolve, reject) => {
    const file = bucket.file(destination);

    const req = request(url);
    req.pause();
    req.on("response", res => {
      if (res.statusCode !== 200) {
        return reject(
          new Error(
            `Failed to request file from url: ${url}, status code: ${res.statusCode}`
          )
        );
      }

      req
        .pipe(
          file.createWriteStream({
            resumable: false,
            public: true,
            metadata: {
              contentType: res.headers["content-type"]
            }
          })
        )
        .on("error", err => {
          reject(
            new Error(
              `Failed to upload ${url} to ${destinationDir}: ${err.message}`
            )
          );
        })
        .on("finish", () => {
          console.log(`Successfully uploaded ${url} to ${destination}`);
          resolve(file);
        });
      req.resume();
    });
  });
}

Answer 2

I think you can not pass a URL directly to bucket.upload() But You can download a file and pipe it to upload function as below:

import { join, parse } from "path";
import { generate } from "shortid";
import { URL } from "url";
import * as request from "request";
import { getBucket } from "./helpers";
export async function uploadFile(url, destinationDir, options = {
    generateUniqueFilename: true
 }){

    return new Promise(function(resolve, reject) {
        const pathname = new URL(url).pathname;
        const { ext, base } = parse(pathname);
        let destination = join(destinationDir, base);
        let filename;
        if (options.generateUniqueFilename) {
            const shortId = generate();
            filename = `${shortId}${ext}`;
            destination = join(destinationDir, filename);
        }
        let req = request(FILE_URL);
        req.pause();
        req.on('response', res => {
            if (res.statusCode !== 200) {
                reject(new Error("unable to download file from url"));
            }
            const bucket = await getBucket();
            const writeStream = bucket.file(filename)
                .createWriteStream({
                public: true,
                destination,
                metadata: {
                    contentType: res.headers['content-type']
                }
            });
            req.pipe(writeStream);
            req.resume(); // resume when pipe is set up    
            req.on('finish', () => {
                console.log('saved');
                resolve(true);
            });
            req.on('error', err => {
                writeStream.end();
                console.error(err);
                reject(err);
            });
        });
    });
}