How to copy folders from blob storage?

Question

The functionality I am trying to implement is copy paste files/folders from one source to another(same container). Can I use same method to copy folders like I do for copy files? startcopyblob() throws error while copy pasting folders.

input:

newFileName:'new folder_copy1' newFilePath:'603487d1e966a91fd86b6c11/spe9_rs_2021-03-17_17-14-38/output' oldFilePath:'603487d1e966a91fd86b6c11/spe9_rs_2021-02-23_11-14-41/output/new folder'

error:

code:'CannotVerifyCopySource'
message:'The specified blob does not exist.
name:'StorageError'
requestId:'4a8a76bf-701e-0078-17c8-1b4439000000'
stack:'StorageError: The specified blob does not exist.
statusCode:404

Uncaught Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client

code:

var host = sasurl.substring(0, sasurl.lastIndexOf("/"));
var containerName = sasurl.substring(sasurl.lastIndexOf("/"), sasurl.indexOf("?")).split("/")[1];
var saskey = sasurl.substring(sasurl.indexOf("?"), sasurl.length);
var blobService = storage.createBlobServiceWithSas(host, saskey);
pasteFiles.forEach(elem => {
var storageuri = host + "/" + containerName + "/" + elem["oldFilePath"] + saskey;
var blobName = elem["newFilePath"] + "/" + elem["newFileName"];
blobService.startCopyBlob(storageuri, containerName, blobName, err => {
if (err) {
console.log(err)
return res.status(500).json({
message: 'error',
status: err
})

..... ..

Answer 1

Azure blob storage has a 2 level hierarchy - blob container and blob. It is based on a flat storage scheme, not a hierarchical scheme. It does not have a directory structure. We just can specify a character or string delimiter within a blob name to create a virtual hierarchy. So if we want to copy one folder with azure node blob sdk, we need to copy blobs in the folder one by one.

For example sdk

npm i @azure/storage-blob

code

const {
  BlobServiceClient,
  StorageSharedKeyCredential,
  generateBlobSASQueryParameters,
  ContainerSASPermissions,
} = require("@azure/storage-blob");

const accountName = "andyprivate";
const accountKey =
  "";
const creds = new StorageSharedKeyCredential(accountName, accountKey);
const blobServiceClient = new BlobServiceClient(
  `https://${accountName}.blob.core.windows.net`,
  creds
);
async function test() {
  try {
    const sourceContainerClient = blobServiceClient.getContainerClient("input");
    const desContainerClient = blobServiceClient.getContainerClient("output");
    const blobSAS = generateBlobSASQueryParameters(
      {
        expiresOn: new Date(new Date().valueOf() + 86400000),
        containerName: sourceContainerClient.containerName,
        permissions: ContainerSASPermissions.parse("rl"),
      },
      creds
    ).toString();
    for await (const response of sourceContainerClient
      .listBlobsFlat({ prefix: "<your folder name>/" })
      .byPage()) {
      for (const blob of response.segment.blobItems) {
        console.log(`Blob name : ${blob.name}`);
        const sourceBlob = sourceContainerClient.getBlobClient(blob.name);
        const sourceUrl = sourceBlob.url + "?" + blobSAS;
        const res = await (
          await desContainerClient
            .getBlobClient(blob.name)
            .beginCopyFromURL(sourceUrl)
        ).pollUntilDone();

        console.log(res.copyStatus);
      }
    }
  } catch (error) {
    console.log(error);
  }
}
test();

Besides, if you want to directly copy one folder from one container from another container, we can use azcopy implement it. For more details, please refer to here and here

For example

npm i @azure/storage-blob @azure-tools/azcopy-node @azure-tools/azcopy-<your system win32 linux win64>
 

code

const {
  StorageSharedKeyCredential,
  generateAccountSASQueryParameters,
  AccountSASPermissions,
  AccountSASResourceTypes,
  AccountSASServices,
} = require("@azure/storage-blob");

const accountName = "andyprivate";
const accountKey =
  "";
const creds = new StorageSharedKeyCredential(accountName, accountKey);
//create account sas token
const accountSas = generateAccountSASQueryParameters(
  {
    startsOn: new Date(new Date().valueOf() - 8640),
    expiresOn: new Date(new Date().valueOf() + 86400000),
    resourceTypes: AccountSASResourceTypes.parse("sco").toString(),
    permissions: AccountSASPermissions.parse("rwdlacup").toString(),
    services: AccountSASServices.parse("b").toString(),
  },
  creds
).toString();
const { AzCopyClient } = require("@azure-tools/azcopy-node");
let copyClient = new AzCopyClient();

async function copy() {


  try {
    let jobId = await copyClient.copy(
      {
        type: "RemoteSas",
        resourceUri: "https://<>.blob.core.windows.net/input",
        sasToken: accountSas,
        path: "/<folder name>",
      },
      {
        type: "RemoteSas",
        resourceUri: "https://<>.blob.core.windows.net/outcontainer",
        sasToken: accountSas,
        path: "",
      },
      { recursive: true }
    );
    let status;
    while (!status || status.StatusType !== "EndOfJob") {
      let jobInfo = await copyClient.getJobInfo(jobId);
      status = jobInfo.latestStatus;
      await new Promise((resolve, reject) => setTimeout(resolve, 1000));
    }
    console.log("OK");
  } catch (error) {
    console.log(error);
  }
}

copy();