CODEWITHSUNDEEP
azuregoazure-storageazure-blob-storage
howie
howie

Reputation: 2685

Can I copy azure blob folder without traverse each blob file?

Currently I use golang to implement copy file from one storage account to another storage account.

For example:

https://storage1.blob.core.windows.net/container/A/B/file1.mp4
https://storage1.blob.core.windows.net/container/A/B/file2.mp4
https://storage1.blob.core.windows.net/container/A/B/file3.mp4


Copy from

https://storage1.blob.core.windows.net/container/A/B/
to 
https://storage2.blob.core.windows.net/container/A/B/

Can I copy entire folder without traverse all files in folder A/B?

Here is my sample code



func (css StorageOperator) NestedListBlobs(ctx context.Context, srcFolderPath string) ([]azblob.BlobURL, error) {

    containerName, folderPath := css.getContainerNameAndPath(srcFolderPath)
    containerRawUrl := "https://" + css.StorageName + "." + lib.AzureBlobUrl + "/" + containerName
    containerUrl, _ := url.Parse(containerRawUrl)
    po := azblob.NewPipeline(css.sharedKeyCredential, css.createPipelineOptions())
    container := azblob.NewContainerURL(*containerUrl, po)

    var blobUrls []azblob.BlobURL

    // List the blob(s) in our container; since a container may hold millions of blobs, this is done 1 segment at a time.
    for marker := (azblob.Marker{}); marker.NotDone(); {
        // The parens around Marker{} are required to avoid compiler error.
        // Get a result segment starting with the blob indicated by the current Marker.
        listBlob, err := container.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{
            Prefix: folderPath,
        })
        if err != nil {
            return nil, err
        }
        // IMPORTANT: ListBlobs returns the start of the next segment; you MUST use this to get
        // the next segment (after processing the current result segment).
        marker = listBlob.NextMarker

        // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute)
        for _, blobInfo := range listBlob.Segment.BlobItems {
            css.logger.Debugf("Blob name: " + containerRawUrl + "/" + blobInfo.Name)

            blobUrl, _ := url.Parse(containerRawUrl + "/" + blobInfo.Name)
            blobUrls = append(blobUrls, azblob.NewBlobURL(*blobUrl, po))
        }
    }

    return blobUrls, nil
}

func (css StorageOperator) parallelCopyFileToDst(ctx context.Context, po pipeline.Pipeline, dstRawUrl string, srcBlobs []azblob.BlobURL) error {


for _, srcBlob := range srcBlobs {


dstUrl, _ := url.Parse(dstRawUrl + css.extractBlobPath(srcBlob))

dstBlobURL := azblob.NewBlobURL(*dstUrl, po)
srcSASUrl, err := css.createSASURL(css.lukeSharedKeyCredential, srcBlob.String())
if err != nil {
    css.logger.Errorf("createSASURL fail:%v", err)
    return
}
startCopy, err := dstBlobURL.StartCopyFromURL(ctx,
                *srcSASUrl,
                azblob.Metadata{},
                azblob.ModifiedAccessConditions{},
                azblob.BlobAccessConditions{})

if err != nil {
    css.logger.Errorf("startCopy fail:%v", err)
    return
}

if err = css.checkCopyStatus(ctx, dstBlobURL, startCopy); err != nil {
                css.logger.Errorf("checkCopyStatus fail:%v", err)
                return
}


}//for
    return nil
}

Upvotes: 0

Views: 983

Answers (1)

Adam Smith - Microsoft Azure
Adam Smith - Microsoft Azure

Reputation: 2513

Folder structure within azure blobs doesn't exist, instead it's using a path name, so if you want to copy blobs within a "virtual folder", all you need to do is use the path, below is an example in python listing blobs:

  ['Virtual-Folder/Boards.png', 'storage2.gif']

So in order to transfer the Boards.png blob, you'd have to append the folder name before it and not look within the folder. I wrote some tutorials in python pertaining to this example/issue: https://github.com/adamsmith0016/Azure-storage

Upvotes: 1

Related Questions