Zip File Unsupported Format while uploading it through PUT Block Azure Blob Storage

Question

I am using the below code to upload a zip file to my azure container using the PUT block request type in Azure Blob Storage.

const request = require('request')
const fs = require('fs')
const account = 'Enter your account here'
const containerName = 'stream-test-container'
const blobName = 'sampleBlob.zip'
const strTime = new Date().toUTCString()
const _ = require('lodash')
const uuid = require('uuid')

const requestBlockIdArray = []
const responseBlockIdArray = []
let readStreamEnded = false
const token = 'Enter your token here'
const readStream = fs.createReadStream('fiveMb.csv.zip')
let blobContent = ''
let contentLength = 0
const thresholdSize = 512 * 1024

readStream.on('data', function (chunk) {
  blobContent = blobContent + chunk
  contentLength = contentLength + chunk.length
  if (contentLength >= thresholdSize) {
    console.log('content length is:', contentLength)
    putBlock(blobContent, contentLength)
    contentLength = 0
    blobContent = ''
  }
})

readStream.on('end', () => {
  if (contentLength) {
    putBlock(blobContent, contentLength)
  }
  readStreamEnded = true
})

function callbackPutBlock (error, response, body) {
  console.log(response.statusCode, response.statusMessage, 'Block created')
  responseBlockIdArray.push(response.headers['x-ms-client-request-id'])
  console.log(response.headers['x-ms-client-request-id'], requestBlockIdArray[requestBlockIdArray.length - 1])
  if (readStreamEnded && responseBlockIdArray.length === requestBlockIdArray.length) {
    putBlockList()
  }
  if (error) {
    console.log(error)
  }
}

function callbackPutBlockList (error, response, body) {
  console.log(response.statusCode, response.statusMessage)
  if (response.statusCode === 201) {
    console.log('Blob Created')
  }
  if (error) {
    console.log(error)
  }
}

function putBlock (blobContent, contentLength) {
  const blockId = Buffer.from(uuid.v4().replace(/-/g, '')).toString('base64')
  requestBlockIdArray.push(blockId)
  const optionsPutBlock = {
    url: `https://${account}.blob.core.windows.net/${containerName}/${blobName}?comp=block&blockid=${blockId}`,
    headers: {
      Authorization:
      `Bearer ${token}`,
      'x-ms-date': strTime,
      'x-ms-version': '2019-02-02',
      'Content-Length': contentLength,
      'x-ms-client-request-id': blockId
    },
    body: blobContent
  }
  request.put(optionsPutBlock, callbackPutBlock)
}

function putBlockList () {
  const xmlBlockList = '<?xml version="1.0" encoding="utf-8"?><BlockList>' + (_.map(requestBlockIdArray, (id) => { return `<Latest>${id}</Latest>` })).join('') + '</BlockList>'
  const xmlBlockListLength = new TextEncoder().encode(xmlBlockList).length
  const optionsPutBlockList = {
    url: `https://${account}.blob.core.windows.net/${containerName}/${blobName}?comp=blocklist`,
    headers: {
      Authorization:
      `Bearer ${token}`,
      'x-ms-date': strTime,
      'x-ms-version': '2019-02-02',
      'Content-Length': xmlBlockListLength,
      'Content-Type': 'application/text-plain'
    },
    body: xmlBlockList
  }
  request.put(optionsPutBlockList, callbackPutBlockList)
}

But when I try to download the uploaded zip file from azure and try to open it in my system it says that the file is in an unsupported format.

I am not sure what the problem is, Does it have to do something with not passing content-type. As I am not passing it as in documentation in headers as it is not mentioned.

References:

https://learn.microsoft.com/en-us/rest/api/storageservices/put-block

https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list

Answer 1

Please try by changing the following lines of code:

let blobContent = ''
let contentLength = 0
const thresholdSize = 512 * 1024

readStream.on('data', function (chunk) {
  blobContent = blobContent + chunk
  contentLength = contentLength + chunk.length
  if (contentLength >= thresholdSize) {
    console.log('content length is:', contentLength)
    putBlock(blobContent, contentLength)
    contentLength = 0
    blobContent = ''
  }
})

to

let blobContent = Buffer.from([]);
let contentLength = 0
const thresholdSize = 512 * 1024

readStream.on('data', function (chunk) {
  blobContent = Buffer.concat([blobContent, chunk], blobContent.length + chunk.length);
  contentLength = contentLength + chunk.length
  if (contentLength >= thresholdSize) {
    console.log('content length is: ', contentLength)
    putBlock(blobContent, contentLength)
    contentLength = 0
    blobContent = Buffer.from([]);
  }
})

Essentially I have changed the way you're treating the blob content. I am keeping the data as buffer and using appropriate buffer functions to add the content.