My CheckSumSHA256 i create in javascript for S3 ManagedUpload is always wrong

Question

sha256 is the SHA256 Hash of a file. I use then btoa(256) to base64 it.

The result is always InvalidRequest: Value for x-amz-checksum-sha256 header is invalid.

The AWS Documentation says,

ChecksumSHA256 — (String) The base64-encoded, 256-bit SHA-256 digest of the object. This will only be present if it was uploaded with the object. With multipart uploads, this may not be a checksum value of the object. For more information about how checksums are calculated with multipart uploads, see Checking object integrity in the Amazon S3 User Guide. Checking object integrity - Amazon Simple Storage Service Verify the integrity of objects uploaded and downloaded to Amazon S3.

let sha256conv = btoa(sha256);

const params = {
        Bucket: process.env.BUCKET,
        Key: path.basename(task.file),
        Body: fileData,
        ContentType: ContentType || 'application/octet-stream',
        CacheControl: "max-age=172800",
        ChecksumAlgorithm: 'sha256',
        ChecksumSHA256: sha256conv
    
    };

const upload = new AWS.S3.ManagedUpload({
        service: s3,
        params
    });

the sha256 is generated like this:

export async function getFileSha256(fileName, fileSize, onProgress) {
    return new Promise((resolve, reject) => {
        const hash = crypto.createHash('sha256');

// change to 'binary' if you want a binary hash.
        hash.setEncoding('hex');
        const bar1 = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
        bar1.start(fileSize, 0);

        const fd = fs.createReadStream(fileName);

        fd.on('data', async chunk => {
            let processedBytes = bar1.value + chunk.length;
            bar1.update(processedBytes);
            await onProgress(processedBytes)
        })

        fd.on('end', function () {
            bar1.stop();
            hash.end();
            return resolve(hash.read())
        });

// read all file and pipe it (write it) to the hash object
        fd.pipe(hash);
    });
}

Answer 1

      (async()=>{
      
            var message   = 'helloworld';
            var hex       = await digest(message);


            //  hmmm i certainly went round the houses here
            //  var b64       = b64encode(hex);                

            var b64       = btoa(hex);
            
            console.log(b64);
        
      })();      

      
      async function digest(message) {
      
            var msgUint8      = new TextEncoder().encode(message);                            // encode as (utf-8) Uint8Array
            var hashBuffer    = await crypto.subtle.digest('SHA-256',msgUint8);               // hash the message
            var hashArray     = Array.from(new Uint8Array(hashBuffer));                       // convert buffer to byte array
            var hashHex       = hashArray.map(b=>b.toString(16).padStart(2,'0')).join('');    // convert bytes to hex string
            return hashHex;
        
      }//digest


        function b64encode(input){
        
              input         = utf8_encode(input);
              var output    = '';
              
              var keyStr    = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';
              
              var i         = 0;              
              while(i<input.length){
              
                      var chr1    = input.charCodeAt(i++);
                      var chr2    = input.charCodeAt(i++);
                      var chr3    = input.charCodeAt(i++);
                      var enc1    = chr1>>2;
                      var enc2    = ((chr1&3)<<4)|(chr2>>4);
                      var enc3    = ((chr2&15)<<2)|(chr3>>6);
                      var enc4    = chr3&63;
                      
                      if(isNaN(chr2)){
                            enc3    = enc4    = 64;
                      }else{
                            if(isNaN(chr3)){
                                  enc4    = 64;
                            }
                      }
                      
                      output    = output                +
                                  keyStr.charAt(enc1)   +
                                  keyStr.charAt(enc2)   +
                                  keyStr.charAt(enc3)   +
                                  keyStr.charAt(enc4)
                      ;
                      
              }//while
              
              return output;
              
        }//encode

        function utf8_encode(string){
        
              string        = string.replace(/\r\n/g,'\n');
              var utftext   = "";
              
              for(var n=0;n<string.length;n++){
              
                    var c   = string.charCodeAt(n);
                    if(c<128){
                        utftext  += String.fromCharCode(c);
                    }else{
                          if((c>127)&&(c<2048)){
                                utftext += String.fromCharCode((c>>6)|192);
                                utftext += String.fromCharCode((c&63)|128);
                          }else{
                                utftext += String.fromCharCode((c>>12)|224);
                                utftext += String.fromCharCode(((c>>6)&63)|128);
                                utftext += String.fromCharCode((c&63)|128);
                          }
                    }
                    
              }//for
              
              return utftext;
              
        }//utf8_encode

this code produces the same output as

sha-256 hash generator

Base64 encode

hope this helps

Answer 2

SHA-256 takes bytes (not text) as input and produces bytes (not text) as output. Of course, dealing with raw bytes is not always convenient so checksum bytes are often somehow converted to text, and there're multiple ways to do so.

I'm no JavaScript expert but it appears that when calculating SHA-256 checksum you automatically encode it as Hex:

hash.setEncoding('hex');

Then you encode it further to Base64:

let sha256conv = btoa(sha256);

So the entire chain looks like: bytes -> Hex -> Base64.

It might solve the issue if you change it to bytes -> Base64 by using hash.setEncoding('binary').