Convert Canvas to audio in Browser for datamoshing

Question

I want to do some datamoshing in the browser and would like to load the ImageData from a Canvas into AudioData, so I can use some audio libraries (like tuna) to add effects on the Data.

This is what I have tried so far

const canvasToBlob = (canvas) => {
  let dataUrl = canvas.toDataURL('image/png');
  return dataURItoBlob(dataUrl);
};

const dataURItoBlob = (dataURI) => {
  // convert base64 to raw binary data held in a string
  // doesn't handle URLEncoded DataURIs - see SO answer #6850276 for code that does this
  let byteString = atob(dataURI.split(',')[1]);

  // separate out the mime component
  let mime = dataURI.split(',')[0].split(':')[1].split(';')[0];
  console.log(mime);

  // write the bytes of the string to an ArrayBuffer
  let ab = new ArrayBuffer(byteString.length);
  let ia = new Uint8Array(ab);
  for (let i = 0; i < byteString.length; i++) {
    ia[i] = byteString.charCodeAt(i);
  };

  // write the ArrayBuffer to a blob, and you're done
  let blob = new Blob([ab], {type: 'audio/wav'});
  return blob;
};

export default function audioConvert (canvas) {
  const imageBlob = canvasToBlob(canvas);
  let audioCtx = new AudioContext();
  let fileReader = new FileReader();

  fileReader.onloadend = () => {
    console.log(fileReader.result);
    audioCtx.decodeAudioData(fileReader.result).then((decodedData) => {
      console.log(decodedData);
    });
  };
  fileReader.readAsArrayBuffer(imageBlob);
};

Unfortunately I only get the error

Uncaught (in promise) DOMException: Unable to decode audio data

How could I change my code to get an AudioContext.

Or is an AudioContext even the correct way?

Some effects that I would like to add to the Image are 'Echoing', or 'Wah-Wah' as found f.e. in Audacity.

Blindman67 · Accepted Answer

Image to Canvas to Audio context soundbuffer example

No answer yet so I have just added a very quick load image and convert to soundbuffer and play.

// Stereo
var log = (data)=>console.log(data);
var audioCtx = new AudioContext(); // get audio context
var pixelPlayTime,sBuffers,ctx,can,startTime,source;  // sBuffers is the wav form array
// load image
var image = new Image;
image.setAttribute('crossOrigin', 'anonymous');
image.src = "https://upload.wikimedia.org/wikipedia/commons/5/5a/Hoverfly07.jpg";

// when loaded convert to wave
image.onload = function(){
    displayit.innerHTML = "";

    // create a canvas to read pixel data
    can = document.createElement("canvas");
    can.width = this.width;
    can.height = this.height;
    ctx = can.getContext("2d");
    ctx.drawImage(this,0,0);
    var data = ctx.getImageData(0,0,this.width,this.height).data;

    // get image size and workout sound buffer sizes
    var pixelCount = this.width * this.height;
    pixelPlayTime = pixelCount / audioCtx.sampleRate;
    sBuffers = audioCtx.createBuffer(2, pixelCount, audioCtx.sampleRate);

    // left gets red full vol and blue half
    var channel1 = sBuffers.getChannelData(0); // get channel 0
    var i = 0;
    while(i < pixelCount){
        // get red and some of green for this channel
        var sample = ((data[i*4]/128) - 1 + (data[i*4+2]/255) - 0.5) / 1.5;
        channel1[i] = sample < -1 ? -1 : sample > 1 ? 1 : sample;
        i ++
    }

    // Right gets green full vol and blue half
    var channel2 = sBuffers.getChannelData(1); // get channel 1
    i = 0;
    while(i < pixelCount){
        // get blue and some of green for this channel
        var sample = ((data[i*4 + 1]/128) - 1 + (data[i*4+2]/255) - 0.5) / 1.5;
        channel2[i] = sample < -1 ? -1 : sample > 1 ? 1 : sample;
        i ++
    }

    // sound buffers created prep to play
    source = audioCtx.createBufferSource(); // for playing the buffer
    source.buffer = sBuffers;              // point to the buffer
    source.connect(audioCtx.destination);       // connect

    // show image as canvas
    can.style.width = Math.floor(this.width / 4) + "px"
    can.style.height = Math.floor(this.height / 4) + "px"
    displayit.innerHTML = "Click image to hear it
";
    displayit.appendChild(can);

    // Wait for clien click
    can.addEventListener("click",()=>{
      requestAnimationFrame(startSounds);
     });

}
// start playing the sount
function startSounds(time){
    if(startTime === undefined){
      startTime = time;
      pixelPlayTime *= 1000; // convert from seconds to micro seconds
      source.start();                             // and play dat phunky fly
      requestAnimationFrame(update);
      ctx.fillStyle = "white";
    }
}

// display where the sound data is on the image.
// Basic timing use to show at what line the sound is being generated from.
function update(time){
    var pos = (time - startTime) / pixelPlayTime;
    var posY = Math.floor(pos * can.height);
    ctx.globalCompositeOperation = "lighter";
    ctx.fillRect(0,posY,can.width,4);
    ctx.globalCompositeOperation = "source-over";
    ctx.drawImage(image,0,posY-4,can.width,4,0,posY-4,can.width,4);
    if(pos < 1){
        requestAnimationFrame(update);
    }
}

Please wait loading image


fir0002 | flagstaffotos.com.au



Image attribution



This file is published under the following Creative Commons license:

Attribution NonCommercial Unported 3.0

Convert Canvas to audio in Browser for datamoshing

Answers (1)

Image to Canvas to Audio context soundbuffer example

Related Questions