CODEWITHSUNDEEP
webassemblyweb-audio-apiaudio-worklet
Thierry GAYET
Thierry GAYET

Reputation: 41

Audio crackling using an audio worklet

I'm working on an audio/video application using WebAssembly and I'm encountering issues with sound playback. I capture audio from the microphone via getUserMedia, and I've created a proof of concept to transmit the captured audio in raw PCM format. The audio is sent to the main thread via a post message, and then forwarded to a second audio worklet through a direct connection for playback. While the chain works and the audio chunks are received, the sound is distorted with crackling, making it inaudible.

Main thread in javascript :

<!DOCTYPE html>
<html lang="en">
<head>
    <title>Audio worklet capture/renderer</title>
</head>
<body>
    <h2>Audio worklet</h2>
    <img alt="" src="HP.gif">
    <button id="cmdStart"></button>
    <button id="cmdStop" disabled></button>
  <script>
    const cmdStart = document.getElementById('cmdStart');
    const cmdStop = document.getElementById('cmdStop');

    let hasUserGesture = false;
    let FRate = 48000;
    let captureNode;
    let rendererNode;
    let mediaStream;
    let audioContext;
    
    async function loadWorklets() {
      await audioContext.audioWorklet.addModule('AudioWorkletProcessor-capture.js');
      console.log('AudioWorkletProcessor-capture loaded');

      await audioContext.audioWorklet.addModule('AudioWorkletProcessor-renderer.js');
      console.log('AudioWorkletProcessor-renderer loaded');
    }
    
    function sendPCMBuffer(pcmData) {
      // Create an AudioBuffer from the PCM data
      const audioBuffer = audioContext.createBuffer(1, pcmData.length, audioContext.sampleRate);
      audioBuffer.copyToChannel(pcmData, 0);

      // Create an AudioBufferSourceNode and connect it to the audio worklet
      const source = audioContext.createBufferSource();
      source.buffer = audioBuffer;
      source.connect(rendererNode);
      source.start();
    }

    cmdStop.textContent = "Stop";
    cmdStop.disabled = true;
    cmdStop.addEventListener('click', stopAudio);
    cmdStart.textContent = "Start";

    cmdStart.addEventListener('click', startAudio);
    
    async function startAudio() {
      console.log('Start button clicked');

      hasUserGesture = true;
      
      cmdStart.disabled = true;
      cmdStop.disabled = false;

      if (!audioContext) audioContext = new AudioContext(); /*{
        latencyHint: 'high',
        sampleRate: 44100
      });*/

      // Check if AudioContext is suspended and resume if necessary
      if (audioContext.state === 'suspended' && hasUserGesture) {
        await audioContext.resume();
        console.log('AudioContext resumed');
      }

      // Load the worklets before creating the nodes
      await loadWorklets();

      rendererNode = new AudioWorkletNode(audioContext, 'audioworklet-audio-renderer');
      rendererNode.port.start();

      captureNode = new AudioWorkletNode(audioContext, 'audioworklet-audio-capture');
      if (captureNode) {
        if (rendererNode) {
            rendererNode.port.postMessage(1);
        }  
        captureNode.port.onmessage = (event) => {
          console.log(`Received data from capture processor: ${event.data}`);
          if (rendererNode) {
            console.log(typeof event.data);
            var audio_chunk = event.data;
            //rendererNode.port.postMessage(audio_chunk.slice(0));
            if ((audio_chunk.length>0) && (audio_chunk[0].length>0))  {
              for (var i=0; i< audio_chunk[0].length; i++){
                var fa = audio_chunk[0][i];
                sendPCMBuffer(fa);
              }
            }
            //rendererNode.port.postMessage(event.data);
          }
        };
        captureNode.port.start();
      } else {
        console.error('Error: captureNode is undefined');
      }

      mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
      const sourceNode = audioContext.createMediaStreamSource(mediaStream);
      sourceNode.connect(captureNode);
      captureNode.connect(rendererNode);
      rendererNode.connect(audioContext.destination);
    }

   function stopAudio() {
   cmdStart.disabled = false;
   cmdStop.disabled = true;
   if (captureNode) {
       captureNode.port.postMessage('stop');
       captureNode = undefined; // Release the reference for garbage collection
   }
      if (rendererNode) {
        rendererNode.port.postMessage('stop');
        rendererNode = undefined;  // Release the reference for garbage collection
      }
      
      // Stop each track individually
      const tracks = mediaStream.getTracks();
      tracks.forEach(track => { track.stop(); });      

      audioContext.suspend();
    }
  </script>
  </script>
</body>
</html>

Audio worklet used for the capture (AudioWorkletProcessor-capture.js) :

class AudioWorkletProcessorRawPCMCapture extends AudioWorkletProcessor 
{
  constructor() 
  {
    super();
    this.shouldProcess = true;
    this.port.onmessage = (event) => {
      if (event.data === 'stop') {
        this.shouldProcess = false;
      } 
    };
  }

  process(inputs, outputs, parameters)
  {
    if (!this.shouldProcess) {
      return false;
    }
    const input = inputs[0];
    const output = outputs[0];
    if (inputs) {
    this.port.postMessage( inputs.slice(0) ); // Slice is to avoid detached buffer !
    } 
    return true;
  }
}
registerProcessor('audioworklet-audio-capture', AudioWorkletProcessorRawPCMCapture);

Audio worklet used for the audio rendering (AudioWorkletProcessor-renderer.js) :

class AudioWorkletProcessorRawPCMRenderer extends AudioWorkletProcessor 
    {
        constructor() 
        {
            super();
    
            this.shouldProcess = true;
            this.chunkQueue = [];
            this.gain = 1.0; // Initialize gain to 1.0 (no gain)
            
            // Listen message that contain PCM audio chunks from the main thread
            this.port.onmessage = (event) => 
            {
                this.port.onmessage = (event) => {
                    if (event.data) {                
                        if (typeof event.data === 'number') {
                            console.log(`AudioWorkletProcessorRawPCMRenderer / Receiving new audio gain : ${event.data} `);
                            this.gain = event.data; // Update gain if a number is received
                        } else if (typeof event.data === 'string') {
                            if (event.data === 'stop') {
                                console.log('AudioWorkletProcessorRawPCMRenderer / Stopping audio processing');
                                this.shouldProcess = false; // disable the audio worklet
                            }
                        }
                    }
                };
            };
        }
    
        process(inputs, outputs, parameters) 
        {
            if (!this.shouldProcess) {
              return false; // Stop the audio processor
            }
            if (!inputs || inputs.length === 0) {
              return true; // Continue processing even if no input
            }
            const inputBuffer = inputs[0][0];
            const outputChannel = outputs[0][0];
            if (inputBuffer.length > 0) {
              console.log("AudioWorkletProcessorRawPCMRenderer / Process the input buffer.");
              outputs[0][0].set(inputBuffer);
            } /*else {              
              outputs[0][0].fill(0); // Example: Fill with silence
            }*/
            return true; // Continue the audio process
        }
    }
    registerProcessor('audioworklet-audio-renderer', AudioWorkletProcessorRawPCMRenderer);

Any ideas on how to fix this problem? Could it be related to real-time processing? The audio chunks received by the audio worklet (renderer) might be playing too quickly, not waiting for the previous one to finish, which could cause crackling due to overlapping or other issues.

Upvotes: 2

Views: 76

Answers (0)

Related Questions