Swapnil Phadke
Swapnil Phadke

Reputation: 1

Real-time Audio Streaming chunk is not playing sequentially using javascript + elevenlabs

I am trying to play the audio stream chunks I receive from ElevenLabs in real time. When the next chunk is received, the audio player should preload the audio so that once the first chunk finishes, the second chunk automatically starts without any gap or overlap. I am using Plyr player to get the base64 and play the audio.

--App.js--

let player = new Plyr('#audioPlayer', {
        autoplay: false,
        muted: false,
        controls: ['play', 'progress', 'current-time', 'duration', 'volume', 'mute', 'fullscreen']
    });

    const ELEVENLABS_WS_URL = "wss://api.elevenlabs.io/v1/text-to-speech/language-id/stream-input?    model_id=eleven_multilingual_v1&inactivity_timeout=20&language=English_American";

    let ws = new WebSocket(ELEVENLABS_WS_URL);
    let audioQueue = [];
    let isPlaying = false;
    let countofchunk = 1;

    async function base64ToArrayBuffer(base64) {
        const binaryString = window.atob(base64);
        const len = binaryString.length;
        const bytes = new Uint8Array(len);
        for (let i = 0; i < len; i++) {
            bytes[i] = binaryString.charCodeAt(i);
        }
        return bytes.buffer;
    }

    async function decodeAndQueueAudio(base64Audio) {
        console.log("loop ", countofchunk);
        const audioBuffer = await base64ToArrayBuffer(base64Audio);
        const blob = new Blob([audioBuffer], { type: 'audio/mpeg' });
        const audioUrl = URL.createObjectURL(blob);
        await audioQueue.push(audioUrl); // Always push to the queue
        console.log("audioQueue ", audioQueue);
        countofchunk++;
        if (!isPlaying) { // Only call playNextChunk if not currently playing
            playNextChunk();
        } else {
            console.log("Audio is already playing, buffering new chunk");
        }
    }

    function playNextChunk() {
    if (audioQueue.length > 0 && !isPlaying) {
      isPlaying = true;
      const audioUrl = audioQueue.shift();

      player.source = {
          type: 'audio',
          sources: [{
              src: audioUrl,
              type: 'audio/mpeg'
          }]
      };

      const playPromise = player.play();

      if (playPromise !== undefined) {
        playPromise.then(() => {
              player.on('ended', () => {
                  console.error("Autoplay was ended:", countofchunk, audioQueue);
                  isPlaying = false;
                  URL.revokeObjectURL(audioUrl);
                  playNextChunk();
              });
          }).catch(error => {
              console.error("Autoplay was prevented:", error, audioQueue);
              console.error("Autoplay was prevented count:", countofchunk, audioQueue);
              isPlaying = false;
              URL.revokeObjectURL(audioUrl);
              playNextChunk(); //Try to play next chunk even if current fails
          });
      } else {
          console.log("Audio started playing (no promise)");
          player.on('ended', () => {
              isPlaying = false;
              URL.revokeObjectURL(audioUrl);
              playNextChunk();
          });
      }
    } else {
      console.log("No more audio chunks in the queue or already playing.", audioQueue);
    }
    }

    ws.onmessage = async function (event) {
      const audioData = JSON.parse(event.data);
      console.log("Audio chunk received", audioData);
      if (audioData && audioData.audio) {
          const base64Audio = audioData.audio;
          await decodeAndQueueAudio(base64Audio);
      } else if (audioData.detail && audioData.detail.reason === "Stream timed out") {
          console.log("Stream timed out");
          if (player.playing) {
          player.stop();
          }
          audioQueue = [];
          isPlaying = false;
      } else if (event.data === '{"isFinal": true}'){
          console.log("Stream is final");
      } else {
          console.error("No audio data received or unexpected message:", event.data);
      }
    };

    document.getElementById("sendTextButton").addEventListener("click", function () {
      const inputText = document.getElementById("inputText").value.trim();
      if (inputText !== "") {
          const message = {
              text: inputText,
              voice_settings: { stability: 1, similarity_boost: true, optimize_streaming_latency: 0 },
              xi_api_key: "", // Replace with your API key
              streaming: true
          };
          if (ws.readyState === WebSocket.OPEN) {
              ws.send(JSON.stringify(message));
          } else {
              console.error("WebSocket is not open.");
          }
      }
    });

    ws.onopen = () => console.log("Connected to ElevenLabs WebSocket.");
    ws.onerror = error => console.error("WebSocket error:", error);
    ws.onclose = () => {
      console.log("Disconnected from ElevenLabs WebSocket. Attempting to reconnect...");
      setTimeout(() => {
          ws = new WebSocket(ELEVENLABS_WS_URL);
          ws.onopen = () => console.log("Reconnected to ElevenLabs WebSocket.");
          ws.onerror = error => console.error("WebSocket error:", error);
          ws.onmessage = ws.onmessage;
          ws.onclose = ws.onclose;
      }, 5000);
    };

--index.html--

    <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/plyr.css" rel="stylesheet" />
    <h1>Live TTS Streaming with ElevenLabs and Plyr.js</h1>
    <!-- Text Input and Send Button -->
    <div>
        <input type="text" id="inputText" placeholder="Enter text for TTS" value="Hi I have a front end application in react js. I using Elevenlabs to convert the text into audio. I am using streaming API of Elevenlabs "/>
        <button id="sendTextButton">Send Text</button>
    </div>
    <!-- Audio Player -->
    <audio id="audioPlayer" controls></audio>
    <!-- Messages Section -->
    <div>
        <h3>Messages Received:</h3>
        <div id="receive-box"></div>
    </div>
    <!-- WebSocket and Audio Handling Scripts -->
    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/plyr.min.js"></script>
    <script src="app.js"></script>

The audio player should be able to play each chunk as it is received, with preloading, without any gaps or overlap issues.

Upvotes: 0

Views: 84

Answers (0)

Related Questions