WebRTC: Unable to to receive remote video or audio

Question

The goal is to have a webrtc video/audio p2p connection. I did take some examples and put them together and manage to get far enough that "most" exchange (via eventsource on NodeJS) is done, just remote stream is not received.

I found that Chrome browser had a bug in which the request is send twice (any fix for this?).

But even when I use firefox (on laptop and phone), there is just no stream received. with no error on both sides.

This is the full script:

    if (!console || !console.log) {
       var console = {
        log: function() {}
      };
    }

    // Ugh, globals.
    // const signaling = new SignalingChannel(); hier moet een module voor komen, als een EventServerSide en/of Websocket...
    var peerc;
    var source = new EventSource("events");
    var selfView = document.getElementById('localvideo');
    var remoteView = document.getElementById('remotevideo');
    //var TextHistory = ["apple", "orange", "cherry"]; //[ ];
    var audio = null;
    var audioSendTrack = null;
    var video = null;
    var videoSendTrack = null;
    var chatbox = document.getElementById("chatbox");
    var constraints = {video:true, audio:true};
    var configuration = { 
      iceServers: [{
        'urls': 'stun:stun.l.google.com:19302'
      }]
    };
    var started = false;
    var pc;


    $("#incomingCall").modal();
    $("#incomingCall").modal("hide");

    $("#incomingCall").on("hidden", function() {
      document.getElementById("incomingRing").pause();
    });

    source.addEventListener("ping", function(e) {}, false);

    source.addEventListener("userjoined", function(e) {
      appendUser(e.data);
    }, false);

    source.addEventListener("userleft", function(e) {
      removeUser(e.data);
    }, false);

    source.addEventListener("offer", async function(e) {
      log("offer received");
      if (!pc) {
        log("PeerConnection ontbreekt");
        warmup(false); 
      } else {
        warmup(true); 
      }

      try {
        const message = JSON.parse(e.data);  
        if (message.desc) {
          log("Offer: Desc");
          const desc = message.desc;

          // if we get an offer, we need to reply with an answer
          if (desc.type == 'offer') {
            log("Offer: Desc offer");
            var connectionState = pc.connectionState;
            log(connectionState);
            await pc.setRemoteDescription(JSON.parse(desc));  // Parse nodig?
            await pc.setLocalDescription(await pc.createAnswer());
            var connectionState = pc.connectionState;
            log(connectionState);
            //signaling.send(JSON.stringify({desc: pc.localDescription}));
              jQuery.post(
                "offer", {
                  to: message.from,
                  from: document.getElementById("user").innerHTML,
                  desc: JSON.stringify(pc.localDescription)
                },
                function() { log("Offer sent!"); }
              ).error("desc", error);
          } else {
            log("Offer: Desc answer");
            await pc.setRemoteDescription(JSON.parse(desc));
          }
        } else if (message.start) {
          log("Offer: Start received from: " + message.from);
          started = true;
          voor = message.from
          van  = message.to
          if (audio && audioSendTrack) {
            log("Wacht op audio");
            await audio.sender.replaceTrack(audioSendTrack);
          }
          if (video && videoSendTrack) {
            log("Wacht op video");
            await video.sender.replaceTrack(videoSendTrack);
          }
          log("Offer: Started....wat next???");
        } else {
          log("Offer: wacht op candidate");
          log(message.candidate);
          await pc.addIceCandidate(message.candidate);
        }
      } catch (err) {
        console.log(err);
        log("Error in Offer event: " + err);
      }
    }, false);

    source.addEventListener("message", function(e) {
      var message = JSON.parse(e.data);
      text = "User: " + message.from + " - Say: " + message.text;
      WriteChat(text);
    }, false);

    source.addEventListener("answer", function(e) {
      var answer = JSON.parse(e.data);
      peerc.setRemoteDescription(new RTCSessionDescription(JSON.parse(answer.answer)), function() {
        console.log("Call established!");
      }, error);
    }, false);

    function log(info) {
      var d = document.getElementById("debug");
      d.innerHTML += info + "\n\n";
    }

    function appendUser(user) {
      // If user already exists, ignore it
      var index = users.indexOf(user);
      if (index > -1)
        return;

      users.push(user);
      console.log("appendUser: user = " + user + ", users.length = " + users.length);


      var d = document.createElement("div");
      d.setAttribute("id", btoa(user));

      var a = document.createElement("button");
      a.setAttribute("class", "vertical-align");
      a.setAttribute("onclick", "initiateCall('" + user + "');"); //Dit moet dus prive chat worden.
      a.innerHTML = "<i class='icon-user icon-white'></i> " + user;

      d.appendChild(a);
      d.appendChild(document.createElement("br"));
      document.getElementById("users").appendChild(d);
    }

    function removeUser(user) {
      // If user already exists, ignore it
      var index = users.indexOf(user);
      if (index == -1)
        return;

      users.splice(index, 1)

      var d = document.getElementById(btoa(user));
      if (d) {
        document.getElementById("users").removeChild(d);
      }
    }

    function sendPrive(user) {
      log("Prive message"); // + JSON.stringify(offer));
      if(!user) { user = "niets!"};
      jQuery.post(
        "message", {
          to: user,
          from: document.getElementById("user").innerHTML,
          text: JSON.stringify(message)
        },
        function() { console.log("Message sent!"); }
      ).error("privemsg",error);

    }

    function offer(data){
      log("Offer to send message: " + JSON.stringify(data));
    // start
    // desc + type
    // candidate


      jQuery.post(
        "offer", {data: JSON.stringify(data)},
        function() { console.log("Message sent!"); }
      ).error("offer",error);

    }

    function BroadcastMessage() {
      log("Broadcast Message"); // + JSON.stringify(offer));  //function uitbreiden met argumentinvoer
      message = document.getElementById("text_input").value;
      jQuery.post(
        "message", {
          to: user,
          from: document.getElementById("user").innerHTML,
          text: JSON.stringify(message)
        },
        function() { console.log("Message sent!"); }
      ).error("BroadcastMessage",error);
      //WriteChat(msg);
    }

    // Dit is een interne, dit moet dus via events gaan!!
    function WriteChat(text){
      // kan nog user en tijd bijkomen...
      chatbox.innerHTML += text + "<br>";   // deze werkt dus niet meer, canvas wil andere methode
    }

    // Call warmup() to warm-up ICE, DTLS, and media, but not send media yet.
    async function warmup(isAnswerer) {
      log("Warming up...");
      pc = new RTCPeerConnection(configuration);
      if (!isAnswerer) {                          //uitzoeken waarom deze uitgeschakelen...
        audio = pc.addTransceiver('audio');
        video = pc.addTransceiver('video');
      }

      // send any ice candidates to the other peer
      pc.onicecandidate = (event) => {
        log("Offer: onicecandidate...");
        //signaling.send(JSON.stringify({candidate: event.candidate}));
        log(event.candidate)
        if(event.candidate){
          jQuery.post(
            "offer", {
            to: voor,
            from: document.getElementById("user").innerHTML,
            data: event.candidate,              // debugging...
            candidate: event.candidate
          }, // hier zijn we nu...candidate blijft leeg????
          function() { log("Offer: onicecandidate sent!"); }
        ).error("onicecandidate",error);
        } else {
          log("geen candidate");
        }
      };

      pc.onnegotiationneeded = function() {
        log("negotiation nodig...");
        //var connectionState = RTCPeerConnection.connectionState;
        pc.createOffer().then(function(aanbod) {
          var connectionState = pc.connectionState;
          log(connectionState);
          log(JSON.stringify(aanbod));
          return pc.setLocalDescription(aanbod);
        })
        .then(function() {
          log(JSON.stringify(pc.localDescription));
          jQuery.post(
            "offer", {
              to: document.getElementById("user").innerHTML,
              from: voor,
              desc: JSON.stringify(pc.localDescription)
            },
            function() { log("Offer: localDescription sent!"); }
          ).error("onnegotiationneeded",error);
        })
        .catch(error);
      }

    log("Remote video");
      // once media for the remote track arrives, show it in the remote video element
      pc.ontrack = async (event) => {
        log("start ontrack...");
        remoteView.srcObject = event.streams[0];
        remoteView.play();
        selfView.play();
        log("oude ontrack...");
        try {
          log(event.track.kind);
          if (event.track.kind == 'audio') {
            log("Track heeft audio");
            if (isAnswerer) {
              log("beantwoord audio");
              audio = event.transceiver;
              audio.direction = 'sendrecv';
              if (started && audioSendTrack) {
                await audio.sender.replaceTrack(audioSendTrack);
              }
            }
          }
        } catch (err) {
          console.log(err);
          log("Error in audio ontrack: " + err);
        }
        try {
          log(event.track.kind);
          if (event.track.kind == 'video') {
            log("Track heeft video");
            if (isAnswerer) {
              log("beantwoord video");
              video = event.transceiver;
              video.direction = 'sendrecv';
              if (started && videoSendTrack) {
                await video.sender.replaceTrack(videoSendTrack);
              }
            }
          }
        } catch (err) {
          console.log(err);
          log("Error in video ontrack: " + err);
        }
        try {
          // don't set srcObject again if it is already set.
          if (!remoteView.srcObject) {
            log("Nog geen remote video...");
            remoteView.srcObject = new MediaStream();
          }
          log("Voeg remote video toe...");
          log(event.track);
          remoteView.srcObject.addTrack(event.track);
        } catch (err) {
          console.log(err);
          log("Error in last ontrack: " + err);
        }
      };
    log("Local video & audio");
      try {
        // get a local stream, show it in a self-view and add it to be sent
        const stream = await navigator.mediaDevices.getUserMedia(constraints);
        selfView.srcObject = stream;
        audioSendTrack = stream.getAudioTracks()[0];
        if (started) {
          log("Started = True => audio");
          log(audio)
          await audio.sender.replaceTrack(audioSendTrack);
        }
        videoSendTrack = stream.getVideoTracks()[0];
        if (started) {
          log("Started = True => video");
          log(video)
          await video.sender.replaceTrack(videoSendTrack);
        }
      } catch (err) {
        console.log(err);
        log("Error in local video & audio: " + err);
      }
      log("Warmup completed");
    }


    function initiateCall(user) {
      log("Start van video verzoek");
      // Verander UI
      //document.getElementById("main").style.display = "none";
      //document.getElementById("call").style.display = "block";
      log("Voor gebruiker: " + user);
      voor = user;
      log("Van gebruiker:  " + document.getElementById("user").innerHTML);
      van = document.getElementById("user").innerHTML;
      started = true;
      //offer(JSON.stringify({start: true}));
      jQuery.post(
        "offer", {
          to: user,
          from: document.getElementById("user").innerHTML,
          start: true
        },
        function() { console.log("Offer sent!"); }
      ).error("initiateCall",error);
    }


    function endCall() {
      log("Ending call");
      document.getElementById("call").style.display = "none";
      document.getElementById("main").style.display = "block";

      document.getElementById("localvideo").mozSrcObject.stop();
      document.getElementById("localvideo").mozSrcObject = null;
      document.getElementById("remotevideo").mozSrcObject = null;

      peerc.close();
      peerc = null;
    }

    function error(from,e) {
      if (typeof e == typeof {}) {
        //alert("Oh no! " + JSON.stringify(e));
        log("Oh no! " + from + " ERROR: " + JSON.stringify(e));
      } else {
        alert("Oh no!!!! " + from + " ERROR: " + e);
      }
      //endCall();
    }

    var users = [];
    users.push(document.getElementById("user").innerHTML);

Please note that I combined examples for:

• developer.mozilla.org
• w3c.github.io
• www.html5rocks.com So there might be unnecessary code If required, I can put all code in repository and share that for full view.

Additional logging shows the following errors: DOMException: "Cannot add ICE candidate when there is no remote SDP" InvalidModificationError: Cannot set local offer when createOffer has not been called.

Answer 1

I found the problem and solution.

First: The STUN/TURN server was not a problem, this
urls: 'stun:stun.l.google.com:19302' Is enough for local network development. But I did install turnserver (ubuntu 18) which seems fine.

Second: My problem was first in de message exchange, you need to make sure the OFFER/ANSWER go to right clients.

Last: The negotiation needs to be stable before sending/receiving media. I modified my code that connection and start video sending are separated. Also this allows for just one side to send media or not.

Answer 2

You should try to use a TURN-Server. Even for usage in development. The traffic will be relayed if a direct P2P-connection can't be stablished.

Have a look at Coturn. I can't say that this will be the solution for the current problem, but in my opinion most of the issues will be solved - and for production use, you will need it definitely.