Audio Stream ends up with additional chriping and noise in TCP server written in python

Question

We are working on a mobile app where we have a server on python3 and are accommodating the streams through this server.

But when android clients connect to the server the voice is heard with 5 seconds of delay or so and also becomes too much noisy and contains chirping, we have our best on the server side and android side, need someone to guide through the way.

Here is the code for instance:

import socket import time from threading import Thread from multiprocessing.dummy import Pool as ThreadPool

class VoiP(Thread):

    def __init__(self):
        Thread.__init__(self)


    def server_start(self, TCP_IP = '0.0.0.0', TCP_PORT = 2004 ,BUFFER_SIZE = 1024, Connections = 4):

        self.Server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.Server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.Server.bind((TCP_IP, TCP_PORT))

        self.Server.listen(Connections)
        print("Server is starting.")


    def client_connection(self):
        try:
            print("Waiting for Room1 Client A \n")
            (self.Room1a, address) = self.Server.accept()
            print (type(self.Room1a))
            print("Room1 Client A connected \n")
            print("Waiting for Room1 Client B \n")
            (self.Room1b, address) = self.Server.accept()
            print("Room1 Client B connected \n")
            print("Waiting for Room2 Client A \n")
            (self.Room2a, address) = self.Server.accept()
            print("Room2 Client A connected \n")
            print("Waiting for Room2 Client B \n")
            (self.Room2b, address) = self.Server.accept()
            print("Room2 Client B connected \n")
            print("Print All clients connected")

        except Exception as e:
            print("Error While Connecting to client")

    def byte_logic(self, byte, data):
        if byte is not b'':
            data = byte + data
            byte = b''

        if len(data) % 2 is not 0:
            byte = (data[-1])
            data = data[0:-1]
        return np.frombuffer(data, np.int16), byte

This is the adder to add client voices def array_adder(self, a, b):

        if len(a) < len(b):
            c = b.copy()
            c[:len(a)] += a
        else:
            c = a.copy()
            c[:len(b)] += b

        return bytes(c)

    def room1a_transcev(self):

        while True:
            try:
                self.data1a = self.Room1a.recv(2048)
                self.np_r1a, self.extra_byte_1 = self.byte_logic(self.extra_byte_1, self.data1a)
                self.room1_stream = self.array_adder(self.np_r1a, self.np_r1b)

                self.Room2a.sendall(self.room1_stream)
                self.Room2b.sendall(self.room1_stream)
            except Exception as e:
                print(e, "room1a_transcev error")

    def room1b_rec(self):
        while True:
            try:
                self.data1b = self.Room1b.recv(2048)
                self.np_r1b, self.extra_byte_2 = self.byte_logic(self.extra_byte_2, self.data1b)

            except Exception as e:
                print(e, "room1b_transcev error")

    def room2a_transcev(self):
        while True:
            try:
                self.data2a = self.Room2a.recv(2048)
                self.np_r2a, self.extra_byte_3 = self.byte_logic(self.extra_byte_3, self.data2a)
                self.room2_stream = self.array_adder(self.np_r2a, self.np_r2b)

                self.Room1a.sendall(self.room2_stream)
                self.Room1b.sendall(self.room2_stream)
            except Exception as e:
                print(e, "room2a_transcev error")

    def room2b_rec(self):
        while True:
            try:
                self.data2b = self.Room2b.recv(2048)
                self.np_r2b, self.extra_byte_4 = self.byte_logic(self.extra_byte_4, self.data2b)

            except Exception as e:
                print(e, "room2b_rec error")

if __name__ == "__main__":

    objVOIP = VoiP()
    objVOIP.server_start()
    objVOIP.client_connection()

    # asyncio.run(objVOIP.main())



    thread1 = Thread(target=objVOIP.room1a_transcev)
    thread2 = Thread(target=objVOIP.room1b_rec)
    thread3 = Thread(target=objVOIP.room2a_transcev)
    thread4 = Thread(target=objVOIP.room2b_rec)




    thread1.setDaemon = True
    thread2.setDaemon = True
    thread3.setDaemon = True
    thread4.setDaemon = True

    thread1.start()
    thread2.start()
    thread3.start()
    thread4.start()

I need the ideal case where there are minimal delay and noise. And mixing of two voices.

Answer 1

Well, this is a complex matter and I can't give a definitive answer, but I can give some tips.

First, this is called jitter, and usually happens when packets are not received in order or some of them take more time than others. So:

• First, discard network problems by executing a test with local connections only, or in a controlled network where you know there is no traffic. I guess you have already done this.
• Second, consider using UDP instead of TCP. Yeah, UDP can lose some packets if the network is busy and you have to control yourself their order (they can be received in a different order that they are sent, you will have to put an order number and reorder if necessary), but it's waaaay faster than TCP, and in a call it isn't really that important if a millisecond or two of audio are lost.
• And third: For audio processing make sure you are using the C bind functions of your library correctly. That can make a huge difference in performance.