UDP video stream C#

Question

I use UDP for video streaming, but I must divided into segments main array and send a few packets because a frame's size bigger than udp max size . It's ok, but it's udp, so arrays received often not in the order in which were on client side(something like this: first segment of the first frame, third segment of the second frame etc...). I can numerate segments, but often segments received in wrong frames(first segment of the first frame, second segment of the second frame, third segment of the second frame). Send:

while(true){
int c = CvInvoke.cvWaitKey(33);
if (c == 27)
break;
    Byte[] byteData1 = rgb32Image.Bytes;
    Byte[] byte1 = new Byte[65000];
    Byte[] byte2 = new Byte[65000];
    Byte[] byte3 = new Byte[65000];
    Byte[] byte4 = new Byte[65000];
    byte1[0] = 1;
    byte2[0] = 2;
    byte3[0] = 3;
    byte4[0] = 4;
    System.Buffer.BlockCopy(byteData1, 0, byte1, 1, 64999);
    obj.socket.Send(byte1, byte1.Length);
    System.Buffer.BlockCopy(byteData1, 64999, byte2, 1, 64999);
    obj.socket.Send(byte2, byte2.Length);
    System.Buffer.BlockCopy(byteData1, 64999 * 2, byte3, 1, 64999);
    obj.socket.Send(byte3, byte3.Length);
    System.Buffer.BlockCopy(byteData1, 64999 * 3, byte4, 1, byteData1.Length - (3 * 64999);
    obj.socket.Send(byte4, byte4.Length);
}

Receive:

VideoReceive(IAsyncResult ar)
{
StateObject stateobj = (StateObject)ar.AsyncState;
UdpClient client = stateobj.workSocket;
IPEndPoint senderIPEndPoint = new IPEndPoint(IPAddress.Any, 0);
Byte[] byte1 = client.EndReceive(ar, ref senderIPEndPoint);
Byte[] byte2 = client.Receive(ref senderIPEndPoint);
Byte[] byte3 = client.Receive(ref senderIPEndPoint);
Byte[] byte4 = client.Receive(ref senderIPEndPoint);
switch (byte1[0])
{
case 1:
System.Buffer.BlockCopy(byte1, 1, buffer, 0, 64999);
break;
case 2:
System.Buffer.BlockCopy(byte1, 1, buffer, 64999, 64999);
break;
case 3:
System.Buffer.BlockCopy(byte1, 1, buffer, 64999 * 2, 64999);
break;
case 4:
System.Buffer.BlockCopy(byte1, 1, buffer, 64999 * 3, 64999);
break;
}
switch (byte2[0])
{
case 1:
System.Buffer.BlockCopy(byte2, 1, buffer, 0, 64999);
break;
case 2:
System.Buffer.BlockCopy(byte2, 1, buffer, 64999, 64999);
break;
case 3:
System.Buffer.BlockCopy(byte2, 1, buffer, 64999 * 2, 64999);
break;
case 4:
System.Buffer.BlockCopy(byte2, 1, buffer, 64999 * 3, 64999);
break;
}
switch (byte3[0])
{
case 1:
System.Buffer.BlockCopy(byte3, 1, buffer, 0, 64999);
break;
case 2:
System.Buffer.BlockCopy(byte3, 1, buffer, 64999, 64999);
break;
case 3:
System.Buffer.BlockCopy(byte3, 1, buffer, 64999 * 2, 64999);
break;
case 4:
System.Buffer.BlockCopy(byte3, 1, buffer, 64999 * 3, 64999);
break;
}
switch (byte4[0])
{
case 1:
System.Buffer.BlockCopy(byte4, 1, buffer, 0, 64999);
break;
case 2:
System.Buffer.BlockCopy(byte4, 1, buffer, 64999, 64999);
break;
case 3:
System.Buffer.BlockCopy(byte4, 1, buffer, 64999 * 2, 64999);
break;
case 4:
System.Buffer.BlockCopy(byte4, 1, buffer, 64999 * 3, 64999);
break;
}

stateobj.workSocket.BeginReceive(new System.AsyncCallback(VideoReceive), stateobj);
mask2.Bytes = buffer;
pictureBox1.BeginInvoke(new MethodInvoker(() => { pictureBox1.Image = mask2.ToBitmap(); }));

}

So, how I can stream video via udp sockets(without any additional library)? P.S. You may answer by code or algorithm.

Answer 1

I did this exact thing a while ago, my solution was to split the image into columns and send each one. Initially I was concerned that there would be excessive frame tearing but with a high fps camera and stream it work well.

Sender:

if(imageAvailable) {
    try {
       image = //get image (I am using a BufferedImage image)
               // here I sent the image size through a tcp connection
       ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
       DataOutputStream stream = new DataOutputStream(byteStream);
       for(int a = 0; a < image.getWidth(); a++) {
          stream.writeInt(a);
          for(int b = 0; b < image.getHeight(); b++) {
               int rgb = image.getRGB(a, b);
               byte red = (byte) ((rgb >> 16) & 0x000000FF);
               byte green = (byte) ((rgb >> 8 ) & 0x000000FF);
               byte blue = (byte) ((rgb) & 0x000000FF);
               stream.writeByte(red);
               stream.writeByte(green);
               stream.writeByte(blue);
           }
                DatagramPacket packet = new DatagramPacket(byteStream.toByteArray(), 0,   
                                       byteStream.toByteArray().length, dsAddresses); //dsAddresses is the ip of the client.
                udpSocket.send(packet);
       }
       byteStream = new ByteArrayOutputStream();
       stream = new DataOutputStream(byteStream);
   }

receiver:

 public class DataReceiver implements Runnable {
        private DatagramSocket socket;
        byte [][][] packets;
        public DataReceiver(DatagramSocket socket) {
             this.socket = socket;
             packets = new byte[640][480][3];
        }

        @Override
        public void run() {
        while(true) {
              DatagramPacket packet = new DatagramPacket(new byte[1444], 1444);
              try {
                    socket.receive(packet);
              } catch (IOException e) {
                  e.printStackTrace();
              }
              try{
                 byte[] data = packet.getData();
                 DataInputStream inputStream = new DataInputStream(new          
                                               ByteArrayInputStream(data));
                 int colum = inputStream.readInt();
                 for (int a = 0; a < packets[0].length; a++){
                     packets[colum][a][0] = inputStream.readByte();
                     packets[colum][a][1] = inputStream.readByte() ;
                     packets[colum][a][2] = inputStream.readByte();
                 }
                 }catch(Exception e){
                    e.printStackTrace();
                 }

         }
         }
    public byte[][][] getData(){
         return packets;
    }
}

Basically send each column in a packet and as soon as the packet is received write it to an array of data. When its time to render the frame grab a copy of the data.