How to create TCP message framing for the stream

Question

Here is how my Client connects to the server:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using System.Net.Sockets;
using System.IO;
using System;
using System.Text.RegularExpressions;
using UnityEngine.SceneManagement;
using Newtonsoft.Json;
using System.Linq;

public class ClientWorldServer : MonoBehaviour {

    public bool socketReady;
    public static TcpClient socket;
    public static NetworkStream stream;
    public static StreamWriter writer;
    public static StreamReader reader;

    public void ConnectToWorldServer()
    {
        if (socketReady)
        {
            return;
        }
        //Default host and port values;
        string host = "127.0.0.1";
        int port = 8080;

        try
        {

            socket = new TcpClient(host, port);
            stream = socket.GetStream();
            writer = new StreamWriter(stream);
            reader = new StreamReader(stream);
            socketReady = true;
        }
        catch (Exception e)
        {
            Debug.Log("Socket error : " + e.Message);
        }

    }
}

Here is how i send data to the server using my Send function:

public void Send(string header, Dictionary<string, string> data)
{

    if (stream.CanRead)
    {
        socketReady = true;
    }

    if (!socketReady)
    {
        return;
    }
    JsonData SendData = new JsonData();
    SendData.header = "1x" + header;
    foreach (var item in data)
    {
        SendData.data.Add(item.Key.ToString(), item.Value.ToString());
    }
    SendData.connectionId = connectionId;

    string json = JsonConvert.SerializeObject(SendData);
    var howManyBytes = json.Length * sizeof(Char);
    writer.WriteLine(json);
    writer.Flush();

    Debug.Log("Client World:" + json);
}

As you can see i'm sending the data to the Stream like a string not like a byte array. As far as i know i should send the data as byte array prepending the size of the message and following the message. On the server side i have no clue how i can read that data.

Here is how i read it now(it works for now, but it will not work if i try to send more messages at once):

class WorldServer
{
    public List<ServerClient> clients = new List<ServerClient>();
    public List<ServerClient> disconnectList;


    public List<CharactersOnline> charactersOnline = new List<CharactersOnline>();

    public int port = 8080;
    private TcpListener server;
    private bool serverStarted;

    private int connectionIncrementor;

    private string mysqlConnectionString = @"server=xxx;userid=xxx;password=xxx;database=xx";

    private MySqlConnection mysqlConn = null;
    private MySqlDataReader mysqlReader;

    static void Main(string[] args)
    {
        WorldServer serverInstance = new WorldServer();

        Console.WriteLine("Starting World Server...");

        try
        {
            serverInstance.mysqlConn = new MySqlConnection(serverInstance.mysqlConnectionString);
            serverInstance.mysqlConn.Open();
            Console.WriteLine("Connected to MySQL version: " + serverInstance.mysqlConn.ServerVersion + "\n");
        }
        catch (Exception e)
        {
            Console.WriteLine("MySQL Error: " + e.ToString());
        }
        finally
        {
            if (serverInstance.mysqlConn != null)
            {
                serverInstance.mysqlConn.Close();
            }
        }

        serverInstance.clients = new List<ServerClient>();
        serverInstance.disconnectList = new List<ServerClient>();

        try
        {
            serverInstance.server = new TcpListener(IPAddress.Any, serverInstance.port);
            serverInstance.server.Start();

            serverInstance.StartListening();
            serverInstance.serverStarted = true;


            Console.WriteLine("Server has been started on port: " + serverInstance.port);
        }
        catch (Exception e)
        {
            Console.WriteLine("Socket error: " + e.Message);
        }

        new Thread(() =>
        {
            Thread.CurrentThread.IsBackground = true;
            /* run your code here */
            while (true)
            {
                string input = Console.ReadLine();
                string[] commands = input.Split(':');
                if (commands[0] == "show online players")
                {
                    Console.WriteLine("Showing connections\n");
                    foreach (CharactersOnline c in serverInstance.charactersOnline)
                    {
                        Console.WriteLine("Character name: " + c.characterName + "Character ID: " + c.characterId + "Connection id: " + c.connectionId + "\n");
                    }
                }
                continue;
            }

        }).Start();

        while (true)
        {
            serverInstance.Update();
        }
    }

    private void Update()
    {
        //Console.WriteLine("Call");
        if (!serverStarted)
        {
            return;
        }

        foreach (ServerClient c in clients.ToList())
        {
            // Is the client still connected?
            if (!IsConnected(c.tcp))
            {
                c.tcp.Close();
                disconnectList.Add(c);
                Console.WriteLine(c.connectionId + " has disconnected.");
                CharacterLogout(c.connectionId);
                continue;
                //Console.WriteLine("Check for connection?\n");
            }
            else
            {



                // Check for message from Client.
                NetworkStream s = c.tcp.GetStream();
                if (s.DataAvailable)
                {
                    StreamReader reader = new StreamReader(s, true);
                    string data = reader.ReadLine();

                    if (data != null)
                    {
                        OnIncomingData(c, data);
                    }

                }
                //continue;
            }
        }

        for (int i = 0; i < disconnectList.Count - 1; i++)
        {
            clients.Remove(disconnectList[i]);
            disconnectList.RemoveAt(i);
        }


    }

   private void OnIncomingData(ServerClient c, string data)
    {
        Console.WriteLine(data);
        dynamic json = JsonConvert.DeserializeObject(data);

        string header = json.header;
        //Console.WriteLine("Conn ID:" + json.connectionId);

        string connId = json.connectionId;
        int.TryParse(connId, out int connectionId);

        string prefix = header.Substring(0, 2);

        if (prefix != "1x")
        {
            Console.WriteLine("Unknown packet: " + data + "\n");
        }
        else
        {
            string HeaderPacket = header.Substring(2);
            switch (HeaderPacket)
            {
                default:
                    Console.WriteLine("Unknown packet: " + data + "\n");
                    break;
                case "004":
                    int accountId = json.data["accountId"];
                    SelectAccountCharacters(accountId, connectionId);
                    break;
                case "005":
                    int characterId = json.data["characterId"];
                    getCharacterDetails(characterId, connectionId);
                    break;
                case "006":
                    int charId = json.data["characterId"];
                    SendDataForSpawningOnlinePlayers(charId, connectionId);
                    break;
                case "008":
                    Dictionary<string, string> dictObj = json.data.ToObject<Dictionary<string, string>>();
                    UpdateCharacterPosition(dictObj, connectionId);
                    break;
            }
        }

    private bool IsConnected(TcpClient c)
    {
        try
        {
            if (c != null && c.Client != null && c.Client.Connected)
            {
                if (c.Client.Poll(0, SelectMode.SelectRead))
                {
                    return !(c.Client.Receive(new byte[1], SocketFlags.Peek) == 0);
                }

                return true;
            }
            else
            {
                return false;
            }
        }
        catch
        {
            return false;
        }
    }

    private void StartListening()
    {
        server.BeginAcceptTcpClient(OnConnection, server);
    }

    private void OnConnection(IAsyncResult ar)
    {
        connectionIncrementor++;
        TcpListener listener = (TcpListener)ar.AsyncState;
        clients.Add(new ServerClient(listener.EndAcceptTcpClient(ar)));
        clients[clients.Count - 1].connectionId = connectionIncrementor;
        StartListening();


        //Send a message to everyone, say someone has connected!
        Dictionary<string, string> SendDataBroadcast = new Dictionary<string, string>();
        SendDataBroadcast.Add("connectionId", clients[clients.Count - 1].connectionId.ToString());

        Broadcast("001", SendDataBroadcast, clients, clients[clients.Count - 1].connectionId);
        Console.WriteLine(clients[clients.Count - 1].connectionId + " has connected.");
    }

And this is how the server send back data to the client:

    private void Send(string header, Dictionary<string, string> data, int cnnId)
    {
        foreach (ServerClient c in clients.ToList())
        {

            if (c.connectionId == cnnId)
            {
                try
                {
                    //Console.WriteLine("Sending...");
                    StreamWriter writer = new StreamWriter(c.tcp.GetStream());
                    if (header == null)
                    {
                        header = "000";
                    }
                    JsonData SendData = new JsonData();
                    SendData.header = "0x" + header;
                    foreach (var item in data)
                    {
                        SendData.data.Add(item.Key.ToString(), item.Value.ToString());
                    }
                    SendData.connectionId = cnnId;

                    string JSonData = JsonConvert.SerializeObject(SendData);

                    writer.WriteLine(JSonData);
                    writer.Flush();
                    //Console.WriteLine("Trying to send data to connection id: " + cnnId + " data:" + sendData);
                }
                catch (Exception e)
                {
                    Console.WriteLine("Write error : " + e.Message + " to client " + c.connectionId);
                }
            }
        }
    }

Here is my ServerClient class:

public class ServerClient
{
    public TcpClient tcp;
    public int accountId;
    public int connectionId;
    public ServerClient(TcpClient clientSocket)
    {
        tcp = clientSocket;
    }
}

Can you please show me how i should modify my Send function on the client to send the data as byte array so i can create "TCP Message Framing" and how should i change my the following part on the server:

        foreach (ServerClient c in clients.ToList())
        {
            // Is the client still connected?
            if (!IsConnected(c.tcp))
            {
                c.tcp.Close();
                disconnectList.Add(c);
                Console.WriteLine(c.connectionId + " has disconnected.");
                CharacterLogout(c.connectionId);
                continue;
                //Console.WriteLine("Check for connection?\n");
            }
            else
            {



                // Check for message from Client.
                NetworkStream s = c.tcp.GetStream();
                if (s.DataAvailable)
                {
                    StreamReader reader = new StreamReader(s, true);
                    string data = reader.ReadLine();

                    if (data != null)
                    {
                        OnIncomingData(c, data);
                    }

                }
                //continue;
            }
        }

which is responsible for receving data on the server ?

Is it possible to change only these parts from the Client and on the Server and make it continue to work but this time properly with TCP Message Framing ?

Of course the listener on the client and the Send function of the server i'll remake once i understand how this framing should look like.

Answer 1

Your frames are already defined by cr/lf - so that much already exists; what you need to do is to keep a back buffer per stream - something like a MemoryStream might be sufficient, depending on how big you need to scale; then essentially what you're looking to do is something like:

while (s.DataAvailable)
{
    // try to read a chunk of data from the inbound stream
    int bytesRead = s.Read(someBuffer, 0, someBuffer.Length);
    if(bytesRead > 0) {
         // append to our per-socket back-buffer
         perSocketStream.Position = perSocketStream.Length;
         perSocketStream.Write(someBuffer, 0, bytesRead);


         int frameSize; // detect any complete frame(s)
         while((frameSize = DetectFirstCRLF(perSocketStream)) >= 0) {
              // decode it as text
              var backBuffer = perSocketStream.GetBuffer(); 
              string message = encoding.GetString(
                    backBuffer, 0, frameSize);
              // remove the frame from the start by copying down and resizing
              Buffer.BlockCopy(backBuffer, frameSize, backBuffer, 0,
                  (int)(backBuffer.Length - frameSize));
              perSocketStream.SetLength(backBuffer.Length - frameSize);

              // process it
              ProcessMessage(message);
         }
    }
}