Asynchronous Socket Server Hangs Intermittently

Question

I have an Asynchronous server application in c# tweaked slightly from the msdn example: msdn example

I was searching and came across this post Similar Situation

I've been monitoring our server for the past few days tweaking my code, so the server will close the socket when an exception occurs like a malformed packet, or the bytes read < 0, but every so often a bunch of "CLOSE_WAITS" occur, using netstat command to look at my specific port, and other than closing the socket, what other options might I have?

Additionally when my close_wait count starts exceeding 80-100 connections, I start to see the "An established connection was aborted by the software in your host machine" error, which I think indicates that windows is killing them. <-- big guess on my part.

Are there any ways to monitor for the "close_wait" status and abort those sockets if they don't close on their own? The more devices we add to send data to the server, the less intermittently this is occurring so I'm trying to determine a solution before this becomes more of a nightmare.

Answer 1

So this issue turned out to be network specific. We are using a satellite modem that sends data to a hub, the hub connects to our server to send the data from the sat modem. The hub for our satellite provider is not closing the connection to the server after no more data is available from the socket.

My solution was to add a timer to my state object for my asynchronous socket server and close it if no data has been received after a period of time.

public class StateObject : IDisposable
    {
        // Client  socket.
        public Socket workSocket = null;
        // Size of receive buffer.
        public static int BufferSize = int.Parse(System.Configuration.ConfigurationManager.AppSettings["ListenerBufferSize"]); // 32768;  //32kb buffer
        // Receive buffer.
        public byte[] buffer = new byte[BufferSize];
        //debug string.
        public StringBuilder DebugData = new StringBuilder();
        //string of Ipaddress belonging to socket
        public string IpAddress = string.Empty;

        public int ByteCountReceived = 0;

        //Statistics
        public DateTime TimeSocketConnected = DateTime.Now;
        public DateTime TimeSocketDisconnected;
        public DateTime TimeLastDataPacketWasReceived;
        public DateTime TimeParsingRoutineStarted;
        public DateTime TimeParsingRoutineFinished;

        public double TotalSecondsConnected
        {
            get { return TimeSocketDisconnected.Subtract(TimeSocketConnected).TotalSeconds; }
        }

        public int ReceiveTimeout = int.Parse(System.Configuration.ConfigurationManager.AppSettings["TCPListenerV3_ReceiveTimeout"]); //15;
        private System.Timers.Timer _timer = new System.Timers.Timer();
        public bool IsDisposed { get; private set; }

        public void StartReceive()
        {
            _timer.Interval = ReceiveTimeout * 1000;
            _timer.Elapsed += _timer_Elapsed;
            _timer.Start();
        }

        public void ResetReceiveTimer()
        {
            _timer.Stop();
            _timer.Start();
        }

        private void _timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
        {
            try
            {
                DebugData.AppendLine(string.Format("[SOCKET RECEIVE TIMEOUT OCCURRED] - Ip: {0} Has not Sent any data for {1} seconds, and didn't disconnect on it's own. Byte Count Received: {2}",IpAddress, ReceiveTimeout, ByteCountReceived));

                    if (!IsDisposed)
                    {
                        DebugData.AppendLine(string.Format("[SOCKET STATISTICS*] - Ip: {0}, Time Socket Connected: {1}, Time Socket Disconnected: {2}, Time Last Packet Received: {3}, Total Bytes Recvd: {4}, Total Seconds Connected: {5}"
                           , IpAddress, TimeSocketConnected, TimeSocketDisconnected, TimeLastDataPacketWasReceived, ByteCountReceived, TotalSecondsConnected));
                        workSocket.Disconnect(false);
                        workSocket.Shutdown(SocketShutdown.Both);
                        workSocket.Close();
                    }

                else
                {
                    //socket isn't connected, stop the timer
                    _timer.Dispose();
                }
            }
            catch (Exception ex)
            {
                //removed for reading purposes, just logged message to event log
            }
        }

        public void Dispose()
        {
            Dispose(true);
            GC.SuppressFinalize(this);
        }

        protected virtual void Dispose(bool disposing)
        {
            try
            {
                if (!IsDisposed)
                {
                    if (disposing)
                    {
                        if (workSocket != null)
                        {
                            try //added on 6/10/2013
                            {
                                _timer.Dispose();
                                if (ME3ProsoftDataStreamService.listen.SocketConnected(workSocket))
                                {
                                    TimeSocketDisconnected = DateTime.Now;
                                    workSocket.Disconnect(false);
                                    workSocket.Shutdown(SocketShutdown.Both);
                                    workSocket.Close();
                                }

                            }
                            catch (Exception ex1)
                            {
                                //removed for reading purposes, just logged message to event log
                            }
                        }
                    }
                }
                workSocket = null;
                buffer = null;
                //DebugData.Length = 0;
                IpAddress = null;
                IsDisposed = true;
            }
            catch (Exception ex)
            {
                //removed for reading purposes, just logged message to event log
            }
        }
    }