WSASend completion routine has never been called

Question

I'm playing around with Overlapped IO and suddenly found out that it looks like I'm the only one who can't encourage Completion callback to work (All claims was about: it works and I don't like it).

The idea of my application is: a client (telnet localhost 27015) connects to the server and server starts pushing huge amount of data to the the client. And I've never had CompletionCallback called.

Here is the code:

#include <winsock2.h>
#include <ws2tcpip.h>
#include <atomic>

#pragma comment(lib, "ws2_32.lib")
#define DATA_BUFSIZE 16384

class CSync
{
private:
    CRITICAL_SECTION  m_cs;

public:
    CSync()     { ZeroMemory(&m_cs, sizeof(m_cs)); InitializeCriticalSection(&m_cs); }
    ~CSync()    { DeleteCriticalSection(&m_cs);   ZeroMemory(&m_cs, sizeof(m_cs)); }
    inline void Lock()   { EnterCriticalSection(&m_cs); }
    inline void Unlock() { LeaveCriticalSection(&m_cs); }
    inline BOOL WINAPI TryLock() { return TryEnterCriticalSection(&m_cs); }
};

class ScopedLock
{
public:
    ScopedLock(CSync& lock) : m_lock(lock) { m_lock.Lock(); }
    ~ScopedLock() { m_lock.Unlock(); }

private:
    CSync m_lock;
};


class SendServer
{
private:
    SOCKET                socket;
    //  std::atomic<bool>     busy;
    char                  buffer[2][DATA_BUFSIZE];
    CSync                 syncer;
    WSABUF                wsabuf;
    OVERLAPPED            overlapped;
    //  HANDLE                socketEvent;
    std::atomic_flag      busy;
    char                  toBuffer; // 0 or 1
    DWORD                 sent;

public:
    SendServer(SOCKET& sock);
    virtual ~SendServer();
    bool Write(char* buff);
};


static void __stdcall Produce(SendServer *server);
void CALLBACK CompletionCallback(DWORD dwError, DWORD cbTransferred, LPWSAOVERLAPPED lpOverlapped, DWORD dwFlags);



static bool run = 1;

int main(int argc, char* argv[])
{
    WSADATA wsd;

    struct addrinfo *result = NULL;
    struct addrinfo hints;

    SOCKET ListenSocket = INVALID_SOCKET;
    SOCKET AcceptSocket = INVALID_SOCKET;

    int err = 0;
    int rc;

    // Load Winsock
    rc = WSAStartup((2, 2), &wsd);
    if (rc != 0) {
        printf("Unable to load Winsock: %d\n", rc);
        return 1;
    }

    // Make sure the hints struct is zeroed out
    SecureZeroMemory((PVOID)& hints, sizeof(struct addrinfo));

    // Initialize the hints to obtain the 
    // wildcard bind address for IPv4
    hints.ai_family = AF_INET;
    hints.ai_socktype = SOCK_STREAM;
    hints.ai_protocol = IPPROTO_TCP;
    hints.ai_flags = AI_PASSIVE;

    rc = getaddrinfo(NULL, "27015", &hints, &result);
    if (rc != 0) {
        printf("getaddrinfo failed with error: %d\n", rc);
        return 1;
    }

    ListenSocket = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
        //socket(result->ai_family, result->ai_socktype, result->ai_protocol);
    if (ListenSocket == INVALID_SOCKET) {
        printf("socket failed with error: %d\n", WSAGetLastError());
        freeaddrinfo(result);
        return 1;
    }

    rc = bind(ListenSocket, result->ai_addr, (int)result->ai_addrlen);
    if (rc == SOCKET_ERROR) {
        printf("bind failed with error: %d\n", WSAGetLastError());
        freeaddrinfo(result);
        closesocket(ListenSocket);
        return 1;
    }

    rc = listen(ListenSocket, 1);
    if (rc == SOCKET_ERROR) {
        printf("listen failed with error: %d\n", WSAGetLastError());
        freeaddrinfo(result);
        closesocket(ListenSocket);
        return 1;
    }
    // Accept an incoming connection request
    AcceptSocket = accept(ListenSocket, NULL, NULL);
    if (AcceptSocket == INVALID_SOCKET) {
        printf("accept failed with error: %d\n", WSAGetLastError());
        freeaddrinfo(result);
        closesocket(ListenSocket);
        return 1;
    }

    printf("Client Accepted...\n");

    SendServer server(AcceptSocket);
    HANDLE h[1];

    h[0] = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)&Produce, &server, 0, NULL);

    getchar();
    run = 0;
    WaitForMultipleObjects(1, h, TRUE, INFINITE);

    return 0;
}

void __stdcall Produce(SendServer *server)
{
    char buf[] = "------------------------------------------------------------------------------------------";
    char s = 0;

    while (run) {
        buf[0] = '0' + s++;

        if (s > 9)
            s = 0;

        server->Write(buf);
        Sleep(10);
    }
}

void CALLBACK CompletionCallback(DWORD dwError, DWORD cbTransferred, LPWSAOVERLAPPED lpOverlapped, DWORD dwFlags)
{
    ((SendServer*)(lpOverlapped->hEvent))->Write(NULL);
}

SendServer::SendServer(SOCKET& sock) : toBuffer(0)
{
    socket = sock;
    ZeroMemory(buffer, DATA_BUFSIZE << 1);
    busy.clear();
}

SendServer::~SendServer()
{
    shutdown(socket, 2);
    closesocket(socket);
}

bool SendServer::Write(char* buff)
{
    ScopedLock lock(syncer);
    int size = strlen(buffer[toBuffer]), toAdd = 0;

    if (buff == NULL) {
        busy.clear();
        SecureZeroMemory(buffer[!toBuffer], DATA_BUFSIZE);
    }
    else {
        toAdd = strlen(buff);
        if (size + toAdd < DATA_BUFSIZE) {
            memcpy_s(buffer[toBuffer] + size, toAdd, buff, toAdd);
            size += toAdd;
            buffer[toBuffer][size] = 0;
            return TRUE;
        }
        else {
            printf("\nCan't add anymore!\n");
        }
    }

    if (size > 0 && !busy.test_and_set()) {
        wsabuf.buf = (char*)buffer[toBuffer];
        wsabuf.len = size;

        SecureZeroMemory(&overlapped, sizeof OVERLAPPED);
        overlapped.hEvent = this;

        toBuffer = !toBuffer;
        size = WSASend(socket, &wsabuf, 1, &sent, 0, &overlapped, CompletionCallback);
        if (size == 0) {
            //return Write(NULL);
        }
        if (WSA_IO_PENDING != WSAGetLastError()) {
            return FALSE;
        }
    }
    return TRUE;
}

Thank you.

Answer 1

Completion callbacks are invoked during alertable wait. You have no alertable wait, so the completion callbacks get queued up but never get a chance to run.

Change WaitForMultipleObjects to a loop with WaitForMultipleObjectsEx and Sleep to SleepEx, and pass TRUE as the bAlertable parameter.

This is explained right in the WSASend documentation

The completion routine follows the same rules as stipulated for Windows file I/O completion routines. The completion routine will not be invoked until the thread is in an alertable wait state such as can occur when the function WSAWaitForMultipleEvents with the fAlertable parameter set to TRUE is invoked.