multi threaded TCP server in C crashes

Question

I wrote a multi threaded TCP server in C. It crashes when I run multiple test client against it. I get 2 type of crashes which both seems to have same root cause. The attached crash happens in main. In the other flavor, I get the crash in client part when it wants to lock in order to close the socket. Can somebody please tell me what causes the crash?

TCP Server code:

#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <unistd.h>
#include <arpa/inet.h> //inet_addr
#include <pthread.h> //for threading , link with lpthread

#define BUFSIZE     2048 // TODO
#define MAXWORKERS 10

pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
const pthread_cond_t condInit = PTHREAD_COND_INITIALIZER;
pthread_cond_t idle = PTHREAD_COND_INITIALIZER;

typedef struct {
    pthread_t       tid;
    int             sd;
    pthread_cond_t  cond;
    uint8_t         num;
    uint8_t         state;
} worker_t;

worker_t     workers[MAXWORKERS]; //  reference to worker

void error(const char *msg)
{
    perror(msg);
    exit(1);
}

/* worker thread */
void *handle_client(void *arg)
{
    worker_t* worker = (worker_t *) arg;
    int n;
    char buf[BUFSIZE];

    /* By default a new thread is joinable, we don't
     really want this (unless we do something special we
     end up with the thread equivalent of zombies). So
     we explicitly change the thread type to detached
     */

    pthread_detach(pthread_self());

    printf("Thread %ld started for client number %d (sd %d)\n", pthread_self(),
            worker->num, worker->sd);

    /* worker thread */
    while (1)
    {
        /* wait for work to do */
        while (worker->state == 0)
        {
            pthread_cond_wait(&worker->cond, &mtx);
        }
        int sd = worker->sd; /* get the updated socket fd */
        pthread_mutex_unlock(&mtx);

         n = read(sd, buf, BUFSIZE);
         if (n < 0)
             error("ERROR reading from socket");

         n = write(sd, "I got your message",18);
         if (n < 0)
             error("ERROR writing to socket");

        /* work done - set itself idle assumes that read returned EOF */
        pthread_mutex_lock(&mtx);
        close(sd);
        worker->state = 0;
        printf("Worker %d has completed work \n", worker->num);
        pthread_cond_signal(&idle); /* notifies dispatcher*/
        pthread_mutex_unlock(&mtx);
    } /* end while */

}

int main() { /* Dispatcher */
    int ld, sd;
    struct sockaddr_in skaddr;
    struct sockaddr_in from;
    int addrlen, length;
    int i;

    if ((ld = socket( AF_INET, SOCK_STREAM, 0)) < 0) {
        perror("Problem creating socket\n");
        exit(1);
    }

    skaddr.sin_family = AF_INET;
    skaddr.sin_addr.s_addr = htonl(INADDR_ANY);

    //skaddr.sin_port = htons(0);
    skaddr.sin_port = htons(41332);

    if (bind(ld, (struct sockaddr *) &skaddr, sizeof(skaddr)) < 0) {
        perror("Problem binding\n");
        exit(0);
    }

    /* find out what port we were assigned and print it out */

    length = sizeof(skaddr);
    if (getsockname(ld, (struct sockaddr *) &skaddr, &length) < 0) {
        perror("Error getsockname\n");
        exit(1);
    }
    in_port_t pport = ntohs(skaddr.sin_port);
    printf("%d\n", pport);

    /* put the socket into passive mode (waiting for connections) */

    if (listen(ld, 5) < 0) {
        perror("Error calling listen\n");
        exit(1);
    }

    /* do some initialization */

    for (i = 0; i < MAXWORKERS; i++) {
        workers[i].state    = 0;
        workers[i].num      = i;
        workers[i].sd       = 0;
        workers[i].cond     = condInit;
        pthread_create(&workers[i].tid, NULL, handle_client, (void *) &workers[i]);
    }

    /* Dispatcher now processes incoming connections forever ... */
    while (1) {
        printf("Ready for a connection...\n");
        addrlen = sizeof(skaddr);

        printf("trying to accept a new connection\n");
        if ((sd = accept(ld, (struct sockaddr*) &from, &addrlen)) < 0) {
            perror("Problem with accept call\n");
            exit(1);
        }
        printf("Got a connection - processing...\n");
        for (i = 0; i < MAXWORKERS; i++) {
            pthread_mutex_lock(&mtx);
            if (workers[i].state == 0) /* worker i is idle – dispatch him to work */
            {
                printf("dispatch to worker number: %d \n", i);
                pthread_mutex_unlock(&mtx);
                break;
            }
            printf("worker number: %d was busy\n", i);

            pthread_mutex_unlock(&mtx);
        }

        if (i == MAXWORKERS) {
            /* all workers busy */
            pthread_mutex_lock(&mtx);
            pthread_cond_wait(&idle, &mtx); /* wait for one idle; */
            pthread_mutex_unlock(&mtx);
        } else { /* dispatch worker */
            pthread_mutex_lock(&mtx);
            workers[i].state = 1;
            workers[i].sd = sd;
            pthread_cond_signal(&workers[i].cond); /* wake up worker */
            pthread_mutex_unlock(&mtx);
        }
    }
}

crash:

Thread 11 (Thread 0x7fae90004700 (LWP 16318)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000401024 in handle_client (arg=0x602408 <workers+648>) at main.c:87
#2  0x00007fae94bdf6ba in start_thread (arg=0x7fae90004700) at pthread_create.c:333
#3  0x00007fae949153dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 10 (Thread 0x7fae90805700 (LWP 16317)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000401024 in handle_client (arg=0x6023c0 <workers+576>) at main.c:87
#2  0x00007fae94bdf6ba in start_thread (arg=0x7fae90805700) at pthread_create.c:333
#3  0x00007fae949153dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 9 (Thread 0x7fae9400c700 (LWP 16310)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000401024 in handle_client (arg=0x6021c8 <workers+72>) at main.c:87
#2  0x00007fae94bdf6ba in start_thread (arg=0x7fae9400c700) at pthread_create.c:333
#3  0x00007fae949153dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 8 (Thread 0x7fae91006700 (LWP 16316)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000401024 in handle_client (arg=0x602378 <workers+504>) at main.c:87
#2  0x00007fae94bdf6ba in start_thread (arg=0x7fae91006700) at pthread_create.c:333
#3  0x00007fae949153dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 7 (Thread 0x7fae91807700 (LWP 16315)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000401024 in handle_client (arg=0x602330 <workers+432>) at main.c:87
#2  0x00007fae94bdf6ba in start_thread (arg=0x7fae91807700) at pthread_create.c:333
#3  0x00007fae949153dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 6 (Thread 0x7fae9300a700 (LWP 16312)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000401024 in handle_client (arg=0x602258 <workers+216>) at main.c:87
#2  0x00007fae94bdf6ba in start_thread (arg=0x7fae9300a700) at pthread_create.c:333
#3  0x00007fae949153dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 5 (Thread 0x7fae92008700 (LWP 16314)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000401024 in handle_client (arg=0x6022e8 <workers+360>) at main.c:87
#2  0x00007fae94bdf6ba in start_thread (arg=0x7fae92008700) at pthread_create.c:333
#3  0x00007fae949153dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 4 (Thread 0x7fae92809700 (LWP 16313)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000401024 in handle_client (arg=0x6022a0 <workers+288>) at main.c:87
#2  0x00007fae94bdf6ba in start_thread (arg=0x7fae92809700) at pthread_create.c:333
#3  0x00007fae949153dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 3 (Thread 0x7fae9380b700 (LWP 16311)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000401024 in handle_client (arg=0x602210 <workers+144>) at main.c:87
#2  0x00007fae94bdf6ba in start_thread (arg=0x7fae9380b700) at pthread_create.c:333
#3  0x00007fae949153dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 2 (Thread 0x7fae9480d700 (LWP 16309)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000401024 in handle_client (arg=0x602180 <workers>) at main.c:87
#2  0x00007fae94bdf6ba in start_thread (arg=0x7fae9480d700) at pthread_create.c:333
#3  0x00007fae949153dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 1 (Thread 0x7fae94ff3700 (LWP 16308)):
#0  0x00007fae94843428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
---Type <return> to continue, or q <return> to quit---
#1  0x00007fae9484502a in __GI_abort () at abort.c:89
#2  0x00007fae9483bbd7 in __assert_fail_base (fmt=<optimized out>, assertion=assertion@entry=0x7fae94beb015 "mutex->__data.__owner == 0", file=file@entry=0x7fae94beaff8 "../nptl/pthread_mutex_lock.c", line=line@entry=81, 
    function=function@entry=0x7fae94beb180 <__PRETTY_FUNCTION__.8623> "__pthread_mutex_lock") at assert.c:92
#3  0x00007fae9483bc82 in __GI___assert_fail (assertion=assertion@entry=0x7fae94beb015 "mutex->__data.__owner == 0", file=file@entry=0x7fae94beaff8 "../nptl/pthread_mutex_lock.c", line=line@entry=81, 
    function=function@entry=0x7fae94beb180 <__PRETTY_FUNCTION__.8623> "__pthread_mutex_lock") at assert.c:101
#4  0x00007fae94be1f68 in __GI___pthread_mutex_lock (mutex=mutex@entry=0x602140 <mtx>) at ../nptl/pthread_mutex_lock.c:81
#5  0x0000000000400cfa in main () at main.c:173

Answer 1

In your worker thread, the first thing that happens is either a pthread_cond_wait without having first locked the mutex or a pthread_mutex_unlock without having first locked the mutex. In both cases, this is undefined behaviour according to the POSIX docs.

Your worker function should start thus:

    /* worker thread */
while (1)
{
    /* wait for work to do */
    pthread_mutex_lock(&mtx);
    while (worker->state == 0)
    {
        pthread_cond_wait(&worker->cond, &mtx);
    }
    int sd = worker->sd; /* get the updated socket fd */
    pthread_mutex_unlock(&mtx);