epoll-based server fails (mutual authentication)

Question

Based on the examples at http://simplestcodings.blogspot.com/2010/08/secure-server-client-using-openssl-in-c.html I have written the following epoll server code to replace the regular tcp sockets with SSL. My code is slightly modified to do mutual authentication and verify the certs since I add

SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL); SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);.

My OS is Ubuntu 12.04.

I have also generated server cert, client cert, server key, client key, and of course cacert.pem files from the IBM tutorial http://pic.dhe.ibm.com/infocenter/lnxinfo/v3r0m0/index.jsp?topic=%2Fliaat%2Fliaatseccreatecskeycert.htm

I ran the single server - single client code and it runs perfectly fine. Inspired by this I wanted to see how the epoll server code would behave but then I ran into issues. Here is the code for the server

Code that does not work

//(c) 2014 enthusiasticgeek for stackoverflow - epollserver.cc
//============================================================================

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/epoll.h>
#include <errno.h>
#include <iostream>
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <resolv.h>
#include "openssl/ssl.h"
#include "openssl/err.h"

#define FAIL    -1

#define MAXEVENTS 64


SSL_CTX* InitServerCTX(void)
{   const SSL_METHOD *method;
    SSL_CTX *ctx;

    OpenSSL_add_all_algorithms();  /* load & register all cryptos, etc. */
    SSL_load_error_strings();   /* load all error messages */
    method = SSLv23_server_method();  /* create new server-method instance */
    ctx = SSL_CTX_new(method);   /* create new context from method */
    if ( ctx == NULL )
    {
    ERR_print_errors_fp(stderr);
    abort();
    }
    return ctx;
}

void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{

    SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
    SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);

    /* set the local certificate from CertFile */
    if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0 )
    {
    ERR_print_errors_fp(stderr);
    abort();
    }
    /* set the private key from KeyFile (may be the same as CertFile) */
    if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
    {
    ERR_print_errors_fp(stderr);
    abort();
    }
    /* verify private key */
    if ( !SSL_CTX_check_private_key(ctx) )
    {
    fprintf(stderr, "Private key does not match the public certificate\n");
    abort();
    }
}

void ShowCerts(SSL* ssl)
{   X509 *cert;
    char *line;

    cert = SSL_get_peer_certificate(ssl); /* Get certificates (if available) */
    if ( cert != NULL )
    {
    printf("Server certificates:\n");
    line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
    printf("Subject: %s\n", line);
    free(line);
    line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
    printf("Issuer: %s\n", line);
    free(line);
    X509_free(cert);
    }
    else
    printf("No certificates.\n");
}

static int
AibSocketNonBlocking (int sfd)
{
    int flags, s;

    flags = fcntl (sfd, F_GETFL, 0);
    if (flags == -1)
    {
    perror ("fcntl");
    return -1;
    }

    flags |= O_NONBLOCK;
    s = fcntl (sfd, F_SETFL, flags);
    if (s == -1)
    {
    perror ("fcntl");
    return -1;
    }

    return 0;
}

static int
AibCreateAndBind (char *port)
{
    struct addrinfo hints;
    struct addrinfo *result, *rp;
    int s, sfd;

    memset (&hints, 0, sizeof (struct addrinfo));
    hints.ai_family = AF_UNSPEC;     /* Return IPv4 and IPv6 choices */
    hints.ai_socktype = SOCK_STREAM; /* We want a TCP socket */
    hints.ai_flags = AI_PASSIVE;     /* All interfaces */

    s = getaddrinfo (NULL, port, &hints, &result);
    if (s != 0)
    {
    fprintf (stderr, "getaddrinfo: %s\n", gai_strerror (s));
    return -1;
    }

    for (rp = result; rp != NULL; rp = rp->ai_next)
    {
    sfd = socket (rp->ai_family, rp->ai_socktype, rp->ai_protocol);
    if (sfd == -1)
        continue;

    s = bind (sfd, rp->ai_addr, rp->ai_addrlen);
    if (s == 0)
    {
        /* We managed to bind successfully! */
        break;
    }

    close (sfd);
    }

    if (rp == NULL)
    {
    fprintf (stderr, "Could not bind\n");
    return -1;
    }

    freeaddrinfo (result);

    return sfd;
}

int
main (int argc, char *argv[])
{
    SSL_CTX *ctx;
    SSL_library_init();
    ctx = InitServerCTX();        /* initialize SSL */
    LoadCertificates(ctx, "/home/test/ssl/cert_files/servercert.pem", "/home/test/ssl/cert_files/serverkey.pem"); /* load certs */

    int sfd, s;
    int efd;
    struct epoll_event aibevent;
    struct epoll_event *aibevents;
    if (argc != 2) {
    fprintf (stderr, "Usage: %s [port]\n", argv[0]);
    exit (EXIT_FAILURE);
    }

    char portt[sizeof (unsigned int)];
    snprintf(portt,sizeof portt + 1,"%u",atoi(argv[1]));

    printf("sizeof %s argv[1] = %d\n",argv[1], sizeof(argv[1]));
    printf("sizeof %s portt = %d\n",portt, sizeof(portt));

    sfd = AibCreateAndBind (portt);//argv[1]);
    if (sfd == -1) {
    abort ();
    }
    s = AibSocketNonBlocking (sfd);
    if (s == -1) {
    abort ();
    }
    s = listen (sfd, SOMAXCONN);
    if (s == -1) {
    perror ("listen");
    abort ();
    }
    efd = epoll_create1 (0);
    if (efd == -1) {
    perror ("epoll_create");
    abort ();
    }
    aibevent.data.fd = sfd;
    aibevent.events = EPOLLIN | EPOLLET;
    s = epoll_ctl (efd, EPOLL_CTL_ADD, sfd, &aibevent);
    if (s == -1) {
    perror ("epoll_ctl");
    abort ();
    }

    // Buffer where events are returned
    //events = static_cast<epoll_event*>(calloc (MAXEVENTS, sizeof event));
    //aibevents = static_cast<epoll_event*>(malloc (MAXEVENTS * sizeof aibevent));
    aibevents = new epoll_event[MAXEVENTS * sizeof aibevent];

    // The event loop
    while (true)
    {
    int n, i;

    n = epoll_wait (efd, aibevents, MAXEVENTS, -1);
    for (i = 0; i < n; i++)
    {
        if ((aibevents[i].events & EPOLLERR) ||
                (aibevents[i].events & EPOLLHUP) ||
                (!(aibevents[i].events & EPOLLIN)))
        {
            // An error has occured on this fd, or the socket is not
            // ready for reading (why were we notified then?)
            fprintf (stderr, "epoll error\n");
            close (aibevents[i].data.fd);
            continue;
        } else if (sfd == aibevents[i].data.fd) {
            // We have a notification on the listening socket, which
            // means one or more incoming connections.
            while (1)
            {

                struct sockaddr in_addr;
                socklen_t in_len;
                int infd;
                char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];

                in_len = sizeof in_addr;
                infd = accept (sfd, &in_addr, &in_len);
                if (infd == -1)
                {
                    if ((errno == EAGAIN) ||(errno == EWOULDBLOCK)) {
                        // We have processed all incoming
                        // connections.
                        break;
                    } else {
                        perror ("accept");
                        break;
                    }
                }

                s = getnameinfo (&in_addr, in_len,
                                 hbuf, sizeof hbuf,
                                 sbuf, sizeof sbuf,
                                 NI_NUMERICHOST | NI_NUMERICSERV);
                if (s == 0) {
                    printf("Accepted connection on descriptor %d "
                           "(host=%s, port=%s)\n", infd, hbuf, sbuf);
                }

                // Make the incoming socket non-blocking and add it to the
                // list of fds to monitor.
                s = AibSocketNonBlocking (infd);
                if (s == -1) {
                    abort ();
                }
                aibevent.data.fd = infd;
                aibevent.events = EPOLLIN | EPOLLET;
                s = epoll_ctl (efd, EPOLL_CTL_ADD, infd, &aibevent);
                if (s == -1) {
                    perror ("epoll_ctl");
                    abort ();
                }



            }
            continue;
        } else {
            // We have data on the fd waiting to be read. Read and
            // display it. We must read whatever data is available
            // completely, as we are running in edge-triggered mode
            // and won't get a notification again for the same
            // data.
            int done = 0;
            int sd;

            SSL *ssl;

            while (1)
            {
                ssize_t count;
                char buf[1024];
                char reply[1024];

                printf("Performing exchange.\n");

                ssl = SSL_new(ctx);              /* get new SSL state with context */
                SSL_set_fd(ssl, aibevents[i].data.fd);      /* set connection socket to SSL state */

                printf("Performing exchange 1.\n");

                const char* HTMLecho="<html><body><pre>%s</pre></body></html>\n\n";

                if ( SSL_accept(ssl) == FAIL ) {    /* do SSL-protocol accept */
                    ERR_print_errors_fp(stderr);
                    printf("Performing exchange Error 1.\n");
                    done = 1;
                    break;
                } else {
                    ShowCerts(ssl);        /* get any certificates */
                    count = SSL_read(ssl, buf, sizeof(buf)); /* get request */
                    if ( count > 0 )
                    {
                        buf[count] = 0;
                        printf("Client msg: \"%s\"\n", buf);
                        sprintf(reply, HTMLecho, buf);   /* construct reply */
                        SSL_write(ssl, reply, strlen(reply)); /* send reply */
                    } else {
                        ERR_print_errors_fp(stderr);
                        printf("Performing exchange Error 2.\n");
                        done = 1;
                        break;
                    }
                }
                sd = SSL_get_fd(ssl);       /* get socket connection */
                /*
                                    count = read (aibevents[i].data.fd, buf, sizeof buf);
                                    if (count == -1)
                                    {
                                        // If errno == EAGAIN, that means we have read all
                                        // data. So go back to the main loop.
                                        if (errno != EAGAIN)
                                        {
                                            perror ("read");
                                            done = 1;
                                        }
                                        break;
                                    }
                                    else if (count == 0)
                                    {
                                        // End of file. The remote has closed the
                                        // connection.
                                        done = 1;
                                        break;
                                    }
                                    // Write the buffer to standard output
                                    s = write (1, buf, count);
                                    if (s == -1)
                                    {
                                        perror ("write");
                                        abort ();
                                    }
                                    printf(" read correctly (n > 0) n==%d\n",s);
                                    printf("msg: %s\n", buf);
                                    write(aibevents[i].data.fd, buf,s);
                                    memset(buf,'\0', s);
                */
            }
            if (done)
            {
                printf("Freeing data.\n");
                SSL_free(ssl);         /* release SSL state */
                close(sd);          /* close connection */

                //printf ("Closed connection on descriptor %d\n",
                //        aibevents[i].data.fd);
                // Closing the descriptor will make epoll remove it
                // from the set of descriptors which are monitored.
                //close (aibevents[i].data.fd);
            }
        }
    }
    }
    //free (aibevents);
    delete[] aibevents;
    close (sfd);


    SSL_CTX_free(ctx);         /* release context */

    return EXIT_SUCCESS;
}

The single server- single client code are listed below for reference. Note that I am using the same client code to interact with the epoll server.

Single server code. Works

//SSL-Single Server code that works ! used as a reference for epoll server
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <resolv.h>
#include "openssl/ssl.h"
#include "openssl/err.h"

#define FAIL    -1

int OpenListener(int port)
{   int sd;
    struct sockaddr_in addr;

    sd = socket(PF_INET, SOCK_STREAM, 0);
    bzero(&addr, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port = htons(port);
    addr.sin_addr.s_addr = INADDR_ANY;
    if ( bind(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0 )
    {
    perror("can't bind port");
    abort();
    }
    if ( listen(sd, 10) != 0 )
    {
    perror("Can't configure listening port");
    abort();
    }
    return sd;
}

SSL_CTX* InitServerCTX(void)
{   const SSL_METHOD *method;
    SSL_CTX *ctx;

    OpenSSL_add_all_algorithms();  /* load & register all cryptos, etc. */
    SSL_load_error_strings();   /* load all error messages */
    method = SSLv23_server_method();  /* create new server-method instance */
    ctx = SSL_CTX_new(method);   /* create new context from method */
    if ( ctx == NULL )
    {
    ERR_print_errors_fp(stderr);
    abort();
    }
    return ctx;
}

void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{

    SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
    SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);

    /* set the local certificate from CertFile */
    if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0 )
    {
    ERR_print_errors_fp(stderr);
    abort();
    }
    /* set the private key from KeyFile (may be the same as CertFile) */
    if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
    {
    ERR_print_errors_fp(stderr);
    abort();
    }
    /* verify private key */
    if ( !SSL_CTX_check_private_key(ctx) )
    {
    fprintf(stderr, "Private key does not match the public certificate\n");
    abort();
    }
}

void ShowCerts(SSL* ssl)
{   X509 *cert;
    char *line;

    cert = SSL_get_peer_certificate(ssl); /* Get certificates (if available) */
    if ( cert != NULL )
    {
    printf("Server certificates:\n");
    line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
    printf("Subject: %s\n", line);
    free(line);
    line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
    printf("Issuer: %s\n", line);
    free(line);
    X509_free(cert);
    }
    else
    printf("No certificates.\n");
}

void Servlet(SSL* ssl) /* Serve the connection -- threadable */
{   char buf[1024];
    char reply[1024];
    int sd, bytes;
    const char* HTMLecho="<html><body><pre>%s</pre></body></html>\n\n";

    if ( SSL_accept(ssl) == FAIL )     /* do SSL-protocol accept */
    ERR_print_errors_fp(stderr);
    else
    {
    ShowCerts(ssl);        /* get any certificates */
    bytes = SSL_read(ssl, buf, sizeof(buf)); /* get request */
    if ( bytes > 0 )
    {
        buf[bytes] = 0;
        printf("Client msg: \"%s\"\n", buf);
        sprintf(reply, HTMLecho, buf);   /* construct reply */
        SSL_write(ssl, reply, strlen(reply)); /* send reply */
    }
    else
        ERR_print_errors_fp(stderr);
    }
    sd = SSL_get_fd(ssl);       /* get socket connection */
    SSL_free(ssl);         /* release SSL state */
    close(sd);          /* close connection */
}

int main(int count, char *strings[])
{   SSL_CTX *ctx;
    int server;
    char *portnum;

    if ( count != 2 )
    {
    printf("Usage: %s <portnum>\n", strings[0]);
    exit(0);
    }
    SSL_library_init();
    portnum = strings[1];
    ctx = InitServerCTX();        /* initialize SSL */
    LoadCertificates(ctx, "/home/test/ssl/cert_files/servercert.pem", "/home/test/ssl/cert_files/serverkey.pem"); /* load certs */
    server = OpenListener(atoi(portnum));    /* create server socket */
    while (1)
    {   struct sockaddr_in addr;
    socklen_t len = sizeof(addr);
    SSL *ssl;

    int client = accept(server, (struct sockaddr*)&addr, &len);  /* accept connection as usual */
    printf("Connection: %s:%d\n",inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));
    ssl = SSL_new(ctx);              /* get new SSL state with context */
    SSL_set_fd(ssl, client);      /* set connection socket to SSL state */
    Servlet(ssl);         /* service connection */
    }
    close(server);          /* close server socket */
    SSL_CTX_free(ctx);         /* release context */
} 

Single client code Works

// Single Client Code that works!
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <sys/socket.h>
#include <resolv.h>
#include <netdb.h>
#include <openssl/ssl.h>
#include <openssl/err.h>

#define FAIL    -1


//Added the LoadCertificates how in the server-side makes.
void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{

    SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
    SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);
    /* set the local certificate from CertFile */
    if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0 )
    {
    ERR_print_errors_fp(stderr);
    abort();
    }
    /* set the private key from KeyFile (may be the same as CertFile) */
    if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
    {
    ERR_print_errors_fp(stderr);
    abort();
    }
    /* verify private key */
    if ( !SSL_CTX_check_private_key(ctx) )
    {
    fprintf(stderr, "Private key does not match the public certificate\n");
    abort();
    }
}

int OpenConnection(const char *hostname, int port)
{   int sd;
    struct hostent *host;
    struct sockaddr_in addr;

    if ( (host = gethostbyname(hostname)) == NULL )
    {
    perror(hostname);
    abort();
    }
    sd = socket(PF_INET, SOCK_STREAM, 0);
    bzero(&addr, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port = htons(port);
    addr.sin_addr.s_addr = *(long*)(host->h_addr);
    if ( connect(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0 )
    {
    close(sd);
    perror(hostname);
    abort();
    }
    return sd;
}

SSL_CTX* InitCTX(void)
{   const SSL_METHOD *method;
    SSL_CTX *ctx;

    OpenSSL_add_all_algorithms(); /* Load cryptos, et.al. */
    SSL_load_error_strings(); /* Bring in and register error messages */
    method = SSLv23_client_method(); /* Create new client-method instance */
    ctx = SSL_CTX_new(method); /* Create new context */
    if ( ctx == NULL )
    {
    ERR_print_errors_fp(stderr);
    abort();
    }
    return ctx;
}

void ShowCerts(SSL* ssl)
{   X509 *cert;
    char *line;

    cert = SSL_get_peer_certificate(ssl); /* get the server's certificate */
    if ( cert != NULL )
    {
    printf("Server certificates:\n");
    line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
    printf("Subject: %s\n", line);
    free(line); /* free the malloc'ed string */
    line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
    printf("Issuer: %s\n", line);
    free(line); /* free the malloc'ed string */
    X509_free(cert); /* free the malloc'ed certificate copy */
    }
    else
    printf("No certificates.\n");
}

int main(int count, char *strings[])
{
    SSL_CTX *ctx;
    int server;
    SSL *ssl;
    char buf[1024];
    int bytes;
    if ( count != 3 )
    {
    printf("Usage: %s <host/ip> <portnum>\n", strings[0]);
    exit(0);
    }

    char* hostname=strings[1];//"127.0.0.1";
    char* portnum=strings[2];//"3334";
    char CertFile[] = "/home/test/ssl/cert_files/clientcert.pem";
    char KeyFile[] = "/home/test/ssl/cert_files/clientkey.pem";

    SSL_library_init();

    ctx = InitCTX();
    LoadCertificates(ctx, CertFile, KeyFile);
    server = OpenConnection(hostname, atoi(portnum));
    ssl = SSL_new(ctx); /* create new SSL connection state */
    SSL_set_fd(ssl, server); /* attach the socket descriptor */
    if ( SSL_connect(ssl) == FAIL ) /* perform the connection */
    ERR_print_errors_fp(stderr);
    else
    {   char *msg = "Hello???";

    printf("Connected with %s encryption\n", SSL_get_cipher(ssl));
    ShowCerts(ssl); /* get any certs */
    SSL_write(ssl, msg, strlen(msg)); /* encrypt & send message */
    bytes = SSL_read(ssl, buf, sizeof(buf)); /* get reply & decrypt */
    buf[bytes] = 0;
    printf("Received: \"%s\"\n", buf);
    SSL_free(ssl); /* release connection state */
    }
    close(server); /* close socket */
    SSL_CTX_free(ctx); /* release context */
    return 0;
}

I did compilation using g++ -g <src>.cc -o <src> -lssl -lcrypto for each of the three files (I had a Makefile)

The error that see is

./tcpserver_epoll 3334 sizeof 3334 argv[1] = 4 sizeof 3334 portt = 4 Accepted connection on descriptor 5 (host=127.0.0.1, port=44716) Performing exchange. Performing exchange 1. Performing exchange Error 1. Freeing data. Performing exchange. Performing exchange 1. 3072792824:error:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol:s23_srvr.c:628: Performing exchange Error 1. Freeing data.

I am wondering if I have to create multiple instances of SSL_CTX *ctx;. I am a bit lost. Any help is appreciated. Thanks.

Update: Thanks to everyone for guidance. I found an example here that closely matches what I want http://bendecplusplus.googlecode.com/svn/trunk/ssl_mycode/epoll_ssl/server.c http://bendecplusplus.googlecode.com/svn/trunk/ssl_mycode/epoll_ssl/client.c

Answer 1

I am wondering if I have to create multiple instances of SSL_CTX *ctx;

No, you can use a single SSL_CTX*.

The SSL_CTX* is reference counted. It will be destroyed after the last SSL_free.

Since you are using a common SSL_CTX*, you should tune each SSL session with the non-context API calls. For example, SSL_set_cipher_list rather than SSL_CTX_set_cipher_list or SSL_use_certificate rather than SSL_CTX_use_certificate (unless, of course, its common).

Answer 2

Your code is very hard to read (how about proper indentation?) but I think the main problem is, that you consider a return of -1 from SSL_accept to be a fatal error.

According to the man page (which is true in this case) -1 can happen on fatal error or "It can also occur of action is need to continue the operation for non-blocking BIOs. Call SSL_get_error() with the return value ret to find out the reason.". So you have to check the error and if its SSL_ERROR_WANT_READ you have to wait the socket is readable and on SSL_ERROR_WANT_WRITE until its writable.