SSL Memory not released after peers disconnection

Question

I'm developing a simple multi threaded HTTP/s server using non-blocking socket to handle HTTP request based on Linux Epoll . It creates 4/8 threads (struct SSLWorker), each one can accept and handle connections, no shared data between connections besides SSL_CTX (struct SSLContext).

With every new connection the memory grows and after disconnection, memory is not released ever. I can't figure out the reason. It seems that it related to SSL code because it doesn't happens when using HTTP request.

Using valgrind, heaptrack and leak sanitizers doesn't help since it seems that the memory is still reachable and no leak was detected. How would be the proper way and order to clean Connection struct SSL and BIO data ? (freeSsl() ). Any help on what I'm missing ?

NOTES: - Tried Openssl 1.1 and 1.1.1 from Debian and self build, they behave the same. - Memory growth is more noticeable when clients use TLS 1.2 - Tried disabling internal SSL SESSION caching and setting SSL_MODE_RELEASE_BUFFERS. - Disconnection start by clients side.

struct SSLContext{   //Shared between worker threads
    SSL_CTX *ssl_ctx{nullptr};
    SSLContext(){
        ERR_load_crypto_strings();
        ERR_load_SSL_strings();
        SSL_load_error_strings();
        OpenSSL_add_all_algorithms();
        int r = SSL_library_init();
        ssl_ctx = SSL_CTX_new(SSLv23_method());
        int r = SSL_CTX_use_certificate_file(ssl_ctx, global::cert_file.c_str(),
            SSL_FILETYPE_PEM); 
        r = SSL_CTX_use_PrivateKey_file(ssl_ctx, global::key_file.c_str(), SSL_FILETYPE_PEM); 
        r = SSL_CTX_check_private_key(ssl_ctx); 
    }

    virtual ~SSLContext(){
        SSL_CTX_free(ssl_ctx);
        ERR_free_strings();
    }
};

struct Connection //Data associated to every connection
{
    int sock_fd;
    SSL *ssl{nullptr}; 
    BIO *sbio{nullptr};// socket bio  
    BIO *io{nullptr};// buffer bio  
    BIO *ssl_bio{nullptr};// ssl bio
    ~Connection(){ //free ssl objects
        if (ssl != nullptr) {
            SSL_set_shutdown(ssl, SSL_RECEIVED_SHUTDOWN);
            freeSsl();
            if(sock_fd > 0) ::close(sock_fd);
        }
    }
    void freeSsl(){ //free connection       
        //SSL_shutdown(ssl); it seems to be done by bellow free call.
        //SSL_free(ssl); it seems to be done by bellow free call.
        if(sbio != nullptr) {
            BIO_free(sbio);
            sbio = nullptr;
        }
        if(io != nullptr) {
            BIO_flush(io);
            BIO_free(io);
            io = nullptr;
        }
        if(ssl_bio != nullptr) {
            BIO_flush(ssl_bio);
            BIO_free(ssl_bio);     
            ssl_bio = nullptr; 
        }
        ssl = nullptr;
    }
    bool enableReadEvent(); //enable socket read events, set EPOLLIN | EPOLLET
    bool enableWriteEvent();//enable socket write event, set EPOLLOUT | EPOLLET 
};

struct SSLConnectionManager //ssl operations hanldler.
{
    static SSLContext ssl_context; 

    bool handleHandshake(Connection &ssl_connection) //Initialize connection and do hanshake
    {
        if(ssl_connection.ssl != nullptr) ssl_connection.freeSsl();     
        ssl_connection.ssl = SSL_new(ssl_context->ssl_ctx);
        ssl_connection.sbio = BIO_new_socket(ssl_connection.sock_fd, BIO_CLOSE);
        SSL_set_bio(ssl_connection.ssl, ssl_connection.sbio, ssl_connection.sbio);
        ssl_connection.io = BIO_new(BIO_f_buffer());
        ssl_connection.ssl_bio = BIO_new(BIO_f_ssl());
        BIO_set_ssl(ssl_connection.ssl_bio, ssl_connection.ssl, BIO_CLOSE);
        BIO_push(ssl_connection.io, ssl_connection.ssl_bio);
        SSL_set_accept_state(ssl_connection.ssl);

        int r = SSL_do_handshake(ssl_connection.ssl); 
        if (r == 1) {
            ssl_connection.ssl_connected = true;   
            ssl_connection.enableReadEvent();
            return true;
        }

        int err = SSL_get_error(ssl_connection.ssl, r);
        if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) {  
            ssl_connection.enableReadEvent();
            return true;
        } 
        //"SSL_do_handshake error, abort connection
        return false;    

    }

    IO::IO_RESULT SSLConnectionManager::handleRead(Connection &ssl_connection) {
        if (!ssl_connection.ssl_connected) {
            return IO::IO_RESULT::SSL_NEED_HANDSHAKE;
        }
        int rc = -1;
        int bytes_read = 0;
        for (;;) {
            rc = BIO_read(ssl_connection.io,
                ssl_connection.buffer + ssl_connection.buffer_size,
                static_cast(MAX_DATA_SIZE - ssl_connection.buffer_size));        
            if (rc == 0) {
                return bytes_read > 0 ? IO::IO_RESULT::SUCCESS : IO::IO_RESULT::ZERO_DATA_READ;            
            }else if (rc < 0) {
                if (BIO_should_retry(ssl_connection.io)) {
                    return IO::IO_RESULT::DONE_TRY_AGAIN;
                }
                return IO::IO_RESULT::ERROR;
            }
            bytes_read += rc;
            ssl_connection.buffer_size += static_cast(rc);
        }
        return IO::IO_RESULT::SUCCESS;
    }

    IO::IO_RESULT SSLConnectionManager::handleWrite(Connection &ssl_connection,
        const char *data, size_t data_size, size_t &written) {
        if (!ssl_connection.ssl_connected) {
            return IO::IO_RESULT::SSL_NEED_HANDSHAKE; // after we call handleHanshake
        }

        IO::IO_RESULT result;
        int rc = -1;
        written = 0;
        for (;;) {
            rc = BIO_write(ssl_connection.io, data + written, static_cast(data_size - written));        
            if (rc == 0) {
                result = IO::IO_RESULT::DONE_TRY_AGAIN;
                break;
            } else if (rc < 0) {
                if (BIO_should_retry(ssl_connection.io)) {
                    result = IO::IO_RESULT::DONE_TRY_AGAIN;             
                    break;
                } else {
                    return IO::IO_RESULT::ERROR;                    
                }
            } else {
                written += rc;
                if ((data_size - written) == 0) {
                    result = IO::IO_RESULT::SUCCESS;
                    break;
                };
            }
        }
        BIO_flush(ssl_connection.io);  
        return result;
    }   
};

struct SSLWorker : EpollManager{ //Thread worker task.

    SSLConnectionManager ssl_connection_manager;

    bool onConnectEvent(Connection &ssl_connection){
        auto sock_fd  = accept(...);    
        Connection * new_connection = new  Connection(sock_fd);
        addToEventManager(*new_connection, EV_READ);
    } 

    void doWork() {
        is_running = true;
        int res = 0;
        epoll_event events[1024];
        while (is_running) {
            res = ::epoll_wait(epoll_fd, events, 1024, -1);
            if (res < 0)
                return;
            for (int i = 0; i < res; i++) {
                auto conn = static_cast(events[i].data.ptr);
                if (events[i].events & (EPOLLHUP | EPOLLERR | EPOLLRDHUP)) {
                    delete conn; //remote closed connection, free all
                } else {
                    if (events[i].events & EPOLLIN) {
                        if (conn->fd == global::listen_fd) {            
                            onConnectEvent();
                            continue;
                        } else {            
                            ssl_connection_manager.handleRead(*conn);
                            processRequest();
                            conn.enableWriteEvent();
                        }
                    }
                    if (events[i].events & EPOLLOUT) {   
                        ssl_connection_manager.handleWrite(*conn);
                    }
                }
            }
        }
    }


};

Here you can see after a brief execution, what heaptrack report leaking.

leak stack

SSL Memory not released after peers disconnection

Answers (1)

Related Questions