Openssl API Client Hello callback function is never called

Question

I am trying to use the openssl api to implement the JA3 method (TLS fingerprinting method) on a personnal project. To do that I need to get some informations on the Client Hello packet and I am trying to do that with the openssl api https://www.openssl.org/docs/man1.1.1/man3/ . Firstly I downloaded a code which set up a TLS connection between my computer and a website and retrieved the certificat information. This program is running well, the tls connection is using TLS 1.3 and the certificat information are ok. Then I tried to retrieved the Client Hello packet to start implement the JA3 method. After few researches I found many function that may allow me to get the information I need : https://www.openssl.org/docs/man1.1.1/man3/SSL_client_hello_cb_fn.html . All of those function can only be call on the client hello callback function. This function can be called thanks to the SSL_CTX_set_client_hello_cb function. But in my case my function is never called. I am starting to be hopeless it's why I need your help. I really searched a lot to debug that but I didn't find a flag that can allow the callback function or something like that. Here is the program :

#include <sys/socket.h>
#include <resolv.h>
#include <netdb.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <string.h>

#include <openssl/bio.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/pem.h>
#include <openssl/x509.h>
#include <openssl/x509_vfy.h>

int create_socket(char[], BIO *);

int callback(SSL *s, int *al, void *arg)
{
  int * number = arg;
  *number = (*number) + 1;
  printf("\nWe are in the callback function !\n");
  return SSL_CLIENT_HELLO_SUCCESS;
}

int main() {

  char           dest_url[] = "https://www.hp.com";
  BIO              *certbio = NULL;
  BIO               *outbio = NULL;
  X509                *cert = NULL;
  X509_NAME       *certname = NULL;
  const SSL_METHOD *method;
  SSL_CTX *ctx;
  struct ssl_st *ssl;
  int server = 0;

  OpenSSL_add_all_algorithms();
  ERR_load_BIO_strings();
  ERR_load_crypto_strings();
  SSL_load_error_strings();

  certbio = BIO_new(BIO_s_file());
  outbio  = BIO_new_fp(stdout, BIO_NOCLOSE);

  if(SSL_library_init() < 0)
    BIO_printf(outbio, "Could not initialize the OpenSSL library !\n");

  method = TLS_client_method();

  if ( (ctx = SSL_CTX_new(method)) == NULL)
    BIO_printf(outbio, "Unable to create a new SSL context structure.\n");

  int hope = 12;
  SSL_CTX_set_client_hello_cb(ctx, &callback, (void *)&hope);

  ssl = SSL_new(ctx);
  
  server = create_socket(dest_url, outbio);
  if(server != 0)
    BIO_printf(outbio, "Successfully made the TCP connection to: %s.\n", dest_url);

  SSL_set_fd(ssl, server);

  if ( SSL_connect(ssl) != 1 )
    BIO_printf(outbio, "Error: Could not build a SSL session to: %s.\n", dest_url);
  else
    BIO_printf(outbio, "Successfully enabled SSL/TLS session to: %s.\n", dest_url);
    
  cert = SSL_get_peer_certificate(ssl);
  if (cert == NULL)
    BIO_printf(outbio, "Error: Could not get a certificate from: %s.\n", dest_url);
  else
    BIO_printf(outbio, "Retrieved the server's certificate from: %s.\n", dest_url);

  certname = X509_NAME_new();
  certname = X509_get_subject_name(cert);

  BIO_printf(outbio, "Displaying the certificate subject data:\n");
  X509_NAME_print_ex(outbio, certname, 0, 0);
  BIO_printf(outbio, "\n");

  SSL_free(ssl);
  close(server);
  X509_free(cert);
  SSL_CTX_free(ctx);
  BIO_printf(outbio, "Finished SSL/TLS connection with server: %s.\n", dest_url);

  printf("Hope value : %d \n", hope);

  return(0);
}

int create_socket(char url_str[], BIO *out) {
  int sockfd;
  char hostname[256] = "";
  char    portnum[6] = "443";
  char      proto[6] = "";
  char      *tmp_ptr = NULL;
  int           port;
  struct hostent *host;
  struct sockaddr_in dest_addr;

  if(url_str[strlen(url_str)] == '/')
    url_str[strlen(url_str)] = '\0';

  strncpy(proto, url_str, (strchr(url_str, ':')-url_str));
  strncpy(hostname, strstr(url_str, "://")+3, sizeof(hostname));

  if(strchr(hostname, ':')) {
    tmp_ptr = strchr(hostname, ':');
    /* the last : starts the port number, if avail, i.e. 8443 */
    strncpy(portnum, tmp_ptr+1,  sizeof(portnum));
    *tmp_ptr = '\0';
  }

  port = atoi(portnum);

  if ( (host = gethostbyname(hostname)) == NULL ) {
    BIO_printf(out, "Error: Cannot resolve hostname %s.\n",  hostname);
    abort();
  }

  sockfd = socket(AF_INET, SOCK_STREAM, 0);

  dest_addr.sin_family=AF_INET;
  dest_addr.sin_port=htons(port);
  dest_addr.sin_addr.s_addr = *(long*)(host->h_addr);

  memset(&(dest_addr.sin_zero), '\0', 8);

  tmp_ptr = inet_ntoa(dest_addr.sin_addr);

  if ( connect(sockfd, (struct sockaddr *) &dest_addr,
                              sizeof(struct sockaddr)) == -1 ) {
    BIO_printf(out, "Error: Cannot connect to host %s [%s] on port %d.\n",
             hostname, tmp_ptr, port);
  }

  return sockfd;
}

I am waiting for your answer thanks guys.

Answer 1

The hint for what is going wrong is from the first line of the description in the doc that you linked to:

SSL_CTX_set_client_hello_cb() sets the callback function, which is automatically called during the early stages of ClientHello processing on the server.

This is called by a server when processing the received ClientHello. You have written a client and therefore it is never called.

As an alternative I suggest you look at SSL_CTX_set_msg_callback():

https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_msg_callback.html

From the docs:

SSL_CTX_set_msg_callback() or SSL_set_msg_callback() can be used to define a message callback function cb for observing all SSL/TLS protocol messages (such as handshake messages) that are received or sent, as well as other events that occur during processing.