Can't connect to web server from http C client Raspberry Pi Pico W lwIP

Question

I just try for already 3 days to make a http GET request, used a lot of examples, but all of them do not work.

The code below, I checked, succesfully connects to WiFi, and resolves dns - obtains ip address.

The problem is in connection to server: it passes tls_client_open, then dns_gethostbyname also succesfully, then tls_client_connect_to_server_ip as callback from tls_client_dns_found.

Then in tls_client_connect_to_server_ip it calls altcp_connect, and it returns ERR_OK, but tls_client_connected callback is never called, so it just runs waiting loop (in run_TLS_CLIENT_Test function) infinitelly.

What am I doing wrong?

---

The current code is:

picohttps.c

#define WIFI_SSID "wifi"
#define WIFI_PASSWORD "pass"

#include <string.h>
#include <time.h>

#include "hardware/structs/rosc.h"
#include "pico/stdlib.h"
#include "pico/cyw43_arch.h"
#include "lwip/pbuf.h"
#include "lwip/altcp_tcp.h"
#include "lwip/altcp_tls.h"
#include "lwip/dns.h"


#define TLS_CLIENT_SERVER        "google.com"
#define TLS_CLIENT_HTTP_REQUEST  "GET /about HTTP/1.1\r\n" \
                                "Host: " TLS_CLIENT_SERVER "\r\n" \
                                "Connection: close\r\n" \
                                "\r\n"
#define TLS_CLIENT_TIMEOUT_SECS  15


typedef struct TLS_CLIENT_T_ {
    struct altcp_pcb *pcb;
    bool complete;
} TLS_CLIENT_T;

static struct altcp_tls_config *tls_config = NULL;

void on()
{
    cyw43_arch_gpio_put(CYW43_WL_GPIO_LED_PIN, 1);
}

void off()
{
    cyw43_arch_gpio_put(CYW43_WL_GPIO_LED_PIN, 0);
}

void fail(int times)
{
    for(int i = 0; i < times; i++)
    {
    cyw43_arch_gpio_put(CYW43_WL_GPIO_LED_PIN, 1);
    sleep_ms(100);
    cyw43_arch_gpio_put(CYW43_WL_GPIO_LED_PIN, 0);
    sleep_ms(100);
    }
    sleep_ms(1000);
}


void ok()
{
    cyw43_arch_gpio_put(CYW43_WL_GPIO_LED_PIN, 1);
    sleep_ms(1000);
cyw43_arch_gpio_put(CYW43_WL_GPIO_LED_PIN, 0);
    sleep_ms(1000);
}


/* Function to feed mbedtls entropy. May be better to move it to pico-sdk */
int _mbedtls_hardware_poll(void *data, unsigned char *output, size_t len, size_t *olen) {
    /* Code borrowed from pico_lwip_random_byte(), which is static, so we cannot call it directly */
    static uint8_t byte;

    for(int p=0; p<len; p++) {
        for(int i=0;i<32;i++) {
            // picked a fairly arbitrary polynomial of 0x35u - this doesn't have to be crazily uniform.
            byte = ((byte << 1) | rosc_hw->randombit) ^ (byte & 0x80u ? 0x35u : 0);
            // delay a little because the random bit is a little slow
            busy_wait_at_least_cycles(30);
        }
        output[p] = byte;
    }

    *olen = len;
    return 0;
}


static err_t tls_client_close(void *arg) {
    TLS_CLIENT_T *state = (TLS_CLIENT_T*)arg;
    err_t err = ERR_OK;

    state->complete = true;
    if (state->pcb != NULL) {

        altcp_arg(state->pcb, NULL);
        altcp_poll(state->pcb, NULL, 0);
        altcp_recv(state->pcb, NULL);
        altcp_err(state->pcb, NULL);
        err = altcp_close(state->pcb);
        if (err != ERR_OK) {
            printf("close failed %d, calling abort\n", err);
            altcp_abort(state->pcb);
            err = ERR_ABRT;
        }
        state->pcb = NULL;
    }
    return err;
}

static err_t tls_client_connected(void *arg, struct altcp_pcb *pcb, err_t err) {

    TLS_CLIENT_T *state = (TLS_CLIENT_T*)arg;
    if (err != ERR_OK) {
        printf("connect failed %d\n", err);
        return tls_client_close(state);
    }

    printf("connected to server, sending request\n");
    err = altcp_write(state->pcb, TLS_CLIENT_HTTP_REQUEST, strlen(TLS_CLIENT_HTTP_REQUEST), TCP_WRITE_FLAG_COPY);
    if (err != ERR_OK) {
        printf("error writing data, err=%d", err);
        return tls_client_close(state);
    }

    return ERR_OK;
}

static err_t tls_client_poll(void *arg, struct altcp_pcb *pcb) {
    printf("timed out");
    return tls_client_close(arg);
}

static void tls_client_err(void *arg, err_t err) {
    TLS_CLIENT_T *state = (TLS_CLIENT_T*)arg;
    printf("tls_client_err %d\n", err);
    state->pcb = NULL; /* pcb freed by lwip when _err function is called */
}

static err_t tls_client_recv(void *arg, struct altcp_pcb *pcb, struct pbuf *p, err_t err) {
    TLS_CLIENT_T *state = (TLS_CLIENT_T*)arg;
    if (!p) {
        printf("connection closed\n");
        return tls_client_close(state);
    }

    if (p->tot_len > 0) {
        /* For simplicity this examples creates a buffer on stack the size of the data pending here, 
        and copies all the data to it in one go.
        Do be aware that the amount of data can potentially be a bit large (TLS record size can be 16 KB),
        so you may want to use a smaller fixed size buffer and copy the data to it using a loop, if memory is a concern */
        char buf[p->tot_len + 1];

        pbuf_copy_partial(p, buf, p->tot_len, 0);
        buf[p->tot_len] = 0;

        printf("***\nnew data received from server:\n***\n\n%s\n", buf);

        altcp_recved(pcb, p->tot_len);
    }
    pbuf_free(p);

    return ERR_OK;
}

static void tls_client_connect_to_server_ip(const ip_addr_t *ipaddr, TLS_CLIENT_T *state)
{
    err_t err;
    u16_t port = 443;

    
    printf("connecting to server IP %s port %d\n", ipaddr_ntoa(ipaddr), port);
    err = altcp_connect(state->pcb, ipaddr, port, tls_client_connected);
    
    if (err != ERR_OK)
    {
        fail(8);
        fprintf(stderr, "error initiating connect, err=%d\n", err);
        tls_client_close(state);
    }
}

static void tls_client_dns_found(const char* hostname, const ip_addr_t *ipaddr, void *arg)
{
    if (ipaddr)
    {
        printf("DNS resolving complete\n");
        tls_client_connect_to_server_ip(ipaddr, (TLS_CLIENT_T *) arg);
    }
    else
    {
        printf("error resolving hostname %s\n", hostname);
        tls_client_close(arg);
    }
}


static bool tls_client_open(const char *hostname, void *arg) {
    err_t err;
    ip_addr_t server_ip;
    TLS_CLIENT_T *state = (TLS_CLIENT_T*)arg;

    state->pcb = altcp_tls_new(tls_config, IPADDR_TYPE_ANY);
    if (!state->pcb) {
        printf("failed to create pcb\n");
        return false;
    }

    altcp_arg(state->pcb, state);
    altcp_poll(state->pcb, tls_client_poll, TLS_CLIENT_TIMEOUT_SECS * 2);
    altcp_recv(state->pcb, tls_client_recv);
    altcp_err(state->pcb, tls_client_err);

    /* Set SNI */
    mbedtls_ssl_set_hostname(altcp_tls_context(state->pcb), hostname);

    printf("resolving %s\n", hostname);

    // cyw43_arch_lwip_begin/end should be used around calls into lwIP to ensure correct locking.
    // You can omit them if you are in a callback from lwIP. Note that when using pico_cyw_arch_poll
    // these calls are a no-op and can be omitted, but it is a good practice to use them in
    // case you switch the cyw43_arch type later.
    //cyw43_arch_lwip_begin();

    err = dns_gethostbyname(hostname, &server_ip, tls_client_dns_found, state);
    
    if (err == ERR_OK)
    {
        /* host is in DNS cache */

        tls_client_connect_to_server_ip(&server_ip, state);
    }
    else if (err != ERR_INPROGRESS)
    {
        fail(5);
        printf("error initiating DNS resolving, err=%d\n", err);
        tls_client_close(state->pcb);
    }

    //cyw43_arch_lwip_end();

    return err == ERR_OK || err == ERR_INPROGRESS;
}

// Perform initialisation
static TLS_CLIENT_T* tls_client_init(void) {

    TLS_CLIENT_T *state = calloc(1, sizeof(TLS_CLIENT_T));

    if (!state) {
        printf("failed to allocate state\n");
        return NULL;
    }

    return state;
}

void run_TLS_CLIENT_Test(void) {
    /* No CA certificate checking */
    tls_config = altcp_tls_create_config_client(NULL, 0);


    TLS_CLIENT_T *state = tls_client_init();

    if (!state) {
        return;
    }

    if (!tls_client_open(TLS_CLIENT_SERVER, state)) {
        return;
    }


    while(!state->complete) {
        // the following #ifdef is only here so this same example can be used in multiple modes;
        // you do not need it in your code
    
#if PICO_CYW43_ARCH_POLL
        // if you are using pico_cyw43_arch_poll, then you must poll periodically from your
        // main loop (not from a timer) to check for WiFi driver or lwIP work that needs to be done.
        cyw43_arch_poll();
        sleep_ms(1);
        ok();
#else
        // if you are not using pico_cyw43_arch_poll, then WiFI driver and lwIP work
        // is done via interrupt in the background. This sleep is just an example of some (blocking)
        // work you might be doing.
        
#endif
    }


    free(state);
    altcp_tls_free_config(tls_config);
}

int main() {
    stdio_init_all();

    if (cyw43_arch_init()) {
        printf("failed to initialise\n");
        return 1;
    }
    cyw43_arch_enable_sta_mode();

    if (cyw43_arch_wifi_connect_timeout_ms(WIFI_SSID, WIFI_PASSWORD, CYW43_AUTH_WPA2_AES_PSK, 30000)) {
        printf("failed to connect\n");
        return 1;
    }


    run_TLS_CLIENT_Test();



    /* sleep a bit to let usb stdio write out any buffer to host */
    sleep_ms(100);

    cyw43_arch_deinit();
    return 0;
}

mbedtls_config.h

/* Workaround for some mbedtls source files using INT_MAX without including limits.h */
#include <limits.h>

#define MBEDTLS_NO_PLATFORM_ENTROPY
#define MBEDTLS_ENTROPY_HARDWARE_ALT

#define MBEDTLS_SSL_OUT_CONTENT_LEN    2048

#define MBEDTLS_ALLOW_PRIVATE_ACCESS
#define MBEDTLS_HAVE_TIME

#define MBEDTLS_CIPHER_MODE_CBC
#define MBEDTLS_ECP_DP_SECP192R1_ENABLED
#define MBEDTLS_ECP_DP_SECP224R1_ENABLED
#define MBEDTLS_ECP_DP_SECP256R1_ENABLED
#define MBEDTLS_ECP_DP_SECP384R1_ENABLED
#define MBEDTLS_ECP_DP_SECP521R1_ENABLED
#define MBEDTLS_ECP_DP_SECP192K1_ENABLED
#define MBEDTLS_ECP_DP_SECP224K1_ENABLED
#define MBEDTLS_ECP_DP_SECP256K1_ENABLED
#define MBEDTLS_ECP_DP_BP256R1_ENABLED
#define MBEDTLS_ECP_DP_BP384R1_ENABLED
#define MBEDTLS_ECP_DP_BP512R1_ENABLED
#define MBEDTLS_ECP_DP_CURVE25519_ENABLED
#define MBEDTLS_KEY_EXCHANGE_RSA_ENABLED
#define MBEDTLS_PKCS1_V15
#define MBEDTLS_SHA256_SMALLER
#define MBEDTLS_SSL_SERVER_NAME_INDICATION
#define MBEDTLS_AES_C
#define MBEDTLS_ASN1_PARSE_C
#define MBEDTLS_BIGNUM_C
#define MBEDTLS_CIPHER_C
#define MBEDTLS_CTR_DRBG_C
#define MBEDTLS_ENTROPY_C
#define MBEDTLS_ERROR_C
#define MBEDTLS_MD_C
#define MBEDTLS_MD5_C
#define MBEDTLS_OID_C
#define MBEDTLS_PKCS5_C
#define MBEDTLS_PK_C
#define MBEDTLS_PK_PARSE_C
#define MBEDTLS_PLATFORM_C
#define MBEDTLS_RSA_C
#define MBEDTLS_SHA1_C
#define MBEDTLS_SHA224_C
#define MBEDTLS_SHA256_C
#define MBEDTLS_SHA512_C
#define MBEDTLS_SSL_CLI_C
#define MBEDTLS_SSL_SRV_C
#define MBEDTLS_SSL_TLS_C
#define MBEDTLS_X509_CRT_PARSE_C
#define MBEDTLS_X509_USE_C
#define MBEDTLS_AES_FEWER_TABLES

/* TLS 1.2 */
#define MBEDTLS_SSL_PROTO_TLS1_2
#define MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED
#define MBEDTLS_GCM_C
#define MBEDTLS_ECDH_C
#define MBEDTLS_ECP_C
#define MBEDTLS_ECDSA_C
#define MBEDTLS_ASN1_WRITE_C

lwiopts.h

#ifndef _LWIPOPTS_H
#define _LWIPOPTS_H

/* lwIP configuration for Pico HTTPS example **********************************
*                                                                            *
*  Configuration for the lwIP network library included in the Pico SDK and   *
*  required for the Pico HTTPS example.                                      *
*                                                                            *
*  N.b. Not all options are strictly required; this is just an example       *
*  configuration.                                                            *
*                                                                            *
*  https://www.nongnu.org/lwip/2_1_x/group__lwip__opts.html                  *
*  https://github.com/lwip-tcpip/lwip/blob/master/src/include/lwip/opt.h     *
*                                                                            *
******************************************************************************/

#ifndef _LWIPOPTS_EXAMPLE_COMMONH_H
#define _LWIPOPTS_EXAMPLE_COMMONH_H

/* System options *************************************************************/

// Run without OS
//
//  No OS on Pico W platform
//
#define NO_SYS                      1



/* Memory options *************************************************************/

// Disable native C-library malloc
//
//  Incompatible with `pico_cyw43_arch_lwip_threadsafe_background`
//
#define MEM_LIBC_MALLOC             0

// Byte alignment
#define MEM_ALIGNMENT               4       // bytes

// Heap size
#define MEM_SIZE                    4000    // bytes



/* Memory pool options ********************************************************/

// Max queued ARP packets
#define MEMP_NUM_ARP_QUEUE          10
//
// Max queued TCP segments
#define MEMP_NUM_TCP_SEG            32



/* ARP options ****************************************************************/

// Enable ARP support
//
//  Required for IP layer of network stack
//
#define LWIP_ARP                    1



/* ICMP options ***************************************************************/

// Enable ICMP support
//
//  Probably required for IP layer of network stack?
//
#define LWIP_ICMP                   1



/* IP options *****************************************************************/

// Enable IPv4 support
#define LWIP_IPV4                   1



/* DHCP options ***************************************************************/

// Enable DHCP support
//
//  Required for connecting to wireless network
//
#define LWIP_DHCP                   1

// Disable address conflict detection
#define LWIP_DHCP_DOES_ACD_CHECK    0

// Disable ARP check
#define DHCP_DOES_ARP_CHECK         0

/* DNS options ****************************************************************/

// Enable DNS support
//
//  Required for hostname resolution
//
#define LWIP_DNS                    1



/* UDP options ****************************************************************/

// Enable UDP support
//
//  Probably required for DNS queries?
//
#define LWIP_UDP                    1



/* TCP options ****************************************************************/

// Enable TCP support
#define LWIP_TCP                    1

// Max segment size
#define TCP_MSS                     1460

// Window size
#define TCP_WND                     (8 * TCP_MSS)

// Send buffer size
#define TCP_SND_BUF                 (8 * TCP_MSS)

// Send queue length
#define TCP_SND_QUEUELEN            ((4 * (TCP_SND_BUF) + (TCP_MSS - 1)) / (TCP_MSS))

// TCP options
#define LWIP_TCP_KEEPALIVE          1


/* ALTCP options **************************************************************/

// Enable ALTCP support
//
//  ALTCP is lwIP interface for TCP + X. In the case of the Pico HTTPS example,
//  X should be TLS, as required for HTTPS.
//
#define LWIP_ALTCP                  1

// Enable ALTCP-compatible TLS interface
//
//  i.e. Set X to TLS in ALTCP = TCP + X
//
#define LWIP_ALTCP_TLS              1

// Enable ALTCP-compatible TLS interface
//
//  A port of the Mbed-TLS library is included in lwIP. __N.b. this is not a
//  full MbedTLS distribution__, but rather simply provides an lwIP compatible
//  interface to Mbed-TLS.
//
#define LWIP_ALTCP_TLS_MBEDTLS      1



/* Mbed-TLS options ***********************************************************/

// Require TLS authentication (certificate)
//
//  Cause ignoring certificate errors leads to bad things…
//
#define ALTCP_MBEDTLS_AUTHMODE      MBEDTLS_SSL_VERIFY_REQUIRED



/* Network interface options **************************************************/

// Disable NETIF API support
//
//  Not needed. Sequential API, and therefore for platforms with OSes only.
//
#define LWIP_NETIF_API              0

// Set interface name from hostname
#define LWIP_NETIF_HOSTNAME         1

// Enable callback on interface state change
#define LWIP_NETIF_STATUS_CALLBACK  1

// Enable callback on link state change
#define LWIP_NETIF_LINK_CALLBACK    1

// Try to put all TX data in single pbuf
#define LWIP_NETIF_TX_SINGLE_PBUF   1



/* Sequntial API options ******************************************************/

// Disable socket support
//
//  Not needed. Sequential API, and therefore for platforms with OSes only.
//
#define LWIP_SOCKET                 0

// Disable netconn support
//
//  Not needed. Sequential API, and therefore for platforms with OSes only.
//
#define LWIP_NETCONN                0



/* Statistics options *********************************************************/

// Enable statistics
#define LWIP_STATS                  1

// Enable statistics display function
#define LWIP_STATS_DISPLAY          1

// Enable memory stats
#define MEM_STATS                   1

// Disable system stats
#define SYS_STATS                   0

// Disable memory pool stats
#define MEMP_STATS                  0

// Disable link stats
#define LINK_STATS                  0



/* Debug options **************************************************************/

// Enable debugging
#define LWIP_DEBUG                  1


#endif //_LWIPOPTS_EXAMPLE_COMMONH_H


/* TCP WND must be at least 16 kb to match TLS record size
or you will get a warning "altcp_tls: TCP_WND is smaller than the RX decrypion buffer, connection RX might stall!" */
#undef TCP_WND
#define TCP_WND  16384

#define LWIP_ALTCP               1
#define LWIP_ALTCP_TLS           1
#define LWIP_ALTCP_TLS_MBEDTLS   1

#define LWIP_DEBUG 1
#define ALTCP_MBEDTLS_DEBUG  LWIP_DBG_ON

#endif

CMakeLists.txt

# CMake configuration for Pico HTTPS example ###################################
#                                                                              #
#   Configuration for the CMake build system used to build the Pico HTTPS      #
#   example.                                                                   #
#                                                                              #
#   To be used with the Pico SDK.                                              #
#                                                                              #
################################################################################

cmake_minimum_required(VERSION 3.13)

# Include Pico SDK CMake macros
#
#   Adjust the path below to the local Pico SDK installation
#
#   https://github.com/raspberrypi/pico-sdk/blob/master/pico_sdk_init.cmake
#
include("C:/Program Files/Raspberry Pi/Pico SDK v1.5.1/pico-sdk/pico_sdk_init.cmake")

# Declare CMake project
project(picohttps)

# Initialize Pico SDK
#
#   Defined in Pico SDK macros — must be called after macro include.
#
#   Requires CMake project — must be called after project declaration.
#
pico_sdk_init()

# Define build inputs/outputs
#
#   https://cmake.org/cmake/help/latest/command/add_executable.html
#

add_executable(https
        picohttps.c
        )
target_compile_definitions(https PRIVATE
        WIFI_SSID=\"${WIFI_SSID}\"
        WIFI_PASSWORD=\"${WIFI_PASSWORD}\"
        )
target_include_directories(https PRIVATE
        ${CMAKE_CURRENT_LIST_DIR}
        ${CMAKE_CURRENT_LIST_DIR}/.. # for our common lwipopts
        )
target_link_libraries(https
        pico_cyw43_arch_lwip_poll
        pico_lwip_mbedtls
        pico_mbedtls
        pico_stdlib
        )
pico_add_extra_outputs(https)

pico_enable_stdio_usb(https 1)
pico_enable_stdio_uart(https 0)

Examples I tried:

• https://rfwumcu.blogspot.com/2023/01/raspberry-pi-pico-w-c-sdk-lwip-ep-2.html - just hangs on httpc_get_file_dns; also do not know, where to get ntp_time.h and correspondning functions ntp_time_init and get_ntp_time, so I just removed them;
• https://github.com/marceloalcocer/picohttps/ - as I remember, same problem - infinite loop, waiting for server connection. In that example, the CA ROOT x509 certificate needed, whithout it, as I remember some function just fail, before the loop, but with (as I think) correct certificate it goes to infinite loop.

Answer 1

Currently I "solved it".

I used pico w tls example, but this time, in root directory I also created .vscode folder, put there setting.json with the next content:

    {
    "files.associations": {
        "dhcpserver.h": "c",
        "pbuf.h": "c",
        "cyw43_arch.h": "c",
        "stdlib.h": "c",
        "lwipopts_examples_common.h": "c",
        "async_context.h": "c"
    },
    "cmake.configureSettings": {
        "PICO_BOARD": "pico_w",
        "WIFI_SSID": "somewifi",
        "WIFI_PASSWORD": "somepass"
    }
}

I do not know for sure, why specifying WIFI_SSID and WIFI_PASSWORD in cmake files did not work. Perhaps, there could be dependecies on this constants even somewhere in SDK (that will be insane).

However, that code makes get request very long (about 10 seconds), but this is an another question.