Reusing a single socket for multiple HTTPS requests (HTTPS client example) with nRF9160 DK over LTE connection

[Ana_]

I was wondering how I can update my current code (see below) to reuse a single socket for multiple HTTPS requests, rather than opening and closing a socket for each HTTPS request. The code is a slightly modified version of the Nordic SDK HTTPS client sample code which can be found under /nrf/samples/net/https_client. It should be noted that I am using an nRF9160 DK over an LTE connection. 

Alternatively, if there is any way to speed up the current code as it is taking me at least 3-4 seconds to run a single request. 

Any help is appreciated. 

main.c 

#include <string.h>
#include <zephyr/kernel.h>
#include <stdlib.h>
#include <zephyr/net/socket.h>
#include <zephyr/net/conn_mgr_monitor.h>
#include <zephyr/net/conn_mgr_connectivity.h>
#include <zephyr/net/tls_credentials.h>

#if defined(CONFIG_POSIX_API)
#include <zephyr/posix/arpa/inet.h>
#include <zephyr/posix/netdb.h>
#include <zephyr/posix/unistd.h>
#include <zephyr/posix/sys/socket.h>
#endif

#if CONFIG_MODEM_KEY_MGMT
#include <modem/modem_key_mgmt.h>
#endif

#define HTTPS_PORT "443"
#define HTTPS_HOSTNAME "[insert url here]"
#define HTTP_GET_PATH "/?mac=[insert mac address here]"
#define HTTP_POST_PATH "/[...]/[...]/[insert mac address here]"
#define HTTP_POST_BODY "[insert body here]"
#define USE_POST 1

#define RECV_BUF_SIZE 2048
#define TLS_SEC_TAG 42

#define L4_EVENT_MASK (NET_EVENT_L4_CONNECTED | NET_EVENT_L4_DISCONNECTED)
#define CONN_LAYER_EVENT_MASK (NET_EVENT_CONN_IF_FATAL_ERROR)

static char send_buf[1024];
static char recv_buf[RECV_BUF_SIZE];
static K_SEM_DEFINE(network_connected_sem, 0, 1);

static const char cert[] = {
    #include "DigiCertGlobalRootG2.pem.inc"
    IF_ENABLED(CONFIG_TLS_CREDENTIALS, (0x00))
};

static struct net_mgmt_event_callback l4_cb;
static struct net_mgmt_event_callback conn_cb;

BUILD_ASSERT(sizeof(cert) < KB(4), "Certificate too large");

int cert_provision(void)
{
    int err;
    printk("Provisioning certificate\n");
#if CONFIG_MODEM_KEY_MGMT
    bool exists;
    int mismatch;
    err = modem_key_mgmt_exists(TLS_SEC_TAG, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, &exists);
    if (err) return err;
    if (exists) {
        mismatch = modem_key_mgmt_cmp(TLS_SEC_TAG, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, cert, sizeof(cert));
        if (!mismatch) return 0;
        modem_key_mgmt_delete(TLS_SEC_TAG, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN);
    }
    err = modem_key_mgmt_write(TLS_SEC_TAG, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, cert, sizeof(cert));
#else
    err = tls_credential_add(TLS_SEC_TAG, TLS_CREDENTIAL_CA_CERTIFICATE, cert, sizeof(cert));
    if (err == -EEXIST) err = 0;
#endif
    return err;
}

int tls_setup(int fd)
{
    int err;
    int verify = 2;
    const sec_tag_t tls_sec_tag[] = { TLS_SEC_TAG };

    err = setsockopt(fd, SOL_TLS, TLS_PEER_VERIFY, &verify, sizeof(verify));
    if (err) return err;

    err = setsockopt(fd, SOL_TLS, TLS_SEC_TAG_LIST, tls_sec_tag, sizeof(tls_sec_tag));
    if (err) return err;

    err = setsockopt(fd, SOL_TLS, TLS_HOSTNAME, HTTPS_HOSTNAME, strlen(HTTPS_HOSTNAME));
    return err;
}

static void on_net_event_l4_disconnected(void) { printk("Disconnected from the network\n"); }
static void on_net_event_l4_connected(void) { k_sem_give(&network_connected_sem); }

static void l4_event_handler(struct net_mgmt_event_callback *cb, uint32_t event, struct net_if *iface)
{
    if (event == NET_EVENT_L4_CONNECTED) on_net_event_l4_connected();
    else if (event == NET_EVENT_L4_DISCONNECTED) on_net_event_l4_disconnected();
}

static void connectivity_event_handler(struct net_mgmt_event_callback *cb, uint32_t event, struct net_if *iface)
{
    if (event == NET_EVENT_CONN_IF_FATAL_ERROR)
        printk("Fatal error received from the connectivity layer\n");
}

static int send_http_request(void)
{
    int err, fd, bytes;
    size_t off = 0;
    struct addrinfo *res;
    struct addrinfo hints = {
        .ai_flags = AI_NUMERICSERV,
        .ai_socktype = SOCK_STREAM,
    };
    char peer_addr[INET6_ADDRSTRLEN];

    printk("Looking up %s\n", HTTPS_HOSTNAME);
    err = getaddrinfo(HTTPS_HOSTNAME, HTTPS_PORT, &hints, &res);
    if (err) return;

    inet_ntop(res->ai_family, &((struct sockaddr_in *)(res->ai_addr))->sin_addr,
              peer_addr, INET6_ADDRSTRLEN);
    printk("Resolved %s (%s)\n", peer_addr, net_family2str(res->ai_family));

    fd = socket(res->ai_family, SOCK_STREAM, IPPROTO_TLS_1_2);
    if (fd == -1) goto clean_up;

    if (tls_setup(fd)) goto clean_up;

    printk("Connecting to %s\n", HTTPS_HOSTNAME);
    if (connect(fd, res->ai_addr, res->ai_addrlen)) goto clean_up;

    if (USE_POST) {
        snprintf(send_buf, sizeof(send_buf),
            "POST %s HTTP/1.1\r\n"
            "Host: %s\r\n"
            "User-Agent: slm\r\n"
            "Accept: */*\r\n"
            "Content-Type: application/json\r\n"
            "Content-Length: %zu\r\n"
            "Connection: close\r\n\r\n%s",
            HTTP_POST_PATH, HTTPS_HOSTNAME,
            strlen(HTTP_POST_BODY), HTTP_POST_BODY);
    } else {
        snprintf(send_buf, sizeof(send_buf),
            "GET %s HTTP/1.1\r\n"
            "Host: %s\r\n"
            "Connection: close\r\n\r\n",
            HTTP_GET_PATH, HTTPS_HOSTNAME);
    }

    size_t total_len = strlen(send_buf);
    off = 0;
    do {
        bytes = send(fd, &send_buf[off], total_len - off, 0);
        if (bytes < 0) {
            printk("send() failed: errno=%d\n", errno);
            goto clean_up;
        }
        off += bytes;
    } while (off < total_len);

    printk("Sent %d bytes\n", off);

    // clear the receive buffer before use
    memset(recv_buf, 0, sizeof(recv_buf));

    off = 0;
    do {
        bytes = recv(fd, &recv_buf[off], RECV_BUF_SIZE - off - 1, 0);
        if (bytes < 0) {
            printk("recv() failed: errno=%d\n", errno);
            goto clean_up;
        }
        off += bytes;

        // prevent buffer overflow
        if (off >= RECV_BUF_SIZE - 1) {
            printk("Warning: response too large, truncating\n");
            break;
        }
    } while (bytes != 0);  // 0 means peer closed connection

    recv_buf[off] = '\0';
    printk("\nFull response:\n%s\n", recv_buf);

    freeaddrinfo(res);
    close(fd);
    return 0;

clean_up:
    if (res) freeaddrinfo(res);
    if (fd >= 0) close(fd);
    return -1;
}

int main(void)
{
    int err;
	int request_count = 0;

    printk("HTTPS client sample started\n\r");

    net_mgmt_init_event_callback(&l4_cb, l4_event_handler, L4_EVENT_MASK);
    net_mgmt_add_event_callback(&l4_cb);
    net_mgmt_init_event_callback(&conn_cb, connectivity_event_handler, CONN_LAYER_EVENT_MASK);
    net_mgmt_add_event_callback(&conn_cb);

    printk("Bringing network interface up\n");
    if (conn_mgr_all_if_up(true)) return -1;

    if (cert_provision()) return -1;

    printk("Connecting to the network\n");
    if (conn_mgr_all_if_connect(true)) return -1;

    k_sem_take(&network_connected_sem, K_FOREVER);
    
	while (1) {
        request_count++;
        printk("\n>>> Sending POST request #%d\n", request_count);

        err = send_http_request();  // return 0 on success, <0 on fail
        if (err < 0) {
            printk("POST request #%d failed. Exiting loop.\n", request_count);
            break;
        }

        // k_sleep(K_MSEC(100));  // optional: short delay between requests
    }

    printk("Disconnecting network after %d requests\n", request_count);

    k_sleep(K_SECONDS(1));
    conn_mgr_all_if_disconnect(true);
    conn_mgr_all_if_down(true);

    return 0;
}

prj.conf

# General
CONFIG_HEAP_MEM_POOL_SIZE=4096
CONFIG_MAIN_STACK_SIZE=8192

# Logging
CONFIG_LOG=y

# Network
CONFIG_NETWORKING=y
CONFIG_NET_SOCKETS=y
CONFIG_POSIX_API=y
CONFIG_NET_IPV4=y
CONFIG_NET_IPV6=y
CONFIG_NET_CONNECTION_MANAGER=y
CONFIG_NET_CONNECTION_MANAGER_MONITOR_STACK_SIZE=4600

[Sigurd Hellesvik]

Hi,

I was wondering how I can update my current code (see below) to reuse a single socket for multiple HTTPS requests, rather than opening and closing a socket for each HTTPS request.

Try to split the send_http_request function into different parts, for example like this. Then you just close the connection once you are done communicating.

I think this is known as TCP Connection Pooling.

Regards,
Sigurd Hellesvik

[Ana_]

Hi Sugurd, 

When using a single socket for multiple http requests I have been running into issues where the modem disconnects after one request is sent. I am using an Azure website and read somewhere that Azure automatically closes the connection after each request. Is this true? Is there anything I can do to surpass this? Alternatively is there any way of speeding up the current code as each request is taking 3-4 seconds to connect to the server.

Thank you.

[Sigurd Hellesvik]

Try to change

Connection: close

to

Connection: keep-alive

[Ana_]

I have already tried that, but no luck.

[Ana_]

I was wondering if there is anyway I can reduce the time spent per request, with each request opening a new socket?

[Sigurd Hellesvik]

I think it is still worth it to test if Azure closes the connection for you or not.

Let us try to divide the problem into Azure vs nRF.

Can you try one or both of the following:

1. Test the nRF with another HTTPS page, to check if this still disconnects.

2. Make a python script or CURL or something similar with Azure, to see if that can keep the connection.