Reusing a single socket for multiple HTTPS requests (HTTPS client example) with nRF9160 DK over LTE connection

[Ana_]

I was wondering how I can update my current code (see below) to reuse a single socket for multiple HTTPS requests, rather than opening and closing a socket for each HTTPS request. The code is a slightly modified version of the Nordic SDK HTTPS client sample code which can be found under /nrf/samples/net/https_client. It should be noted that I am using an nRF9160 DK over an LTE connection. 

Alternatively, if there is any way to speed up the current code as it is taking me at least 3-4 seconds to run a single request. 

Any help is appreciated. 

main.c 

#include <string.h>
#include <zephyr/kernel.h>
#include <stdlib.h>
#include <zephyr/net/socket.h>
#include <zephyr/net/conn_mgr_monitor.h>
#include <zephyr/net/conn_mgr_connectivity.h>
#include <zephyr/net/tls_credentials.h>

#if defined(CONFIG_POSIX_API)
#include <zephyr/posix/arpa/inet.h>
#include <zephyr/posix/netdb.h>
#include <zephyr/posix/unistd.h>
#include <zephyr/posix/sys/socket.h>
#endif

#if CONFIG_MODEM_KEY_MGMT
#include <modem/modem_key_mgmt.h>
#endif

#define HTTPS_PORT "443"
#define HTTPS_HOSTNAME "[insert url here]"
#define HTTP_GET_PATH "/?mac=[insert mac address here]"
#define HTTP_POST_PATH "/[...]/[...]/[insert mac address here]"
#define HTTP_POST_BODY "[insert body here]"
#define USE_POST 1

#define RECV_BUF_SIZE 2048
#define TLS_SEC_TAG 42

#define L4_EVENT_MASK (NET_EVENT_L4_CONNECTED | NET_EVENT_L4_DISCONNECTED)
#define CONN_LAYER_EVENT_MASK (NET_EVENT_CONN_IF_FATAL_ERROR)

static char send_buf[1024];
static char recv_buf[RECV_BUF_SIZE];
static K_SEM_DEFINE(network_connected_sem, 0, 1);

static const char cert[] = {
    #include "DigiCertGlobalRootG2.pem.inc"
    IF_ENABLED(CONFIG_TLS_CREDENTIALS, (0x00))
};

static struct net_mgmt_event_callback l4_cb;
static struct net_mgmt_event_callback conn_cb;

BUILD_ASSERT(sizeof(cert) < KB(4), "Certificate too large");

int cert_provision(void)
{
    int err;
    printk("Provisioning certificate\n");
#if CONFIG_MODEM_KEY_MGMT
    bool exists;
    int mismatch;
    err = modem_key_mgmt_exists(TLS_SEC_TAG, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, &exists);
    if (err) return err;
    if (exists) {
        mismatch = modem_key_mgmt_cmp(TLS_SEC_TAG, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, cert, sizeof(cert));
        if (!mismatch) return 0;
        modem_key_mgmt_delete(TLS_SEC_TAG, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN);
    }
    err = modem_key_mgmt_write(TLS_SEC_TAG, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, cert, sizeof(cert));
#else
    err = tls_credential_add(TLS_SEC_TAG, TLS_CREDENTIAL_CA_CERTIFICATE, cert, sizeof(cert));
    if (err == -EEXIST) err = 0;
#endif
    return err;
}

int tls_setup(int fd)
{
    int err;
    int verify = 2;
    const sec_tag_t tls_sec_tag[] = { TLS_SEC_TAG };

    err = setsockopt(fd, SOL_TLS, TLS_PEER_VERIFY, &verify, sizeof(verify));
    if (err) return err;

    err = setsockopt(fd, SOL_TLS, TLS_SEC_TAG_LIST, tls_sec_tag, sizeof(tls_sec_tag));
    if (err) return err;

    err = setsockopt(fd, SOL_TLS, TLS_HOSTNAME, HTTPS_HOSTNAME, strlen(HTTPS_HOSTNAME));
    return err;
}

static void on_net_event_l4_disconnected(void) { printk("Disconnected from the network\n"); }
static void on_net_event_l4_connected(void) { k_sem_give(&network_connected_sem); }

static void l4_event_handler(struct net_mgmt_event_callback *cb, uint32_t event, struct net_if *iface)
{
    if (event == NET_EVENT_L4_CONNECTED) on_net_event_l4_connected();
    else if (event == NET_EVENT_L4_DISCONNECTED) on_net_event_l4_disconnected();
}

static void connectivity_event_handler(struct net_mgmt_event_callback *cb, uint32_t event, struct net_if *iface)
{
    if (event == NET_EVENT_CONN_IF_FATAL_ERROR)
        printk("Fatal error received from the connectivity layer\n");
}

static int send_http_request(void)
{
    int err, fd, bytes;
    size_t off = 0;
    struct addrinfo *res;
    struct addrinfo hints = {
        .ai_flags = AI_NUMERICSERV,
        .ai_socktype = SOCK_STREAM,
    };
    char peer_addr[INET6_ADDRSTRLEN];

    printk("Looking up %s\n", HTTPS_HOSTNAME);
    err = getaddrinfo(HTTPS_HOSTNAME, HTTPS_PORT, &hints, &res);
    if (err) return;

    inet_ntop(res->ai_family, &((struct sockaddr_in *)(res->ai_addr))->sin_addr,
              peer_addr, INET6_ADDRSTRLEN);
    printk("Resolved %s (%s)\n", peer_addr, net_family2str(res->ai_family));

    fd = socket(res->ai_family, SOCK_STREAM, IPPROTO_TLS_1_2);
    if (fd == -1) goto clean_up;

    if (tls_setup(fd)) goto clean_up;

    printk("Connecting to %s\n", HTTPS_HOSTNAME);
    if (connect(fd, res->ai_addr, res->ai_addrlen)) goto clean_up;

    if (USE_POST) {
        snprintf(send_buf, sizeof(send_buf),
            "POST %s HTTP/1.1\r\n"
            "Host: %s\r\n"
            "User-Agent: slm\r\n"
            "Accept: */*\r\n"
            "Content-Type: application/json\r\n"
            "Content-Length: %zu\r\n"
            "Connection: close\r\n\r\n%s",
            HTTP_POST_PATH, HTTPS_HOSTNAME,
            strlen(HTTP_POST_BODY), HTTP_POST_BODY);
    } else {
        snprintf(send_buf, sizeof(send_buf),
            "GET %s HTTP/1.1\r\n"
            "Host: %s\r\n"
            "Connection: close\r\n\r\n",
            HTTP_GET_PATH, HTTPS_HOSTNAME);
    }

    size_t total_len = strlen(send_buf);
    off = 0;
    do {
        bytes = send(fd, &send_buf[off], total_len - off, 0);
        if (bytes < 0) {
            printk("send() failed: errno=%d\n", errno);
            goto clean_up;
        }
        off += bytes;
    } while (off < total_len);

    printk("Sent %d bytes\n", off);

    // clear the receive buffer before use
    memset(recv_buf, 0, sizeof(recv_buf));

    off = 0;
    do {
        bytes = recv(fd, &recv_buf[off], RECV_BUF_SIZE - off - 1, 0);
        if (bytes < 0) {
            printk("recv() failed: errno=%d\n", errno);
            goto clean_up;
        }
        off += bytes;

        // prevent buffer overflow
        if (off >= RECV_BUF_SIZE - 1) {
            printk("Warning: response too large, truncating\n");
            break;
        }
    } while (bytes != 0);  // 0 means peer closed connection

    recv_buf[off] = '\0';
    printk("\nFull response:\n%s\n", recv_buf);

    freeaddrinfo(res);
    close(fd);
    return 0;

clean_up:
    if (res) freeaddrinfo(res);
    if (fd >= 0) close(fd);
    return -1;
}

int main(void)
{
    int err;
	int request_count = 0;

    printk("HTTPS client sample started\n\r");

    net_mgmt_init_event_callback(&l4_cb, l4_event_handler, L4_EVENT_MASK);
    net_mgmt_add_event_callback(&l4_cb);
    net_mgmt_init_event_callback(&conn_cb, connectivity_event_handler, CONN_LAYER_EVENT_MASK);
    net_mgmt_add_event_callback(&conn_cb);

    printk("Bringing network interface up\n");
    if (conn_mgr_all_if_up(true)) return -1;

    if (cert_provision()) return -1;

    printk("Connecting to the network\n");
    if (conn_mgr_all_if_connect(true)) return -1;

    k_sem_take(&network_connected_sem, K_FOREVER);
    
	while (1) {
        request_count++;
        printk("\n>>> Sending POST request #%d\n", request_count);

        err = send_http_request();  // return 0 on success, <0 on fail
        if (err < 0) {
            printk("POST request #%d failed. Exiting loop.\n", request_count);
            break;
        }

        // k_sleep(K_MSEC(100));  // optional: short delay between requests
    }

    printk("Disconnecting network after %d requests\n", request_count);

    k_sleep(K_SECONDS(1));
    conn_mgr_all_if_disconnect(true);
    conn_mgr_all_if_down(true);

    return 0;
}

prj.conf

# General
CONFIG_HEAP_MEM_POOL_SIZE=4096
CONFIG_MAIN_STACK_SIZE=8192

# Logging
CONFIG_LOG=y

# Network
CONFIG_NETWORKING=y
CONFIG_NET_SOCKETS=y
CONFIG_POSIX_API=y
CONFIG_NET_IPV4=y
CONFIG_NET_IPV6=y
CONFIG_NET_CONNECTION_MANAGER=y
CONFIG_NET_CONNECTION_MANAGER_MONITOR_STACK_SIZE=4600

[Sigurd Hellesvik]

Hi,

I was wondering how I can update my current code (see below) to reuse a single socket for multiple HTTPS requests, rather than opening and closing a socket for each HTTPS request.

Try to split the send_http_request function into different parts, for example like this. Then you just close the connection once you are done communicating.

I think this is known as TCP Connection Pooling.

Regards,
Sigurd Hellesvik

[Ana_]

Hi Sugurd, 

When using a single socket for multiple http requests I have been running into issues where the modem disconnects after one request is sent. I am using an Azure website and read somewhere that Azure automatically closes the connection after each request. Is this true? Is there anything I can do to surpass this? Alternatively is there any way of speeding up the current code as each request is taking 3-4 seconds to connect to the server.

Thank you.

[Sigurd Hellesvik]

Try to change

Connection: close

to

Connection: keep-alive

[Ana_]

I have already tried that, but no luck.

[Ana_]

I was wondering if there is anyway I can reduce the time spent per request, with each request opening a new socket?