CoAP with DTLS on nRF9151

[Tom01]

Dear Nordic team,

I am struggling to set up CoAP with DTLS for a Thingsboard CoAP backend. The connection works just fine when using plain UDP, but when I set up a socket with DTLS I run into an error, where the CoAP client request responds with error 127. I think this means that the CoAP client library identifies the socket as already being connected, but I am not sure why. To break this down some more, I've returned to the official "coap_client" sample and adjusted it to communicate with the Thingsboard Cloud backend. I have confirmed that the sample works using plain UDP. The GET request with the specified path (NOTE: access token is redacted below) works flawlessly. This would be the expected log:

[00:02:10.370,086] <inf> coap_client_sample: CoAP response: code: 0x45, payload: {"shared":{"sdlConfigTs":6}}

From there I modified the sample to provision the required certificate (validity checked with CoAP client on my host machine), use port 5684 and then adjust socket type and options. The modified sample is:

/*
 * Copyright (c) 2019 Nordic Semiconductor ASA
 *
 * SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
 */

#include <stdio.h>
#include <string.h>

#if defined(CONFIG_POSIX_API)
#include <zephyr/posix/arpa/inet.h>
#include <zephyr/posix/netdb.h>
#include <zephyr/posix/sys/socket.h>
#include <zephyr/posix/poll.h>
#else
#include <zephyr/net/socket.h>
#endif /* CONFIG_POSIX_API */

#if CONFIG_MODEM_KEY_MGMT
#include <modem/modem_key_mgmt.h>
#endif

#include <zephyr/kernel.h>
#include <zephyr/sys/reboot.h>
#include <zephyr/net/coap.h>
#include <zephyr/net/socket.h>
#include <zephyr/net/conn_mgr_connectivity.h>
#include <zephyr/net/conn_mgr_monitor.h>
#include <zephyr/random/random.h>
#include <zephyr/net/coap_client.h>
#include <zephyr/logging/log.h>
#include <zephyr/logging/log_ctrl.h>
#include <zephyr/net/tls_credentials.h>

LOG_MODULE_REGISTER(coap_client_sample, CONFIG_COAP_CLIENT_SAMPLE_LOG_LEVEL);

#define COAP_HOSTNAME "coap.eu.thingsboard.cloud"
#define COAP_PORT 5684
#define COMODO_SEC_TAG 43


static const char comodo_cert[] = {
#include "tb-cloud-root-ca.pem.inc"
	IF_ENABLED(CONFIG_TLS_CREDENTIALS, (0x00))
};

BUILD_ASSERT(sizeof(comodo_cert) < KB(4), "Comodo certificate too large");

/**
 * @brief Provisions a single CA certificate to a specific security tag.
 * @param[in] sec_tag The security tag that the modem will use for the encrypted socket (choose
 * different tags for different DNS addresses).
 * @param[in] cert_buf The root certificate to provision.
 * @param[in] cert_len Length of the certificate.
 * @returns 0 or a negative errno on error
 */
static int32_t tls_cert_provision_single(sec_tag_t sec_tag, const char* cert_buf, size_t cert_len)
{
	int32_t err;

	printk("Processing certificate provisioning for sec tag: %d\n", sec_tag);

#if CONFIG_MODEM_KEY_MGMT
	bool exists;
	int32_t mismatch;

	err = modem_key_mgmt_exists(sec_tag, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, &exists);
	if (err) {
		printk("Failed to check for certificate on tag %d, err %d\n", sec_tag, err);
		return err;
	}

	if (exists) {
		mismatch = modem_key_mgmt_cmp(
		    sec_tag, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, cert_buf, cert_len);
		if (!mismatch) {
			printk(
			    "Certificate on tag %d matches expected content. Skipping.\n", sec_tag);
			return 0;
		}

		printk("Certificate mismatch detected on tag %d. Overwriting...\n", sec_tag);
		err = modem_key_mgmt_delete(sec_tag, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN);
		if (err) {
			printk(
			    "Failed to delete old certificate on tag %d, err %d\n", sec_tag, err);
		}
	}

	printk("Writing certificate to modem storage (tag %d)...\n", sec_tag);
	err = modem_key_mgmt_write(sec_tag, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, cert_buf, cert_len);
	if (err) {
		printk("Failed to write certificate to tag %d, err %d\n", sec_tag, err);
		return err;
	}
#else /* CONFIG_MODEM_KEY_MGMT */
	err = tls_credential_add(sec_tag, TLS_CREDENTIAL_CA_CERTIFICATE, cert_buf, cert_len);
	if (err == -EEXIST) {
		printk("CA certificate already exists on application core, sec tag: %d\n", sec_tag);
	} else if (err < 0) {
		printk("Failed to register CA certificate to app core: %d\n", err);
		return err;
	}
#endif /* !CONFIG_MODEM_KEY_MGMT */

	printk("Successfully provisioned tag %d\n", sec_tag);
	return 0;
}

/**
 * @brief Provision all certificates to the modem
 */
int32_t tls_cert_provision_all(void)
{
	int32_t err = tls_cert_provision_single(COMODO_SEC_TAG, comodo_cert, sizeof(comodo_cert));
	if (err) {
		printk("Failed to provision Comodo certificate chain\n");
		return err;
	}

	printk("All certificates provisioned successfully.\n");
	return 0;
}


/* Macros used to subscribe to specific Zephyr NET management events. */
#define L4_EVENT_MASK (NET_EVENT_L4_CONNECTED | NET_EVENT_L4_DISCONNECTED)
#define CONN_LAYER_EVENT_MASK (NET_EVENT_CONN_IF_FATAL_ERROR)

/* Macro called upon a fatal error, reboots the device. */
#define FATAL_ERROR()					\
	LOG_ERR("Fatal error! Rebooting the device.");	\
	LOG_PANIC();					\
	IF_ENABLED(CONFIG_REBOOT, (sys_reboot(0)))

/* Zephyr NET management event callback structures. */
static struct net_mgmt_event_callback l4_cb;
static struct net_mgmt_event_callback conn_cb;

/* Variable used to indicate if network is connected. */
static bool is_connected;

/* Mutex and conditional variable used to signal network connectivity. */
K_MUTEX_DEFINE(network_connected_lock);
K_CONDVAR_DEFINE(network_connected);

static int server_resolve(struct sockaddr_storage *server)
{
	int err;
	struct addrinfo *result;
	struct addrinfo hints = {
		.ai_family = AF_INET,
		.ai_socktype = SOCK_DGRAM
	};
	char ipv4_addr[NET_IPV4_ADDR_LEN];

	err = getaddrinfo(COAP_HOSTNAME, NULL, &hints, &result);
	if (err) {
		LOG_ERR("getaddrinfo, error: %d", err);
		return err;
	}

	if (result == NULL) {
		LOG_ERR("Address not found");
		return -ENOENT;
	}

	/* IPv4 Address. */
	struct sockaddr_in *server4 = ((struct sockaddr_in *)server);

	server4->sin_addr.s_addr = ((struct sockaddr_in *)result->ai_addr)->sin_addr.s_addr;
	server4->sin_family = AF_INET;
	server4->sin_port = htons(COAP_PORT);

	inet_ntop(AF_INET, &server4->sin_addr.s_addr, ipv4_addr, sizeof(ipv4_addr));

	LOG_INF("IPv4 Address found %s", ipv4_addr);

	/* Free the address. */
	freeaddrinfo(result);

	return 0;
}

static void wait_for_network(void)
{
	k_mutex_lock(&network_connected_lock, K_FOREVER);

	if (!is_connected) {
		LOG_INF("Waiting for network connectivity");
		k_condvar_wait(&network_connected, &network_connected_lock, K_FOREVER);
	}

	k_mutex_unlock(&network_connected_lock);
}

static void response_cb(const struct coap_client_response_data *data, void *user_data)
{
	if (data->result_code >= 0) {
		LOG_INF("CoAP response: code: 0x%x, payload: %s",
			data->result_code, data->payload);
	} else {
		LOG_INF("Response received with error code: %d", data->result_code);
	}
}

static int periodic_coap_request_loop(void)
{
	int err, sock;
	struct sockaddr_storage server = { 0 };
	struct coap_client coap_client = { 0 };
	struct coap_client_request req = {
		.method = COAP_METHOD_GET,
		.confirmable = true,
		.fmt = COAP_CONTENT_FORMAT_TEXT_PLAIN,
		.payload = NULL,
		.cb = response_cb,
		.len = 0,
		.path = "/api/v1/ACCESS_TOKEN_REDACTED/attributes?sharedKeys=sdlConfigTs",
	};

	err = server_resolve(&server);
	if (err) {
		LOG_ERR("Failed to resolve server name");
		return err;
	}

	sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_DTLS_1_2);
	// sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
	if (sock < 0) {
		LOG_ERR("Failed to create CoAP socket: %d.", -errno);
		return -errno;
	}

	/* Require peer (server) certificate verification */
	enum {
		NONE = 0,
		OPTIONAL = 1,
		REQUIRED = 2,
	};
	int verify = REQUIRED;

	err = zsock_setsockopt(sock, SOL_TLS, TLS_PEER_VERIFY, &verify, sizeof(verify));
	if (err) {
		LOG_ERR("Failed to setup peer verification, errno %d", errno);
		return -errno;
	}

	/* Set the server hostname for SNI / certificate validation */
	err = zsock_setsockopt(sock, SOL_TLS, TLS_HOSTNAME, COAP_HOSTNAME, strlen(COAP_HOSTNAME));
	if (err) {
		LOG_ERR("Failed to setup TLS hostname (%s), errno %d", COAP_HOSTNAME, errno);
		return -errno;
	}

	/* Attach the provisioned CA certificate via its security tag */
	sec_tag_t sec_tag_list[] = { COMODO_SEC_TAG };

	err = zsock_setsockopt(sock, SOL_TLS, TLS_SEC_TAG_LIST, sec_tag_list,
	    sizeof(sec_tag_t) * ARRAY_SIZE(sec_tag_list));
	if (err) {
		LOG_ERR("Failed to setup socket security tag, errno %d", errno);
		return -errno;
	}

	LOG_INF("Initializing CoAP client");

	err = coap_client_init(&coap_client, NULL);
	if (err) {
		LOG_ERR("Failed to initialize CoAP client: %d", err);
		return err;
	}

	while (true) {
		wait_for_network();

		/* Send request */
		err = coap_client_req(&coap_client, sock, (struct sockaddr *)&server, &req, NULL);
		if (err) {
			LOG_ERR("Failed to send request: %d", err);
			return err;
		}

		LOG_INF("CoAP GET request sent sent to %s, resource: %s",
			CONFIG_COAP_SAMPLE_SERVER_HOSTNAME, CONFIG_COAP_SAMPLE_RESOURCE);

		k_sleep(K_SECONDS(CONFIG_COAP_SAMPLE_REQUEST_INTERVAL_SECONDS));
	}
}

static void l4_event_handler(struct net_mgmt_event_callback *cb,
			     uint64_t event,
			     struct net_if *iface)
{
	switch (event) {
	case NET_EVENT_L4_CONNECTED:
		LOG_INF("Network connectivity established");
		k_mutex_lock(&network_connected_lock, K_FOREVER);
		is_connected = true;
		k_condvar_signal(&network_connected);
		k_mutex_unlock(&network_connected_lock);
		break;
	case NET_EVENT_L4_DISCONNECTED:
		LOG_INF("Network connectivity lost");
		k_mutex_lock(&network_connected_lock, K_FOREVER);
		is_connected = false;
		k_mutex_unlock(&network_connected_lock);
		break;
	default:
		/* Don't care */
		return;
	}
}
static void connectivity_event_handler(struct net_mgmt_event_callback *cb,
						uint64_t event,
						struct net_if *iface)
{
	if (event == NET_EVENT_CONN_IF_FATAL_ERROR) {
		LOG_ERR("NET_EVENT_CONN_IF_FATAL_ERROR");
		FATAL_ERROR();
		return;
	}
}

int main(void)
{
	int err;

	LOG_INF("The CoAP client sample started");

	/* Setup handler for Zephyr NET Connection Manager events and Connectivity layer. */
	net_mgmt_init_event_callback(&l4_cb, l4_event_handler, L4_EVENT_MASK);
	net_mgmt_add_event_callback(&l4_cb);

	net_mgmt_init_event_callback(&conn_cb, connectivity_event_handler, CONN_LAYER_EVENT_MASK);
	net_mgmt_add_event_callback(&conn_cb);

	/* Bring all network interfaces up.
	 * Wi-Fi or LTE depending on the board that the sample was built for.
	 */
	LOG_INF("Bringing network interface up and connecting to the network");

	err = conn_mgr_all_if_up(true);
	if (err) {
		LOG_ERR("conn_mgr_all_if_up, error: %d", err);
		FATAL_ERROR();
		return err;
	}

	err = conn_mgr_all_if_connect(true);
	if (err) {
		LOG_ERR("conn_mgr_all_if_connect, error: %d", err);
		FATAL_ERROR();
		return err;
	}

	/* Resend connection status if the sample is built for NATIVE_SIM.
	 * This is necessary because the network interface is automatically brought up
	 * at SYS_INIT() before main() is called.
	 * This means that NET_EVENT_L4_CONNECTED fires before the
	 * appropriate handler l4_event_handler() is registered.
	 */
	if (IS_ENABLED(CONFIG_BOARD_NATIVE_SIM)) {
		conn_mgr_mon_resend_status();
	}

	wait_for_network();

	err = periodic_coap_request_loop();
	if (err) {
		LOG_ERR("periodic_coap_request_loop, error: %d", err);
		FATAL_ERROR();
		return err;
	}

	return 0;
}

This yields the following logs.

*** Booting nRF Connect SDK v3.4.0-99553055607b ***
*** Using Zephyr OS v4.4.0-bf801e4e3d19 ***
[00:00:00.254,058] <inf> coap_client_sample: The CoAP client sample started
[00:00:00.254,089] <inf> coap_client_sample: Bringing network interface up and connecting to the network
[00:00:00.586,791] <inf> coap_client_sample: Waiting for network connectivity
[00:00:04.264,648] <inf> coap_client_sample: Network connectivity established
[00:00:04.364,440] <inf> coap_client_sample: IPv4 Address found 3.127.76.36
[00:00:04.364,807] <inf> coap_client_sample: Initializing CoAP client
[00:00:04.366,149] <err> coap_client_sample: Failed to send request: -127
[00:00:04.366,180] <err> coap_client_sample: periodic_coap_request_loop, error: -127
[00:00:04.366,180] <err> coap_client_sample: Fatal error! Rebooting the device.

I have also tried manually connecting the socket and then passing NULL as the address pointer for the request:

/*
 * Copyright (c) 2019 Nordic Semiconductor ASA
 *
 * SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
 */

#include <stdio.h>
#include <string.h>

#if defined(CONFIG_POSIX_API)
#include <zephyr/posix/arpa/inet.h>
#include <zephyr/posix/netdb.h>
#include <zephyr/posix/sys/socket.h>
#include <zephyr/posix/poll.h>
#else
#include <zephyr/net/socket.h>
#endif /* CONFIG_POSIX_API */

#if CONFIG_MODEM_KEY_MGMT
#include <modem/modem_key_mgmt.h>
#endif

#include <zephyr/kernel.h>
#include <zephyr/sys/reboot.h>
#include <zephyr/net/coap.h>
#include <zephyr/net/socket.h>
#include <zephyr/net/conn_mgr_connectivity.h>
#include <zephyr/net/conn_mgr_monitor.h>
#include <zephyr/random/random.h>
#include <zephyr/net/coap_client.h>
#include <zephyr/logging/log.h>
#include <zephyr/logging/log_ctrl.h>
#include <zephyr/net/tls_credentials.h>

LOG_MODULE_REGISTER(coap_client_sample, CONFIG_COAP_CLIENT_SAMPLE_LOG_LEVEL);

#define COAP_HOSTNAME "coap.eu.thingsboard.cloud"
#define COAP_PORT 5684
#define COMODO_SEC_TAG 43


static const char comodo_cert[] = {
#include "tb-cloud-root-ca.pem.inc"
	IF_ENABLED(CONFIG_TLS_CREDENTIALS, (0x00))
};

BUILD_ASSERT(sizeof(comodo_cert) < KB(4), "Comodo certificate too large");

/**
 * @brief Provisions a single CA certificate to a specific security tag.
 * @param[in] sec_tag The security tag that the modem will use for the encrypted socket (choose
 * different tags for different DNS addresses).
 * @param[in] cert_buf The root certificate to provision.
 * @param[in] cert_len Length of the certificate.
 * @returns 0 or a negative errno on error
 */
static int32_t tls_cert_provision_single(sec_tag_t sec_tag, const char* cert_buf, size_t cert_len)
{
	int32_t err;

	printk("Processing certificate provisioning for sec tag: %d\n", sec_tag);

#if CONFIG_MODEM_KEY_MGMT
	bool exists;
	int32_t mismatch;

	err = modem_key_mgmt_exists(sec_tag, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, &exists);
	if (err) {
		printk("Failed to check for certificate on tag %d, err %d\n", sec_tag, err);
		return err;
	}

	if (exists) {
		mismatch = modem_key_mgmt_cmp(
		    sec_tag, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, cert_buf, cert_len);
		if (!mismatch) {
			printk(
			    "Certificate on tag %d matches expected content. Skipping.\n", sec_tag);
			return 0;
		}

		printk("Certificate mismatch detected on tag %d. Overwriting...\n", sec_tag);
		err = modem_key_mgmt_delete(sec_tag, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN);
		if (err) {
			printk(
			    "Failed to delete old certificate on tag %d, err %d\n", sec_tag, err);
		}
	}

	printk("Writing certificate to modem storage (tag %d)...\n", sec_tag);
	err = modem_key_mgmt_write(sec_tag, MODEM_KEY_MGMT_CRED_TYPE_CA_CHAIN, cert_buf, cert_len);
	if (err) {
		printk("Failed to write certificate to tag %d, err %d\n", sec_tag, err);
		return err;
	}
#else /* CONFIG_MODEM_KEY_MGMT */
	err = tls_credential_add(sec_tag, TLS_CREDENTIAL_CA_CERTIFICATE, cert_buf, cert_len);
	if (err == -EEXIST) {
		printk("CA certificate already exists on application core, sec tag: %d\n", sec_tag);
	} else if (err < 0) {
		printk("Failed to register CA certificate to app core: %d\n", err);
		return err;
	}
#endif /* !CONFIG_MODEM_KEY_MGMT */

	printk("Successfully provisioned tag %d\n", sec_tag);
	return 0;
}

/**
 * @brief Provision all certificates to the modem
 */
int32_t tls_cert_provision_all(void)
{
	int32_t err = tls_cert_provision_single(COMODO_SEC_TAG, comodo_cert, sizeof(comodo_cert));
	if (err) {
		printk("Failed to provision Comodo certificate chain\n");
		return err;
	}

	printk("All certificates provisioned successfully.\n");
	return 0;
}


/* Macros used to subscribe to specific Zephyr NET management events. */
#define L4_EVENT_MASK (NET_EVENT_L4_CONNECTED | NET_EVENT_L4_DISCONNECTED)
#define CONN_LAYER_EVENT_MASK (NET_EVENT_CONN_IF_FATAL_ERROR)

/* Macro called upon a fatal error, reboots the device. */
#define FATAL_ERROR()					\
	LOG_ERR("Fatal error! Rebooting the device.");	\
	LOG_PANIC();					\
	IF_ENABLED(CONFIG_REBOOT, (sys_reboot(0)))

/* Zephyr NET management event callback structures. */
static struct net_mgmt_event_callback l4_cb;
static struct net_mgmt_event_callback conn_cb;

/* Variable used to indicate if network is connected. */
static bool is_connected;

/* Mutex and conditional variable used to signal network connectivity. */
K_MUTEX_DEFINE(network_connected_lock);
K_CONDVAR_DEFINE(network_connected);

static int server_resolve(struct sockaddr_storage *server)
{
	int err;
	struct addrinfo *result;
	struct addrinfo hints = {
		.ai_family = AF_INET,
		.ai_socktype = SOCK_DGRAM
	};
	char ipv4_addr[NET_IPV4_ADDR_LEN];

	err = getaddrinfo(COAP_HOSTNAME, NULL, &hints, &result);
	if (err) {
		LOG_ERR("getaddrinfo, error: %d", err);
		return err;
	}

	if (result == NULL) {
		LOG_ERR("Address not found");
		return -ENOENT;
	}

	/* IPv4 Address. */
	struct sockaddr_in *server4 = ((struct sockaddr_in *)server);

	server4->sin_addr.s_addr = ((struct sockaddr_in *)result->ai_addr)->sin_addr.s_addr;
	server4->sin_family = AF_INET;
	server4->sin_port = htons(COAP_PORT);

	inet_ntop(AF_INET, &server4->sin_addr.s_addr, ipv4_addr, sizeof(ipv4_addr));

	LOG_INF("IPv4 Address found %s", ipv4_addr);

	/* Free the address. */
	freeaddrinfo(result);

	return 0;
}

static void wait_for_network(void)
{
	k_mutex_lock(&network_connected_lock, K_FOREVER);

	if (!is_connected) {
		LOG_INF("Waiting for network connectivity");
		k_condvar_wait(&network_connected, &network_connected_lock, K_FOREVER);
	}

	k_mutex_unlock(&network_connected_lock);
}

static void response_cb(const struct coap_client_response_data *data, void *user_data)
{
	if (data->result_code >= 0) {
		LOG_INF("CoAP response: code: 0x%x, payload: %s",
			data->result_code, data->payload);
	} else {
		LOG_INF("Response received with error code: %d", data->result_code);
	}
}

static int periodic_coap_request_loop(void)
{
	int err, sock;
	struct sockaddr_storage server = { 0 };
	struct coap_client coap_client = { 0 };
	struct coap_client_request req = {
		.method = COAP_METHOD_GET,
		.confirmable = true,
		.fmt = COAP_CONTENT_FORMAT_TEXT_PLAIN,
		.payload = NULL,
		.cb = response_cb,
		.len = 0,
		.path = "/api/v1/ACCESS_TOKEN_REDACTED/attributes?sharedKeys=sdlConfigTs",
	};

	err = server_resolve(&server);
	if (err) {
		LOG_ERR("Failed to resolve server name");
		return err;
	}

	sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_DTLS_1_2);
	// sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
	if (sock < 0) {
		LOG_ERR("Failed to create CoAP socket: %d.", -errno);
		return -errno;
	}

	/* Require peer (server) certificate verification */
	enum {
		NONE = 0,
		OPTIONAL = 1,
		REQUIRED = 2,
	};
	int verify = REQUIRED;

	err = zsock_setsockopt(sock, SOL_TLS, TLS_PEER_VERIFY, &verify, sizeof(verify));
	if (err) {
		LOG_ERR("Failed to setup peer verification, errno %d", errno);
		return -errno;
	}

	/* Set the server hostname for SNI / certificate validation */
	err = zsock_setsockopt(sock, SOL_TLS, TLS_HOSTNAME, COAP_HOSTNAME, strlen(COAP_HOSTNAME));
	if (err) {
		LOG_ERR("Failed to setup TLS hostname (%s), errno %d", COAP_HOSTNAME, errno);
		return -errno;
	}

	/* Attach the provisioned CA certificate via its security tag */
	sec_tag_t sec_tag_list[] = { COMODO_SEC_TAG };

	err = zsock_setsockopt(sock, SOL_TLS, TLS_SEC_TAG_LIST, sec_tag_list,
	    sizeof(sec_tag_t) * ARRAY_SIZE(sec_tag_list));
	if (err) {
		LOG_ERR("Failed to setup socket security tag, errno %d", errno);
		return -errno;
	}

	socklen_t addr_len = (((struct sockaddr *)&server)->sa_family == AF_INET) ? sizeof(struct sockaddr_in)
								 : sizeof(struct sockaddr_in6);

	err = connect(sock, (struct sockaddr *)&server, addr_len);
	if (err < 0) {
		LOG_ERR("DTLS transport connect failed, errno %d", errno);
		return -errno;
	}

	LOG_INF("Initializing CoAP client");

	err = coap_client_init(&coap_client, NULL);
	if (err) {
		LOG_ERR("Failed to initialize CoAP client: %d", err);
		return err;
	}

	while (true) {
		wait_for_network();

		/* Send request */
		err = coap_client_req(&coap_client, sock, NULL, &req, NULL);
		if (err) {
			LOG_ERR("Failed to send request: %d", err);
			return err;
		}

		LOG_INF("CoAP GET request sent sent to %s, resource: %s",
			CONFIG_COAP_SAMPLE_SERVER_HOSTNAME, CONFIG_COAP_SAMPLE_RESOURCE);

		k_sleep(K_SECONDS(CONFIG_COAP_SAMPLE_REQUEST_INTERVAL_SECONDS));
	}
}

static void l4_event_handler(struct net_mgmt_event_callback *cb,
			     uint64_t event,
			     struct net_if *iface)
{
	switch (event) {
	case NET_EVENT_L4_CONNECTED:
		LOG_INF("Network connectivity established");
		k_mutex_lock(&network_connected_lock, K_FOREVER);
		is_connected = true;
		k_condvar_signal(&network_connected);
		k_mutex_unlock(&network_connected_lock);
		break;
	case NET_EVENT_L4_DISCONNECTED:
		LOG_INF("Network connectivity lost");
		k_mutex_lock(&network_connected_lock, K_FOREVER);
		is_connected = false;
		k_mutex_unlock(&network_connected_lock);
		break;
	default:
		/* Don't care */
		return;
	}
}
static void connectivity_event_handler(struct net_mgmt_event_callback *cb,
						uint64_t event,
						struct net_if *iface)
{
	if (event == NET_EVENT_CONN_IF_FATAL_ERROR) {
		LOG_ERR("NET_EVENT_CONN_IF_FATAL_ERROR");
		FATAL_ERROR();
		return;
	}
}

int main(void)
{
	int err;

	LOG_INF("The CoAP client sample started");

	/* Setup handler for Zephyr NET Connection Manager events and Connectivity layer. */
	net_mgmt_init_event_callback(&l4_cb, l4_event_handler, L4_EVENT_MASK);
	net_mgmt_add_event_callback(&l4_cb);

	net_mgmt_init_event_callback(&conn_cb, connectivity_event_handler, CONN_LAYER_EVENT_MASK);
	net_mgmt_add_event_callback(&conn_cb);

	/* Bring all network interfaces up.
	 * Wi-Fi or LTE depending on the board that the sample was built for.
	 */
	LOG_INF("Bringing network interface up and connecting to the network");

	err = conn_mgr_all_if_up(true);
	if (err) {
		LOG_ERR("conn_mgr_all_if_up, error: %d", err);
		FATAL_ERROR();
		return err;
	}

	err = conn_mgr_all_if_connect(true);
	if (err) {
		LOG_ERR("conn_mgr_all_if_connect, error: %d", err);
		FATAL_ERROR();
		return err;
	}

	/* Resend connection status if the sample is built for NATIVE_SIM.
	 * This is necessary because the network interface is automatically brought up
	 * at SYS_INIT() before main() is called.
	 * This means that NET_EVENT_L4_CONNECTED fires before the
	 * appropriate handler l4_event_handler() is registered.
	 */
	if (IS_ENABLED(CONFIG_BOARD_NATIVE_SIM)) {
		conn_mgr_mon_resend_status();
	}

	wait_for_network();

	err = periodic_coap_request_loop();
	if (err) {
		LOG_ERR("periodic_coap_request_loop, error: %d", err);
		FATAL_ERROR();
		return err;
	}

	return 0;
}

That produces a different application log:

*** Booting nRF Connect SDK v3.4.0-99553055607b ***
*** Using Zephyr OS v4.4.0-bf801e4e3d19 ***
[00:00:00.385,070] <inf> coap_client_sample: The CoAP client sample started
[00:00:00.385,101] <inf> coap_client_sample: Bringing network interface up and connecting to the network
[00:00:00.720,092] <inf> coap_client_sample: Waiting for network connectivity
[00:00:05.107,818] <inf> coap_client_sample: Network connectivity established
[00:00:05.241,302] <inf> coap_client_sample: IPv4 Address found 52.58.111.18
[00:00:07.060,394] <inf> coap_client_sample: Initializing CoAP client
[00:00:07.063,079] <inf> coap_client_sample: CoAP GET request sent sent to californium.eclipseprojects.io, resource: obs
[00:00:07.161,773] <inf> coap_client_sample: Response received with error code: -106
[00:00:10.102,813] <inf> coap_client_sample: Response received with error code: -106

Is the following definition if the 106 error the one being bubbled up into the callback?

#define EAFNOSUPPORT 106	/* Address family not supported by protocol family */

This leaves me very confused. I also tried to rule out certificate issues by setting the following.

/* Require peer (server) certificate verification */
	enum {
		NONE = 0,
		OPTIONAL = 1,
		REQUIRED = 2,
	};
	int verify = NONE;

	err = zsock_setsockopt(sock, SOL_TLS, TLS_PEER_VERIFY, &verify, sizeof(verify));
	if (err) {
		LOG_ERR("Failed to setup peer verification, errno %d", errno);
		return -errno;
	}

This yields the same results.

Is the CoAP client library compatible with DTLS and is the intended use to let the library connect the socket or should the socket be manually connected? 

Help is greatly appreciated!

Best,

Tom

EDIT: I was able to confirm that the request can be successfully carried out when using the low-level CoAP methods. Therefore, I can rule out certificate or DTLS handshake issues. I am probably misunderstanding something about the high-level CoAP client library...

[Syed Maysum Abbas Zaidi]

Hi Tom,

Thanks for the detailed CoAP debug log, it matches what we see here.

On nRF9151 with your approach (DTLS, connect(), NULL address to coap_client_req()), we see the same behaviour on NCS v3.4.0 the request and response complete, then sending the empty ACK fails (Error sending a CoAP ACK-message, -106 in the callback). The same application on NCS v3.4.1 works without that error.

Your tests with non-confirmable requests fit this, that path does not hit the same ACK handling. This does not indicate that confirmable CoAP is unsupported on the modem, I think it points to a CoAP client issue on v3.4.0 when acknowledging a confirmable response on a connected DTLS socket. So recommendation is to move to NCS v3.4.1. Do let me know if it runs without error on it.

[Tom01]

Hey Syed, 

thanks for the response. I just updated to NCS v3.4.1 and now confirmable requests work! Thanks!

For the record and for the AI: CoAP with DTLS works when the socket is created, socket options for DTLS are applied and the socket is then connected manually. When using the CoAP client API, it is important to then pass NULL as the "addr" argument in the "coap_client_req()" method. This works on NCS v3.4.1.

Furthermore I have two additional questions:

1. Our application will probably have to renew the socket regularly and I am not yet sure if we will be able to use CID for session resumptions. For general TLS session resumption with HTTPS the following seems to work:

int32_t cache_enabled = 1;
err = setsockopt(
    fd, NRF_SOL_SECURE, NRF_SO_SEC_SESSION_CACHE, &cache_enabled, sizeof(cache_enabled));
if (err) {
	printk("Failed to enable TLS session cache, err %d\n", errno);
	return err;
}

Would this also apply to DTLS session resumption in the CoAP case?

2. I was wondering how PSM and eDRX settings affect the inner workings of the CoAP client. We do not plan to use observe functionality. Would settings like

# PSM
CONFIG_LTE_LC_PSM_MODULE=y
CONFIG_LTE_PSM_REQ=y
# PSM Periodic TAU (2 hours)
CONFIG_LTE_PSM_REQ_RPTAU="00100010"
# PSM Active Time (22 seconds)
CONFIG_LTE_PSM_REQ_RAT="00001011"
# eDRX
CONFIG_LTE_LC_EDRX_MODULE=y
CONFIG_LTE_EDRX_REQ=y
# eDRX Cycle: "0000" = 5.12 s, "0010" = 20.48 s
CONFIG_LTE_EDRX_REQ_VALUE_LTE_M="0010"
CONFIG_LTE_EDRX_REQ_VALUE_NBIOT="0010"
# Paging Time Window (PTW): "0011" = 5.12 s for LTE-M / 10.24 s for NB-IoT
CONFIG_LTE_PTW_VALUE_LTE_M="0011"
CONFIG_LTE_PTW_VALUE_NBIOT="0011"

cause conflicts with the timeout and retry behavior of the CoAP client if the modem successfully negotiates them?

Thanks again and best,

Tom

[Achim Kraus]

> Our application will probably have to renew the socket regularly 

Why?

> I am not yet sure if we will be able to use CID for session resumptions.

CID is not related to session resumption. CID rather obsoletes session resumption in the most cases.

The nasty point in Nordic's DTLS modem implementation is, that the DTLS state is bound to the socket. If that socket closes, the state is usually gone. I don't know, If there is a way to preserve it, but Nordic will know it. 

[Syed Maysum Abbas Zaidi]

Hi Tom,

Good to know it worked.

I tested your pattern, TLS_SESSION_CACHE on a new DTLS socket, connect(), then coap_client_req close and repeat. However TLS_DTLS_HANDSHAKE_STATUS was full on two connects, so I did not see session resumption on my test server (not ThingsBoard). The cache option applies to DTLS like HTTPS, but I think resumption depends on the server side so I would suggest you to try on your ThingsBoard.

I am checking details about your second question and will get back to you by tomorrow. Thanks

[Achim Kraus]

If you provide an ip-capture/wireshark, we will see, what happens exactly.

[Tom01]

Hey Achim,

thanks for your responses, this and the previous one. 

Achim Kraus said:

> Our application will probably have to renew the socket regularly 

Why?

It might well be that my understanding of UDP sockets is not correct: Our application will have situations in which the socket will only be used every few hours, potentially even once a day. My assumption so far was that we cannot leave the socket open and inactive for long periods, because the cellular provider might close the session. Though, now that I am thinking about it, this might come down to how PSM and eDRX settings affect the session state. Do you have any insights and advice on how long the socket could remain open but inactive?

Achim Kraus said:

> I am not yet sure if we will be able to use CID for session resumptions.

CID is not related to session resumption. CID rather obsoletes session resumption in the most cases.

My wording is probably not correct, because this is the first time that I work with CoAP in-depth. My understanding is that the CoAP server and especially the load balancer need to be CID aware when multiple server instances are involved. I haven't yet checked whether this is possible for us. Maybe you know this better: What happens to CIDs if for example load balancer or server instances are restarted or scaled down?

Best,

Tom

[Achim Kraus]

> Do you have any insights and advice on how long the socket could remain open but inactive?

I use PSM and in some cases eDRX. My UDP sockets stay open for days and weeks. They mainly get closed by the modem itself, when that lost the network.

> My understanding is that the CoAP server and especially the load balancer need to be CID aware when multiple server instances are involved. 

CID is used to overcome the widely used NATs. The depends a lot on your network setup, but frequently some NATs are on the path, say at the MNO outgress or your cloud ingress. In quite a lot of cases that's even not known. For UDP unfortunately the most NATs uses pretty small timeouts, e.g. 30s or 120s. That means, if your device sleeps (doesn't send a UDP packet), the next UDP packet may show a different public IP address to the server. Usually not bad for UDP, but the DTLS context is related to that address and if that changes, then the server will not be able to find the right DTLS context for that packet.

CID simply replaces the ip-address by a much more stable Connection ID.

Before the CID, one strategy to overcome the address change was to use the (D)TLS session ID to resume a session. That takes less message than a full handshake, but still additional messages.

But "load balancer" is then something, which in many cases stops both, CID and session resumption, because the UDP load balancer will also be affected by the ip-address changes. And the nodes usually don't share the DTLS contexts. 

There some ideas to overcome that. One is to use multiple DNS entries and stick after the handshake to the chosen address. An other approach is more complex, "record forwarding" and I only know Eclipse/Californium (I'm one of the developers of that), which comes with such an implementation. And I still don't know, if UDP loadbalancers are out, which would support routing by CID.

(On more interest, you may read some pages at https://github.com/eclipse-californium/californium/wiki)

> What happens to CIDs if for example load balancer or server instances are restarted or scaled down?

Let's focus on the "multiple DNS entries". If a node crashes, it depends on your client to detect that. If the mobile network is healthy and no response nor ack is received, then a new DNS lookup and a new handshake is required. If a "down node" is chosen, the handshake times out and the client needs to check the next addresses.

There are not that many implementations, which offers a DTLS graceful restart, I mainly know Eclipse/Californium. With that, a planed server update usually takes a few seconds. The most device will not recognize that at all and a few ones will need a normal CoAP retry.