Error -22 in mqtt_connect() - nRF52840dk with Azure IoT Hub using OpenThread and TCP

Question

I am attempting to connect a nRF52840dk to Azure IoT Hub using OpenThread and TCP. I combined the azure_iot_hub and azure_fota samples into one project, which ran successfully on an nRF9160dk using Connect SDK v1.9.1. I have modified that project for a nRF52840dk, OpenThread, TCP, and Connect SDK 2.0.0. I think that I am close, but I am getting error -22 ("invalid argument") in azure_iot_hub.c's mqtt_connect(). 
 Wireshark capture using nRF sniffer: 
 
 Serial output (I added the "Error in zsock_connect!" log in mqtt_transport_socket_tls.c): 
 
 I've repurposed CONFIG_AZURE_IOT_HUB_STATIC_IPV4 to be an ipv6 address as seen in my prj further below. The getaddrinfo() DNS resolver is working, but I'd have to add a conversion from the returned ipv4 to ipv6 and I'd prefer to bypass that for now unless this is causing the issue. In azure_iot_hub.c's broket_init(), I changed &broker to &broker4 to resolve an error and made some other updates to switch from ipv4 to ipv6. These are the only changes I made in azure_iot_hub.c. The IoT Hub setup is kicked off in main() via err = azure_iot_hub_connect(); 
 #if defined(CONFIG_AZURE_IOT_HUB_STATIC_IPV4)
static int broker_init(bool dps)
{
	//TB changed sockaddr's from sockaddr_in to sockaddr_in6
	struct sockaddr_in6 *broker4 =
 ((struct sockaddr_in6 *)&broker);

//TB changed "AF_INET" to "AF_INET6" twice and "&broker" to "&broker4" on 7/20/22
	inet_pton(AF_INET6, CONFIG_AZURE_IOT_HUB_STATIC_IPV4_ADDR,
 &broker4->sin6_addr);//&broker->sin_addr);
	broker4->sin6_family = AF_INET6;
	broker4->sin6_port = htons(CONFIG_AZURE_IOT_HUB_PORT);

//TB added:
	char ipv6_addr[NET_IPV6_ADDR_LEN];
	inet_ntop(AF_INET6, &broker4->sin6_addr.s6_addr, ipv6_addr,
 sizeof(ipv6_addr));
	LOG_DBG("IPv6 address set in broker_init to %s", log_strdup(ipv6_addr));

return 0;
} 
 prj.conf: 
 #
# Copyright (c) 2020 Nordic Semiconductor ASA
#
# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
#
# General config
CONFIG_REBOOT=y
CONFIG_DEBUG=y

# Heap and stacks
CONFIG_HEAP_MEM_POOL_SIZE=6144
CONFIG_MAIN_STACK_SIZE=8192

# Log
CONFIG_LOG=y
CONFIG_PRINTK=y
CONFIG_SERIAL=y
CONFIG_CONSOLE=y
CONFIG_UART_CONSOLE=y
CONFIG_LOG_BACKEND_UART=y
CONFIG_LOG_PROCESS_THREAD=y
CONFIG_LOG_MODE_IMMEDIATE=y
CONFIG_LOG_STRDUP_MAX_STRING=128
CONFIG_LOG_STRDUP_BUF_COUNT=50
CONFIG_LOG_PROCESS_THREAD_STACK_SIZE=8096

##### booting and bootloader #####
CONFIG_BOOT_DELAY=1000
CONFIG_BOOT_BANNER=y
CONFIG_BOOTLOADER_MCUBOOT=y

##### DFU #####
CONFIG_DFU_TARGET=y
CONFIG_DFU_TARGET_MCUBOOT=y
CONFIG_IMG_MANAGER=y
CONFIG_MCUBOOT_IMG_MANAGER=y
CONFIG_IMG_ERASE_PROGRESSIVELY=y

##### for external flash support ####
CONFIG_NORDIC_QSPI_NOR=y
CONFIG_NORDIC_QSPI_NOR_FLASH_LAYOUT_PAGE_SIZE=4096
CONFIG_NORDIC_QSPI_NOR_STACK_WRITE_BUFFER_SIZE=16
CONFIG_PM_EXTERNAL_FLASH_MCUBOOT_SECONDARY=y

##### FLASH #####
CONFIG_FLASH=y
CONFIG_FLASH_MAP=y
CONFIG_STREAM_FLASH=y
CONFIG_STREAM_FLASH_ERASE=y
CONFIG_FLASH_PAGE_LAYOUT=y

CONFIG_MPU_ALLOW_FLASH_WRITE=y

# LED control
CONFIG_DK_LIBRARY=y
#CONFIG_DK_LIBRARY_INVERT_LEDS=n #not available in v2.0.0

#Openthread
CONFIG_OPENTHREAD_JOINER=y
CONFIG_NET_L2_OPENTHREAD=y
CONFIG_OPENTHREAD_SHELL=n
CONFIG_OPENTHREAD_CUSTOM_PARAMETERS="OPENTHREAD_CONFIG_JOINER_ENABLE=1"
CONFIG_OPENTHREAD_JOINER_AUTOSTART=y
CONFIG_OPENTHREAD_JOINER_PSKD="J01NME"
CONFIG_OPENTHREAD_THREAD_VERSION_1_2=y
#CONFIG_MBEDTLS_SHA1_C=n #TB commented
CONFIG_FPU=y

# TLS configuration #TB commented
#CONFIG_MBEDTLS_ENABLE_HEAP=y
#CONFIG_MBEDTLS_HEAP_SIZE=10240
#CONFIG_MBEDTLS_SSL_MAX_CONTENT_LEN=2048
#CONFIG_MBEDTLS=y
#CONFIG_MBEDTLS_BUILTIN=n
#CONFIG_MBEDTLS_KEY_EXCHANGE_SOME_PSK_ENABLED=y
#CONFIG_MBEDTLS_KEY_EXCHANGE_PSK_ENABLED=y
#CONFIG_NET_SOCKETS_SOCKOPT_TLS=y

##### OPENTHREAD #####
CONFIG_OPENTHREAD_NORDIC_LIBRARY_MASTER=y
CONFIG_OPENTHREAD_FTD=n
CONFIG_OPENTHREAD_MTD=y
CONFIG_OPENTHREAD_MTD_SED=n
CONFIG_OPENTHREAD_THREAD_STACK_SIZE=10240
CONFIG_OPENTHREAD_DEBUG=y
CONFIG_OPENTHREAD_L2_DEBUG=y
CONFIG_OPENTHREAD_MANUAL_START=y
# Enable Thread 1.2 features
CONFIG_OPENTHREAD_THREAD_VERSION_1_2=y
CONFIG_OPENTHREAD_DUA=y
CONFIG_OPENTHREAD_MLR=y
CONFIG_OPENTHREAD_BACKBONE_ROUTER=y
CONFIG_OPENTHREAD_LINK_METRICS_INITIATOR=y
CONFIG_OPENTHREAD_LINK_METRICS_SUBJECT=y
CONFIG_OPENTHREAD_CSL_RECEIVER=y

# Network
CONFIG_NETWORKING=y
CONFIG_NET_L2_OPENTHREAD=y
CONFIG_NET_IPV6_NBR_CACHE=n
CONFIG_NET_IPV6_MLD=n
# CONFIG_NET_RAW_MODE=n
CONFIG_NET_IPV6=y
CONFIG_NET_IPV4=n
CONFIG_NET_CONFIG_NEED_IPV4=n
CONFIG_NET_CONFIG_NEED_IPV6=y
# Network sockets
CONFIG_NET_SOCKETS=y
CONFIG_NET_SOCKETS_POSIX_NAMES=y
CONFIG_NET_SOCKETS_POLL_MAX=4
CONFIG_NET_SOCKETS_SOCKOPT_TLS=y
CONFIG_NET_SOCKETS_TLS_MAX_CONTEXTS=2
# Enable TCP support
CONFIG_NET_TCP=y # Required for SOCKET STREAM
#CONFIG_NET_UDP=y # Testing UDP connection #TB commented
CONFIG_OPENTHREAD_TCP_ENABLE=n
#^from https://github.com/openthread/openthread/discussions/7784

# disable external crystal
CONFIG_CLOCK_CONTROL_NRF_K32SRC_XTAL=n
# enable synth crystal for powered devices
CONFIG_CLOCK_CONTROL_NRF_K32SRC_SYNTH=y
# enable RC crystal for battery devices
# CONFIG_CLOCK_CONTROL_NRF_K32SRC_RC is not set
#^from https://github.com/openthread/openthread/discussions/7784

CONFIG_NET_PKT_RX_COUNT=8
CONFIG_NET_PKT_TX_COUNT=8
CONFIG_NET_BUF_RX_COUNT=32
CONFIG_NET_BUF_TX_COUNT=32
#^from https://github.com/openthread/openthread/discussions/7784

# Network buffers
#CONFIG_NET_PKT_RX_COUNT=10
#CONFIG_NET_PKT_TX_COUNT=16
#CONFIG_NET_BUF_RX_COUNT=16
#CONFIG_NET_BUF_TX_COUNT=16
#^old values

# Kernel options
CONFIG_INIT_STACKS=y

# Increase set for threads with meta-irq priority
CONFIG_NUM_METAIRQ_PRIORITIES=1

# Logging
CONFIG_NET_LOG=y #POWERSAVING

# Disable certain parts of Zephyr IPv6 stack
CONFIG_NET_IPV6_NBR_CACHE=n
CONFIG_NET_IPV6_MLD=n

# Stack sizes configuration
CONFIG_NET_TX_STACK_SIZE=1200
CONFIG_NET_RX_STACK_SIZE=1500
CONFIG_SYSTEM_WORKQUEUE_STACK_SIZE=4096

# L2 OpenThread enabling
CONFIG_OPENTHREAD_L2_LOG_LEVEL_INF=y

# Enable ping sender support
CONFIG_OPENTHREAD_PING_SENDER=y

# Configure dependencies
CONFIG_NRF_802154_ENCRYPTION=y
CONFIG_IEEE802154_2015=y
CONFIG_IEEE802154_CSL_ENDPOINT=y
CONFIG_NET_PKT_TXTIME=y
CONFIG_NET_PKT_TIMESTAMP=y
CONFIG_OPENTHREAD_MAC_SOFTWARE_TX_SECURITY_ENABLE=n

# CSL configuration
CONFIG_OPENTHREAD_CSL_RECEIVE_TIME_AHEAD=3000
CONFIG_OPENTHREAD_CSL_MIN_RECEIVE_ON=300

# Azure IoT Hub library
CONFIG_AZURE_IOT_HUB=y
CONFIG_AZURE_IOT_HUB_DEVICE_ID="mynrf52840dk"
# Host name must be configured if DPS is not used
CONFIG_AZURE_IOT_HUB_HOSTNAME="my-iot-hub.azure-devices.net"
# Change the security tag to the tag where relevant certificates are provisioned
CONFIG_AZURE_IOT_HUB_SEC_TAG=42
# Uncomment to get more verbose logging when debugging
CONFIG_AZURE_IOT_HUB_LOG_LEVEL_DBG=y
CONFIG_AZURE_IOT_HUB_LOG_LEVEL_WRN=y
#Use manual certificates
CONFIG_USE_MANUAL_IOTHUB_CERTS=y
CONFIG_AZURE_IOT_HUB_PROVISION_CERTIFICATES=y
CONFIG_AZURE_IOT_HUB_STATIC_IPV4=y
CONFIG_AZURE_IOT_HUB_STATIC_IPV4_ADDR="64:ff9b::myio:thub"
CONFIG_AZURE_IOT_HUB_NATIVE_TLS=n
#CONFIG_AZURE_IOT_HUB_CERTIFICATES_FILE keep = default

# Azure FOTA
# Download Client
CONFIG_DOWNLOAD_CLIENT=y
CONFIG_DOWNLOAD_CLIENT_HTTP_FRAG_SIZE_1024=y
CONFIG_DOWNLOAD_CLIENT_STACK_SIZE=4096
CONFIG_DOWNLOAD_CLIENT_LOG_LEVEL_INF=y
CONFIG_DOWNLOAD_CLIENT_BUF_SIZE=2300
# DFU Target
CONFIG_DFU_TARGET=y
# Application update support
CONFIG_BOOTLOADER_MCUBOOT=y
# Image manager
CONFIG_IMG_MANAGER=y
CONFIG_IMG_ERASE_PROGRESSIVELY=y
# FOTA Download
CONFIG_FOTA_DOWNLOAD=y
CONFIG_FOTA_DOWNLOAD_PROGRESS_EVT=y

# Azure FOTA
CONFIG_CJSON_LIB=y
#CONFIG_ZEPHYR_CJSON_MODULE=y
CONFIG_AZURE_FOTA=y
CONFIG_AZURE_FOTA_APP_VERSION_AUTO=y
CONFIG_AZURE_FOTA_TLS=y
CONFIG_FW_INFO=y
# Change the security tag to the tag where the certificates are provisioned
# for the server where the FOTA image is hosted
CONFIG_AZURE_FOTA_SEC_TAG=42
# Uncomment the below line to get more debug logging
# CONFIG_AZURE_FOTA_LOG_LEVEL_DBG=y

CONFIG_NEWLIB_LIBC_FLOAT_PRINTF=y
CONFIG_NEWLIB_LIBC=y
CONFIG_EXTERNAL_LIBC=n
CONFIG_CJSON_LIB=y

CONFIG_NORDIC_SECURITY_BACKEND=y
CONFIG_NRF_SECURITY=y 
CONFIG_MBEDTLS_BUILTIN=n

CONFIG_MBEDTLS=y
CONFIG_MBEDTLS_CFG_FILE="config-tls-generic.h"
CONFIG_NET_TCP_ISN_RFC6528=n

CONFIG_OPENTHREAD_MBEDTLS_CHOICE=n

# Select OpenThread nRF Security backends
CONFIG_OPENTHREAD_NRF_SECURITY_CHOICE=n

# Generic networking options
CONFIG_NET_CONNECTION_MANAGER=n

CONFIG_NET_TCP_LOG_LEVEL_DBG=y
CONFIG_LOG_STRDUP_BUF_COUNT=20

# DNS Settings
CONFIG_DNS_RESOLVER=y
CONFIG_DNS_SERVER_IP_ADDRESSES=y
CONFIG_DNS_SERVER1="64:ff9b::0808:0808"
#CONFIG_OPENTHREAD_DNS_CLIENT=y

CONFIG_MBEDTLS_CIPHER_MODE_CBC=y 
 Certificate format: 
 "-----BEGIN CA CERTIFICATE-----
"
"abcd+efgh
"
"ijkl/mnop
"
"-----END CA CERTIFICATE-----
" 
 "-----BEGIN CLIENT CERTIFICATE-----
"
"abcd+efgh
"
"ijkl/mnop
"
"-----END CLIENT CERTIFICATE-----
" 
 "-----BEGIN PRIVATE KEY-----
"
"abcd+efgh
"
"ijkl/mnop
"
"-----END PRIVATE KEY-----
" 
 Per this DevZone ticket and this Microsoft documentation, maybe I am missing something with the CBC ciphers and MBedTLS? TCP messages appear to be going back and forth between the IoT Hub MQTT and my OpenThread end device in the Wireshark sniffer trace, so hopefully I am close to the finish line. Thank you in advance for the assistance.

brown27 · Accepted Answer

I made some further configuration changes and it is working now! I am even sending dummy temperature values to IoT Hub in JSON format, which are then forwarded to blob storage. 
 To summarize, I needed to select the right combination of OpenThread, security, MBEDTLS, cipher, and key configurations in my prj.conf file. I have copied the relevant sections of my prj.conf file below. Pretty much everything that is commented out is something that I tried without success. I'm certain that there are some unnecessary configurations in there, but I have not yet attempted to whittle those down to the essentials. Interestingly, it was not necessary for me to set the MBEDTLS platform time to get this to work. 
 Thank you for the help! 
 #Openthread
# Select OpenThread nRF Security backends
CONFIG_CUSTOM_OPENTHREAD_SECURITY=y
#CONFIG_OPENTHREAD_NRF_SECURITY_CHOICE=n
#CONFIG_MBEDTLS_LEGACY_CRYPTO_C=y

##### OPENTHREAD #####
CONFIG_OPENTHREAD_NORDIC_LIBRARY_FTD=y
#CONFIG_OPENTHREAD_FTD=n
#CONFIG_OPENTHREAD_MTD=y
#CONFIG_OPENTHREAD_MTD_SED=n
CONFIG_OPENTHREAD_THREAD_STACK_SIZE=10240
CONFIG_OPENTHREAD_DEBUG=y
CONFIG_OPENTHREAD_L2_DEBUG=y
CONFIG_OPENTHREAD_MANUAL_START=y

# Enable Thread 1.2 features
CONFIG_OPENTHREAD_DUA=y
CONFIG_OPENTHREAD_MLR=y
#CONFIG_OPENTHREAD_BACKBONE_ROUTER=y
CONFIG_OPENTHREAD_LINK_METRICS_INITIATOR=y
CONFIG_OPENTHREAD_LINK_METRICS_SUBJECT=y
CONFIG_OPENTHREAD_CSL_RECEIVER=y

# Network
CONFIG_NETWORKING=y
CONFIG_NET_L2_OPENTHREAD=y
CONFIG_NET_IPV6_NBR_CACHE=n
CONFIG_NET_IPV6_MLD=n
CONFIG_NET_IPV6=y
CONFIG_NET_IPV4=n
CONFIG_NET_CONFIG_SETTINGS=y
CONFIG_NET_CONFIG_NEED_IPV4=n
CONFIG_NET_CONFIG_NEED_IPV6=y
# Network sockets
CONFIG_NET_SOCKETS=y
CONFIG_NET_SOCKETS_POSIX_NAMES=y
CONFIG_NET_SOCKETS_POLL_MAX=4
# Enable TCP support
CONFIG_NET_TCP=y # Required for SOCKET STREAM
CONFIG_NET_UDP=y # Required for getting UTC time 
CONFIG_OPENTHREAD_TCP_ENABLE=n
#^from https://github.com/openthread/openthread/discussions/7784

CONFIG_NET_SOCKETS_TLS_MAX_CONTEXTS=10
#^from https://devzone.nordicsemi.com/f/nordic-q-a/86632/openthread-and-mqtt-over-tls-is-single-program

#Socket settings
CONFIG_NET_SOCKETS_SOCKOPT_TLS=y
CONFIG_NET_SOCKETS_ENABLE_DTLS=n
CONFIG_POSIX_MAX_FDS=8

#MBEDTLS and security configuration 
#CONFIG_NORDIC_SECURITY_BACKEND=y
#CONFIG_NRF_SECURITY=y
CONFIG_MBEDTLS_CFG_FILE="config-tls-generic.h"
CONFIG_NET_TCP_ISN_RFC6528=n

# Enable all crypto
#CONFIG_MBEDTLS_KEY_EXCHANGE_ALL_ENABLED=y
CONFIG_MBEDTLS_CIPHER_ALL_ENABLED=y
CONFIG_MBEDTLS_ECP_ALL_ENABLED=y
CONFIG_MBEDTLS_MAC_ALL_ENABLED=y
CONFIG_MBEDTLS_GENPRIME_ENABLED=y
CONFIG_MBEDTLS_HMAC_DRBG_ENABLED=y

#MBEDTLS Configuration cont'd
CONFIG_OPENTHREAD_MBEDTLS_CHOICE=y
#^ y allows use of MBEDTLS_BUILTIN
#^ n does not allow MBEDTLS_BUILTIN and uses NRF_SECURITY
CONFIG_MBEDTLS_TLS_VERSION_1_2=y
#CONFIG_MBEDTLS_KEY_EXCHANGE_PSK_ENABLED=y
#CONFIG_MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED=y
#CONFIG_MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED=y
CONFIG_MBEDTLS_CIPHER_MODE_CBC_ENABLED=y
#^from https://devzone.nordicsemi.com/f/nordic-q-a/86632/openthread-and-mqtt-over-tls-is-single-program

CONFIG_MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED=n
CONFIG_MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED=y
CONFIG_MBEDTLS_KEY_EXCHANGE_RSA_ENABLED=y
CONFIG_MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED=y
CONFIG_MBEDTLS_CIPHER_AES_ENABLED=y
CONFIG_MBEDTLS_CIPHER_CCM_ENABLED=y
CONFIG_MBEDTLS_CIPHER_GCM_ENABLED=y
CONFIG_MBEDTLS_GCM_C=y

#Following needed to set CONFIG_MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED=y
CONFIG_MBEDTLS_DHM_C=y
CONFIG_MBEDTLS_RSA_C=y
CONFIG_MBEDTLS_X509_CRT_PARSE_C=y
CONFIG_MBEDTLS_PKCS1_V15=y

#CONFIG_MBEDTLS_BIGNUM_C=y
#CONFIG_MBEDTLS_OID_C=y
#CONFIG_MBEDTLS_ASN1_PARSE_C=y
#CONFIG_MBEDTLS_PK_PARSE_C=y
#CONFIG_MBEDTLS_SSL_PROTO_DTLS=y
CONFIG_MBEDTLS_TLS_LIBRARY=y
CONFIG_MBEDTLS_SHA1_C=y
CONFIG_MBEDTLS_SHA256_C=y
CONFIG_MBEDTLS_SHA512_C=y
CONFIG_MBEDTLS_PSA_CRYPTO_C=y
CONFIG_MBEDTLS_SSL_PROTO_TLS1_2=y

CONFIG_MBEDTLS_X509_LIBRARY=y
CONFIG_MBEDTLS_X509_USE_C=y

# TLS configuration
CONFIG_MBEDTLS=y
CONFIG_MBEDTLS_BUILTIN=y
CONFIG_MBEDTLS_ENABLE_HEAP=y
CONFIG_MBEDTLS_HEAP_SIZE=64000
# certificate must fit into one message, fragmenting is not supported
CONFIG_MBEDTLS_SSL_MAX_CONTENT_LEN=10240
#CONFIG_MBEDTLS_PEM_CERTIFICATE_FORMAT=y
CONFIG_MBEDTLS_PK_WRITE_C=y
CONFIG_MBEDTLS_CTR_DRBG_C=y
CONFIG_MBEDTLS_ECDSA_C=y
CONFIG_MBEDTLS_SHA256_C=y
#CONFIG_MBEDTLS_RSA_C=y #defined above already
CONFIG_MBEDTLS_AES_C=y
CONFIG_MBEDTLS_PKCS1_V21=y
#Credentials
CONFIG_TLS_CREDENTIALS=y
CONFIG_TLS_MAX_CREDENTIALS_NUMBER=4

Error -22 in mqtt_connect() - nRF52840dk with Azure IoT Hub using OpenThread and TCP

Top Replies