nRF7002 randomly unable to connect to Wifi

[Rocky]

I have an application that syncs via the internet using a nRF5340 and nRF7002.  I am using the thingy53 with a nRF7002EB for now while the custom circuit boards are being manufactured.  The application connects via wifi every 4 minutes to transmit data.  It then disconnects from the Wifi network to save power between sync intervals.  The application is currently using nRF Connect v3.0.1 and is based on the Azure IoT sample.

The application is in early field testing with around 20 users.  It works well most of the time, but randomly, after 1-2 weeks the system will stop being able to connect to the Wifi network.  A Wifi scan is initiated, but a connection event is never received.  I have also discovered that the system will start working again if I issue the new diagnostic command nrf70 util rpu_recovery_test via a shell connection.

My suspicion is that this is a bug in the nRF7002 where it can somehow get into a semi-unresponsive state.  The only way to recover is to externally reset the nRF7002.  It is difficult to know for sure because this issue takes a week or two to reproduce.  I do have CONFIG_NRF_WIFI_RPU_RECOVERY set, but this doesn't seem to be working.  I am considering updating the application firmware to add a custom nRF7002 watchdog timer that can reset it if it hangs.  However, I wanted to check with the Nordic team to make sure there aren't any known issues or other suggested troubleshooting steps.

[Rocky]

Adding wifi connect/disconnect code for reference.

Fullscreen wifi.c Download

#include "wifi.h"
#include "zephyr/net/wifi_credentials.h"
#include <net/wifi_ready.h>
#include <stdio.h>
#include <stdlib.h>
#include <zephyr/kernel.h>
#include <zephyr/logging/log.h>
#include <zephyr/net/conn_mgr_connectivity.h>
#include <zephyr/net/conn_mgr_monitor.h>
#include <zephyr/net/wifi_mgmt.h>
#include <zephyr/sys/reboot.h>
#include <zephyr/shell/shell.h>

LOG_MODULE_REGISTER(wifi, LOG_LEVEL_INF);

/* Macros used to subscribe to specific Zephyr net management events. */
#define CONN_LAYER_EVENT_MASK   (NET_EVENT_CONN_IF_FATAL_ERROR)
#define CONN_TIMEOUT            K_SECONDS(500)

/* Zephyr net management event callback structures. */
static struct net_mgmt_event_callback conn_cb;
static struct net_mgmt_event_callback ipv4_cb;

static connect_callback_t m_connect_cb = NULL;
static bool is_connected = false;
static bool wifi_ready_status;

static void conn_work_handler(struct k_work* work)
{
    LOG_WRN("Connection timeout");
    WIFI_disconnect();
}

K_WORK_DEFINE(conn_work, conn_work_handler);
static void conn_timeout_handler(struct k_timer* dummy)
{
    if (is_connected)
    {
        k_work_submit(&conn_work);
    }
}

K_TIMER_DEFINE(conn_timer, conn_timeout_handler, NULL);
int32_t WIFI_connect(connect_callback_t cb)
{
    LOG_INF("Connecting to Wi-Fi...");
    k_timer_stop(&conn_timer);
    if (!wifi_ready_status)
    {
        LOG_ERR("nRF7002 not ready for connect");
        return -1;
    }
    if (wifi_credentials_is_empty())
    {
        LOG_WRN("No Wi-Fi credentials found");
        return -1;
    }

    k_timer_start(&conn_timer, CONN_TIMEOUT, K_NO_WAIT);

    int32_t err = conn_mgr_all_if_connect(true);
    if (err)
    {
        LOG_ERR("conn_mgr_all_if_connect, error: %d", err);
        return err;
    }
    m_connect_cb = cb;
    return err;
}

int32_t WIFI_disconnect(void)
{
    int32_t err = conn_mgr_all_if_disconnect(true);
    if (err)
    {
        LOG_ERR("conn_mgr_all_if_disconnect, error: %d", err);
        return err;
    }
    return err;
}

bool WIFI_is_connected(void)
{
    return is_connected;
}

static void connectivity_event_handler(struct net_mgmt_event_callback* cb, uint32_t event,
                                       struct net_if* iface)
{
    if (event == NET_EVENT_CONN_IF_FATAL_ERROR)
    {
        LOG_ERR("Fatal error received from the connectivity layer, rebooting");
        k_sleep(K_SECONDS(1));
        sys_reboot(SYS_REBOOT_COLD);
        CODE_UNREACHABLE;
    }
}

static void ipv4_event_handler(struct net_mgmt_event_callback* cb, uint32_t event,
                               struct net_if* iface)
{
    switch (event)
    {
    case NET_EVENT_IPV4_ADDR_ADD: {
        struct wifi_iface_status status = {0};
        if (net_mgmt(NET_REQUEST_WIFI_IFACE_STATUS, iface, &status,
                     sizeof(struct wifi_iface_status)))
        {
            LOG_ERR("Status request failed");
        }
        else
        {
            LOG_INF("DHCP addr assigned, SSID: %s, RSSI: %d", status.ssid, status.rssi);
        }
        is_connected = true;

        k_timer_stop(&conn_timer);
        k_timer_start(&conn_timer, CONN_TIMEOUT, K_NO_WAIT);

        if (m_connect_cb != NULL)
        {
            m_connect_cb();
        }
    }
    break;
    case NET_EVENT_IPV4_ADDR_DEL:
        LOG_INF("DHCP addr removed");
        is_connected = false;
        k_timer_stop(&conn_timer);
        break;
    default:
        LOG_DBG("Unknown event: 0x%08X", event);
        break;
    }
}

static void wifi_ready_cb(bool wifi_ready)
{
    LOG_INF("nRF7002 ready?: %s", wifi_ready ? "yes" : "no");
    wifi_ready_status = wifi_ready;
}

int32_t WIFI_init(void)
{
    net_mgmt_init_event_callback(&conn_cb, connectivity_event_handler, CONN_LAYER_EVENT_MASK);
    net_mgmt_add_event_callback(&conn_cb);
    net_mgmt_init_event_callback(&ipv4_cb, ipv4_event_handler,
                                 NET_EVENT_IPV4_ADDR_ADD | NET_EVENT_IPV4_ADDR_DEL);
    net_mgmt_add_event_callback(&ipv4_cb);

    wifi_ready_callback_t cb;
    struct net_if* iface = net_if_get_first_wifi();

    if (!iface)
    {
        LOG_ERR("Failed to get Wi-Fi interface");
        return -1;
    }

    cb.wifi_ready_cb = wifi_ready_cb;

    LOG_DBG("Registering Wi-Fi ready callbacks");
    int32_t ret = register_wifi_ready_callback(cb, iface);
    if (ret)
    {
        LOG_ERR("Failed to register Wi-Fi ready callbacks %s", strerror(ret));
        return ret;
    }

    return 0;
}

SHELL_SUBCMD_SET_CREATE(wifi_commands, (wifi));
SHELL_CMD_REGISTER(wifi, &wifi_commands, "Wi-Fi commands", NULL);