Memory Leak in Wi-Fi Reconnect Loop Leading to UMAC Firmware Load Failure

[Hemakumar Arumugam]

int wifi_connect(void)
{
struct net_if *iface = net_if_get_default();
static struct wifi_connect_req_params cnx_params;

context.connected = false;
context.connect_result = false;
__wifi_args_to_params(&cnx_params);

if (net_mgmt(NET_REQUEST_WIFI_CONNECT, iface,
&cnx_params, sizeof(struct wifi_connect_req_params)))
{
LOG_ERR("Connection request failed");

PRINT_LOG_BLE(BLE_LOG_ENABLE, "Connection request failed");
uplinkData.ible_data_status_flags |= WIFI_CONNECT_FAILED;

return -ENOEXEC;
}

LOG_INF("Connection request success");
PRINT_LOG_BLE(BLE_LOG_ENABLE, "Connection request");
k_sem_reset(&wait_for_next);
int ret = k_sem_take(&wait_for_next, K_SECONDS(flash_config_buf.wifi_mod.wifi_conn_timeout));
LOG_INF("Sem return value:%d\n", ret);

return 0;
}


int wifi_disconnect(void)
{
struct net_if *iface = net_if_get_default();
int status;

context.disconnect_requested = true;

status = net_mgmt(NET_REQUEST_WIFI_DISCONNECT, iface, NULL, 0);

if (status)
{
context.disconnect_requested = false;

if (status == -EALREADY)
{
LOG_INF("Already disconnected");
PRINT_LOG_BLE(BLE_LOG_ENABLE, "Already disconnected");
}
else
{
LOG_ERR("Disconnect request failed");
PRINT_LOG_BLE(BLE_LOG_ENABLE, "Disconnect request failed");
uplinkData.ible_data_status_flags |= WIFI_DISCONNECT_FAILED;

return -ENOEXEC;
}
}
else
{
LOG_INF("Disconnect requested");
}

return 0;
}


int shutdown_wifi(struct net_if *iface)
{
int ret;

if (!net_if_is_admin_up(iface))
{
return 0;
}

ret = net_if_down(iface);
if (ret)
{
LOG_ERR("Cannot bring down iface (%d)", ret);
PRINT_LOG_BLE(BLE_LOG_ENABLE, "Cannot bring down iface (%d)", ret);
uplinkData.ible_data_status_flags |= WIFI_MODULE_OFF_FAILED;

return ret;
}

LOG_INF("Interface down");
PRINT_LOG_BLE(BLE_LOG_ENABLE, "Interface down");

return 0;
}

int startup_wifi(struct net_if *iface)
{
int ret;

if (!net_if_is_admin_up(iface))
{
ret = net_if_up(iface);
if (ret)
{
LOG_ERR("Cannot bring up iface (%d)", ret);
PRINT_LOG_BLE(BLE_LOG_ENABLE, "Cannot bring up iface (%d)", ret);
uplinkData.ible_data_status_flags |= WIFI_MODULE_ON_FAILED;

return ret;
}

LOG_INF("Interface up");
PRINT_LOG_BLE(BLE_LOG_ENABLE, "Interface up");
}

return 0;
}

void internet_module_power_off(void)
{
struct net_if *iface = net_if_get_default();

log_heap_stats("shutdown_before");
shutdown_wifi(iface);
log_heap_stats("shutdown_after");
}

void internet_module_power_on(void)
{
struct net_if *iface = net_if_get_default();

log_heap_stats("startup_before");
startup_wifi(iface);
log_heap_stats("startup_after");
}


void log_heap_stats(const char *label)
{
struct sys_memory_stats stats;
memset(&stats, 0, sizeof(stats));
sys_heap_runtime_stats_get(&_system_heap, &stats);

printk("[%s] heap_free=%u, heap_alloc=%u, max_alloc=%u\n",
label,
(unsigned)stats.free_bytes,
(unsigned)stats.allocated_bytes,
(unsigned)stats.max_allocated_bytes);
}

In the function mentioned above, I repeatedly perform the following operations in a loop at a specified interval:

1. Power on the internet module

2. Connect to Wi-Fi

3. Send TCP data

4. Disconnect from Wi-Fi

5. Power off the internet module

However, after running for a day, the process stops and I see the following error:

result: 0x23[02:53:06.607,543] <err> wifi_nrf: hal_fw_patch_load: Failed to allocate memory for patch UMAC-bin: chunk 1/6, size: 8192
[02:53:06.607,604] <err> wifi_nrf: nrf_wifi_fmac_fw_load: UMAC patch load failed

[02:53:06.607,604] <err> wifi_nrf: nrf_wifi_fw_load: nrf_wifi_fmac_fw_load failed
[02:53:06.607,635] <err> wifi_nrf: nrf_wifi_fmac_dev_add_zep: nrf_wifi_fw_load failed
[02:53:06.607,849] <err> wifi_nrf: nrf_wifi_if_start_zep: nrf_wifi_fmac_dev_add_zep failed
[02:53:06.607,879] <err> sta: Cannot bring up iface (-1)

To investigate, I added heap memory statistics in internet_module_power_on and internet_module_power_off to check for memory leaks during each loop iteration.

From the logs, I observed that free heap memory decreases with every cycle:

[shutdown_before] heap_free=30588, heap_alloc=122164, max_alloc=126604
[00:03:49.094,024] <inf> sta: Interface down
[shutdown_after] heap_free=118556, heap_alloc=34852, max_alloc=126604

[shutdown_before] heap_free=30448, heap_alloc=122280, max_alloc=126672
[00:13:49.104,766] <inf> sta: Interface down
[shutdown_after] heap_free=118428, heap_alloc=34956, max_alloc=126672

[shutdown_before] heap_free=29684, heap_alloc=122988, max_alloc=126776
[00:23:49.161,804] <inf> sta: Interface down
[shutdown_after] heap_free=118308, heap_alloc=35060, max_alloc=126776

[shutdown_before] heap_free=29608, heap_alloc=123056, max_alloc=126880
[00:33:49.138,122] <inf> sta: Interface down
[shutdown_after] heap_free=118184, heap_alloc=35164, max_alloc=126880

[shutdown_before] heap_free=30116, heap_alloc=122580, max_alloc=126984
[00:43:49.105,224] <inf> sta: Interface down
[shutdown_after] heap_free=118064, heap_alloc=35268, max_alloc=126984

[shutdown_before] heap_free=29768, heap_alloc=122916, max_alloc=127088
[00:53:49.163,665] <inf> sta: Interface down
[shutdown_after] heap_free=117944, heap_alloc=35372, max_alloc=127088

[shutdown_before] heap_free=28708, heap_alloc=123916, max_alloc=127192
[01:03:49.135,223] <inf> sta: Interface down
[shutdown_after] heap_free=117824, heap_alloc=35476, max_alloc=127192

[shutdown_before] heap_free=29540, heap_alloc=123124, max_alloc=127296
[01:13:49.127,929] <inf> sta: Interface down
[shutdown_after] heap_free=117708, heap_alloc=35580, max_alloc=127296

[shutdown_before] heap_free=28752, heap_alloc=123860, max_alloc=127400
[01:23:49.114,288] <inf> sta: Interface down
[shutdown_after] heap_free=117588, heap_alloc=35684, max_alloc=127400

[shutdown_before] heap_free=19152, heap_alloc=132584, max_alloc=135876
[15:06:14.258,819] <inf> sta: Interface down
[shutdown_after] heap_free=108068, heap_alloc=44160, max_alloc=135876

[shutdown_before] heap_free=18788, heap_alloc=132848, max_alloc=137020
[16:56:14.236,358] <inf> sta: Interface down
[shutdown_after] heap_free=106808, heap_alloc=45304, max_alloc=137020

[shutdown_before] heap_free=14392, heap_alloc=136880, max_alloc=140556
[22:36:14.255,523] <inf> sta: Interface down
[shutdown_after] heap_free=102896, heap_alloc=48840, max_alloc=140556

This indicates that some memory is not being freed properly in each cycle, eventually leading to failure in loading the Wi-Fi firmware patch.

Please guide me on how to resolve this issue.

 

[runsiv]

Hi

Which version of NCS are you using? This issue sounds familiar with an older version of the SDK 

Regards

Runar

[Hemakumar Arumugam]

Iam using NCS version 2.6.0

[runsiv]

Thanks

If you are planning to stay on NCS 2.6 I would strongly recommend that upgrade to 2.6.4 which have several wifi fixes including the issue you are experiencing. 

Regards

Runar