Central adds peripheral to the whitelist, but won't reconnect

Question

I am developing an application which uses 2 nrf52 running sdk 12.3 and softdevice 132 V3. 
 The central device appears to be saving and whitelisting the two peers, but it won't connect to them when I actually enable the whitelist - disable the whitelist and it works fine. 
 static void scan_start(bool refreshWhitelist)
{
uint32_t flash_busy;

(void) fs_queued_op_count_get(&flash_busy);

if (flash_busy != 0)
{
	m_memory_access_in_progress = true;
	return;
}

ble_gap_addr_t whitelist_addrs[maxWhitelist];
ble_gap_irk_t whitelist_irks[maxWhitelist];

memset(whitelist_addrs, 0x00, sizeof(whitelist_addrs));
memset(whitelist_addrs, 0x00, sizeof(whitelist_irks));

uint32_t addr_cnt = (sizeof(whitelist_addrs) / sizeof(ble_gap_addr_t));
uint32_t irk_cnt = (sizeof(whitelist_irks) / sizeof(ble_gap_irk_t));

#if (NRF_SD_BLE_API_VERSION == 2)

ble_gap_addr_t * p_whitelist_addrs[maxWhitelist];
ble_gap_irk_t * p_whitelist_irks[maxWhitelist];

for (uint32_t i = 0; i < maxWhitelist; i++)
{
 p_whitelist_addrs[i] = &whitelist_addrs[maxWhitelist];
 p_whitelist_irks[i] = &whitelist_irks[maxWhitelist];
}

ble_gap_whitelist_t whitelist =
{
 .pp_addrs = p_whitelist_addrs,
 .pp_irks = p_whitelist_irks,
};

#endif

if(refreshWhitelist) whitelist_load();

ret_code_t ret;

ret = pm_whitelist_get(whitelist_addrs, &addr_cnt,
 whitelist_irks, &irk_cnt);

m_scan_params.active = 1;
m_scan_params.interval = SCAN_INTERVAL;
m_scan_params.window = SCAN_WINDOW;

if (((addr_cnt == 0) && (irk_cnt == 0)) ||
 (m_whitelist_disabled))
{
 // Don't use whitelist.
 #if (NRF_SD_BLE_API_VERSION == 2)
 m_scan_params.selective = 0;
 m_scan_params.p_whitelist = NULL;
 #endif
 #if (NRF_SD_BLE_API_VERSION == 3)
 m_scan_params.use_whitelist = 0;
 m_scan_params.adv_dir_report = 0;
 #endif
 m_scan_params.timeout = 0x0000; // No timeout.
}
else
{
 // Use whitelist.
 #if (NRF_SD_BLE_API_VERSION == 2)
 whitelist.addr_count = addr_cnt;
 whitelist.irk_count = irk_cnt;
 m_scan_params.selective = 1;
 m_scan_params.p_whitelist = &whitelist;
 #endif
 #if (NRF_SD_BLE_API_VERSION == 3)
 m_scan_params.use_whitelist = 1;
 m_scan_params.adv_dir_report = 1;
 #endif
 m_scan_params.timeout = 0x0000;
}

ret = sd_ble_gap_scan_start(&m_scan_params);
APP_ERROR_CHECK(ret);
}
 
 The whitelist_load() and peer_list_get() functions are the same as the examples they are taken from. In debugger (I am developing this using a Rigado dev board & Eclipse) I see that sd_ble_gap_scan_start() returns NRF_SUCCESS whether or not I start it with the whitelist enabled or disabled. 
 My peripheral device is running peer manager, but never sets a whitelist. These are my security parameters for both devices. 
 sec_param.bond = true;
sec_param.mitm = false;
sec_param.lesc = 0;
sec_param.keypress = 0;
sec_param.io_caps = BLE_GAP_IO_CAPS_DISPLAY_ONLY;
sec_param.oob = false;
sec_param.min_key_size = 7;
sec_param.max_key_size = 16;
sec_param.kdist_own.enc = 1;
sec_param.kdist_own.id = 1;
sec_param.kdist_peer.enc = 1;
sec_param.kdist_peer.id = 1;
 
 I'm kind of stuck here.