Hello everyone,
I am having a random I2C communication failure.
There are two things that communicate. One is master which is nRF52840 and the other is ADS7128.
Those two ICs on different boards and the communication goes over differential I2C IC (PCA9616).
I attached the schematics for both sides.
And the problem is, we lose communication randomly sometimes for a short time sometimes for a long time like hours even days.
In the code, we reinit the I2C if it fails but we don't do any error check.
On the Nordic side, there are two 2.2k pull-ups. Below is the code from driver's side.
// Registers #define REG_SYSTEM_STATUS 0x00 #define REG_GENERAL_CFG 0x01 #define REG_DATA_CFG 0x02 #define REG_OSR_CFG 0x03 #define REG_OPMODE_CFG 0x04 #define REG_PIN_CFG 0x05 #define REG_GPIO_CFG 0x07 #define REG_GPO_DRIVE_CFG 0x09 #define REG_SEQUENCE_CFG 0x10 #define REG_CHANNEL_SEL 0x11 #define REG_AUTO_SEQ_CH_SEL 0x12 #define REG_RMS_CFG 0xC0 #define REG_RMS_LSB 0xC1 #define REG_RMS_MSB 0xC2 #define REG_MAX_CH0_LSB 0x60 #define REG_MIN_CH0_LSB 0x80 #define REG_RECENT_CH0_LSB 0xA0 #define CURRENT_DEFAULT_FREQUENCY 16.0f #define CURRENT_DEFAULT_MEDIAN_WINDOW 3 #define CURRENT_DEFAULT_MEDIAN_REMOVE 1 #define CURRENT_DEFAULT_LOWPASS_ORDER 2 #define CURRENT_DEFAULT_LOWPASS_FREQUENCY 4.0f #define CURRENT_DEFAULT_DECIMATION 2 #define CURRENT_DEFAULT_PRECISION 12 INIT_SAMPLER(current, CHANNEL_CURRENT, 1, NULL, NULL, NULL); // Multiply by that to convert to uV // gain = 3.3V * 1000000 uV/V / 65536 LSB = 50.354 uV / LSB #define GAIN_NUM_UV 19487 #define GAIN_DEN_UV 387 #define SAMPLE_LSB_TO_UV(s) (( ((uint32_t) (s)) * GAIN_NUM_UV) / GAIN_DEN_UV) enum { CONFIG_MODE_AUTOSEQUENCE, //CONFIG_MODE_RMS_SWITCHED_AC, CONFIG_MODE_RMS_CURRENT }; static bool adsConfigDone = false; static uint8_t adsConfigMode = 0; static uint32_t adsConfigTicks = 0; #define MONITOR_PERIOD_MS 1000 #define MONITOR_SAMPLE_DELAY_MS 40 static bool monitorEnabled; static TimerHandle_t monitorTimerHandle; static StaticTimer_t monitorTimer; QUEUE_STATIC_DECLARATION(monitorTipSwik, 1, sizeof(int32_t)); QUEUE_STATIC_DECLARATION(monitorDc5V, 1, sizeof(int32_t)); //QUEUE_STATIC_DECLARATION(monitorControlFET, 1, sizeof(int32_t)); QUEUE_STATIC_DECLARATION(monitorCurrent, 1, sizeof(uint32_t)); QUEUE_STATIC_DECLARATION(monitorSSRVoltage, 1, sizeof(uint32_t)); QUEUE_STATIC_DECLARATION(monitorControlPostFET, 1, sizeof(int32_t)); // ======== TWI RELATED STUFF ======== //0x10 is what we use/want, addr is tied to gnd, which isn't defined by datasheet, but 0x10 seems to b default #define ADS_ADDRESS 0x10 // Define TWI instance that we'll be using static const nrf_drv_twi_t m_twi = NRF_DRV_TWI_INSTANCE(ADS_TWI_INSTANCE); static bool twi_initialized = false; #define TWI_NUMBER_TRIES_MAX 1 #define TWI_DEFAULT_TIMEOUT_US 1000 #define TWI_DELAY_BEFORE_US 50 #if TWI_DELAY_BEFORE_US > 0 #define TWI_DELAY_BEFORE_FCT nrf_delay_us(TWI_DELAY_BEFORE_US) #else #define TWI_DELAY_BEFORE_FCT #endif static volatile bool twi_failure = false; static volatile bool twi_done = false; static uint8_t i2cBuffer[32]; static void ads_twi_handler(nrf_drv_twi_evt_t const *event, void *context) { twi_done = true; twi_failure = (event->type != NRF_DRV_TWI_EVT_DONE); } static void ads_twi_init(void) { //NRF_LOG_INFO("twi init"); ret_code_t err_code; const nrf_drv_twi_config_t twi_config = { .scl = ADS_SCL_PIN, .sda = ADS_SDA_PIN, .frequency = NRF_DRV_TWI_FREQ_400K, .interrupt_priority = APP_IRQ_PRIORITY_LOW, .clear_bus_init = true }; // Initialize TWI instance with config err_code = nrf_drv_twi_init(&m_twi, &twi_config, ads_twi_handler, NULL); // Enable instance nrf_drv_twi_enable(&m_twi); if (err_code == NRF_SUCCESS) { twi_initialized = true; twi_failure = false; } else { NRF_LOG_INFO("ads_twi_init failure"); } } static void ads_twi_uninit(void) { nrf_drv_twi_disable(&m_twi); nrf_drv_twi_uninit(&m_twi); twi_initialized = false; adsConfigDone = false; } static void ads_twi_retry(void) { if (twi_initialized && twi_failure) { ads_twi_uninit(); } if (!twi_initialized) { ads_twi_init(); } twi_failure = false; } // Function to write data to ADS static void ads_write(uint8_t const * data, uint8_t len, bool nostop) { if (twi_initialized && !twi_failure) { ret_code_t err_code; for (uint8_t nTry = 0; nTry < TWI_NUMBER_TRIES_MAX; ++nTry) { twi_failure = false; twi_done = false; err_code = nrf_drv_twi_tx(&m_twi, ADS_ADDRESS, data, len, nostop); if (err_code != 0) { twi_done = true; twi_failure = true; //NRF_LOG_INFO("w t %u r %u", nTry, err_code); return; } uint16_t countdown = TWI_DEFAULT_TIMEOUT_US; while (!twi_done) { if (--countdown == 0) { twi_done = true; twi_failure = true; break; } nrf_delay_us(1); } if (!twi_failure) { return; } else { //NRF_LOG_INFO("w t %u r %u d %u f %u c %u", nTry, err_code, twi_done, twi_failure, countdown); } } } } // Function to read data from ADS static void ads_read(uint8_t* data, uint8_t len) { if (twi_initialized && !twi_failure) { ret_code_t err_code; for (uint8_t nTry = 0; nTry < TWI_NUMBER_TRIES_MAX; ++nTry) { twi_failure = false; twi_done = false; err_code = nrf_drv_twi_rx(&m_twi, ADS_ADDRESS, data, len); if (err_code != 0) { twi_done = true; twi_failure = true; //NRF_LOG_INFO("r t %u r %u", nTry, err_code); return; } uint16_t countdown = TWI_DEFAULT_TIMEOUT_US; while (!twi_done) { if (--countdown == 0) { twi_done = true; twi_failure = true; break; } nrf_delay_us(1); } //NRF_LOG_INFO("r t %u r %u d %u f %u c %u", nTry, err_code, twi_done, twi_failure, countdown); if (!twi_failure) { return; } } } if (twi_failure) { memset(data, 0, len); } } /* ads7128 has 'opcodes' which is a layer above i2c, it seems a bit redundant to me.. so you send the addr, then the opcode, then the reg then data. relevant opcodes: 00010000b = register read =16 00001000b = register write also, 7128 has 8 bit regs, instead of 16 */ #define OP_CODE_READ 0b00010000 #define OP_CODE_READ_MANY 0b00110000 #define OP_CODE_WRITE 0b00001000 #define OP_CODE_WRITE_MANY 0b00101000 static uint8_t adsReadRegister(uint8_t reg) { TWI_DELAY_BEFORE_FCT; i2cBuffer[0] = OP_CODE_READ; i2cBuffer[1] = reg; ads_write(i2cBuffer, 2, false); ads_read(i2cBuffer, 1); return i2cBuffer[0]; } static uint16_t adsReadRegister2(uint8_t reg) { TWI_DELAY_BEFORE_FCT; i2cBuffer[0] = OP_CODE_READ_MANY; i2cBuffer[1] = reg; ads_write(i2cBuffer, 2, false); ads_read(i2cBuffer, 2); return ((uint16_t)i2cBuffer[0]) | (((uint16_t)i2cBuffer[1] << 8)); } static void adsWriteRegister(uint8_t reg, uint8_t value) { TWI_DELAY_BEFORE_FCT; i2cBuffer[0] = OP_CODE_WRITE; i2cBuffer[1] = reg; i2cBuffer[2] = value; ads_write(i2cBuffer, 3, false); } static void adsSetConfig(uint8_t newConfigMode, bool uninit) { if (uninit) { if (twi_initialized) { ads_twi_uninit(); } } //NRF_LOG_INFO("Set config %u", newConfigMode); // Try again ads_twi_retry(); switch (newConfigMode) { case CONFIG_MODE_AUTOSEQUENCE: { // Set all analog io, STATS disabled to erase MIN and MAX registers adsWriteRegister(REG_GENERAL_CFG, 0b00000100); // all io set as analog in adsWriteRegister(REG_PIN_CFG, 0b00000000); // Only enable the used channels adsWriteRegister(REG_AUTO_SEQ_CH_SEL, 0b11010101); // Enable sequencing mode adsWriteRegister(REG_SEQUENCE_CFG, 0b00000001); // Enable autonomous mode, with sequencing // Clock to 1 MHz adsWriteRegister(REG_OPMODE_CFG, 0b00100000); // Average 128x adsWriteRegister(REG_OSR_CFG, 0b00000111); // Enable DWC adsWriteRegister(REG_GENERAL_CFG, 0b00110100); // Start the sequencer / sampling adsWriteRegister(REG_SEQUENCE_CFG, 0b00010001); break; } //case CONFIG_MODE_RMS_SWITCHED_AC: case CONFIG_MODE_RMS_CURRENT: { uint8_t channel = ADS_CHANNEL_CURRENT; // Set all analog io, STATS disabled to erase MIN and MAX registers adsWriteRegister(REG_GENERAL_CFG, 0b00000100); // Channel mask adsWriteRegister(REG_AUTO_SEQ_CH_SEL, 0b00000001 << channel); // Enable sequencing mode adsWriteRegister(REG_SEQUENCE_CFG, 0b00000001); // No average adsWriteRegister(REG_OSR_CFG, 0b00000000); // Enable autonomous mode, with sequencing // Clock to 1 MHz adsWriteRegister(REG_OPMODE_CFG, 0b00100000); // Set RMS module, 16384 samples for 16ms, DC component removed adsWriteRegister(REG_RMS_CFG, (channel << 4) | 0b00000110); // Enable DWC, enable RMS adsWriteRegister(REG_GENERAL_CFG, 0b10110100); // Start the sequencer / sampling adsWriteRegister(REG_SEQUENCE_CFG, 0b00010001); // Clear RMS done flag adsWriteRegister(REG_SYSTEM_STATUS, 0b00010000); break; } } if (twi_failure) { //NRF_LOG_INFO("Config Failure"); adsConfigDone = false; } else { adsConfigDone = true; adsConfigMode = newConfigMode; adsConfigTicks = xTaskGetTickCount(); } } static int32_t adsReadSample(uint8_t channel, uint8_t regBase, uint16_t* sample) { uint8_t lsbReg = regBase + 2*channel; *sample = adsReadRegister2(lsbReg); return twi_failure? -1 : 0; } static int32_t adsReadRms(uint16_t* sample) { uint8_t systemStatus = adsReadRegister(REG_SYSTEM_STATUS); if (!twi_failure && (systemStatus & 0b00010000)) { *sample = adsReadRegister2(REG_RMS_LSB); return 0; } NRF_LOG_INFO("rms invalid %u %u %u", twi_failure, systemStatus, adsConfigMode); return -1; } void currentCallback(TimerHandle_t timer) { uint32_t ticksElapsed = xTaskGetTickCount() - adsConfigTicks; int16_t sample = SAMPLE_INVALID; if (!adsConfigDone || adsConfigMode != CONFIG_MODE_RMS_CURRENT || ticksElapsed < pdMS_TO_TICKS(20)) { if (adsConfigDone && adsConfigMode == CONFIG_MODE_RMS_CURRENT) { //NRF_LOG_INFO("/!\\ ADS CONFIG %u /!\\", ticksElapsed); } adsSetConfig(CONFIG_MODE_RMS_CURRENT, false); } else { ads_twi_retry(); //NRF_LOG_INFO("ticks %u", ticksElapsed); uint16_t reading16; int32_t reading32; int32_t err_code = adsReadRms(&reading16); adsSetConfig(CONFIG_MODE_RMS_CURRENT, false); if (err_code >= 0) { reading32 = ((int32_t) reading16) * 2 - 32768; if (reading32 > SAMPLE_VALID_MAX) { sample = SAMPLE_VALID_MAX; } else if (reading32 < SAMPLE_VALID_MIN) { sample = SAMPLE_VALID_MIN; } else { sample = reading32; } #if DEBUG_PRINT_AVERAGE { static int32_t TEST_counter = 0; static int32_t TEST_sum = 0; for (uint8_t i = 0; i < currentSampler.numberReads; ++i) { TEST_sum += reading; } ++TEST_counter; if (TEST_counter == currentSampler.frequency) { int32_t average = TEST_sum / TEST_counter / currentSampler.numberReads; //int32_t average_uV = average * gainValues_uV; NRF_LOG_INFO("Current average: %d", average); TEST_counter = 0; TEST_sum = 0; } } #endif } } // Reset the sampler if needed if (currentSampler.mustEmptyRaw) { samplerEmptyRaw(¤tSampler); } if (timeIsValid() && !currentSampler.blockRaw) { // Process a timestamp if needed if (samplerNeedTimestamp(¤tSampler)) { timeMs_t time; timeGetCurrent(&time); samplerAddTimestamp(¤tSampler, &time); } // Add the sample samplerAddSamples(¤tSampler, &sample); //, xTaskGetTickCount()); } } static void monitorCallback(TimerHandle_t timer) { static uint32_t cycle_counter = 0; //uint32_t tickCount = xTaskGetTickCount(); //NRF_LOG_INFO("1 %09u", tickCount); if (!monitorEnabled) { // Reset the cycle cycle_counter = 0; return; } uint8_t monitorConfigMode = 0; //(cycle_counter/2) % 2; if (cycle_counter % 2 == 0) { //NRF_LOG_INFO("2 %09u", tickCount + pdMS_TO_TICKS(MONITOR_SAMPLE_DELAY_MS)); // Time to set the config to the proper mode if (!adsConfigDone || adsConfigMode != monitorConfigMode) { adsSetConfig(monitorConfigMode, monitorConfigMode == 0); } // This function should also start the timer xTimerChangePeriod(monitorTimerHandle, pdMS_TO_TICKS(MONITOR_SAMPLE_DELAY_MS), 0); } else { if (adsConfigDone && adsConfigMode == monitorConfigMode) { ads_twi_retry(); // Add the correct sample to the queue switch (monitorConfigMode) { case CONFIG_MODE_AUTOSEQUENCE: { int32_t err_code; uint16_t reading; uint16_t readingMin; uint16_t readingMax; uint32_t sample_uV; err_code = adsReadSample(ADS_CHANNEL_TIP_SWIK, REG_RECENT_CH0_LSB, &reading); if (err_code >= 0) { sample_uV = SAMPLE_LSB_TO_UV(reading); xQueueOverwrite(monitorTipSwikQueueHandle, &sample_uV); //NRF_LOG_INFO("Tip Swik %04d %07d", reading, sample_uV); } //NRF_LOG_INFO("%u %d %d", twi_failure, reading, sample_uV); err_code = adsReadSample(ADS_CHANNEL_DC_5V, REG_RECENT_CH0_LSB, &reading); if (err_code >= 0) { sample_uV = 2 * SAMPLE_LSB_TO_UV(reading); xQueueOverwrite(monitorDc5VQueueHandle, &sample_uV); behaviorSetState(BEHAVIOR_MODE_5V_CONTROL, (sample_uV > 4000000)? BEHAVIOR_STATE_TRUE : BEHAVIOR_STATE_FALSE); //NRF_LOG_INFO("DC 5V %04d %07d", reading, sample_uV); } err_code = adsReadSample(ADS_CHANNEL_CONTROL_POST_FET, REG_RECENT_CH0_LSB, &reading); if (err_code >= 0) { sample_uV = SAMPLE_LSB_TO_UV(reading); xQueueOverwrite(monitorControlPostFETQueueHandle, &sample_uV); //NRF_LOG_INFO("ControlPostFET %04d %07d", reading, sample_uV); } err_code = adsReadSample(ADS_CHANNEL_SWITCHED_AC, REG_MIN_CH0_LSB, &readingMin); err_code |= adsReadSample(ADS_CHANNEL_SWITCHED_AC, REG_MAX_CH0_LSB, &readingMax); if (err_code >= 0) { // Rms = (max - min) / (2*sqrt(2)) sample_uV = SAMPLE_LSB_TO_UV(readingMax - readingMin) * 204 / 577; xQueueOverwrite(monitorSSRVoltageQueueHandle, &sample_uV); //NRF_LOG_INFO("Min %u Max %u", readingMin, readingMax); } err_code = adsReadSample(ADS_CHANNEL_CURRENT, REG_MIN_CH0_LSB, &readingMin); err_code |= adsReadSample(ADS_CHANNEL_CURRENT, REG_MAX_CH0_LSB, &readingMax); if (err_code >= 0) { // * 2000 / 66.5 (sensor coupling ratio, measurement resistor) // Rms = (max - min) / (2*sqrt(2)) sample_uV = SAMPLE_LSB_TO_UV(readingMax - readingMin) * 319 / 30; xQueueOverwrite(monitorCurrentQueueHandle, &sample_uV); //NRF_LOG_INFO("Min %u Max %u RMS %u", readingMin, readingMax, sample_uV); } break; } /*case CONFIG_MODE_RMS_CURRENT: { uint16_t rms; int32_t err_code = adsReadRms(&rms); if (err_code >= 0) { if (rms < 0) { rms = 0; } // * 2000 / 66.5 (sensor coupling ratio, measurement resistor) uint32_t rms_uA = SAMPLE_LSB_TO_UV(rms) * 30; xQueueOverwrite(monitorCurrentQueueHandle, &rms_uA); } break; }*/ } } // Back to the proper mode adsSetConfig(CONFIG_MODE_RMS_CURRENT, false); currentMonitorSetBehavior(true); } ++cycle_counter; } bool currentGetLastTipSwikValue(bool erase, uint32_t* sample_uV) { if (erase) { return xQueueReceive(monitorTipSwikQueueHandle, sample_uV, 0) == pdTRUE; } return xQueuePeek(monitorTipSwikQueueHandle, sample_uV, 0) == pdTRUE; } bool currentGetLastDc5VValue(bool erase, uint32_t* sample_uV) { if (erase) { return xQueueReceive(monitorDc5VQueueHandle, sample_uV, 0) == pdTRUE; } return xQueuePeek(monitorDc5VQueueHandle, sample_uV, 0) == pdTRUE; } /* bool currentGetLastControlFETValue(bool erase, int32_t* sample_uV) { if (erase) { return xQueueReceive(monitorControlFETQueueHandle, sample_uV, 0) == pdTRUE; } return xQueuePeek(monitorControlFETQueueHandle, sample_uV, 0) == pdTRUE; } */ bool currentGetLastCurrentValue(bool erase, uint32_t* sample_uA) { if (erase) { return xQueueReceive(monitorCurrentQueueHandle, sample_uA, 0) == pdTRUE; } return xQueuePeek(monitorCurrentQueueHandle, sample_uA, 0) == pdTRUE; } bool currentGetLastSSRVoltageValue(bool erase, uint32_t* sample_uV) { if (erase) { return xQueueReceive(monitorSSRVoltageQueueHandle, sample_uV, 0) == pdTRUE; } return xQueuePeek(monitorSSRVoltageQueueHandle, sample_uV, 0) == pdTRUE; } bool currentGetLastControlPostFETValue(bool erase, uint32_t* sample_uV) { if (erase) { return xQueueReceive(monitorControlPostFETQueueHandle, sample_uV, 0) == pdTRUE; } return xQueuePeek(monitorControlPostFETQueueHandle, sample_uV, 0) == pdTRUE; } void currentMonitorSetBehavior(bool enabled) { monitorEnabled = enabled; if (enabled) { if (xTimerIsTimerActive(currentSampler.timerHandle) != pdFALSE) { // Current timer is active, be careful with the timing // We want to wake up the monitor timer in more than MONITOR_PERIOD_MS // But we should also leave room for 2*MONITOR_SAMPLE_DELAY_MS + some safety margin // There must not be any sampler expiry time between ticksStart and ticksEnd uint32_t ticks = xTaskGetTickCount(); uint32_t samplerNextExpiry = xTimerGetExpiryTime(currentSampler.timerHandle); uint32_t samplerPeriod = xTimerGetPeriod(currentSampler.timerHandle); uint32_t dt = ticks + pdMS_TO_TICKS(MONITOR_PERIOD_MS) - samplerNextExpiry; uint32_t k = (dt < 0x80000000) * (dt / samplerPeriod + 1); //NRF_LOG_INFO("3 %09u", samplerNextExpiry + k*samplerPeriod + 5); xTimerChangePeriod(monitorTimerHandle, samplerNextExpiry + k*samplerPeriod - ticks + 5, 0); } else { // This function should also start the timer xTimerChangePeriod(monitorTimerHandle, pdMS_TO_TICKS(MONITOR_PERIOD_MS), 0); } } } sampler_t* currentInit(void) { //NRF_LOG_INFO("Current init sampler size: %u bytes, frequency: %u Hz", sizeof(sampler_t), currentSampler.frequency); // Initialize I2C communication QUEUE_STATIC_DEFINITION(monitorTipSwik); QUEUE_STATIC_DEFINITION(monitorDc5V); //QUEUE_STATIC_DEFINITION(monitorControlFET); QUEUE_STATIC_DEFINITION(monitorCurrent); QUEUE_STATIC_DEFINITION(monitorSSRVoltage); QUEUE_STATIC_DEFINITION(monitorControlPostFET); // function to initialize the timer task, expires after 2 milliseconds and does not auto-reload monitorTimerHandle = xTimerCreateStatic( "monitorTimer", pdMS_TO_TICKS(MONITOR_PERIOD_MS), pdFALSE, (void *) (0), monitorCallback, &monitorTimer); // Initialize load cell by creating sampler samplerInit(¤tSampler, currentCallback); samplerInitParams(¤tSampler, CURRENT_DEFAULT_FREQUENCY, CURRENT_DEFAULT_MEDIAN_WINDOW, CURRENT_DEFAULT_MEDIAN_REMOVE, CURRENT_DEFAULT_LOWPASS_ORDER, CURRENT_DEFAULT_LOWPASS_FREQUENCY, CURRENT_DEFAULT_DECIMATION, CURRENT_DEFAULT_PRECISION); return ¤tSampler; } #endif