External flash getting corrupted after MCU reboot

Hi,

I am using NCS v2.9.0.

I am using sysbuild to build MCUboot and application images. Following are relevant Kconfig options:

mcuboot.conf

CONFIG_MAIN_STACK_SIZE=10240

CONFIG_DISABLE_FLASH_PATCH=y
CONFIG_RETENTION=y
CONFIG_RETAINED_MEM=y
CONFIG_RETENTION_BOOT_MODE=y

CONFIG_BOOT_WATCHDOG_FEED=y

# Used for external Flash
CONFIG_NORDIC_QSPI_NOR=y
CONFIG_SPI_NOR_SFDP_RUNTIME=y
CONFIG_BOOT_MAX_IMG_SECTORS=256
CONFIG_NORDIC_QSPI_NOR_FLASH_LAYOUT_PAGE_SIZE=4096
CONFIG_BOOT_BOOTSTRAP=y
CONFIG_LOG=y
CONFIG_MCUBOOT_UTIL_LOG_LEVEL_DBG=y
CONFIG_MCUBOOT_LOG_LEVEL_DBG=y
CONFIG_FLASH_LOG_LEVEL_DBG=y
CONFIG_LOG_BACKEND_RTT=y
CONFIG_USE_SEGGER_RTT=y
CONFIG_LOG_MODE_IMMEDIATE=y

prj.conf

CONFIG_MAIN_STACK_SIZE=8192
CONFIG_SYSTEM_WORKQUEUE_STACK_SIZE=8192
CONFIG_REBOOT=y
CONFIG_GPIO=y
CONFIG_PINCTRL=y
CONFIG_CRC=y
CONFIG_BASE64=y
CONFIG_ZCBOR=y
CONFIG_KERNEL_SHELL=y

CONFIG_FLASH=y
CONFIG_FLASH_MAP=y
CONFIG_STREAM_FLASH=y
CONFIG_IMG_MANAGER=y
CONFIG_NVS=y
CONFIG_DEBUG_OPTIMIZATIONS=y
CONFIG_LOG=y
CONFIG_APP_LOG_LEVEL_DBG=y
CONFIG_FLASH_LOG_LEVEL_DBG=y
CONFIG_LOG_BACKEND_RTT=y
CONFIG_USE_SEGGER_RTT=y
CONFIG_SEGGER_RTT_INIT_MODE_STRONG_CHECK=y
CONFIG_ASSERT=y
CONFIG_CORTEX_M_DEBUG_MONITOR_HOOK=y
CONFIG_SEGGER_DEBUGMON=y
CONFIG_MCUMGR=y
CONFIG_MCUMGR_TRANSPORT_SHELL=y
CONFIG_MCUMGR_TRANSPORT_SHELL_INPUT_TIMEOUT=y
CONFIG_MCUMGR_TRANSPORT_SHELL_INPUT_TIMEOUT_TIME=500
CONFIG_MCUMGR_TRANSPORT_SHELL_MTU=256
CONFIG_MCUMGR_GRP_SHELL=y
CONFIG_MCUMGR_GRP_IMG=y
CONFIG_MCUMGR_GRP_ENUM=y
CONFIG_MCUMGR_GRP_IMG_VERBOSE_ERR=y
CONFIG_MCUMGR_GRP_IMG_MUTEX=y
CONFIG_MCUMGR_GRP_IMG_DIRECT_UPLOAD=n
CONFIG_MCUMGR_GRP_IMG_ALLOW_ERASE_PENDING=y
CONFIG_MCUMGR_GRP_IMG_ALLOW_CONFIRM_NON_ACTIVE_IMAGE_SECONDARY=y
CONFIG_MCUMGR_GRP_IMG_ALLOW_CONFIRM_NON_ACTIVE_IMAGE_ANY=y
CONFIG_MCUMGR_GRP_OS=y
CONFIG_SERIAL=y
CONFIG_USB_DEVICE_STACK=y
CONFIG_USB_DEVICE_PRODUCT="DynamicQR EndDevice"
CONFIG_USB_DEVICE_INITIALIZE_AT_BOOT=n
CONFIG_USB_CDC_ACM=y
CONFIG_UART_INTERRUPT_DRIVEN=y
CONFIG_UART_LINE_CTRL=y
CONFIG_RETENTION=y
CONFIG_RETENTION_BOOT_MODE=y
CONFIG_RETAINED_MEM=y
CONFIG_NORDIC_QSPI_NOR=y
CONFIG_SPI_NOR_SFDP_RUNTIME=y
CONFIG_NORDIC_QSPI_NOR_FLASH_LAYOUT_PAGE_SIZE=4096
CONFIG_BOOTLOADER_MCUBOOT=y
CONFIG_MCUBOOT_GENERATE_CONFIRMED_IMAGE=y

CONFIG_MCUBOOT_UTIL_LOG_LEVEL_DBG=y
CONFIG_MCUMGR_GRP_IMG_LOG_LEVEL_DBG=y
CONFIG_FLASH_LOG_LEVEL_DBG=y

sysbuild.conf

SB_CONFIG_PARTITION_MANAGER=y
SB_CONFIG_BOOTLOADER_MCUBOOT=y
SB_CONFIG_MCUBOOT_MODE_SWAP_WITHOUT_SCRATCH=y
SB_CONFIG_BOOT_SIGNATURE_KEY_FILE="\${APPLICATION_CONFIG_DIR}/../../../priv-ecdsa256.pem"
SB_CONFIG_BOOT_SIGNATURE_TYPE_ECDSA_P256=y
SB_CONFIG_PM_EXTERNAL_FLASH_MCUBOOT_SECONDARY=y

I am using a custom board.

main board (the relevant part)

/dts-v1/;
#include <nordic/nrf52840_qiaa.dtsi>

/ {
    model = "my_model";
    compatible = "my,board";

    chosen {
        zephyr,sram = &sram0;
        zephyr,flash = &flash0;
        nordic,pm-ext-flash = &mx25l32;
        zephyr,shell-uart = &cdc_acm_uart;
        zephyr,boot-mode = &boot_mode;
        /* Must not have zephyr,settings-partition defined here otherwise it conflicts with static partitioning.
         * It will zephyr_settings_partition to be defined in settings_nvs source file causing device-tree partition to be selected instead of yaml file. */
    };

    sram@2003ffff {
        compatible = "zephyr,memory-region", "mmio-sram";
        reg = <0x2003ffff 0x1>;
        zephyr,memory-region = "RETAINED_RAM";
        status = "okay";

        retained_sram {
            compatible = "zephyr,retained-ram";
            status = "okay";
            #address-cells = <1>;
            #size-cells = <1>;

            boot_mode: boot_mode@0 {
                compatible = "zephyr,retention";
                status = "okay";
                reg = <0x0 0x1>;
            };
        };
    };

    fstab: fstab {
        compatible = "zephyr,fstab";
        lfs: lfs {
            compatible = "zephyr,fstab,littlefs";
            mount-point = "/lfs1";
            partition = <&littlefs_partition>;
            automount;
            read-size = <16>;
            prog-size = <16>;
            cache-size = <64>;
            lookahead-size = <32>;
            block-cycles = <512>;
            /* File system must NOT have read-only option enabled even though app (LVGL) only reads the files
             * because of the design decision on LVGL file system layer file create flag is always set which
             * requires the file system to be writeable */
        };
    };
};

&qspi {
    status = "okay";
    pinctrl-0 = <&qspi_default>;
    pinctrl-1 = <&qspi_sleep>;
    pinctrl-names = "default", "sleep";
    mx25l32: mx25l3233f@0 {
        compatible = "nordic,qspi-nor";
        reg = <0>;
        /* MX25L33 supports only pp and pp4io */
        writeoc = "pp";
        /* MX25L33 supports all readoc options */
        readoc = "read2io";
        sck-frequency = <133000000>;
        jedec-id = [c2 20 16];
        size = <33554432>;
        has-dpd;
        t-enter-dpd = <10000>;
        t-exit-dpd = <100000>;
        partitions {
            compatible = "fixed-partitions";
            #address-cells = <1>;
            #size-cells = <1>;

            slot1_partition: partition@0 {
                label = "image-1";
                reg = <0x0 DT_SIZE_K(928)>;
            };
        };
    };
};

&reg0 {
    status = "okay";
};

&reg1 {
    regulator-initial-mode = <NRF5X_REG_MODE_DCDC>;
};

&gpiote {
    status = "okay";
};

&gpio0 {
    status = "okay";
};

&gpio1 {
    status = "okay";
};

&usbd {
    compatible = "nordic,nrf-usbd";
    status = "okay";
    cdc_acm_uart: cdc_acm_uart {
        compatible = "zephyr,cdc-acm-uart";
    };
};

/* Fixed partitioning is required for sysbuild to pass otherwise a build error is reported:
 *
 * Actual values do not matter for boot and app partitions as they are overridden by static partitioning.
 * However the values for storage partition must match with the values defined in static partitioning file because we are using littleFS. */
&flash0 {
    partitions {
        compatible = "fixed-partitions";
        #address-cells = <1>;
        #size-cells = <1>;

        boot_partition: partition@0 {
            label = "mcuboot";
            reg = <0x00000000 DT_SIZE_K(60)>;
        };

        slot0_partition: partition@f000 {
            label = "image-0";
            reg = <0x0000f000 DT_SIZE_K(936)>;
        };

        littlefs_partition: partition@f9000 {
            label = "littlefs";
            reg = <0x000f9000 DT_SIZE_K(20)>;
        };

        storage_partition: partition@fe000 {
            label = "storage";
            reg = <0x000fe000 DT_SIZE_K(8)>;
        };
    };
};

/* Reduce total SRAM0 size by 1 byte to account for non-init area (used by boot mode) */
&sram0 {
    reg = <0x20000000 0x3ffff>;
};

mcuboot.overlay

/* This file is used to:
 *   1) prevent bootloader\mcuboot\boot\zephyr\app.overlay from being pulled in which expects "boot_partition" DT node to be defined
 *      since we are using static partitioning defined in yml file instead of fixed partitioning via DT,
 *   2) disable unused peripheral by MCUboot preventing linking errors due to non-existing drivers. */

/ {
    chosen {
        /delete-property/ zephyr,shell-uart;
    };

    /delete-node/ fstab;
};

&gpiote {
    status = "disabled";
};

&gpio0 {
    status = "disabled";
};

&gpio1 {
    status = "disabled";
};

&usbd {
    status = "disabled";
    /delete-node/ cdc_acm_uart;
};


I am using static partitioning, pm_static.yml file:

# Flash sector size is 0x1000 (4kB).

external_flash:
  address: 0x000000
  device: DT_CHOSEN(nordic_pm_ext_flash)
  size: 0x400000
  region: external_flash

mcuboot:
  address: 0x0
  size: 0xF000
  region: flash_primary

# This must be set to primary slot start address for the build system to correctly place the app region.
# If this is set to 0x0 the build system would place this padding region at 0x0 and app at 0x200.
mcuboot_pad:
  address: 0xF000
  size: 0x200
  region: flash_primary

app:
  address: 0xF200
  size: 0xE9E00
  region: flash_primary

# Two sectors are required for little FS allocation table
littlefs_storage:
  address: 0xF9000
  size: 0x5000
  region: flash_primary

settings_storage:
  address: 0xFE000
  size: 0x2000
  region: flash_primary

mcuboot_primary:
  address: 0xF000
  size: 0xEA000
  span: [mcuboot_pad, app]
  region: flash_primary

mcuboot_primary_app:
  address: 0xF200
  size: 0xE9E00
  span: [app]
  region: flash_primary

# 2 sectors less than primary slot because primary slot includes 2 sectors for scratch area
mcuboot_secondary:
  address: 0x0
  size: 0xE8000
  region: external_flash

sram_primary:
  address: 0x20000000
  size: 0x40000
  region: sram_primary

I am using mcumgr on application and AuTerm v0.36a sofrware for uploading the image to the MCU, AuTerm->MCUmgr->Img->Upload. Image uploads successfully to the external flash. I then verify the trailer of the slot 1 in the external flash and it looks ok. I then reboot the MCU through AuTerm->MCUmgr->OS->Reset. And then immediately in MCUboot i check the trailer of the slot 1 image in external flash and notice that it is modified. Please see the log below. The first part of the log is in application code, the MCUmgr upload process. Then I print out the trailer, it looks good. Then I reboot the MCU. And I modified MCUboot code to print out the trailer of slot 1 immediately in main.c file and the magic is modified for some reason.

[00:00:14.093,139] <dbg> mcumgr_img_grp: img_mgmt_active_slot: (0) => 0
[00:00:14.093,170] <dbg> mcumgr_img_grp: img_mgmt_active_slot: (0) => 0
[00:00:14.093,566] <inf> mcuboot_util: Image index: 0, Swap type: perm
[00:00:14.093,566] <dbg> mcumgr_img_grp: img_mgmt_get_next_boot_slot: (0, *) => slot = 1, type = 0
[00:00:14.093,597] <dbg> mcumgr_img_grp: img_mgmt_slot_in_use: (0) Allowed erase pending
[00:00:14.093,627] <dbg> mcumgr_img_grp: img_mgmt_active_slot: (0) => 0
[00:00:14.093,627] <dbg> mcumgr_img_grp: img_mgmt_active_slot: (0) => 0
[00:00:14.094,024] <inf> mcuboot_util: Image index: 0, Swap type: perm
[00:00:14.094,055] <dbg> mcumgr_img_grp: img_mgmt_get_next_boot_slot: (0, *) => slot = 1, type = 0
[00:00:14.094,055] <dbg> mcumgr_img_grp: img_mgmt_slot_in_use: (1) Allowed erase pending
[00:00:17.805,084] <inf> mcumgr_img_grp: Erased 0xb9000 bytes of image slot
[00:00:17.835,662] <inf> mcumgr_img_grp: Erased 0x1000 bytes of image slot trailer
[00:00:45.514,007] <dbg> mcumgr_img_grp: img_mgmt_active_slot: (0) => 0
[00:00:45.514,038] <dbg> mcumgr_img_grp: img_mgmt_active_slot: (0) => 0
[00:00:45.514,373] <inf> mcuboot_util: Image index: 0, Swap type: none
[00:00:45.514,404] <dbg> mcumgr_img_grp: img_mgmt_get_next_boot_slot: (0, *) => slot = 0, type = 0
[00:00:45.514,434] <dbg> mcumgr_img_grp: img_mgmt_set_next_boot_slot: (1, confirm)
[00:00:45.514,465] <dbg> mcumgr_img_grp: img_mgmt_set_next_boot_slot: aimg = 0, img = 0, aslot = 0, slot = 1, nbs = 0
[00:00:45.514,801] <dbg> mcuboot_util: boot_write_magic: writing magic; fa_id=2 off=0xe7ff0 (0xe7ff0)
[00:00:45.514,984] <dbg> mcuboot_util: boot_write_image_ok: writing image_ok; fa_id=2 off=0xe7fe8 (0xe7fe8)
[00:00:45.515,136] <dbg> mcuboot_util: boot_write_swap_info: writing swap_info; fa_id=2 off=0xe7fd8 (0xe7fd8), swap_type=0x3 image_num=0x0
[00:00:45.515,319] <dbg> mcumgr_img_grp: img_mgmt_active_slot: (0) => 0
[00:00:45.515,747] <inf> mcuboot_util: Image index: 0, Swap type: perm
[00:00:45.515,747] <dbg> mcumgr_img_grp: img_mgmt_get_next_boot_slot: (0, *) => slot = 1, type = 0
[00:00:45.515,777] <dbg> mcumgr_img_grp: img_mgmt_active_slot: (0) => 0
[00:00:55.052,337] <inf> app: Flash end:
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff 03 ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff 01 ff ff ff ff ff ff ff |........ ........
77 c2 95 f3 60 d2 ef 7f 35 52 50 0f 2c b6 79 80 |w...`... 5RP.,.y.

========================================================================================================
--------------------------------------- MCU REBOOT -----------------------------------------------------
========================================================================================================

[00:00:00.249,938] <dbg> qspi_nor: configure_chip: RDSR 40 QE 1 need 1: no-change
*** Booting MCUboot v2.1.0-dev-12e5ee106034 ***
*** Using nRF Connect SDK v2.9.0-7787b2649840 ***
*** Using Zephyr OS v3.7.99-1f8f3dc29142 ***
[00:00:00.250,854] <inf> mcuboot: BOOT_STARTED
[00:00:00.251,159] <inf> mcuboot: boot flash end:
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff 03 ff ff ff ff ff ff ff |........ ........
ff ff ff ff ff ff ff ff 03 ff ff ff ff ff ff ff |........ ........
ff ca 97 f3 e0 da ef ff 35 5a 50 0f ae b6 fb 80 |........ 5ZP.....
[00:00:00.259,429] <inf> mcuboot: Starting bootloader


I am using the following code to read contents of the trailer of the slot 1 image in external flash:

#include <zephyr/device.h>
#include <zephyr/drivers/flash.h>
#include <zephyr/storage/flash_map.h>
void boot_test_flash(void)
{
    const struct device *flash_dev = FIXED_PARTITION_DEVICE(mcuboot_secondary);

    if (!device_is_ready(flash_dev)) {
        BOOT_LOG_ERR("boot flash device is not ready");
        return;
    }

    uint8_t buf[256];
    uint32_t addr;
    uint32_t end_addr;
    addr = 0xE8000 - 256;
    int rc = flash_read(flash_dev, addr, buf, 256);
    if (rc != 0) {
        BOOT_LOG_ERR("boot flash read failed: %d", rc);
    } else {
        LOG_HEXDUMP_INF(buf, sizeof(buf), "boot flash end:");
    }
}


What is the issue here? I dont have a logic analyzer currently at hand to check what is going on on the QSPI lines. Maybe the problem is with my board and MCU produces some garbage out on the lines on MCU reboot. Or maybe there is some issues with the QSPI driver or the kernel. I am writting to you, to check if you had resolved any similar issues? Or if you have any ideas what could be the problem.

Thanks

Related