Hi
On Tue, Dec 20, 2022 at 11:22 AM Roger Quadros <rog...@kernel.org> wrote:
>
> The BCH detection hardware can generate ECC bytes for multiple
> sectors in one go. Use that feature.
>
> correct() only corrects one sector at a time so we need to call it
> repeatedly for each sector.
>
> Signed-off-by: Roger Quadros <rog...@kernel.org>
> Reviewed-by: Michael Trimarchi <mich...@amarulasolutions.com>
> ---
> drivers/mtd/nand/raw/omap_gpmc.c | 325 +++++++++++++++++++++----------
> 1 file changed, 223 insertions(+), 102 deletions(-)
>
> diff --git a/drivers/mtd/nand/raw/omap_gpmc.c
> b/drivers/mtd/nand/raw/omap_gpmc.c
> index 69fc09be097..e772a914c88 100644
> --- a/drivers/mtd/nand/raw/omap_gpmc.c
> +++ b/drivers/mtd/nand/raw/omap_gpmc.c
> @@ -27,6 +27,9 @@
>
> #define BADBLOCK_MARKER_LENGTH 2
> #define SECTOR_BYTES 512
> +#define ECCSIZE0_SHIFT 12
> +#define ECCSIZE1_SHIFT 22
> +#define ECC1RESULTSIZE 0x1
> #define ECCCLEAR (0x1 << 8)
> #define ECCRESULTREG1 (0x1 << 0)
> /* 4 bit padding to make byte aligned, 56 = 52 + 4 */
> @@ -186,72 +189,35 @@ static int __maybe_unused omap_correct_data(struct
> mtd_info *mtd, uint8_t *dat,
> __maybe_unused
> static void omap_enable_hwecc(struct mtd_info *mtd, int32_t mode)
> {
> - struct nand_chip *nand = mtd_to_nand(mtd);
> - struct omap_nand_info *info = nand_get_controller_data(nand);
> + struct nand_chip *nand = mtd_to_nand(mtd);
> + struct omap_nand_info *info = nand_get_controller_data(nand);
> unsigned int dev_width = (nand->options & NAND_BUSWIDTH_16) ? 1 : 0;
> - unsigned int ecc_algo = 0;
> - unsigned int bch_type = 0;
> - unsigned int eccsize1 = 0x00, eccsize0 = 0x00, bch_wrapmode = 0x00;
> - u32 ecc_size_config_val = 0;
> - u32 ecc_config_val = 0;
> - int cs = info->cs;
> + u32 val;
>
> - /* configure GPMC for specific ecc-scheme */
> - switch (info->ecc_scheme) {
> - case OMAP_ECC_HAM1_CODE_SW:
> - return;
> - case OMAP_ECC_HAM1_CODE_HW:
> - ecc_algo = 0x0;
> - bch_type = 0x0;
> - bch_wrapmode = 0x00;
> - eccsize0 = 0xFF;
> - eccsize1 = 0xFF;
> + /* Clear ecc and enable bits */
> + writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
> +
> + /* program ecc and result sizes */
> + val = ((((nand->ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) |
> + ECC1RESULTSIZE);
> + writel(val, &gpmc_cfg->ecc_size_config);
> +
> + switch (mode) {
> + case NAND_ECC_READ:
> + case NAND_ECC_WRITE:
> + writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
> break;
> - case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
> - case OMAP_ECC_BCH8_CODE_HW:
> - ecc_algo = 0x1;
> - bch_type = 0x1;
> - if (mode == NAND_ECC_WRITE) {
> - bch_wrapmode = 0x01;
> - eccsize0 = 0; /* extra bits in nibbles per sector */
> - eccsize1 = 28; /* OOB bits in nibbles per sector */
> - } else {
> - bch_wrapmode = 0x01;
> - eccsize0 = 26; /* ECC bits in nibbles per sector */
> - eccsize1 = 2; /* non-ECC bits in nibbles per sector
> */
> - }
> - break;
> - case OMAP_ECC_BCH16_CODE_HW:
> - ecc_algo = 0x1;
> - bch_type = 0x2;
> - if (mode == NAND_ECC_WRITE) {
> - bch_wrapmode = 0x01;
> - eccsize0 = 0; /* extra bits in nibbles per sector */
> - eccsize1 = 52; /* OOB bits in nibbles per sector */
> - } else {
> - bch_wrapmode = 0x01;
> - eccsize0 = 52; /* ECC bits in nibbles per sector */
> - eccsize1 = 0; /* non-ECC bits in nibbles per sector
> */
> - }
> + case NAND_ECC_READSYN:
> + writel(ECCCLEAR, &gpmc_cfg->ecc_control);
> break;
> default:
> - return;
> + printf("%s: error: unrecognized Mode[%d]!\n", __func__, mode);
> + break;
> }
> - /* Clear ecc and enable bits */
> - writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
> - /* Configure ecc size for BCH */
> - ecc_size_config_val = (eccsize1 << 22) | (eccsize0 << 12);
> - writel(ecc_size_config_val, &gpmc_cfg->ecc_size_config);
> -
> - /* Configure device details for BCH engine */
> - ecc_config_val = ((ecc_algo << 16) | /* HAM1 | BCHx */
> - (bch_type << 12) | /* BCH4/BCH8/BCH16 */
> - (bch_wrapmode << 8) | /* wrap mode */
> - (dev_width << 7) | /* bus width */
> - (0x0 << 4) | /* number of sectors */
> - (cs << 1) | /* ECC CS */
> - (0x1)); /* enable ECC */
> - writel(ecc_config_val, &gpmc_cfg->ecc_config);
> +
> + /* (ECC 16 or 8 bit col) | ( CS ) | ECC Enable */
> + val = (dev_width << 7) | (info->cs << 1) | (0x1);
> + writel(val, &gpmc_cfg->ecc_config);
> }
>
> /*
> @@ -270,6 +236,124 @@ static void omap_enable_hwecc(struct mtd_info *mtd,
> int32_t mode)
> */
> static int omap_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat,
> uint8_t *ecc_code)
> +{
> + u32 val;
> +
> + val = readl(&gpmc_cfg->ecc1_result);
> + ecc_code[0] = val & 0xFF;
> + ecc_code[1] = (val >> 16) & 0xFF;
> + ecc_code[2] = ((val >> 8) & 0x0F) | ((val >> 20) & 0xF0);
> +
> + return 0;
> +}
> +
> +/* GPMC ecc engine settings for read */
> +#define BCH_WRAPMODE_1 1 /* BCH wrap mode 1 */
> +#define BCH8R_ECC_SIZE0 0x1a /* ecc_size0 = 26 */
> +#define BCH8R_ECC_SIZE1 0x2 /* ecc_size1 = 2 */
> +#define BCH4R_ECC_SIZE0 0xd /* ecc_size0 = 13 */
> +#define BCH4R_ECC_SIZE1 0x3 /* ecc_size1 = 3 */
> +
> +/* GPMC ecc engine settings for write */
> +#define BCH_WRAPMODE_6 6 /* BCH wrap mode 6 */
> +#define BCH_ECC_SIZE0 0x0 /* ecc_size0 = 0, no oob protection
> */
> +#define BCH_ECC_SIZE1 0x20 /* ecc_size1 = 32 */
> +
> +/**
> + * omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation
> + * @mtd: MTD device structure
> + * @mode: Read/Write mode
> + *
> + * When using BCH with SW correction (i.e. no ELM), sector size is set
> + * to 512 bytes and we use BCH_WRAPMODE_6 wrapping mode
> + * for both reading and writing with:
> + * eccsize0 = 0 (no additional protected byte in spare area)
> + * eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area)
> + */
> +static void __maybe_unused omap_enable_hwecc_bch(struct mtd_info *mtd,
> + int mode)
> +{
> + unsigned int bch_type;
> + unsigned int dev_width, nsectors;
> + struct nand_chip *chip = mtd_to_nand(mtd);
> + struct omap_nand_info *info = nand_get_controller_data(chip);
> + u32 val, wr_mode;
> + unsigned int ecc_size1, ecc_size0;
> +
> + /* GPMC configurations for calculating ECC */
> + switch (info->ecc_scheme) {
> + case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
> + bch_type = 1;
> + nsectors = 1;
> + wr_mode = BCH_WRAPMODE_6;
> + ecc_size0 = BCH_ECC_SIZE0;
> + ecc_size1 = BCH_ECC_SIZE1;
> + break;
> + case OMAP_ECC_BCH8_CODE_HW:
> + bch_type = 1;
> + nsectors = chip->ecc.steps;
> + if (mode == NAND_ECC_READ) {
> + wr_mode = BCH_WRAPMODE_1;
> + ecc_size0 = BCH8R_ECC_SIZE0;
> + ecc_size1 = BCH8R_ECC_SIZE1;
> + } else {
> + wr_mode = BCH_WRAPMODE_6;
> + ecc_size0 = BCH_ECC_SIZE0;
> + ecc_size1 = BCH_ECC_SIZE1;
> + }
> + break;
> + case OMAP_ECC_BCH16_CODE_HW:
> + bch_type = 0x2;
> + nsectors = chip->ecc.steps;
> + if (mode == NAND_ECC_READ) {
> + wr_mode = 0x01;
> + ecc_size0 = 52; /* ECC bits in nibbles per sector */
> + ecc_size1 = 0; /* non-ECC bits in nibbles per sector
> */
> + } else {
> + wr_mode = 0x01;
> + ecc_size0 = 0; /* extra bits in nibbles per sector */
> + ecc_size1 = 52; /* OOB bits in nibbles per sector */
> + }
> + break;
> + default:
> + return;
> + }
> +
> + writel(ECCRESULTREG1, &gpmc_cfg->ecc_control);
> +
> + /* Configure ecc size for BCH */
> + val = (ecc_size1 << ECCSIZE1_SHIFT) | (ecc_size0 << ECCSIZE0_SHIFT);
> + writel(val, &gpmc_cfg->ecc_size_config);
> +
> + dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0;
> +
> + /* BCH configuration */
> + val = ((1 << 16) | /* enable BCH */
> + (bch_type << 12) | /* BCH4/BCH8/BCH16 */
> + (wr_mode << 8) | /* wrap mode */
> + (dev_width << 7) | /* bus width */
> + (((nsectors - 1) & 0x7) << 4) | /* number of sectors */
> + (info->cs << 1) | /* ECC CS */
> + (0x1)); /* enable ECC */
> +
> + writel(val, &gpmc_cfg->ecc_config);
> +
> + /* Clear ecc and enable bits */
> + writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
> +}
> +
> +/**
> + * _omap_calculate_ecc_bch - Generate BCH ECC bytes for one sector
> + * @mtd: MTD device structure
> + * @dat: The pointer to data on which ecc is computed
> + * @ecc_code: The ecc_code buffer
> + * @sector: The sector number (for a multi sector page)
> + *
> + * Support calculating of BCH4/8/16 ECC vectors for one sector
> + * within a page. Sector number is in @sector.
> + */
> +static int _omap_calculate_ecc_bch(struct mtd_info *mtd, const u8 *dat,
> + u8 *ecc_code, int sector)
> {
> struct nand_chip *chip = mtd_to_nand(mtd);
> struct omap_nand_info *info = nand_get_controller_data(chip);
> @@ -278,17 +362,11 @@ static int omap_calculate_ecc(struct mtd_info *mtd,
> const uint8_t *dat,
> int8_t i = 0, j;
>
> switch (info->ecc_scheme) {
> - case OMAP_ECC_HAM1_CODE_HW:
> - val = readl(&gpmc_cfg->ecc1_result);
> - ecc_code[0] = val & 0xFF;
> - ecc_code[1] = (val >> 16) & 0xFF;
> - ecc_code[2] = ((val >> 8) & 0x0F) | ((val >> 20) & 0xF0);
> - break;
> #ifdef CONFIG_BCH
> case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
> #endif
> case OMAP_ECC_BCH8_CODE_HW:
> - ptr = &gpmc_cfg->bch_result_0_3[0].bch_result_x[3];
> + ptr = &gpmc_cfg->bch_result_0_3[sector].bch_result_x[3];
> val = readl(ptr);
> ecc_code[i++] = (val >> 0) & 0xFF;
> ptr--;
> @@ -300,23 +378,24 @@ static int omap_calculate_ecc(struct mtd_info *mtd,
> const uint8_t *dat,
> ecc_code[i++] = (val >> 0) & 0xFF;
> ptr--;
> }
> +
> break;
> case OMAP_ECC_BCH16_CODE_HW:
> - val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[2]);
> + val =
> readl(&gpmc_cfg->bch_result_4_6[sector].bch_result_x[2]);
> ecc_code[i++] = (val >> 8) & 0xFF;
> ecc_code[i++] = (val >> 0) & 0xFF;
> - val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[1]);
> + val =
> readl(&gpmc_cfg->bch_result_4_6[sector].bch_result_x[1]);
> ecc_code[i++] = (val >> 24) & 0xFF;
> ecc_code[i++] = (val >> 16) & 0xFF;
> ecc_code[i++] = (val >> 8) & 0xFF;
> ecc_code[i++] = (val >> 0) & 0xFF;
> - val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[0]);
> + val =
> readl(&gpmc_cfg->bch_result_4_6[sector].bch_result_x[0]);
> ecc_code[i++] = (val >> 24) & 0xFF;
> ecc_code[i++] = (val >> 16) & 0xFF;
> ecc_code[i++] = (val >> 8) & 0xFF;
> ecc_code[i++] = (val >> 0) & 0xFF;
> for (j = 3; j >= 0; j--) {
> - val =
> readl(&gpmc_cfg->bch_result_0_3[0].bch_result_x[j]
> + val =
> readl(&gpmc_cfg->bch_result_0_3[sector].bch_result_x[j]
> );
> ecc_code[i++] = (val >> 24) & 0xFF;
> ecc_code[i++] = (val >> 16) & 0xFF;
> @@ -329,18 +408,18 @@ static int omap_calculate_ecc(struct mtd_info *mtd,
> const uint8_t *dat,
> }
> /* ECC scheme specific syndrome customizations */
> switch (info->ecc_scheme) {
> - case OMAP_ECC_HAM1_CODE_HW:
> - break;
> #ifdef CONFIG_BCH
> case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
> -
> + /* Add constant polynomial to remainder, so that
> + * ECC of blank pages results in 0x0 on reading back
> + */
> for (i = 0; i < chip->ecc.bytes; i++)
> - *(ecc_code + i) = *(ecc_code + i) ^
> - bch8_polynomial[i];
> + ecc_code[i] ^= bch8_polynomial[i];
> break;
> #endif
> case OMAP_ECC_BCH8_CODE_HW:
> - ecc_code[chip->ecc.bytes - 1] = 0x00;
> + /* Set 14th ECC byte as 0x0 for ROM compatibility */
> + ecc_code[chip->ecc.bytes - 1] = 0x0;
> break;
> case OMAP_ECC_BCH16_CODE_HW:
> break;
> @@ -350,6 +429,22 @@ static int omap_calculate_ecc(struct mtd_info *mtd,
> const uint8_t *dat,
> return 0;
> }
>
> +/**
> + * omap_calculate_ecc_bch - ECC generator for 1 sector
> + * @mtd: MTD device structure
> + * @dat: The pointer to data on which ecc is computed
> + * @ecc_code: The ecc_code buffer
> + *
> + * Support calculating of BCH4/8/16 ECC vectors for one sector. This is used
> + * when SW based correction is required as ECC is required for one sector
> + * at a time.
> + */
> +static int omap_calculate_ecc_bch(struct mtd_info *mtd,
> + const u_char *dat, u_char *ecc_calc)
-static int omap_calculate_ecc_bch(struct mtd_info *mtd,
+static int __maybe_unused omap_calculate_ecc_bch(struct mtd_info *mtd,
const u_char *dat, u_char *ecc_calc)
Acked-by: Michael Trimarchi <mich...@amarulasolutions.com>
> +{
> + return _omap_calculate_ecc_bch(mtd, dat, ecc_calc, 0);
> +}
> +
> static inline void omap_nand_read_buf(struct mtd_info *mtd, uint8_t *buf,
> int len)
> {
> struct nand_chip *chip = mtd_to_nand(mtd);
> @@ -474,6 +569,35 @@ static void omap_nand_read_prefetch(struct mtd_info
> *mtd, uint8_t *buf, int len)
> #endif /* CONFIG_NAND_OMAP_GPMC_PREFETCH */
>
> #ifdef CONFIG_NAND_OMAP_ELM
> +
> +/**
> + * omap_calculate_ecc_bch_multi - Generate ECC for multiple sectors
> + * @mtd: MTD device structure
> + * @dat: The pointer to data on which ecc is computed
> + * @ecc_code: The ecc_code buffer
> + *
> + * Support calculating of BCH4/8/16 ecc vectors for the entire page in one
> go.
> + */
> +static int omap_calculate_ecc_bch_multi(struct mtd_info *mtd,
> + const u_char *dat, u_char *ecc_calc)
> +{
> + struct nand_chip *chip = mtd_to_nand(mtd);
> + int eccbytes = chip->ecc.bytes;
> + unsigned long nsectors;
> + int i, ret;
> +
> + nsectors = ((readl(&gpmc_cfg->ecc_config) >> 4) & 0x7) + 1;
> + for (i = 0; i < nsectors; i++) {
> + ret = _omap_calculate_ecc_bch(mtd, dat, ecc_calc, i);
> + if (ret)
> + return ret;
> +
> + ecc_calc += eccbytes;
> + }
> +
> + return 0;
> +}
> +
> /*
> * omap_reverse_list - re-orders list elements in reverse order [internal]
> * @list: pointer to start of list
> @@ -626,52 +750,49 @@ static int omap_read_page_bch(struct mtd_info *mtd,
> struct nand_chip *chip,
> {
> int i, eccsize = chip->ecc.size;
> int eccbytes = chip->ecc.bytes;
> + int ecctotal = chip->ecc.total;
> int eccsteps = chip->ecc.steps;
> uint8_t *p = buf;
> uint8_t *ecc_calc = chip->buffers->ecccalc;
> uint8_t *ecc_code = chip->buffers->ecccode;
> uint32_t *eccpos = chip->ecc.layout->eccpos;
> uint8_t *oob = chip->oob_poi;
> - uint32_t data_pos;
> uint32_t oob_pos;
>
> - data_pos = 0;
> /* oob area start */
> oob_pos = (eccsize * eccsteps) + chip->ecc.layout->eccpos[0];
> oob += chip->ecc.layout->eccpos[0];
>
> - for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize,
> - oob += eccbytes) {
> - chip->ecc.hwctl(mtd, NAND_ECC_READ);
> - /* read data */
> - chip->cmdfunc(mtd, NAND_CMD_RNDOUT, data_pos, -1);
> - chip->read_buf(mtd, p, eccsize);
> -
> - /* read respective ecc from oob area */
> - chip->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_pos, -1);
> - chip->read_buf(mtd, oob, eccbytes);
> - /* read syndrome */
> - chip->ecc.calculate(mtd, p, &ecc_calc[i]);
> -
> - data_pos += eccsize;
> - oob_pos += eccbytes;
> - }
> + /* Enable ECC engine */
> + chip->ecc.hwctl(mtd, NAND_ECC_READ);
> +
> + /* read entire page */
> + chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
> + chip->read_buf(mtd, buf, mtd->writesize);
> +
> + /* read all ecc bytes from oob area */
> + chip->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_pos, -1);
> + chip->read_buf(mtd, oob, ecctotal);
> +
> + /* Calculate ecc bytes */
> + omap_calculate_ecc_bch_multi(mtd, buf, ecc_calc);
>
> for (i = 0; i < chip->ecc.total; i++)
> ecc_code[i] = chip->oob_poi[eccpos[i]];
>
> + /* error detect & correct */
> eccsteps = chip->ecc.steps;
> p = buf;
>
> for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
> int stat;
> -
> stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]);
> if (stat < 0)
> mtd->ecc_stats.failed++;
> else
> mtd->ecc_stats.corrected += stat;
> }
> +
> return 0;
> }
> #endif /* CONFIG_NAND_OMAP_ELM */
> @@ -819,9 +940,9 @@ static int omap_select_ecc_scheme(struct nand_chip *nand,
> nand->ecc.strength = 8;
> nand->ecc.size = SECTOR_BYTES;
> nand->ecc.bytes = 13;
> - nand->ecc.hwctl = omap_enable_hwecc;
> + nand->ecc.hwctl = omap_enable_hwecc_bch;
> nand->ecc.correct = omap_correct_data_bch_sw;
> - nand->ecc.calculate = omap_calculate_ecc;
> + nand->ecc.calculate = omap_calculate_ecc_bch;
> /* define ecc-layout */
> ecclayout->eccbytes = nand->ecc.bytes * eccsteps;
> ecclayout->eccpos[0] = BADBLOCK_MARKER_LENGTH;
> @@ -860,9 +981,9 @@ static int omap_select_ecc_scheme(struct nand_chip *nand,
> nand->ecc.strength = 8;
> nand->ecc.size = SECTOR_BYTES;
> nand->ecc.bytes = 14;
> - nand->ecc.hwctl = omap_enable_hwecc;
> + nand->ecc.hwctl = omap_enable_hwecc_bch;
> nand->ecc.correct = omap_correct_data_bch;
> - nand->ecc.calculate = omap_calculate_ecc;
> + nand->ecc.calculate = omap_calculate_ecc_bch;
> nand->ecc.read_page = omap_read_page_bch;
> /* define ecc-layout */
> ecclayout->eccbytes = nand->ecc.bytes * eccsteps;
> @@ -893,9 +1014,9 @@ static int omap_select_ecc_scheme(struct nand_chip *nand,
> nand->ecc.size = SECTOR_BYTES;
> nand->ecc.bytes = 26;
> nand->ecc.strength = 16;
> - nand->ecc.hwctl = omap_enable_hwecc;
> + nand->ecc.hwctl = omap_enable_hwecc_bch;
> nand->ecc.correct = omap_correct_data_bch;
> - nand->ecc.calculate = omap_calculate_ecc;
> + nand->ecc.calculate = omap_calculate_ecc_bch;
> nand->ecc.read_page = omap_read_page_bch;
> /* define ecc-layout */
> ecclayout->eccbytes = nand->ecc.bytes * eccsteps;
> --
> 2.34.1
>
--
Michael Nazzareno Trimarchi
Co-Founder & Chief Executive Officer
M. +39 347 913 2170
mich...@amarulasolutions.com
__________________________________
Amarula Solutions BV
Joop Geesinkweg 125, 1114 AB, Amsterdam, NL
T. +31 (0)85 111 9172
i...@amarulasolutions.com
www.amarulasolutions.com
