[PATCH v6 10/18] mtd: rawnand: sunxi: bound DMA batches by the user-data register bank
From: James Hilliard <hidden>
Date: 2026-09-14 21:10:46
Also in:
linux-arm-kernel, linux-sunxi, lkml
Subsystem:
memory technology devices (mtd), nand flash subsystem, the rest · Maintainers:
Miquel Raynal, Richard Weinberger, Vignesh Raghavendra, Linus Torvalds
The H6/H616 controller concatenates protected user data in a bank of 32
four-byte registers. The default layout can allocate more than 128 bytes
across a page, even though each ECC step fits its individual length limit.
DMA transfers then access beyond the user-data register bank. For example,
a 16 KiB page with 1280 OOB bytes and BCH40/1024 has 160 user-data bytes,
while a 16 KiB page with 1664 OOB bytes can allocate 512 bytes.
Do not cap that allocation: changing the lengths would move ECC offsets
and make existing pages written through PIO incompatible. Instead, split
DMA transfers while retaining the original logical main-data and OOB
offsets. Fill the first batch up to the register-bank limit. Transfer each
remaining logical step separately using hardware slot and user-data
register zero, repositioning the main column and spare-area base first.
PAGE_OP generates the second and subsequent main-data columns from its
hardware slot index, not relative to the initial column. Read and write
probes on H616 confirm this behavior. A multi-slot batch starting partway
through the page would therefore return to the wrong main-data column.
Single-slot later batches avoid those internally generated column changes.
Pages whose user data already fits retain the existing single-batch path.
Pass the user-data register index directly to the protected-OOB helpers
and program length fields from the batch's logical starting step. Collect
each batch's ECC status and protected OOB before reusing the registers.
Keep ECC accounting and trailing-OOB handling page-wide, remembering
whether any batch found an erased chunk. Discard all partial DMA statistics
before a PIO read retry.
Use one page-read setup or program-begin command for the whole page. Only
issue program-end after every batch and any trailing OOB transfer succeed.
Do not retry a write in PIO after a batch has already been transferred.
The allocator, free-OOB layout and raw-access callbacks are unchanged.
Fixes: 54dcd6aa69db ("mtd: rawnand: sunxi: introduce maximize variable user data length")
Signed-off-by: James Hilliard <redacted>
---
drivers/mtd/nand/raw/sunxi_nand.c | 284 ++++++++++++++++++++++----------------
1 file changed, 163 insertions(+), 121 deletions(-)
diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c
index e29638c828ca..18a4d1cc4246 100644
--- a/drivers/mtd/nand/raw/sunxi_nand.c
+++ b/drivers/mtd/nand/raw/sunxi_nand.c@@ -54,6 +54,7 @@ #define NFC_REG_H6_RDATA_1 0x004C #define NFC_REG_A10_USER_DATA 0x0050 #define NFC_REG_H6_USER_DATA 0x0080 +#define NFC_H6_USER_DATA_REGS 32 #define NFC_REG_USER_DATA(nfc, x) (nfc->caps->reg_user_data + ((x) * 4)) #define NFC_REG_H6_USER_DATA_LEN 0x0070 /* A USER_DATA_LEN register can hold the length of 8 USER_DATA registers */
@@ -932,41 +933,47 @@ static u8 sunxi_nfc_user_data_sz(struct sunxi_nand_chip *sunxi_nand, int step) return sunxi_nand->user_data_bytes[step]; } +/* Keep the on-flash layout, but fit each DMA batch in the user-data bank. */ +static int sunxi_nfc_dma_batch_steps(struct nand_chip *nand, + int first_step, int end_step) +{ + struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); + struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); + unsigned int user_data_sz = 0; + int i; + + if (!nfc->caps->reg_user_data_len) + return end_step - first_step; + + /* + * PAGE_OP generates main-data columns from hardware slot zero, not + * from the initial column. Use only slot zero after the first batch, + * so the explicit column change selects each remaining logical step. + */ + if (first_step) + return 1; + + for (i = first_step; i < end_step; i++) { + user_data_sz += sunxi_nfc_user_data_sz(sunxi_nand, i); + if (user_data_sz > NFC_H6_USER_DATA_REGS * sizeof(u32)) + break; + } + + return i - first_step; +} + +/* PIO uses register zero; DMA concatenates user data within each batch. */ static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob, - int step, bool bbm, int page, + unsigned int reg_index, bool bbm, int page, unsigned int user_data_sz) { - struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); u32 user_data; + unsigned int i; - if (!nfc->caps->reg_user_data_len) { - /* - * For A10, the user data for step n is in the nth - * REG_USER_DATA - */ - user_data = readl(nfc->regs + NFC_REG_USER_DATA(nfc, step)); - sunxi_nfc_user_data_to_buf(user_data, oob); - } else { - /* - * For H6 NAND controller, the user data for all steps is - * contained in 32 user data registers, but not at a specific - * offset for each step, they are just concatenated. - */ - unsigned int user_data_off = 0; - unsigned int reg_off; - u8 *ptr = oob; - unsigned int i; - - for (i = 0; i < step; i++) - user_data_off += sunxi_nfc_user_data_sz(sunxi_nand, i); - - user_data_off /= 4; - for (i = 0; i < user_data_sz / 4; i++, ptr += 4) { - reg_off = NFC_REG_USER_DATA(nfc, user_data_off + i); - user_data = readl(nfc->regs + reg_off); - sunxi_nfc_user_data_to_buf(user_data, ptr); - } + for (i = 0; i < user_data_sz / 4; i++) { + user_data = readl(nfc->regs + NFC_REG_USER_DATA(nfc, reg_index + i)); + sunxi_nfc_user_data_to_buf(user_data, oob + i * 4); } /* De-randomize the Bad Block Marker. */
@@ -1022,13 +1029,13 @@ static void sunxi_nfc_set_user_data_len(struct sunxi_nfc *nfc, } static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand, - const u8 *oob, int step, + const u8 *oob, unsigned int reg_index, bool bbm, int page, unsigned int user_data_sz) { struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); - struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); u8 user_data[SUNXI_NFC_MAX_USER_DATA_SZ] = {}; + unsigned int i; /* Randomize the Bad Block Marker. */ if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
@@ -1037,33 +1044,10 @@ static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand, oob = user_data; } - if (!nfc->caps->reg_user_data_len) { - /* - * For A10, the user data for step n is in the nth - * REG_USER_DATA - */ - writel(sunxi_nfc_buf_to_user_data(oob), - nfc->regs + NFC_REG_USER_DATA(nfc, step)); - } else { - /* - * For H6 NAND controller, the user data for all steps is - * contained in 32 user data registers, but not at a specific - * offset for each step, they are just concatenated. - */ - unsigned int user_data_off = 0; - const u8 *ptr = oob; - unsigned int i; - - for (i = 0; i < step; i++) - user_data_off += sunxi_nfc_user_data_sz(sunxi_nand, i); - - user_data_off /= 4; - for (i = 0; i < user_data_sz / 4; i++, ptr += 4) { - writel(sunxi_nfc_buf_to_user_data(ptr), - nfc->regs + NFC_REG_USER_DATA(nfc, user_data_off + i)); - } + for (i = 0; i < user_data_sz / 4; i++) { + writel(sunxi_nfc_buf_to_user_data(oob + i * 4), + nfc->regs + NFC_REG_USER_DATA(nfc, reg_index + i)); } - } static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
@@ -1313,27 +1297,35 @@ static int sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand, return 0; } -static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf, - int oob_required, int page, - int nchunks) +static int sunxi_nfc_hw_ecc_read_batch_dma(struct nand_chip *nand, u8 *buf, + int oob_required, int page, + int first_step, int nchunks, + int *raw_mode) { bool randomized = nand->options & NAND_NEED_SCRAMBLING; struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct mtd_info *mtd = nand_to_mtd(nand); struct nand_ecc_ctrl *ecc = &nand->ecc; - unsigned int corrected = mtd->ecc_stats.corrected; - unsigned int failed = mtd->ecc_stats.failed; + unsigned int reg_index = 0, user_data_sz; unsigned int max_bitflips = 0; - int ret, i, raw_mode = 0; + int ret, i; struct scatterlist sg; u32 status, pattern_found, wait; + if (first_step) { + ret = sunxi_nfc_read_column(nand, page, first_step * ecc->size, + NULL, 0); + if (ret) + return ret; + } + ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); if (ret) return ret; - ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks, + ret = sunxi_nfc_dma_op_prepare(nfc, buf + first_step * ecc->size, + ecc->size, nchunks, DMA_FROM_DEVICE, &sg); if (ret) return ret;
@@ -1341,7 +1333,12 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf sunxi_nfc_hw_ecc_enable(nand); sunxi_nfc_reset_user_data_len(nfc); for (i = 0; i < nchunks; i++) - sunxi_nfc_set_user_data_len(nfc, sunxi_nfc_user_data_sz(sunxi_nand, i), i); + sunxi_nfc_set_user_data_len(nfc, + sunxi_nfc_user_data_sz(sunxi_nand, first_step + i), i); + /* exec_op() restores the page's spare base during column changes. */ + if (first_step) + writel(mtd->writesize + sunxi_get_oob_offset(sunxi_nand, ecc, first_step), + nfc->regs + NFC_REG_SPARE_AREA(nfc)); sunxi_nfc_randomizer_config(nand, page, false); sunxi_nfc_randomizer_enable(nand);
@@ -1374,15 +1371,17 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf pattern_found = readl(nfc->regs + nfc->caps->reg_pat_found); pattern_found = field_get(NFC_ECC_PAT_FOUND_MSK(nfc), pattern_found); - for (i = 0; i < nchunks; i++) { - int data_off = i * ecc->size; - unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i); - int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i); + for (i = 0; i < nchunks; i++, reg_index += user_data_sz / 4) { + int logical_step = first_step + i; + int data_off = logical_step * ecc->size; + int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, logical_step); u8 *data = buf + data_off; u8 *oob = nand->oob_poi + oob_off; bool erased; int bitflips; + user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, logical_step); + bitflips = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL, oob_required ? oob : NULL, i, status, pattern_found,
@@ -1397,40 +1396,64 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf ret = sunxi_nfc_read_column(nand, page, mtd->writesize + oob_off, oob, ecc->bytes + user_data_sz); if (ret) - goto err_stats; + return ret; - sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i, !i, + sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, reg_index, !logical_step, page, user_data_sz); } if (erased) - raw_mode = 1; + *raw_mode = 1; sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, bitflips); } if (status & NFC_ECC_ERR_MSK(nfc)) { for (i = 0; i < nchunks; i++) { - int data_off = i * ecc->size; - unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i); - int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i); + int logical_step = first_step + i; + int data_off = logical_step * ecc->size; + int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, logical_step); u8 *data = buf + data_off; u8 *oob = nand->oob_poi + oob_off; if (!(status & NFC_ECC_ERR(i))) continue; + user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, logical_step); ret = sunxi_nfc_hw_ecc_read_error(nand, data, data_off, oob, mtd->writesize + oob_off, user_data_sz, &max_bitflips, page); if (ret < 0) - goto err_stats; + return ret; if (ret) - raw_mode = 1; + *raw_mode = 1; } } + return max_bitflips; +} + +static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, u8 *buf, + int oob_required, int page, int nchunks) +{ + struct mtd_info *mtd = nand_to_mtd(nand); + unsigned int corrected = mtd->ecc_stats.corrected; + unsigned int failed = mtd->ecc_stats.failed; + unsigned int max_bitflips = 0; + int raw_mode = 0; + int first_step, batch_steps, ret; + + for (first_step = 0; first_step < nchunks; first_step += batch_steps) { + batch_steps = sunxi_nfc_dma_batch_steps(nand, first_step, nchunks); + ret = sunxi_nfc_hw_ecc_read_batch_dma(nand, buf, oob_required, page, + first_step, batch_steps, + &raw_mode); + if (ret < 0) + goto err_stats; + max_bitflips = max_t(unsigned int, max_bitflips, ret); + } + if (oob_required) { ret = sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, NULL, !raw_mode, page);
@@ -1761,69 +1784,88 @@ static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand, { struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); + struct mtd_info *mtd = nand_to_mtd(nand); struct nand_ecc_ctrl *ecc = &nand->ecc; struct scatterlist sg; u32 wait; - int ret, i; + int first_step, batch_steps, ret, i; sunxi_nfc_select_chip(nand, nand->cur_cs); - ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); - if (ret) - return ret; + for (first_step = 0; first_step < ecc->steps; first_step += batch_steps) { + unsigned int reg_index = 0; - ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps, - DMA_TO_DEVICE, &sg); - if (ret) - goto pio_fallback; + batch_steps = sunxi_nfc_dma_batch_steps(nand, first_step, ecc->steps); + ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); + if (ret) + return ret; - sunxi_nfc_reset_user_data_len(nfc); - for (i = 0; i < ecc->steps; i++) { - unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i); - int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i); - const u8 *oob = nand->oob_poi + oob_off; + ret = sunxi_nfc_dma_op_prepare(nfc, buf + first_step * ecc->size, + ecc->size, batch_steps, DMA_TO_DEVICE, &sg); + if (ret) { + /* Only retry before any part of the page has been transferred. */ + if (first_step) + return ret; + goto pio_fallback; + } - sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page, - user_data_sz); - sunxi_nfc_set_user_data_len(nfc, user_data_sz, i); - } + sunxi_nfc_reset_user_data_len(nfc); + for (i = first_step; i < first_step + batch_steps; i++) { + unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i); + int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i); + const u8 *oob = nand->oob_poi + oob_off; - ret = nand_prog_page_begin_op(nand, page, 0, NULL, 0); - if (ret) { - sunxi_nfc_dma_op_abort(nfc); - sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); - return ret; - } + sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, reg_index, !i, + page, user_data_sz); + sunxi_nfc_set_user_data_len(nfc, user_data_sz, i - first_step); + reg_index += user_data_sz / 4; + } - sunxi_nfc_hw_ecc_enable(nand); - sunxi_nfc_randomizer_config(nand, page, false); - sunxi_nfc_randomizer_enable(nand); + if (first_step) + ret = nand_change_write_column_op(nand, first_step * ecc->size, + NULL, 0, false); + else + ret = nand_prog_page_begin_op(nand, page, 0, NULL, 0); + if (ret) { + sunxi_nfc_dma_op_abort(nfc); + sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); + return ret; + } - writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, - nfc->regs + NFC_REG_WCMD_SET); + /* exec_op() restores the page's spare base during column changes. */ + if (first_step) + writel(mtd->writesize + sunxi_get_oob_offset(sunxi_nand, ecc, first_step), + nfc->regs + NFC_REG_SPARE_AREA(nfc)); + sunxi_nfc_hw_ecc_enable(nand); + sunxi_nfc_randomizer_config(nand, page, false); + sunxi_nfc_randomizer_enable(nand); - wait = NFC_CMD_INT_FLAG; + writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, + nfc->regs + NFC_REG_WCMD_SET); - if (nfc->use_mdma) - wait |= NFC_DMA_INT_FLAG; - else - dma_async_issue_pending(nfc->dmac); + wait = NFC_CMD_INT_FLAG; - writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | - NFC_DATA_TRANS | NFC_ACCESS_DIR, - nfc->regs + NFC_REG_CMD); + if (nfc->use_mdma) + wait |= NFC_DMA_INT_FLAG; + else + dma_async_issue_pending(nfc->dmac); - ret = sunxi_nfc_wait_events(nfc, wait, false, 0); - if (ret) - sunxi_nfc_dma_op_abort(nfc); + writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | + NFC_DATA_TRANS | NFC_ACCESS_DIR, + nfc->regs + NFC_REG_CMD); - sunxi_nfc_randomizer_disable(nand); - sunxi_nfc_hw_ecc_disable(nand); + ret = sunxi_nfc_wait_events(nfc, wait, false, 0); + if (ret) + sunxi_nfc_dma_op_abort(nfc); - sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); + sunxi_nfc_randomizer_disable(nand); + sunxi_nfc_hw_ecc_disable(nand); - if (ret) - return ret; + sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); + + if (ret) + return ret; + } if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) { /* TODO: use DMA to transfer extra OOB bytes ? */
--
2.53.0