1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (c) 2016, The Linux Foundation. All rights reserved. 4 */ 5#include <linux/clk.h> 6#include <linux/slab.h> 7#include <linux/bitops.h> 8#include <linux/dma-mapping.h> 9#include <linux/dmaengine.h> 10#include <linux/module.h> 11#include <linux/mtd/rawnand.h> 12#include <linux/mtd/partitions.h> 13#include <linux/of.h> 14#include <linux/of_device.h> 15#include <linux/delay.h> 16#include <linux/dma/qcom_bam_dma.h> 17 18/* NANDc reg offsets */ 19#define NAND_FLASH_CMD 0x00 20#define NAND_ADDR0 0x04 21#define NAND_ADDR1 0x08 22#define NAND_FLASH_CHIP_SELECT 0x0c 23#define NAND_EXEC_CMD 0x10 24#define NAND_FLASH_STATUS 0x14 25#define NAND_BUFFER_STATUS 0x18 26#define NAND_DEV0_CFG0 0x20 27#define NAND_DEV0_CFG1 0x24 28#define NAND_DEV0_ECC_CFG 0x28 29#define NAND_DEV1_ECC_CFG 0x2c 30#define NAND_DEV1_CFG0 0x30 31#define NAND_DEV1_CFG1 0x34 32#define NAND_READ_ID 0x40 33#define NAND_READ_STATUS 0x44 34#define NAND_DEV_CMD0 0xa0 35#define NAND_DEV_CMD1 0xa4 36#define NAND_DEV_CMD2 0xa8 37#define NAND_DEV_CMD_VLD 0xac 38#define SFLASHC_BURST_CFG 0xe0 39#define NAND_ERASED_CW_DETECT_CFG 0xe8 40#define NAND_ERASED_CW_DETECT_STATUS 0xec 41#define NAND_EBI2_ECC_BUF_CFG 0xf0 42#define FLASH_BUF_ACC 0x100 43 44#define NAND_CTRL 0xf00 45#define NAND_VERSION 0xf08 46#define NAND_READ_LOCATION_0 0xf20 47#define NAND_READ_LOCATION_1 0xf24 48#define NAND_READ_LOCATION_2 0xf28 49#define NAND_READ_LOCATION_3 0xf2c 50 51/* dummy register offsets, used by write_reg_dma */ 52#define NAND_DEV_CMD1_RESTORE 0xdead 53#define NAND_DEV_CMD_VLD_RESTORE 0xbeef 54 55/* NAND_FLASH_CMD bits */ 56#define PAGE_ACC BIT(4) 57#define LAST_PAGE BIT(5) 58 59/* NAND_FLASH_CHIP_SELECT bits */ 60#define NAND_DEV_SEL 0 61#define DM_EN BIT(2) 62 63/* NAND_FLASH_STATUS bits */ 64#define FS_OP_ERR BIT(4) 65#define FS_READY_BSY_N BIT(5) 66#define FS_MPU_ERR BIT(8) 67#define FS_DEVICE_STS_ERR BIT(16) 68#define FS_DEVICE_WP BIT(23) 69 70/* NAND_BUFFER_STATUS bits */ 71#define BS_UNCORRECTABLE_BIT BIT(8) 72#define BS_CORRECTABLE_ERR_MSK 0x1f 73 74/* NAND_DEVn_CFG0 bits */ 75#define DISABLE_STATUS_AFTER_WRITE 4 76#define CW_PER_PAGE 6 77#define UD_SIZE_BYTES 9 78#define ECC_PARITY_SIZE_BYTES_RS 19 79#define SPARE_SIZE_BYTES 23 80#define NUM_ADDR_CYCLES 27 81#define STATUS_BFR_READ 30 82#define SET_RD_MODE_AFTER_STATUS 31 83 84/* NAND_DEVn_CFG0 bits */ 85#define DEV0_CFG1_ECC_DISABLE 0 86#define WIDE_FLASH 1 87#define NAND_RECOVERY_CYCLES 2 88#define CS_ACTIVE_BSY 5 89#define BAD_BLOCK_BYTE_NUM 6 90#define BAD_BLOCK_IN_SPARE_AREA 16 91#define WR_RD_BSY_GAP 17 92#define ENABLE_BCH_ECC 27 93 94/* NAND_DEV0_ECC_CFG bits */ 95#define ECC_CFG_ECC_DISABLE 0 96#define ECC_SW_RESET 1 97#define ECC_MODE 4 98#define ECC_PARITY_SIZE_BYTES_BCH 8 99#define ECC_NUM_DATA_BYTES 16 100#define ECC_FORCE_CLK_OPEN 30 101 102/* NAND_DEV_CMD1 bits */ 103#define READ_ADDR 0 104 105/* NAND_DEV_CMD_VLD bits */ 106#define READ_START_VLD BIT(0) 107#define READ_STOP_VLD BIT(1) 108#define WRITE_START_VLD BIT(2) 109#define ERASE_START_VLD BIT(3) 110#define SEQ_READ_START_VLD BIT(4) 111 112/* NAND_EBI2_ECC_BUF_CFG bits */ 113#define NUM_STEPS 0 114 115/* NAND_ERASED_CW_DETECT_CFG bits */ 116#define ERASED_CW_ECC_MASK 1 117#define AUTO_DETECT_RES 0 118#define MASK_ECC (1 << ERASED_CW_ECC_MASK) 119#define RESET_ERASED_DET (1 << AUTO_DETECT_RES) 120#define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES) 121#define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC) 122#define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC) 123 124/* NAND_ERASED_CW_DETECT_STATUS bits */ 125#define PAGE_ALL_ERASED BIT(7) 126#define CODEWORD_ALL_ERASED BIT(6) 127#define PAGE_ERASED BIT(5) 128#define CODEWORD_ERASED BIT(4) 129#define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED) 130#define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED) 131 132/* NAND_READ_LOCATION_n bits */ 133#define READ_LOCATION_OFFSET 0 134#define READ_LOCATION_SIZE 16 135#define READ_LOCATION_LAST 31 136 137/* Version Mask */ 138#define NAND_VERSION_MAJOR_MASK 0xf0000000 139#define NAND_VERSION_MAJOR_SHIFT 28 140#define NAND_VERSION_MINOR_MASK 0x0fff0000 141#define NAND_VERSION_MINOR_SHIFT 16 142 143/* NAND OP_CMDs */ 144#define OP_PAGE_READ 0x2 145#define OP_PAGE_READ_WITH_ECC 0x3 146#define OP_PAGE_READ_WITH_ECC_SPARE 0x4 147#define OP_PROGRAM_PAGE 0x6 148#define OP_PAGE_PROGRAM_WITH_ECC 0x7 149#define OP_PROGRAM_PAGE_SPARE 0x9 150#define OP_BLOCK_ERASE 0xa 151#define OP_FETCH_ID 0xb 152#define OP_RESET_DEVICE 0xd 153 154/* Default Value for NAND_DEV_CMD_VLD */ 155#define NAND_DEV_CMD_VLD_VAL (READ_START_VLD | WRITE_START_VLD | \ 156 ERASE_START_VLD | SEQ_READ_START_VLD) 157 158/* NAND_CTRL bits */ 159#define BAM_MODE_EN BIT(0) 160 161/* 162 * the NAND controller performs reads/writes with ECC in 516 byte chunks. 163 * the driver calls the chunks 'step' or 'codeword' interchangeably 164 */ 165#define NANDC_STEP_SIZE 512 166 167/* 168 * the largest page size we support is 8K, this will have 16 steps/codewords 169 * of 512 bytes each 170 */ 171#define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE) 172 173/* we read at most 3 registers per codeword scan */ 174#define MAX_REG_RD (3 * MAX_NUM_STEPS) 175 176/* ECC modes supported by the controller */ 177#define ECC_NONE BIT(0) 178#define ECC_RS_4BIT BIT(1) 179#define ECC_BCH_4BIT BIT(2) 180#define ECC_BCH_8BIT BIT(3) 181 182#define nandc_set_read_loc(nandc, reg, offset, size, is_last) \ 183nandc_set_reg(nandc, NAND_READ_LOCATION_##reg, \ 184 ((offset) << READ_LOCATION_OFFSET) | \ 185 ((size) << READ_LOCATION_SIZE) | \ 186 ((is_last) << READ_LOCATION_LAST)) 187 188/* 189 * Returns the actual register address for all NAND_DEV_ registers 190 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD) 191 */ 192#define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg)) 193 194/* Returns the NAND register physical address */ 195#define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset)) 196 197/* Returns the dma address for reg read buffer */ 198#define reg_buf_dma_addr(chip, vaddr) \ 199 ((chip)->reg_read_dma + \ 200 ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf)) 201 202#define QPIC_PER_CW_CMD_ELEMENTS 32 203#define QPIC_PER_CW_CMD_SGL 32 204#define QPIC_PER_CW_DATA_SGL 8 205 206#define QPIC_NAND_COMPLETION_TIMEOUT msecs_to_jiffies(2000) 207 208/* 209 * Flags used in DMA descriptor preparation helper functions 210 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma) 211 */ 212/* Don't set the EOT in current tx BAM sgl */ 213#define NAND_BAM_NO_EOT BIT(0) 214/* Set the NWD flag in current BAM sgl */ 215#define NAND_BAM_NWD BIT(1) 216/* Finish writing in the current BAM sgl and start writing in another BAM sgl */ 217#define NAND_BAM_NEXT_SGL BIT(2) 218/* 219 * Erased codeword status is being used two times in single transfer so this 220 * flag will determine the current value of erased codeword status register 221 */ 222#define NAND_ERASED_CW_SET BIT(4) 223 224/* 225 * This data type corresponds to the BAM transaction which will be used for all 226 * NAND transfers. 227 * @bam_ce - the array of BAM command elements 228 * @cmd_sgl - sgl for NAND BAM command pipe 229 * @data_sgl - sgl for NAND BAM consumer/producer pipe 230 * @bam_ce_pos - the index in bam_ce which is available for next sgl 231 * @bam_ce_start - the index in bam_ce which marks the start position ce 232 * for current sgl. It will be used for size calculation 233 * for current sgl 234 * @cmd_sgl_pos - current index in command sgl. 235 * @cmd_sgl_start - start index in command sgl. 236 * @tx_sgl_pos - current index in data sgl for tx. 237 * @tx_sgl_start - start index in data sgl for tx. 238 * @rx_sgl_pos - current index in data sgl for rx. 239 * @rx_sgl_start - start index in data sgl for rx. 240 * @wait_second_completion - wait for second DMA desc completion before making 241 * the NAND transfer completion. 242 * @txn_done - completion for NAND transfer. 243 * @last_data_desc - last DMA desc in data channel (tx/rx). 244 * @last_cmd_desc - last DMA desc in command channel. 245 */ 246struct bam_transaction { 247 struct bam_cmd_element *bam_ce; 248 struct scatterlist *cmd_sgl; 249 struct scatterlist *data_sgl; 250 u32 bam_ce_pos; 251 u32 bam_ce_start; 252 u32 cmd_sgl_pos; 253 u32 cmd_sgl_start; 254 u32 tx_sgl_pos; 255 u32 tx_sgl_start; 256 u32 rx_sgl_pos; 257 u32 rx_sgl_start; 258 bool wait_second_completion; 259 struct completion txn_done; 260 struct dma_async_tx_descriptor *last_data_desc; 261 struct dma_async_tx_descriptor *last_cmd_desc; 262}; 263 264/* 265 * This data type corresponds to the nand dma descriptor 266 * @list - list for desc_info 267 * @dir - DMA transfer direction 268 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by 269 * ADM 270 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM 271 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM 272 * @dma_desc - low level DMA engine descriptor 273 */ 274struct desc_info { 275 struct list_head node; 276 277 enum dma_data_direction dir; 278 union { 279 struct scatterlist adm_sgl; 280 struct { 281 struct scatterlist *bam_sgl; 282 int sgl_cnt; 283 }; 284 }; 285 struct dma_async_tx_descriptor *dma_desc; 286}; 287 288/* 289 * holds the current register values that we want to write. acts as a contiguous 290 * chunk of memory which we use to write the controller registers through DMA. 291 */ 292struct nandc_regs { 293 __le32 cmd; 294 __le32 addr0; 295 __le32 addr1; 296 __le32 chip_sel; 297 __le32 exec; 298 299 __le32 cfg0; 300 __le32 cfg1; 301 __le32 ecc_bch_cfg; 302 303 __le32 clrflashstatus; 304 __le32 clrreadstatus; 305 306 __le32 cmd1; 307 __le32 vld; 308 309 __le32 orig_cmd1; 310 __le32 orig_vld; 311 312 __le32 ecc_buf_cfg; 313 __le32 read_location0; 314 __le32 read_location1; 315 __le32 read_location2; 316 __le32 read_location3; 317 318 __le32 erased_cw_detect_cfg_clr; 319 __le32 erased_cw_detect_cfg_set; 320}; 321 322/* 323 * NAND controller data struct 324 * 325 * @controller: base controller structure 326 * @host_list: list containing all the chips attached to the 327 * controller 328 * @dev: parent device 329 * @base: MMIO base 330 * @base_phys: physical base address of controller registers 331 * @base_dma: dma base address of controller registers 332 * @core_clk: controller clock 333 * @aon_clk: another controller clock 334 * 335 * @chan: dma channel 336 * @cmd_crci: ADM DMA CRCI for command flow control 337 * @data_crci: ADM DMA CRCI for data flow control 338 * @desc_list: DMA descriptor list (list of desc_infos) 339 * 340 * @data_buffer: our local DMA buffer for page read/writes, 341 * used when we can't use the buffer provided 342 * by upper layers directly 343 * @buf_size/count/start: markers for chip->legacy.read_buf/write_buf 344 * functions 345 * @reg_read_buf: local buffer for reading back registers via DMA 346 * @reg_read_dma: contains dma address for register read buffer 347 * @reg_read_pos: marker for data read in reg_read_buf 348 * 349 * @regs: a contiguous chunk of memory for DMA register 350 * writes. contains the register values to be 351 * written to controller 352 * @cmd1/vld: some fixed controller register values 353 * @props: properties of current NAND controller, 354 * initialized via DT match data 355 * @max_cwperpage: maximum QPIC codewords required. calculated 356 * from all connected NAND devices pagesize 357 */ 358struct qcom_nand_controller { 359 struct nand_controller controller; 360 struct list_head host_list; 361 362 struct device *dev; 363 364 void __iomem *base; 365 phys_addr_t base_phys; 366 dma_addr_t base_dma; 367 368 struct clk *core_clk; 369 struct clk *aon_clk; 370 371 union { 372 /* will be used only by QPIC for BAM DMA */ 373 struct { 374 struct dma_chan *tx_chan; 375 struct dma_chan *rx_chan; 376 struct dma_chan *cmd_chan; 377 }; 378 379 /* will be used only by EBI2 for ADM DMA */ 380 struct { 381 struct dma_chan *chan; 382 unsigned int cmd_crci; 383 unsigned int data_crci; 384 }; 385 }; 386 387 struct list_head desc_list; 388 struct bam_transaction *bam_txn; 389 390 u8 *data_buffer; 391 int buf_size; 392 int buf_count; 393 int buf_start; 394 unsigned int max_cwperpage; 395 396 __le32 *reg_read_buf; 397 dma_addr_t reg_read_dma; 398 int reg_read_pos; 399 400 struct nandc_regs *regs; 401 402 u32 cmd1, vld; 403 const struct qcom_nandc_props *props; 404}; 405 406/* 407 * NAND chip structure 408 * 409 * @chip: base NAND chip structure 410 * @node: list node to add itself to host_list in 411 * qcom_nand_controller 412 * 413 * @cs: chip select value for this chip 414 * @cw_size: the number of bytes in a single step/codeword 415 * of a page, consisting of all data, ecc, spare 416 * and reserved bytes 417 * @cw_data: the number of bytes within a codeword protected 418 * by ECC 419 * @use_ecc: request the controller to use ECC for the 420 * upcoming read/write 421 * @bch_enabled: flag to tell whether BCH ECC mode is used 422 * @ecc_bytes_hw: ECC bytes used by controller hardware for this 423 * chip 424 * @status: value to be returned if NAND_CMD_STATUS command 425 * is executed 426 * @last_command: keeps track of last command on this chip. used 427 * for reading correct status 428 * 429 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for 430 * ecc/non-ecc mode for the current nand flash 431 * device 432 */ 433struct qcom_nand_host { 434 struct nand_chip chip; 435 struct list_head node; 436 437 int cs; 438 int cw_size; 439 int cw_data; 440 bool use_ecc; 441 bool bch_enabled; 442 int ecc_bytes_hw; 443 int spare_bytes; 444 int bbm_size; 445 u8 status; 446 int last_command; 447 448 u32 cfg0, cfg1; 449 u32 cfg0_raw, cfg1_raw; 450 u32 ecc_buf_cfg; 451 u32 ecc_bch_cfg; 452 u32 clrflashstatus; 453 u32 clrreadstatus; 454}; 455 456/* 457 * This data type corresponds to the NAND controller properties which varies 458 * among different NAND controllers. 459 * @ecc_modes - ecc mode for NAND 460 * @is_bam - whether NAND controller is using BAM 461 * @is_qpic - whether NAND CTRL is part of qpic IP 462 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset 463 */ 464struct qcom_nandc_props { 465 u32 ecc_modes; 466 bool is_bam; 467 bool is_qpic; 468 u32 dev_cmd_reg_start; 469}; 470 471/* Frees the BAM transaction memory */ 472static void free_bam_transaction(struct qcom_nand_controller *nandc) 473{ 474 struct bam_transaction *bam_txn = nandc->bam_txn; 475 476 devm_kfree(nandc->dev, bam_txn); 477} 478 479/* Allocates and Initializes the BAM transaction */ 480static struct bam_transaction * 481alloc_bam_transaction(struct qcom_nand_controller *nandc) 482{ 483 struct bam_transaction *bam_txn; 484 size_t bam_txn_size; 485 unsigned int num_cw = nandc->max_cwperpage; 486 void *bam_txn_buf; 487 488 bam_txn_size = 489 sizeof(*bam_txn) + num_cw * 490 ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) + 491 (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) + 492 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL)); 493 494 bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL); 495 if (!bam_txn_buf) 496 return NULL; 497 498 bam_txn = bam_txn_buf; 499 bam_txn_buf += sizeof(*bam_txn); 500 501 bam_txn->bam_ce = bam_txn_buf; 502 bam_txn_buf += 503 sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw; 504 505 bam_txn->cmd_sgl = bam_txn_buf; 506 bam_txn_buf += 507 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw; 508 509 bam_txn->data_sgl = bam_txn_buf; 510 511 init_completion(&bam_txn->txn_done); 512 513 return bam_txn; 514} 515 516/* Clears the BAM transaction indexes */ 517static void clear_bam_transaction(struct qcom_nand_controller *nandc) 518{ 519 struct bam_transaction *bam_txn = nandc->bam_txn; 520 521 if (!nandc->props->is_bam) 522 return; 523 524 bam_txn->bam_ce_pos = 0; 525 bam_txn->bam_ce_start = 0; 526 bam_txn->cmd_sgl_pos = 0; 527 bam_txn->cmd_sgl_start = 0; 528 bam_txn->tx_sgl_pos = 0; 529 bam_txn->tx_sgl_start = 0; 530 bam_txn->rx_sgl_pos = 0; 531 bam_txn->rx_sgl_start = 0; 532 bam_txn->last_data_desc = NULL; 533 bam_txn->wait_second_completion = false; 534 535 sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage * 536 QPIC_PER_CW_CMD_SGL); 537 sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage * 538 QPIC_PER_CW_DATA_SGL); 539 540 reinit_completion(&bam_txn->txn_done); 541} 542 543/* Callback for DMA descriptor completion */ 544static void qpic_bam_dma_done(void *data) 545{ 546 struct bam_transaction *bam_txn = data; 547 548 /* 549 * In case of data transfer with NAND, 2 callbacks will be generated. 550 * One for command channel and another one for data channel. 551 * If current transaction has data descriptors 552 * (i.e. wait_second_completion is true), then set this to false 553 * and wait for second DMA descriptor completion. 554 */ 555 if (bam_txn->wait_second_completion) 556 bam_txn->wait_second_completion = false; 557 else 558 complete(&bam_txn->txn_done); 559} 560 561static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip) 562{ 563 return container_of(chip, struct qcom_nand_host, chip); 564} 565 566static inline struct qcom_nand_controller * 567get_qcom_nand_controller(struct nand_chip *chip) 568{ 569 return container_of(chip->controller, struct qcom_nand_controller, 570 controller); 571} 572 573static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset) 574{ 575 return ioread32(nandc->base + offset); 576} 577 578static inline void nandc_write(struct qcom_nand_controller *nandc, int offset, 579 u32 val) 580{ 581 iowrite32(val, nandc->base + offset); 582} 583 584static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc, 585 bool is_cpu) 586{ 587 if (!nandc->props->is_bam) 588 return; 589 590 if (is_cpu) 591 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma, 592 MAX_REG_RD * 593 sizeof(*nandc->reg_read_buf), 594 DMA_FROM_DEVICE); 595 else 596 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma, 597 MAX_REG_RD * 598 sizeof(*nandc->reg_read_buf), 599 DMA_FROM_DEVICE); 600} 601 602static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset) 603{ 604 switch (offset) { 605 case NAND_FLASH_CMD: 606 return ®s->cmd; 607 case NAND_ADDR0: 608 return ®s->addr0; 609 case NAND_ADDR1: 610 return ®s->addr1; 611 case NAND_FLASH_CHIP_SELECT: 612 return ®s->chip_sel; 613 case NAND_EXEC_CMD: 614 return ®s->exec; 615 case NAND_FLASH_STATUS: 616 return ®s->clrflashstatus; 617 case NAND_DEV0_CFG0: 618 return ®s->cfg0; 619 case NAND_DEV0_CFG1: 620 return ®s->cfg1; 621 case NAND_DEV0_ECC_CFG: 622 return ®s->ecc_bch_cfg; 623 case NAND_READ_STATUS: 624 return ®s->clrreadstatus; 625 case NAND_DEV_CMD1: 626 return ®s->cmd1; 627 case NAND_DEV_CMD1_RESTORE: 628 return ®s->orig_cmd1; 629 case NAND_DEV_CMD_VLD: 630 return ®s->vld; 631 case NAND_DEV_CMD_VLD_RESTORE: 632 return ®s->orig_vld; 633 case NAND_EBI2_ECC_BUF_CFG: 634 return ®s->ecc_buf_cfg; 635 case NAND_READ_LOCATION_0: 636 return ®s->read_location0; 637 case NAND_READ_LOCATION_1: 638 return ®s->read_location1; 639 case NAND_READ_LOCATION_2: 640 return ®s->read_location2; 641 case NAND_READ_LOCATION_3: 642 return ®s->read_location3; 643 default: 644 return NULL; 645 } 646} 647 648static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset, 649 u32 val) 650{ 651 struct nandc_regs *regs = nandc->regs; 652 __le32 *reg; 653 654 reg = offset_to_nandc_reg(regs, offset); 655 656 if (reg) 657 *reg = cpu_to_le32(val); 658} 659 660/* helper to configure address register values */ 661static void set_address(struct qcom_nand_host *host, u16 column, int page) 662{ 663 struct nand_chip *chip = &host->chip; 664 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 665 666 if (chip->options & NAND_BUSWIDTH_16) 667 column >>= 1; 668 669 nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column); 670 nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff); 671} 672 673/* 674 * update_rw_regs: set up read/write register values, these will be 675 * written to the NAND controller registers via DMA 676 * 677 * @num_cw: number of steps for the read/write operation 678 * @read: read or write operation 679 */ 680static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read) 681{ 682 struct nand_chip *chip = &host->chip; 683 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 684 u32 cmd, cfg0, cfg1, ecc_bch_cfg; 685 686 if (read) { 687 if (host->use_ecc) 688 cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE; 689 else 690 cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE; 691 } else { 692 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE; 693 } 694 695 if (host->use_ecc) { 696 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) | 697 (num_cw - 1) << CW_PER_PAGE; 698 699 cfg1 = host->cfg1; 700 ecc_bch_cfg = host->ecc_bch_cfg; 701 } else { 702 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) | 703 (num_cw - 1) << CW_PER_PAGE; 704 705 cfg1 = host->cfg1_raw; 706 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE; 707 } 708 709 nandc_set_reg(nandc, NAND_FLASH_CMD, cmd); 710 nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0); 711 nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1); 712 nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg); 713 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg); 714 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus); 715 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus); 716 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 717 718 if (read) 719 nandc_set_read_loc(nandc, 0, 0, host->use_ecc ? 720 host->cw_data : host->cw_size, 1); 721} 722 723/* 724 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor 725 * for BAM. This descriptor will be added in the NAND DMA descriptor queue 726 * which will be submitted to DMA engine. 727 */ 728static int prepare_bam_async_desc(struct qcom_nand_controller *nandc, 729 struct dma_chan *chan, 730 unsigned long flags) 731{ 732 struct desc_info *desc; 733 struct scatterlist *sgl; 734 unsigned int sgl_cnt; 735 int ret; 736 struct bam_transaction *bam_txn = nandc->bam_txn; 737 enum dma_transfer_direction dir_eng; 738 struct dma_async_tx_descriptor *dma_desc; 739 740 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 741 if (!desc) 742 return -ENOMEM; 743 744 if (chan == nandc->cmd_chan) { 745 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start]; 746 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start; 747 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos; 748 dir_eng = DMA_MEM_TO_DEV; 749 desc->dir = DMA_TO_DEVICE; 750 } else if (chan == nandc->tx_chan) { 751 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start]; 752 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start; 753 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos; 754 dir_eng = DMA_MEM_TO_DEV; 755 desc->dir = DMA_TO_DEVICE; 756 } else { 757 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start]; 758 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start; 759 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos; 760 dir_eng = DMA_DEV_TO_MEM; 761 desc->dir = DMA_FROM_DEVICE; 762 } 763 764 sg_mark_end(sgl + sgl_cnt - 1); 765 ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir); 766 if (ret == 0) { 767 dev_err(nandc->dev, "failure in mapping desc\n"); 768 kfree(desc); 769 return -ENOMEM; 770 } 771 772 desc->sgl_cnt = sgl_cnt; 773 desc->bam_sgl = sgl; 774 775 dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng, 776 flags); 777 778 if (!dma_desc) { 779 dev_err(nandc->dev, "failure in prep desc\n"); 780 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir); 781 kfree(desc); 782 return -EINVAL; 783 } 784 785 desc->dma_desc = dma_desc; 786 787 /* update last data/command descriptor */ 788 if (chan == nandc->cmd_chan) 789 bam_txn->last_cmd_desc = dma_desc; 790 else 791 bam_txn->last_data_desc = dma_desc; 792 793 list_add_tail(&desc->node, &nandc->desc_list); 794 795 return 0; 796} 797 798/* 799 * Prepares the command descriptor for BAM DMA which will be used for NAND 800 * register reads and writes. The command descriptor requires the command 801 * to be formed in command element type so this function uses the command 802 * element from bam transaction ce array and fills the same with required 803 * data. A single SGL can contain multiple command elements so 804 * NAND_BAM_NEXT_SGL will be used for starting the separate SGL 805 * after the current command element. 806 */ 807static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read, 808 int reg_off, const void *vaddr, 809 int size, unsigned int flags) 810{ 811 int bam_ce_size; 812 int i, ret; 813 struct bam_cmd_element *bam_ce_buffer; 814 struct bam_transaction *bam_txn = nandc->bam_txn; 815 816 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos]; 817 818 /* fill the command desc */ 819 for (i = 0; i < size; i++) { 820 if (read) 821 bam_prep_ce(&bam_ce_buffer[i], 822 nandc_reg_phys(nandc, reg_off + 4 * i), 823 BAM_READ_COMMAND, 824 reg_buf_dma_addr(nandc, 825 (__le32 *)vaddr + i)); 826 else 827 bam_prep_ce_le32(&bam_ce_buffer[i], 828 nandc_reg_phys(nandc, reg_off + 4 * i), 829 BAM_WRITE_COMMAND, 830 *((__le32 *)vaddr + i)); 831 } 832 833 bam_txn->bam_ce_pos += size; 834 835 /* use the separate sgl after this command */ 836 if (flags & NAND_BAM_NEXT_SGL) { 837 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start]; 838 bam_ce_size = (bam_txn->bam_ce_pos - 839 bam_txn->bam_ce_start) * 840 sizeof(struct bam_cmd_element); 841 sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos], 842 bam_ce_buffer, bam_ce_size); 843 bam_txn->cmd_sgl_pos++; 844 bam_txn->bam_ce_start = bam_txn->bam_ce_pos; 845 846 if (flags & NAND_BAM_NWD) { 847 ret = prepare_bam_async_desc(nandc, nandc->cmd_chan, 848 DMA_PREP_FENCE | 849 DMA_PREP_CMD); 850 if (ret) 851 return ret; 852 } 853 } 854 855 return 0; 856} 857 858/* 859 * Prepares the data descriptor for BAM DMA which will be used for NAND 860 * data reads and writes. 861 */ 862static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read, 863 const void *vaddr, 864 int size, unsigned int flags) 865{ 866 int ret; 867 struct bam_transaction *bam_txn = nandc->bam_txn; 868 869 if (read) { 870 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos], 871 vaddr, size); 872 bam_txn->rx_sgl_pos++; 873 } else { 874 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos], 875 vaddr, size); 876 bam_txn->tx_sgl_pos++; 877 878 /* 879 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag 880 * is not set, form the DMA descriptor 881 */ 882 if (!(flags & NAND_BAM_NO_EOT)) { 883 ret = prepare_bam_async_desc(nandc, nandc->tx_chan, 884 DMA_PREP_INTERRUPT); 885 if (ret) 886 return ret; 887 } 888 } 889 890 return 0; 891} 892 893static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read, 894 int reg_off, const void *vaddr, int size, 895 bool flow_control) 896{ 897 struct desc_info *desc; 898 struct dma_async_tx_descriptor *dma_desc; 899 struct scatterlist *sgl; 900 struct dma_slave_config slave_conf; 901 enum dma_transfer_direction dir_eng; 902 int ret; 903 904 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 905 if (!desc) 906 return -ENOMEM; 907 908 sgl = &desc->adm_sgl; 909 910 sg_init_one(sgl, vaddr, size); 911 912 if (read) { 913 dir_eng = DMA_DEV_TO_MEM; 914 desc->dir = DMA_FROM_DEVICE; 915 } else { 916 dir_eng = DMA_MEM_TO_DEV; 917 desc->dir = DMA_TO_DEVICE; 918 } 919 920 ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir); 921 if (ret == 0) { 922 ret = -ENOMEM; 923 goto err; 924 } 925 926 memset(&slave_conf, 0x00, sizeof(slave_conf)); 927 928 slave_conf.device_fc = flow_control; 929 if (read) { 930 slave_conf.src_maxburst = 16; 931 slave_conf.src_addr = nandc->base_dma + reg_off; 932 slave_conf.slave_id = nandc->data_crci; 933 } else { 934 slave_conf.dst_maxburst = 16; 935 slave_conf.dst_addr = nandc->base_dma + reg_off; 936 slave_conf.slave_id = nandc->cmd_crci; 937 } 938 939 ret = dmaengine_slave_config(nandc->chan, &slave_conf); 940 if (ret) { 941 dev_err(nandc->dev, "failed to configure dma channel\n"); 942 goto err; 943 } 944 945 dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0); 946 if (!dma_desc) { 947 dev_err(nandc->dev, "failed to prepare desc\n"); 948 ret = -EINVAL; 949 goto err; 950 } 951 952 desc->dma_desc = dma_desc; 953 954 list_add_tail(&desc->node, &nandc->desc_list); 955 956 return 0; 957err: 958 kfree(desc); 959 960 return ret; 961} 962 963/* 964 * read_reg_dma: prepares a descriptor to read a given number of 965 * contiguous registers to the reg_read_buf pointer 966 * 967 * @first: offset of the first register in the contiguous block 968 * @num_regs: number of registers to read 969 * @flags: flags to control DMA descriptor preparation 970 */ 971static int read_reg_dma(struct qcom_nand_controller *nandc, int first, 972 int num_regs, unsigned int flags) 973{ 974 bool flow_control = false; 975 void *vaddr; 976 977 vaddr = nandc->reg_read_buf + nandc->reg_read_pos; 978 nandc->reg_read_pos += num_regs; 979 980 if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1) 981 first = dev_cmd_reg_addr(nandc, first); 982 983 if (nandc->props->is_bam) 984 return prep_bam_dma_desc_cmd(nandc, true, first, vaddr, 985 num_regs, flags); 986 987 if (first == NAND_READ_ID || first == NAND_FLASH_STATUS) 988 flow_control = true; 989 990 return prep_adm_dma_desc(nandc, true, first, vaddr, 991 num_regs * sizeof(u32), flow_control); 992} 993 994/* 995 * write_reg_dma: prepares a descriptor to write a given number of 996 * contiguous registers 997 * 998 * @first: offset of the first register in the contiguous block 999 * @num_regs: number of registers to write 1000 * @flags: flags to control DMA descriptor preparation 1001 */ 1002static int write_reg_dma(struct qcom_nand_controller *nandc, int first, 1003 int num_regs, unsigned int flags) 1004{ 1005 bool flow_control = false; 1006 struct nandc_regs *regs = nandc->regs; 1007 void *vaddr; 1008 1009 vaddr = offset_to_nandc_reg(regs, first); 1010 1011 if (first == NAND_ERASED_CW_DETECT_CFG) { 1012 if (flags & NAND_ERASED_CW_SET) 1013 vaddr = ®s->erased_cw_detect_cfg_set; 1014 else 1015 vaddr = ®s->erased_cw_detect_cfg_clr; 1016 } 1017 1018 if (first == NAND_EXEC_CMD) 1019 flags |= NAND_BAM_NWD; 1020 1021 if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1) 1022 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1); 1023 1024 if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD) 1025 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD); 1026 1027 if (nandc->props->is_bam) 1028 return prep_bam_dma_desc_cmd(nandc, false, first, vaddr, 1029 num_regs, flags); 1030 1031 if (first == NAND_FLASH_CMD) 1032 flow_control = true; 1033 1034 return prep_adm_dma_desc(nandc, false, first, vaddr, 1035 num_regs * sizeof(u32), flow_control); 1036} 1037 1038/* 1039 * read_data_dma: prepares a DMA descriptor to transfer data from the 1040 * controller's internal buffer to the buffer 'vaddr' 1041 * 1042 * @reg_off: offset within the controller's data buffer 1043 * @vaddr: virtual address of the buffer we want to write to 1044 * @size: DMA transaction size in bytes 1045 * @flags: flags to control DMA descriptor preparation 1046 */ 1047static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off, 1048 const u8 *vaddr, int size, unsigned int flags) 1049{ 1050 if (nandc->props->is_bam) 1051 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags); 1052 1053 return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false); 1054} 1055 1056/* 1057 * write_data_dma: prepares a DMA descriptor to transfer data from 1058 * 'vaddr' to the controller's internal buffer 1059 * 1060 * @reg_off: offset within the controller's data buffer 1061 * @vaddr: virtual address of the buffer we want to read from 1062 * @size: DMA transaction size in bytes 1063 * @flags: flags to control DMA descriptor preparation 1064 */ 1065static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off, 1066 const u8 *vaddr, int size, unsigned int flags) 1067{ 1068 if (nandc->props->is_bam) 1069 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags); 1070 1071 return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false); 1072} 1073 1074/* 1075 * Helper to prepare DMA descriptors for configuring registers 1076 * before reading a NAND page. 1077 */ 1078static void config_nand_page_read(struct qcom_nand_controller *nandc) 1079{ 1080 write_reg_dma(nandc, NAND_ADDR0, 2, 0); 1081 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0); 1082 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0); 1083 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0); 1084 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 1085 NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL); 1086} 1087 1088/* 1089 * Helper to prepare DMA descriptors for configuring registers 1090 * before reading each codeword in NAND page. 1091 */ 1092static void 1093config_nand_cw_read(struct qcom_nand_controller *nandc, bool use_ecc) 1094{ 1095 if (nandc->props->is_bam) 1096 write_reg_dma(nandc, NAND_READ_LOCATION_0, 4, 1097 NAND_BAM_NEXT_SGL); 1098 1099 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1100 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1101 1102 if (use_ecc) { 1103 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0); 1104 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1, 1105 NAND_BAM_NEXT_SGL); 1106 } else { 1107 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1108 } 1109} 1110 1111/* 1112 * Helper to prepare dma descriptors to configure registers needed for reading a 1113 * single codeword in page 1114 */ 1115static void 1116config_nand_single_cw_page_read(struct qcom_nand_controller *nandc, 1117 bool use_ecc) 1118{ 1119 config_nand_page_read(nandc); 1120 config_nand_cw_read(nandc, use_ecc); 1121} 1122 1123/* 1124 * Helper to prepare DMA descriptors used to configure registers needed for 1125 * before writing a NAND page. 1126 */ 1127static void config_nand_page_write(struct qcom_nand_controller *nandc) 1128{ 1129 write_reg_dma(nandc, NAND_ADDR0, 2, 0); 1130 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0); 1131 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 1132 NAND_BAM_NEXT_SGL); 1133} 1134 1135/* 1136 * Helper to prepare DMA descriptors for configuring registers 1137 * before writing each codeword in NAND page. 1138 */ 1139static void config_nand_cw_write(struct qcom_nand_controller *nandc) 1140{ 1141 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1142 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1143 1144 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1145 1146 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0); 1147 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL); 1148} 1149 1150/* 1151 * the following functions are used within chip->legacy.cmdfunc() to 1152 * perform different NAND_CMD_* commands 1153 */ 1154 1155/* sets up descriptors for NAND_CMD_PARAM */ 1156static int nandc_param(struct qcom_nand_host *host) 1157{ 1158 struct nand_chip *chip = &host->chip; 1159 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1160 1161 /* 1162 * NAND_CMD_PARAM is called before we know much about the FLASH chip 1163 * in use. we configure the controller to perform a raw read of 512 1164 * bytes to read onfi params 1165 */ 1166 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_PAGE_READ | PAGE_ACC | LAST_PAGE); 1167 nandc_set_reg(nandc, NAND_ADDR0, 0); 1168 nandc_set_reg(nandc, NAND_ADDR1, 0); 1169 nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE 1170 | 512 << UD_SIZE_BYTES 1171 | 5 << NUM_ADDR_CYCLES 1172 | 0 << SPARE_SIZE_BYTES); 1173 nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES 1174 | 0 << CS_ACTIVE_BSY 1175 | 17 << BAD_BLOCK_BYTE_NUM 1176 | 1 << BAD_BLOCK_IN_SPARE_AREA 1177 | 2 << WR_RD_BSY_GAP 1178 | 0 << WIDE_FLASH 1179 | 1 << DEV0_CFG1_ECC_DISABLE); 1180 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE); 1181 1182 /* configure CMD1 and VLD for ONFI param probing */ 1183 nandc_set_reg(nandc, NAND_DEV_CMD_VLD, 1184 (nandc->vld & ~READ_START_VLD)); 1185 nandc_set_reg(nandc, NAND_DEV_CMD1, 1186 (nandc->cmd1 & ~(0xFF << READ_ADDR)) 1187 | NAND_CMD_PARAM << READ_ADDR); 1188 1189 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1190 1191 nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1); 1192 nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld); 1193 nandc_set_read_loc(nandc, 0, 0, 512, 1); 1194 1195 write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0); 1196 write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL); 1197 1198 nandc->buf_count = 512; 1199 memset(nandc->data_buffer, 0xff, nandc->buf_count); 1200 1201 config_nand_single_cw_page_read(nandc, false); 1202 1203 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, 1204 nandc->buf_count, 0); 1205 1206 /* restore CMD1 and VLD regs */ 1207 write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0); 1208 write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL); 1209 1210 return 0; 1211} 1212 1213/* sets up descriptors for NAND_CMD_ERASE1 */ 1214static int erase_block(struct qcom_nand_host *host, int page_addr) 1215{ 1216 struct nand_chip *chip = &host->chip; 1217 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1218 1219 nandc_set_reg(nandc, NAND_FLASH_CMD, 1220 OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE); 1221 nandc_set_reg(nandc, NAND_ADDR0, page_addr); 1222 nandc_set_reg(nandc, NAND_ADDR1, 0); 1223 nandc_set_reg(nandc, NAND_DEV0_CFG0, 1224 host->cfg0_raw & ~(7 << CW_PER_PAGE)); 1225 nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw); 1226 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1227 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus); 1228 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus); 1229 1230 write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL); 1231 write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL); 1232 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1233 1234 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1235 1236 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0); 1237 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL); 1238 1239 return 0; 1240} 1241 1242/* sets up descriptors for NAND_CMD_READID */ 1243static int read_id(struct qcom_nand_host *host, int column) 1244{ 1245 struct nand_chip *chip = &host->chip; 1246 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1247 1248 if (column == -1) 1249 return 0; 1250 1251 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_FETCH_ID); 1252 nandc_set_reg(nandc, NAND_ADDR0, column); 1253 nandc_set_reg(nandc, NAND_ADDR1, 0); 1254 nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT, 1255 nandc->props->is_bam ? 0 : DM_EN); 1256 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1257 1258 write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL); 1259 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1260 1261 read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL); 1262 1263 return 0; 1264} 1265 1266/* sets up descriptors for NAND_CMD_RESET */ 1267static int reset(struct qcom_nand_host *host) 1268{ 1269 struct nand_chip *chip = &host->chip; 1270 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1271 1272 nandc_set_reg(nandc, NAND_FLASH_CMD, OP_RESET_DEVICE); 1273 nandc_set_reg(nandc, NAND_EXEC_CMD, 1); 1274 1275 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL); 1276 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); 1277 1278 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL); 1279 1280 return 0; 1281} 1282 1283/* helpers to submit/free our list of dma descriptors */ 1284static int submit_descs(struct qcom_nand_controller *nandc) 1285{ 1286 struct desc_info *desc; 1287 dma_cookie_t cookie = 0; 1288 struct bam_transaction *bam_txn = nandc->bam_txn; 1289 int r; 1290 1291 if (nandc->props->is_bam) { 1292 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) { 1293 r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0); 1294 if (r) 1295 return r; 1296 } 1297 1298 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) { 1299 r = prepare_bam_async_desc(nandc, nandc->tx_chan, 1300 DMA_PREP_INTERRUPT); 1301 if (r) 1302 return r; 1303 } 1304 1305 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) { 1306 r = prepare_bam_async_desc(nandc, nandc->cmd_chan, 1307 DMA_PREP_CMD); 1308 if (r) 1309 return r; 1310 } 1311 } 1312 1313 list_for_each_entry(desc, &nandc->desc_list, node) 1314 cookie = dmaengine_submit(desc->dma_desc); 1315 1316 if (nandc->props->is_bam) { 1317 bam_txn->last_cmd_desc->callback = qpic_bam_dma_done; 1318 bam_txn->last_cmd_desc->callback_param = bam_txn; 1319 if (bam_txn->last_data_desc) { 1320 bam_txn->last_data_desc->callback = qpic_bam_dma_done; 1321 bam_txn->last_data_desc->callback_param = bam_txn; 1322 bam_txn->wait_second_completion = true; 1323 } 1324 1325 dma_async_issue_pending(nandc->tx_chan); 1326 dma_async_issue_pending(nandc->rx_chan); 1327 dma_async_issue_pending(nandc->cmd_chan); 1328 1329 if (!wait_for_completion_timeout(&bam_txn->txn_done, 1330 QPIC_NAND_COMPLETION_TIMEOUT)) 1331 return -ETIMEDOUT; 1332 } else { 1333 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE) 1334 return -ETIMEDOUT; 1335 } 1336 1337 return 0; 1338} 1339 1340static void free_descs(struct qcom_nand_controller *nandc) 1341{ 1342 struct desc_info *desc, *n; 1343 1344 list_for_each_entry_safe(desc, n, &nandc->desc_list, node) { 1345 list_del(&desc->node); 1346 1347 if (nandc->props->is_bam) 1348 dma_unmap_sg(nandc->dev, desc->bam_sgl, 1349 desc->sgl_cnt, desc->dir); 1350 else 1351 dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1, 1352 desc->dir); 1353 1354 kfree(desc); 1355 } 1356} 1357 1358/* reset the register read buffer for next NAND operation */ 1359static void clear_read_regs(struct qcom_nand_controller *nandc) 1360{ 1361 nandc->reg_read_pos = 0; 1362 nandc_read_buffer_sync(nandc, false); 1363} 1364 1365static void pre_command(struct qcom_nand_host *host, int command) 1366{ 1367 struct nand_chip *chip = &host->chip; 1368 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1369 1370 nandc->buf_count = 0; 1371 nandc->buf_start = 0; 1372 host->use_ecc = false; 1373 host->last_command = command; 1374 1375 clear_read_regs(nandc); 1376 1377 if (command == NAND_CMD_RESET || command == NAND_CMD_READID || 1378 command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1) 1379 clear_bam_transaction(nandc); 1380} 1381 1382/* 1383 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our 1384 * privately maintained status byte, this status byte can be read after 1385 * NAND_CMD_STATUS is called 1386 */ 1387static void parse_erase_write_errors(struct qcom_nand_host *host, int command) 1388{ 1389 struct nand_chip *chip = &host->chip; 1390 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1391 struct nand_ecc_ctrl *ecc = &chip->ecc; 1392 int num_cw; 1393 int i; 1394 1395 num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1; 1396 nandc_read_buffer_sync(nandc, true); 1397 1398 for (i = 0; i < num_cw; i++) { 1399 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]); 1400 1401 if (flash_status & FS_MPU_ERR) 1402 host->status &= ~NAND_STATUS_WP; 1403 1404 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) && 1405 (flash_status & 1406 FS_DEVICE_STS_ERR))) 1407 host->status |= NAND_STATUS_FAIL; 1408 } 1409} 1410 1411static void post_command(struct qcom_nand_host *host, int command) 1412{ 1413 struct nand_chip *chip = &host->chip; 1414 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1415 1416 switch (command) { 1417 case NAND_CMD_READID: 1418 nandc_read_buffer_sync(nandc, true); 1419 memcpy(nandc->data_buffer, nandc->reg_read_buf, 1420 nandc->buf_count); 1421 break; 1422 case NAND_CMD_PAGEPROG: 1423 case NAND_CMD_ERASE1: 1424 parse_erase_write_errors(host, command); 1425 break; 1426 default: 1427 break; 1428 } 1429} 1430 1431/* 1432 * Implements chip->legacy.cmdfunc. It's only used for a limited set of 1433 * commands. The rest of the commands wouldn't be called by upper layers. 1434 * For example, NAND_CMD_READOOB would never be called because we have our own 1435 * versions of read_oob ops for nand_ecc_ctrl. 1436 */ 1437static void qcom_nandc_command(struct nand_chip *chip, unsigned int command, 1438 int column, int page_addr) 1439{ 1440 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1441 struct nand_ecc_ctrl *ecc = &chip->ecc; 1442 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1443 bool wait = false; 1444 int ret = 0; 1445 1446 pre_command(host, command); 1447 1448 switch (command) { 1449 case NAND_CMD_RESET: 1450 ret = reset(host); 1451 wait = true; 1452 break; 1453 1454 case NAND_CMD_READID: 1455 nandc->buf_count = 4; 1456 ret = read_id(host, column); 1457 wait = true; 1458 break; 1459 1460 case NAND_CMD_PARAM: 1461 ret = nandc_param(host); 1462 wait = true; 1463 break; 1464 1465 case NAND_CMD_ERASE1: 1466 ret = erase_block(host, page_addr); 1467 wait = true; 1468 break; 1469 1470 case NAND_CMD_READ0: 1471 /* we read the entire page for now */ 1472 WARN_ON(column != 0); 1473 1474 host->use_ecc = true; 1475 set_address(host, 0, page_addr); 1476 update_rw_regs(host, ecc->steps, true); 1477 break; 1478 1479 case NAND_CMD_SEQIN: 1480 WARN_ON(column != 0); 1481 set_address(host, 0, page_addr); 1482 break; 1483 1484 case NAND_CMD_PAGEPROG: 1485 case NAND_CMD_STATUS: 1486 case NAND_CMD_NONE: 1487 default: 1488 break; 1489 } 1490 1491 if (ret) { 1492 dev_err(nandc->dev, "failure executing command %d\n", 1493 command); 1494 free_descs(nandc); 1495 return; 1496 } 1497 1498 if (wait) { 1499 ret = submit_descs(nandc); 1500 if (ret) 1501 dev_err(nandc->dev, 1502 "failure submitting descs for command %d\n", 1503 command); 1504 } 1505 1506 free_descs(nandc); 1507 1508 post_command(host, command); 1509} 1510 1511/* 1512 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read 1513 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS. 1514 * 1515 * when using RS ECC, the HW reports the same erros when reading an erased CW, 1516 * but it notifies that it is an erased CW by placing special characters at 1517 * certain offsets in the buffer. 1518 * 1519 * verify if the page is erased or not, and fix up the page for RS ECC by 1520 * replacing the special characters with 0xff. 1521 */ 1522static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len) 1523{ 1524 u8 empty1, empty2; 1525 1526 /* 1527 * an erased page flags an error in NAND_FLASH_STATUS, check if the page 1528 * is erased by looking for 0x54s at offsets 3 and 175 from the 1529 * beginning of each codeword 1530 */ 1531 1532 empty1 = data_buf[3]; 1533 empty2 = data_buf[175]; 1534 1535 /* 1536 * if the erased codework markers, if they exist override them with 1537 * 0xffs 1538 */ 1539 if ((empty1 == 0x54 && empty2 == 0xff) || 1540 (empty1 == 0xff && empty2 == 0x54)) { 1541 data_buf[3] = 0xff; 1542 data_buf[175] = 0xff; 1543 } 1544 1545 /* 1546 * check if the entire chunk contains 0xffs or not. if it doesn't, then 1547 * restore the original values at the special offsets 1548 */ 1549 if (memchr_inv(data_buf, 0xff, data_len)) { 1550 data_buf[3] = empty1; 1551 data_buf[175] = empty2; 1552 1553 return false; 1554 } 1555 1556 return true; 1557} 1558 1559struct read_stats { 1560 __le32 flash; 1561 __le32 buffer; 1562 __le32 erased_cw; 1563}; 1564 1565/* reads back FLASH_STATUS register set by the controller */ 1566static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt) 1567{ 1568 struct nand_chip *chip = &host->chip; 1569 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1570 int i; 1571 1572 nandc_read_buffer_sync(nandc, true); 1573 1574 for (i = 0; i < cw_cnt; i++) { 1575 u32 flash = le32_to_cpu(nandc->reg_read_buf[i]); 1576 1577 if (flash & (FS_OP_ERR | FS_MPU_ERR)) 1578 return -EIO; 1579 } 1580 1581 return 0; 1582} 1583 1584/* performs raw read for one codeword */ 1585static int 1586qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip, 1587 u8 *data_buf, u8 *oob_buf, int page, int cw) 1588{ 1589 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1590 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1591 struct nand_ecc_ctrl *ecc = &chip->ecc; 1592 int data_size1, data_size2, oob_size1, oob_size2; 1593 int ret, reg_off = FLASH_BUF_ACC, read_loc = 0; 1594 1595 nand_read_page_op(chip, page, 0, NULL, 0); 1596 host->use_ecc = false; 1597 1598 clear_bam_transaction(nandc); 1599 set_address(host, host->cw_size * cw, page); 1600 update_rw_regs(host, 1, true); 1601 config_nand_page_read(nandc); 1602 1603 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1); 1604 oob_size1 = host->bbm_size; 1605 1606 if (cw == (ecc->steps - 1)) { 1607 data_size2 = ecc->size - data_size1 - 1608 ((ecc->steps - 1) * 4); 1609 oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw + 1610 host->spare_bytes; 1611 } else { 1612 data_size2 = host->cw_data - data_size1; 1613 oob_size2 = host->ecc_bytes_hw + host->spare_bytes; 1614 } 1615 1616 if (nandc->props->is_bam) { 1617 nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0); 1618 read_loc += data_size1; 1619 1620 nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0); 1621 read_loc += oob_size1; 1622 1623 nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0); 1624 read_loc += data_size2; 1625 1626 nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1); 1627 } 1628 1629 config_nand_cw_read(nandc, false); 1630 1631 read_data_dma(nandc, reg_off, data_buf, data_size1, 0); 1632 reg_off += data_size1; 1633 1634 read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0); 1635 reg_off += oob_size1; 1636 1637 read_data_dma(nandc, reg_off, data_buf + data_size1, data_size2, 0); 1638 reg_off += data_size2; 1639 1640 read_data_dma(nandc, reg_off, oob_buf + oob_size1, oob_size2, 0); 1641 1642 ret = submit_descs(nandc); 1643 free_descs(nandc); 1644 if (ret) { 1645 dev_err(nandc->dev, "failure to read raw cw %d\n", cw); 1646 return ret; 1647 } 1648 1649 return check_flash_errors(host, 1); 1650} 1651 1652/* 1653 * Bitflips can happen in erased codewords also so this function counts the 1654 * number of 0 in each CW for which ECC engine returns the uncorrectable 1655 * error. The page will be assumed as erased if this count is less than or 1656 * equal to the ecc->strength for each CW. 1657 * 1658 * 1. Both DATA and OOB need to be checked for number of 0. The 1659 * top-level API can be called with only data buf or OOB buf so use 1660 * chip->data_buf if data buf is null and chip->oob_poi if oob buf 1661 * is null for copying the raw bytes. 1662 * 2. Perform raw read for all the CW which has uncorrectable errors. 1663 * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes. 1664 * The BBM and spare bytes bit flip won’t affect the ECC so don’t check 1665 * the number of bitflips in this area. 1666 */ 1667static int 1668check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf, 1669 u8 *oob_buf, unsigned long uncorrectable_cws, 1670 int page, unsigned int max_bitflips) 1671{ 1672 struct nand_chip *chip = &host->chip; 1673 struct mtd_info *mtd = nand_to_mtd(chip); 1674 struct nand_ecc_ctrl *ecc = &chip->ecc; 1675 u8 *cw_data_buf, *cw_oob_buf; 1676 int cw, data_size, oob_size, ret = 0; 1677 1678 if (!data_buf) 1679 data_buf = nand_get_data_buf(chip); 1680 1681 if (!oob_buf) { 1682 nand_get_data_buf(chip); 1683 oob_buf = chip->oob_poi; 1684 } 1685 1686 for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) { 1687 if (cw == (ecc->steps - 1)) { 1688 data_size = ecc->size - ((ecc->steps - 1) * 4); 1689 oob_size = (ecc->steps * 4) + host->ecc_bytes_hw; 1690 } else { 1691 data_size = host->cw_data; 1692 oob_size = host->ecc_bytes_hw; 1693 } 1694 1695 /* determine starting buffer address for current CW */ 1696 cw_data_buf = data_buf + (cw * host->cw_data); 1697 cw_oob_buf = oob_buf + (cw * ecc->bytes); 1698 1699 ret = qcom_nandc_read_cw_raw(mtd, chip, cw_data_buf, 1700 cw_oob_buf, page, cw); 1701 if (ret) 1702 return ret; 1703 1704 /* 1705 * make sure it isn't an erased page reported 1706 * as not-erased by HW because of a few bitflips 1707 */ 1708 ret = nand_check_erased_ecc_chunk(cw_data_buf, data_size, 1709 cw_oob_buf + host->bbm_size, 1710 oob_size, NULL, 1711 0, ecc->strength); 1712 if (ret < 0) { 1713 mtd->ecc_stats.failed++; 1714 } else { 1715 mtd->ecc_stats.corrected += ret; 1716 max_bitflips = max_t(unsigned int, max_bitflips, ret); 1717 } 1718 } 1719 1720 return max_bitflips; 1721} 1722 1723/* 1724 * reads back status registers set by the controller to notify page read 1725 * errors. this is equivalent to what 'ecc->correct()' would do. 1726 */ 1727static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf, 1728 u8 *oob_buf, int page) 1729{ 1730 struct nand_chip *chip = &host->chip; 1731 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1732 struct mtd_info *mtd = nand_to_mtd(chip); 1733 struct nand_ecc_ctrl *ecc = &chip->ecc; 1734 unsigned int max_bitflips = 0, uncorrectable_cws = 0; 1735 struct read_stats *buf; 1736 bool flash_op_err = false, erased; 1737 int i; 1738 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf; 1739 1740 buf = (struct read_stats *)nandc->reg_read_buf; 1741 nandc_read_buffer_sync(nandc, true); 1742 1743 for (i = 0; i < ecc->steps; i++, buf++) { 1744 u32 flash, buffer, erased_cw; 1745 int data_len, oob_len; 1746 1747 if (i == (ecc->steps - 1)) { 1748 data_len = ecc->size - ((ecc->steps - 1) << 2); 1749 oob_len = ecc->steps << 2; 1750 } else { 1751 data_len = host->cw_data; 1752 oob_len = 0; 1753 } 1754 1755 flash = le32_to_cpu(buf->flash); 1756 buffer = le32_to_cpu(buf->buffer); 1757 erased_cw = le32_to_cpu(buf->erased_cw); 1758 1759 /* 1760 * Check ECC failure for each codeword. ECC failure can 1761 * happen in either of the following conditions 1762 * 1. If number of bitflips are greater than ECC engine 1763 * capability. 1764 * 2. If this codeword contains all 0xff for which erased 1765 * codeword detection check will be done. 1766 */ 1767 if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) { 1768 /* 1769 * For BCH ECC, ignore erased codeword errors, if 1770 * ERASED_CW bits are set. 1771 */ 1772 if (host->bch_enabled) { 1773 erased = (erased_cw & ERASED_CW) == ERASED_CW ? 1774 true : false; 1775 /* 1776 * For RS ECC, HW reports the erased CW by placing 1777 * special characters at certain offsets in the buffer. 1778 * These special characters will be valid only if 1779 * complete page is read i.e. data_buf is not NULL. 1780 */ 1781 } else if (data_buf) { 1782 erased = erased_chunk_check_and_fixup(data_buf, 1783 data_len); 1784 } else { 1785 erased = false; 1786 } 1787 1788 if (!erased) 1789 uncorrectable_cws |= BIT(i); 1790 /* 1791 * Check if MPU or any other operational error (timeout, 1792 * device failure, etc.) happened for this codeword and 1793 * make flash_op_err true. If flash_op_err is set, then 1794 * EIO will be returned for page read. 1795 */ 1796 } else if (flash & (FS_OP_ERR | FS_MPU_ERR)) { 1797 flash_op_err = true; 1798 /* 1799 * No ECC or operational errors happened. Check the number of 1800 * bits corrected and update the ecc_stats.corrected. 1801 */ 1802 } else { 1803 unsigned int stat; 1804 1805 stat = buffer & BS_CORRECTABLE_ERR_MSK; 1806 mtd->ecc_stats.corrected += stat; 1807 max_bitflips = max(max_bitflips, stat); 1808 } 1809 1810 if (data_buf) 1811 data_buf += data_len; 1812 if (oob_buf) 1813 oob_buf += oob_len + ecc->bytes; 1814 } 1815 1816 if (flash_op_err) 1817 return -EIO; 1818 1819 if (!uncorrectable_cws) 1820 return max_bitflips; 1821 1822 return check_for_erased_page(host, data_buf_start, oob_buf_start, 1823 uncorrectable_cws, page, 1824 max_bitflips); 1825} 1826 1827/* 1828 * helper to perform the actual page read operation, used by ecc->read_page(), 1829 * ecc->read_oob() 1830 */ 1831static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf, 1832 u8 *oob_buf, int page) 1833{ 1834 struct nand_chip *chip = &host->chip; 1835 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1836 struct nand_ecc_ctrl *ecc = &chip->ecc; 1837 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf; 1838 int i, ret; 1839 1840 config_nand_page_read(nandc); 1841 1842 /* queue cmd descs for each codeword */ 1843 for (i = 0; i < ecc->steps; i++) { 1844 int data_size, oob_size; 1845 1846 if (i == (ecc->steps - 1)) { 1847 data_size = ecc->size - ((ecc->steps - 1) << 2); 1848 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw + 1849 host->spare_bytes; 1850 } else { 1851 data_size = host->cw_data; 1852 oob_size = host->ecc_bytes_hw + host->spare_bytes; 1853 } 1854 1855 if (nandc->props->is_bam) { 1856 if (data_buf && oob_buf) { 1857 nandc_set_read_loc(nandc, 0, 0, data_size, 0); 1858 nandc_set_read_loc(nandc, 1, data_size, 1859 oob_size, 1); 1860 } else if (data_buf) { 1861 nandc_set_read_loc(nandc, 0, 0, data_size, 1); 1862 } else { 1863 nandc_set_read_loc(nandc, 0, data_size, 1864 oob_size, 1); 1865 } 1866 } 1867 1868 config_nand_cw_read(nandc, true); 1869 1870 if (data_buf) 1871 read_data_dma(nandc, FLASH_BUF_ACC, data_buf, 1872 data_size, 0); 1873 1874 /* 1875 * when ecc is enabled, the controller doesn't read the real 1876 * or dummy bad block markers in each chunk. To maintain a 1877 * consistent layout across RAW and ECC reads, we just 1878 * leave the real/dummy BBM offsets empty (i.e, filled with 1879 * 0xffs) 1880 */ 1881 if (oob_buf) { 1882 int j; 1883 1884 for (j = 0; j < host->bbm_size; j++) 1885 *oob_buf++ = 0xff; 1886 1887 read_data_dma(nandc, FLASH_BUF_ACC + data_size, 1888 oob_buf, oob_size, 0); 1889 } 1890 1891 if (data_buf) 1892 data_buf += data_size; 1893 if (oob_buf) 1894 oob_buf += oob_size; 1895 } 1896 1897 ret = submit_descs(nandc); 1898 free_descs(nandc); 1899 1900 if (ret) { 1901 dev_err(nandc->dev, "failure to read page/oob\n"); 1902 return ret; 1903 } 1904 1905 return parse_read_errors(host, data_buf_start, oob_buf_start, page); 1906} 1907 1908/* 1909 * a helper that copies the last step/codeword of a page (containing free oob) 1910 * into our local buffer 1911 */ 1912static int copy_last_cw(struct qcom_nand_host *host, int page) 1913{ 1914 struct nand_chip *chip = &host->chip; 1915 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1916 struct nand_ecc_ctrl *ecc = &chip->ecc; 1917 int size; 1918 int ret; 1919 1920 clear_read_regs(nandc); 1921 1922 size = host->use_ecc ? host->cw_data : host->cw_size; 1923 1924 /* prepare a clean read buffer */ 1925 memset(nandc->data_buffer, 0xff, size); 1926 1927 set_address(host, host->cw_size * (ecc->steps - 1), page); 1928 update_rw_regs(host, 1, true); 1929 1930 config_nand_single_cw_page_read(nandc, host->use_ecc); 1931 1932 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0); 1933 1934 ret = submit_descs(nandc); 1935 if (ret) 1936 dev_err(nandc->dev, "failed to copy last codeword\n"); 1937 1938 free_descs(nandc); 1939 1940 return ret; 1941} 1942 1943/* implements ecc->read_page() */ 1944static int qcom_nandc_read_page(struct nand_chip *chip, uint8_t *buf, 1945 int oob_required, int page) 1946{ 1947 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1948 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1949 u8 *data_buf, *oob_buf = NULL; 1950 1951 nand_read_page_op(chip, page, 0, NULL, 0); 1952 data_buf = buf; 1953 oob_buf = oob_required ? chip->oob_poi : NULL; 1954 1955 clear_bam_transaction(nandc); 1956 1957 return read_page_ecc(host, data_buf, oob_buf, page); 1958} 1959 1960/* implements ecc->read_page_raw() */ 1961static int qcom_nandc_read_page_raw(struct nand_chip *chip, uint8_t *buf, 1962 int oob_required, int page) 1963{ 1964 struct mtd_info *mtd = nand_to_mtd(chip); 1965 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1966 struct nand_ecc_ctrl *ecc = &chip->ecc; 1967 int cw, ret; 1968 u8 *data_buf = buf, *oob_buf = chip->oob_poi; 1969 1970 for (cw = 0; cw < ecc->steps; cw++) { 1971 ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf, 1972 page, cw); 1973 if (ret) 1974 return ret; 1975 1976 data_buf += host->cw_data; 1977 oob_buf += ecc->bytes; 1978 } 1979 1980 return 0; 1981} 1982 1983/* implements ecc->read_oob() */ 1984static int qcom_nandc_read_oob(struct nand_chip *chip, int page) 1985{ 1986 struct qcom_nand_host *host = to_qcom_nand_host(chip); 1987 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 1988 struct nand_ecc_ctrl *ecc = &chip->ecc; 1989 1990 clear_read_regs(nandc); 1991 clear_bam_transaction(nandc); 1992 1993 host->use_ecc = true; 1994 set_address(host, 0, page); 1995 update_rw_regs(host, ecc->steps, true); 1996 1997 return read_page_ecc(host, NULL, chip->oob_poi, page); 1998} 1999 2000/* implements ecc->write_page() */ 2001static int qcom_nandc_write_page(struct nand_chip *chip, const uint8_t *buf, 2002 int oob_required, int page) 2003{ 2004 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2005 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2006 struct nand_ecc_ctrl *ecc = &chip->ecc; 2007 u8 *data_buf, *oob_buf; 2008 int i, ret; 2009 2010 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 2011 2012 clear_read_regs(nandc); 2013 clear_bam_transaction(nandc); 2014 2015 data_buf = (u8 *)buf; 2016 oob_buf = chip->oob_poi; 2017 2018 host->use_ecc = true; 2019 update_rw_regs(host, ecc->steps, false); 2020 config_nand_page_write(nandc); 2021 2022 for (i = 0; i < ecc->steps; i++) { 2023 int data_size, oob_size; 2024 2025 if (i == (ecc->steps - 1)) { 2026 data_size = ecc->size - ((ecc->steps - 1) << 2); 2027 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw + 2028 host->spare_bytes; 2029 } else { 2030 data_size = host->cw_data; 2031 oob_size = ecc->bytes; 2032 } 2033 2034 2035 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size, 2036 i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0); 2037 2038 /* 2039 * when ECC is enabled, we don't really need to write anything 2040 * to oob for the first n - 1 codewords since these oob regions 2041 * just contain ECC bytes that's written by the controller 2042 * itself. For the last codeword, we skip the bbm positions and 2043 * write to the free oob area. 2044 */ 2045 if (i == (ecc->steps - 1)) { 2046 oob_buf += host->bbm_size; 2047 2048 write_data_dma(nandc, FLASH_BUF_ACC + data_size, 2049 oob_buf, oob_size, 0); 2050 } 2051 2052 config_nand_cw_write(nandc); 2053 2054 data_buf += data_size; 2055 oob_buf += oob_size; 2056 } 2057 2058 ret = submit_descs(nandc); 2059 if (ret) 2060 dev_err(nandc->dev, "failure to write page\n"); 2061 2062 free_descs(nandc); 2063 2064 if (!ret) 2065 ret = nand_prog_page_end_op(chip); 2066 2067 return ret; 2068} 2069 2070/* implements ecc->write_page_raw() */ 2071static int qcom_nandc_write_page_raw(struct nand_chip *chip, 2072 const uint8_t *buf, int oob_required, 2073 int page) 2074{ 2075 struct mtd_info *mtd = nand_to_mtd(chip); 2076 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2077 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2078 struct nand_ecc_ctrl *ecc = &chip->ecc; 2079 u8 *data_buf, *oob_buf; 2080 int i, ret; 2081 2082 nand_prog_page_begin_op(chip, page, 0, NULL, 0); 2083 clear_read_regs(nandc); 2084 clear_bam_transaction(nandc); 2085 2086 data_buf = (u8 *)buf; 2087 oob_buf = chip->oob_poi; 2088 2089 host->use_ecc = false; 2090 update_rw_regs(host, ecc->steps, false); 2091 config_nand_page_write(nandc); 2092 2093 for (i = 0; i < ecc->steps; i++) { 2094 int data_size1, data_size2, oob_size1, oob_size2; 2095 int reg_off = FLASH_BUF_ACC; 2096 2097 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1); 2098 oob_size1 = host->bbm_size; 2099 2100 if (i == (ecc->steps - 1)) { 2101 data_size2 = ecc->size - data_size1 - 2102 ((ecc->steps - 1) << 2); 2103 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw + 2104 host->spare_bytes; 2105 } else { 2106 data_size2 = host->cw_data - data_size1; 2107 oob_size2 = host->ecc_bytes_hw + host->spare_bytes; 2108 } 2109 2110 write_data_dma(nandc, reg_off, data_buf, data_size1, 2111 NAND_BAM_NO_EOT); 2112 reg_off += data_size1; 2113 data_buf += data_size1; 2114 2115 write_data_dma(nandc, reg_off, oob_buf, oob_size1, 2116 NAND_BAM_NO_EOT); 2117 reg_off += oob_size1; 2118 oob_buf += oob_size1; 2119 2120 write_data_dma(nandc, reg_off, data_buf, data_size2, 2121 NAND_BAM_NO_EOT); 2122 reg_off += data_size2; 2123 data_buf += data_size2; 2124 2125 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0); 2126 oob_buf += oob_size2; 2127 2128 config_nand_cw_write(nandc); 2129 } 2130 2131 ret = submit_descs(nandc); 2132 if (ret) 2133 dev_err(nandc->dev, "failure to write raw page\n"); 2134 2135 free_descs(nandc); 2136 2137 if (!ret) 2138 ret = nand_prog_page_end_op(chip); 2139 2140 return ret; 2141} 2142 2143/* 2144 * implements ecc->write_oob() 2145 * 2146 * the NAND controller cannot write only data or only OOB within a codeword 2147 * since ECC is calculated for the combined codeword. So update the OOB from 2148 * chip->oob_poi, and pad the data area with OxFF before writing. 2149 */ 2150static int qcom_nandc_write_oob(struct nand_chip *chip, int page) 2151{ 2152 struct mtd_info *mtd = nand_to_mtd(chip); 2153 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2154 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2155 struct nand_ecc_ctrl *ecc = &chip->ecc; 2156 u8 *oob = chip->oob_poi; 2157 int data_size, oob_size; 2158 int ret; 2159 2160 host->use_ecc = true; 2161 clear_bam_transaction(nandc); 2162 2163 /* calculate the data and oob size for the last codeword/step */ 2164 data_size = ecc->size - ((ecc->steps - 1) << 2); 2165 oob_size = mtd->oobavail; 2166 2167 memset(nandc->data_buffer, 0xff, host->cw_data); 2168 /* override new oob content to last codeword */ 2169 mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob, 2170 0, mtd->oobavail); 2171 2172 set_address(host, host->cw_size * (ecc->steps - 1), page); 2173 update_rw_regs(host, 1, false); 2174 2175 config_nand_page_write(nandc); 2176 write_data_dma(nandc, FLASH_BUF_ACC, 2177 nandc->data_buffer, data_size + oob_size, 0); 2178 config_nand_cw_write(nandc); 2179 2180 ret = submit_descs(nandc); 2181 2182 free_descs(nandc); 2183 2184 if (ret) { 2185 dev_err(nandc->dev, "failure to write oob\n"); 2186 return -EIO; 2187 } 2188 2189 return nand_prog_page_end_op(chip); 2190} 2191 2192static int qcom_nandc_block_bad(struct nand_chip *chip, loff_t ofs) 2193{ 2194 struct mtd_info *mtd = nand_to_mtd(chip); 2195 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2196 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2197 struct nand_ecc_ctrl *ecc = &chip->ecc; 2198 int page, ret, bbpos, bad = 0; 2199 2200 page = (int)(ofs >> chip->page_shift) & chip->pagemask; 2201 2202 /* 2203 * configure registers for a raw sub page read, the address is set to 2204 * the beginning of the last codeword, we don't care about reading ecc 2205 * portion of oob. we just want the first few bytes from this codeword 2206 * that contains the BBM 2207 */ 2208 host->use_ecc = false; 2209 2210 clear_bam_transaction(nandc); 2211 ret = copy_last_cw(host, page); 2212 if (ret) 2213 goto err; 2214 2215 if (check_flash_errors(host, 1)) { 2216 dev_warn(nandc->dev, "error when trying to read BBM\n"); 2217 goto err; 2218 } 2219 2220 bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1); 2221 2222 bad = nandc->data_buffer[bbpos] != 0xff; 2223 2224 if (chip->options & NAND_BUSWIDTH_16) 2225 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff); 2226err: 2227 return bad; 2228} 2229 2230static int qcom_nandc_block_markbad(struct nand_chip *chip, loff_t ofs) 2231{ 2232 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2233 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2234 struct nand_ecc_ctrl *ecc = &chip->ecc; 2235 int page, ret; 2236 2237 clear_read_regs(nandc); 2238 clear_bam_transaction(nandc); 2239 2240 /* 2241 * to mark the BBM as bad, we flash the entire last codeword with 0s. 2242 * we don't care about the rest of the content in the codeword since 2243 * we aren't going to use this block again 2244 */ 2245 memset(nandc->data_buffer, 0x00, host->cw_size); 2246 2247 page = (int)(ofs >> chip->page_shift) & chip->pagemask; 2248 2249 /* prepare write */ 2250 host->use_ecc = false; 2251 set_address(host, host->cw_size * (ecc->steps - 1), page); 2252 update_rw_regs(host, 1, false); 2253 2254 config_nand_page_write(nandc); 2255 write_data_dma(nandc, FLASH_BUF_ACC, 2256 nandc->data_buffer, host->cw_size, 0); 2257 config_nand_cw_write(nandc); 2258 2259 ret = submit_descs(nandc); 2260 2261 free_descs(nandc); 2262 2263 if (ret) { 2264 dev_err(nandc->dev, "failure to update BBM\n"); 2265 return -EIO; 2266 } 2267 2268 return nand_prog_page_end_op(chip); 2269} 2270 2271/* 2272 * the three functions below implement chip->legacy.read_byte(), 2273 * chip->legacy.read_buf() and chip->legacy.write_buf() respectively. these 2274 * aren't used for reading/writing page data, they are used for smaller data 2275 * like reading id, status etc 2276 */ 2277static uint8_t qcom_nandc_read_byte(struct nand_chip *chip) 2278{ 2279 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2280 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2281 u8 *buf = nandc->data_buffer; 2282 u8 ret = 0x0; 2283 2284 if (host->last_command == NAND_CMD_STATUS) { 2285 ret = host->status; 2286 2287 host->status = NAND_STATUS_READY | NAND_STATUS_WP; 2288 2289 return ret; 2290 } 2291 2292 if (nandc->buf_start < nandc->buf_count) 2293 ret = buf[nandc->buf_start++]; 2294 2295 return ret; 2296} 2297 2298static void qcom_nandc_read_buf(struct nand_chip *chip, uint8_t *buf, int len) 2299{ 2300 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2301 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); 2302 2303 memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len); 2304 nandc->buf_start += real_len; 2305} 2306 2307static void qcom_nandc_write_buf(struct nand_chip *chip, const uint8_t *buf, 2308 int len) 2309{ 2310 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2311 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start); 2312 2313 memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len); 2314 2315 nandc->buf_start += real_len; 2316} 2317 2318/* we support only one external chip for now */ 2319static void qcom_nandc_select_chip(struct nand_chip *chip, int chipnr) 2320{ 2321 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2322 2323 if (chipnr <= 0) 2324 return; 2325 2326 dev_warn(nandc->dev, "invalid chip select\n"); 2327} 2328 2329/* 2330 * NAND controller page layout info 2331 * 2332 * Layout with ECC enabled: 2333 * 2334 * |----------------------| |---------------------------------| 2335 * | xx.......yy| | *********xx.......yy| 2336 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy| 2337 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy| 2338 * | xx.......yy| | *********xx.......yy| 2339 * |----------------------| |---------------------------------| 2340 * codeword 1,2..n-1 codeword n 2341 * <---(528/532 Bytes)--> <-------(528/532 Bytes)---------> 2342 * 2343 * n = Number of codewords in the page 2344 * . = ECC bytes 2345 * * = Spare/free bytes 2346 * x = Unused byte(s) 2347 * y = Reserved byte(s) 2348 * 2349 * 2K page: n = 4, spare = 16 bytes 2350 * 4K page: n = 8, spare = 32 bytes 2351 * 8K page: n = 16, spare = 64 bytes 2352 * 2353 * the qcom nand controller operates at a sub page/codeword level. each 2354 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively. 2355 * the number of ECC bytes vary based on the ECC strength and the bus width. 2356 * 2357 * the first n - 1 codewords contains 516 bytes of user data, the remaining 2358 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains 2359 * both user data and spare(oobavail) bytes that sum up to 516 bytes. 2360 * 2361 * When we access a page with ECC enabled, the reserved bytes(s) are not 2362 * accessible at all. When reading, we fill up these unreadable positions 2363 * with 0xffs. When writing, the controller skips writing the inaccessible 2364 * bytes. 2365 * 2366 * Layout with ECC disabled: 2367 * 2368 * |------------------------------| |---------------------------------------| 2369 * | yy xx.......| | bb *********xx.......| 2370 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..| 2371 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......| 2372 * | yy xx.......| | bb *********xx.......| 2373 * |------------------------------| |---------------------------------------| 2374 * codeword 1,2..n-1 codeword n 2375 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)-----------> 2376 * 2377 * n = Number of codewords in the page 2378 * . = ECC bytes 2379 * * = Spare/free bytes 2380 * x = Unused byte(s) 2381 * y = Dummy Bad Bock byte(s) 2382 * b = Real Bad Block byte(s) 2383 * size1/size2 = function of codeword size and 'n' 2384 * 2385 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus 2386 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad 2387 * Block Markers. In the last codeword, this position contains the real BBM 2388 * 2389 * In order to have a consistent layout between RAW and ECC modes, we assume 2390 * the following OOB layout arrangement: 2391 * 2392 * |-----------| |--------------------| 2393 * |yyxx.......| |bb*********xx.......| 2394 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..| 2395 * |yyxx.......| |bb*********xx.......| 2396 * |yyxx.......| |bb*********xx.......| 2397 * |-----------| |--------------------| 2398 * first n - 1 nth OOB region 2399 * OOB regions 2400 * 2401 * n = Number of codewords in the page 2402 * . = ECC bytes 2403 * * = FREE OOB bytes 2404 * y = Dummy bad block byte(s) (inaccessible when ECC enabled) 2405 * x = Unused byte(s) 2406 * b = Real bad block byte(s) (inaccessible when ECC enabled) 2407 * 2408 * This layout is read as is when ECC is disabled. When ECC is enabled, the 2409 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob, 2410 * and assumed as 0xffs when we read a page/oob. The ECC, unused and 2411 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is 2412 * the sum of the three). 2413 */ 2414static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section, 2415 struct mtd_oob_region *oobregion) 2416{ 2417 struct nand_chip *chip = mtd_to_nand(mtd); 2418 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2419 struct nand_ecc_ctrl *ecc = &chip->ecc; 2420 2421 if (section > 1) 2422 return -ERANGE; 2423 2424 if (!section) { 2425 oobregion->length = (ecc->bytes * (ecc->steps - 1)) + 2426 host->bbm_size; 2427 oobregion->offset = 0; 2428 } else { 2429 oobregion->length = host->ecc_bytes_hw + host->spare_bytes; 2430 oobregion->offset = mtd->oobsize - oobregion->length; 2431 } 2432 2433 return 0; 2434} 2435 2436static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section, 2437 struct mtd_oob_region *oobregion) 2438{ 2439 struct nand_chip *chip = mtd_to_nand(mtd); 2440 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2441 struct nand_ecc_ctrl *ecc = &chip->ecc; 2442 2443 if (section) 2444 return -ERANGE; 2445 2446 oobregion->length = ecc->steps * 4; 2447 oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size; 2448 2449 return 0; 2450} 2451 2452static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = { 2453 .ecc = qcom_nand_ooblayout_ecc, 2454 .free = qcom_nand_ooblayout_free, 2455}; 2456 2457static int 2458qcom_nandc_calc_ecc_bytes(int step_size, int strength) 2459{ 2460 return strength == 4 ? 12 : 16; 2461} 2462NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes, 2463 NANDC_STEP_SIZE, 4, 8); 2464 2465static int qcom_nand_attach_chip(struct nand_chip *chip) 2466{ 2467 struct mtd_info *mtd = nand_to_mtd(chip); 2468 struct qcom_nand_host *host = to_qcom_nand_host(chip); 2469 struct nand_ecc_ctrl *ecc = &chip->ecc; 2470 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip); 2471 int cwperpage, bad_block_byte, ret; 2472 bool wide_bus; 2473 int ecc_mode = 1; 2474 2475 /* controller only supports 512 bytes data steps */ 2476 ecc->size = NANDC_STEP_SIZE; 2477 wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false; 2478 cwperpage = mtd->writesize / NANDC_STEP_SIZE; 2479 2480 /* 2481 * Each CW has 4 available OOB bytes which will be protected with ECC 2482 * so remaining bytes can be used for ECC. 2483 */ 2484 ret = nand_ecc_choose_conf(chip, &qcom_nandc_ecc_caps, 2485 mtd->oobsize - (cwperpage * 4)); 2486 if (ret) { 2487 dev_err(nandc->dev, "No valid ECC settings possible\n"); 2488 return ret; 2489 } 2490 2491 if (ecc->strength >= 8) { 2492 /* 8 bit ECC defaults to BCH ECC on all platforms */ 2493 host->bch_enabled = true; 2494 ecc_mode = 1; 2495 2496 if (wide_bus) { 2497 host->ecc_bytes_hw = 14; 2498 host->spare_bytes = 0; 2499 host->bbm_size = 2; 2500 } else { 2501 host->ecc_bytes_hw = 13; 2502 host->spare_bytes = 2; 2503 host->bbm_size = 1; 2504 } 2505 } else { 2506 /* 2507 * if the controller supports BCH for 4 bit ECC, the controller 2508 * uses lesser bytes for ECC. If RS is used, the ECC bytes is 2509 * always 10 bytes 2510 */ 2511 if (nandc->props->ecc_modes & ECC_BCH_4BIT) { 2512 /* BCH */ 2513 host->bch_enabled = true; 2514 ecc_mode = 0; 2515 2516 if (wide_bus) { 2517 host->ecc_bytes_hw = 8; 2518 host->spare_bytes = 2; 2519 host->bbm_size = 2; 2520 } else { 2521 host->ecc_bytes_hw = 7; 2522 host->spare_bytes = 4; 2523 host->bbm_size = 1; 2524 } 2525 } else { 2526 /* RS */ 2527 host->ecc_bytes_hw = 10; 2528 2529 if (wide_bus) { 2530 host->spare_bytes = 0; 2531 host->bbm_size = 2; 2532 } else { 2533 host->spare_bytes = 1; 2534 host->bbm_size = 1; 2535 } 2536 } 2537 } 2538 2539 /* 2540 * we consider ecc->bytes as the sum of all the non-data content in a 2541 * step. It gives us a clean representation of the oob area (even if 2542 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit 2543 * ECC and 12 bytes for 4 bit ECC 2544 */ 2545 ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size; 2546 2547 ecc->read_page = qcom_nandc_read_page; 2548 ecc->read_page_raw = qcom_nandc_read_page_raw; 2549 ecc->read_oob = qcom_nandc_read_oob; 2550 ecc->write_page = qcom_nandc_write_page; 2551 ecc->write_page_raw = qcom_nandc_write_page_raw; 2552 ecc->write_oob = qcom_nandc_write_oob; 2553 2554 ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2555 2556 mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops); 2557 2558 nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage, 2559 cwperpage); 2560 2561 /* 2562 * DATA_UD_BYTES varies based on whether the read/write command protects 2563 * spare data with ECC too. We protect spare data by default, so we set 2564 * it to main + spare data, which are 512 and 4 bytes respectively. 2565 */ 2566 host->cw_data = 516; 2567 2568 /* 2569 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes 2570 * for 8 bit ECC 2571 */ 2572 host->cw_size = host->cw_data + ecc->bytes; 2573 bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1; 2574 2575 host->cfg0 = (cwperpage - 1) << CW_PER_PAGE 2576 | host->cw_data << UD_SIZE_BYTES 2577 | 0 << DISABLE_STATUS_AFTER_WRITE 2578 | 5 << NUM_ADDR_CYCLES 2579 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS 2580 | 0 << STATUS_BFR_READ 2581 | 1 << SET_RD_MODE_AFTER_STATUS 2582 | host->spare_bytes << SPARE_SIZE_BYTES; 2583 2584 host->cfg1 = 7 << NAND_RECOVERY_CYCLES 2585 | 0 << CS_ACTIVE_BSY 2586 | bad_block_byte << BAD_BLOCK_BYTE_NUM 2587 | 0 << BAD_BLOCK_IN_SPARE_AREA 2588 | 2 << WR_RD_BSY_GAP 2589 | wide_bus << WIDE_FLASH 2590 | host->bch_enabled << ENABLE_BCH_ECC; 2591 2592 host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE 2593 | host->cw_size << UD_SIZE_BYTES 2594 | 5 << NUM_ADDR_CYCLES 2595 | 0 << SPARE_SIZE_BYTES; 2596 2597 host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES 2598 | 0 << CS_ACTIVE_BSY 2599 | 17 << BAD_BLOCK_BYTE_NUM 2600 | 1 << BAD_BLOCK_IN_SPARE_AREA 2601 | 2 << WR_RD_BSY_GAP 2602 | wide_bus << WIDE_FLASH 2603 | 1 << DEV0_CFG1_ECC_DISABLE; 2604 2605 host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE 2606 | 0 << ECC_SW_RESET 2607 | host->cw_data << ECC_NUM_DATA_BYTES 2608 | 1 << ECC_FORCE_CLK_OPEN 2609 | ecc_mode << ECC_MODE 2610 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH; 2611 2612 host->ecc_buf_cfg = 0x203 << NUM_STEPS; 2613 2614 host->clrflashstatus = FS_READY_BSY_N; 2615 host->clrreadstatus = 0xc0; 2616 nandc->regs->erased_cw_detect_cfg_clr = 2617 cpu_to_le32(CLR_ERASED_PAGE_DET); 2618 nandc->regs->erased_cw_detect_cfg_set = 2619 cpu_to_le32(SET_ERASED_PAGE_DET); 2620 2621 dev_dbg(nandc->dev, 2622 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n", 2623 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg, 2624 host->cw_size, host->cw_data, ecc->strength, ecc->bytes, 2625 cwperpage); 2626 2627 return 0; 2628} 2629 2630static const struct nand_controller_ops qcom_nandc_ops = { 2631 .attach_chip = qcom_nand_attach_chip, 2632}; 2633 2634static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc) 2635{ 2636 if (nandc->props->is_bam) { 2637 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma)) 2638 dma_unmap_single(nandc->dev, nandc->reg_read_dma, 2639 MAX_REG_RD * 2640 sizeof(*nandc->reg_read_buf), 2641 DMA_FROM_DEVICE); 2642 2643 if (nandc->tx_chan) 2644 dma_release_channel(nandc->tx_chan); 2645 2646 if (nandc->rx_chan) 2647 dma_release_channel(nandc->rx_chan); 2648 2649 if (nandc->cmd_chan) 2650 dma_release_channel(nandc->cmd_chan); 2651 } else { 2652 if (nandc->chan) 2653 dma_release_channel(nandc->chan); 2654 } 2655} 2656 2657static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) 2658{ 2659 int ret; 2660 2661 ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32)); 2662 if (ret) { 2663 dev_err(nandc->dev, "failed to set DMA mask\n"); 2664 return ret; 2665 } 2666 2667 /* 2668 * we use the internal buffer for reading ONFI params, reading small 2669 * data like ID and status, and preforming read-copy-write operations 2670 * when writing to a codeword partially. 532 is the maximum possible 2671 * size of a codeword for our nand controller 2672 */ 2673 nandc->buf_size = 532; 2674 2675 nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size, 2676 GFP_KERNEL); 2677 if (!nandc->data_buffer) 2678 return -ENOMEM; 2679 2680 nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs), 2681 GFP_KERNEL); 2682 if (!nandc->regs) 2683 return -ENOMEM; 2684 2685 nandc->reg_read_buf = devm_kcalloc(nandc->dev, 2686 MAX_REG_RD, sizeof(*nandc->reg_read_buf), 2687 GFP_KERNEL); 2688 if (!nandc->reg_read_buf) 2689 return -ENOMEM; 2690 2691 if (nandc->props->is_bam) { 2692 nandc->reg_read_dma = 2693 dma_map_single(nandc->dev, nandc->reg_read_buf, 2694 MAX_REG_RD * 2695 sizeof(*nandc->reg_read_buf), 2696 DMA_FROM_DEVICE); 2697 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) { 2698 dev_err(nandc->dev, "failed to DMA MAP reg buffer\n"); 2699 return -EIO; 2700 } 2701 2702 nandc->tx_chan = dma_request_chan(nandc->dev, "tx"); 2703 if (IS_ERR(nandc->tx_chan)) { 2704 ret = PTR_ERR(nandc->tx_chan); 2705 nandc->tx_chan = NULL; 2706 dev_err_probe(nandc->dev, ret, 2707 "tx DMA channel request failed\n"); 2708 goto unalloc; 2709 } 2710 2711 nandc->rx_chan = dma_request_chan(nandc->dev, "rx"); 2712 if (IS_ERR(nandc->rx_chan)) { 2713 ret = PTR_ERR(nandc->rx_chan); 2714 nandc->rx_chan = NULL; 2715 dev_err_probe(nandc->dev, ret, 2716 "rx DMA channel request failed\n"); 2717 goto unalloc; 2718 } 2719 2720 nandc->cmd_chan = dma_request_chan(nandc->dev, "cmd"); 2721 if (IS_ERR(nandc->cmd_chan)) { 2722 ret = PTR_ERR(nandc->cmd_chan); 2723 nandc->cmd_chan = NULL; 2724 dev_err_probe(nandc->dev, ret, 2725 "cmd DMA channel request failed\n"); 2726 goto unalloc; 2727 } 2728 2729 /* 2730 * Initially allocate BAM transaction to read ONFI param page. 2731 * After detecting all the devices, this BAM transaction will 2732 * be freed and the next BAM tranasction will be allocated with 2733 * maximum codeword size 2734 */ 2735 nandc->max_cwperpage = 1; 2736 nandc->bam_txn = alloc_bam_transaction(nandc); 2737 if (!nandc->bam_txn) { 2738 dev_err(nandc->dev, 2739 "failed to allocate bam transaction\n"); 2740 ret = -ENOMEM; 2741 goto unalloc; 2742 } 2743 } else { 2744 nandc->chan = dma_request_chan(nandc->dev, "rxtx"); 2745 if (IS_ERR(nandc->chan)) { 2746 ret = PTR_ERR(nandc->chan); 2747 nandc->chan = NULL; 2748 dev_err_probe(nandc->dev, ret, 2749 "rxtx DMA channel request failed\n"); 2750 return ret; 2751 } 2752 } 2753 2754 INIT_LIST_HEAD(&nandc->desc_list); 2755 INIT_LIST_HEAD(&nandc->host_list); 2756 2757 nand_controller_init(&nandc->controller); 2758 nandc->controller.ops = &qcom_nandc_ops; 2759 2760 return 0; 2761unalloc: 2762 qcom_nandc_unalloc(nandc); 2763 return ret; 2764} 2765 2766/* one time setup of a few nand controller registers */ 2767static int qcom_nandc_setup(struct qcom_nand_controller *nandc) 2768{ 2769 u32 nand_ctrl; 2770 2771 /* kill onenand */ 2772 if (!nandc->props->is_qpic) 2773 nandc_write(nandc, SFLASHC_BURST_CFG, 0); 2774 nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD), 2775 NAND_DEV_CMD_VLD_VAL); 2776 2777 /* enable ADM or BAM DMA */ 2778 if (nandc->props->is_bam) { 2779 nand_ctrl = nandc_read(nandc, NAND_CTRL); 2780 2781 /* 2782 *NAND_CTRL is an operational registers, and CPU 2783 * access to operational registers are read only 2784 * in BAM mode. So update the NAND_CTRL register 2785 * only if it is not in BAM mode. In most cases BAM 2786 * mode will be enabled in bootloader 2787 */ 2788 if (!(nand_ctrl & BAM_MODE_EN)) 2789 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN); 2790 } else { 2791 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN); 2792 } 2793 2794 /* save the original values of these registers */ 2795 nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1)); 2796 nandc->vld = NAND_DEV_CMD_VLD_VAL; 2797 2798 return 0; 2799} 2800 2801static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc, 2802 struct qcom_nand_host *host, 2803 struct device_node *dn) 2804{ 2805 struct nand_chip *chip = &host->chip; 2806 struct mtd_info *mtd = nand_to_mtd(chip); 2807 struct device *dev = nandc->dev; 2808 int ret; 2809 2810 ret = of_property_read_u32(dn, "reg", &host->cs); 2811 if (ret) { 2812 dev_err(dev, "can't get chip-select\n"); 2813 return -ENXIO; 2814 } 2815 2816 nand_set_flash_node(chip, dn); 2817 mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs); 2818 if (!mtd->name) 2819 return -ENOMEM; 2820 2821 mtd->owner = THIS_MODULE; 2822 mtd->dev.parent = dev; 2823 2824 chip->legacy.cmdfunc = qcom_nandc_command; 2825 chip->legacy.select_chip = qcom_nandc_select_chip; 2826 chip->legacy.read_byte = qcom_nandc_read_byte; 2827 chip->legacy.read_buf = qcom_nandc_read_buf; 2828 chip->legacy.write_buf = qcom_nandc_write_buf; 2829 chip->legacy.set_features = nand_get_set_features_notsupp; 2830 chip->legacy.get_features = nand_get_set_features_notsupp; 2831 2832 /* 2833 * the bad block marker is readable only when we read the last codeword 2834 * of a page with ECC disabled. currently, the nand_base and nand_bbt 2835 * helpers don't allow us to read BB from a nand chip with ECC 2836 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad 2837 * and block_markbad helpers until we permanently switch to using 2838 * MTD_OPS_RAW for all drivers (with the help of badblockbits) 2839 */ 2840 chip->legacy.block_bad = qcom_nandc_block_bad; 2841 chip->legacy.block_markbad = qcom_nandc_block_markbad; 2842 2843 chip->controller = &nandc->controller; 2844 chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USES_DMA | 2845 NAND_SKIP_BBTSCAN; 2846 2847 /* set up initial status value */ 2848 host->status = NAND_STATUS_READY | NAND_STATUS_WP; 2849 2850 ret = nand_scan(chip, 1); 2851 if (ret) 2852 return ret; 2853 2854 if (nandc->props->is_bam) { 2855 free_bam_transaction(nandc); 2856 nandc->bam_txn = alloc_bam_transaction(nandc); 2857 if (!nandc->bam_txn) { 2858 dev_err(nandc->dev, 2859 "failed to allocate bam transaction\n"); 2860 return -ENOMEM; 2861 } 2862 } 2863 2864 ret = mtd_device_register(mtd, NULL, 0); 2865 if (ret) 2866 nand_cleanup(chip); 2867 2868 return ret; 2869} 2870 2871static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc) 2872{ 2873 struct device *dev = nandc->dev; 2874 struct device_node *dn = dev->of_node, *child; 2875 struct qcom_nand_host *host; 2876 int ret = -ENODEV; 2877 2878 for_each_available_child_of_node(dn, child) { 2879 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); 2880 if (!host) { 2881 of_node_put(child); 2882 return -ENOMEM; 2883 } 2884 2885 ret = qcom_nand_host_init_and_register(nandc, host, child); 2886 if (ret) { 2887 devm_kfree(dev, host); 2888 continue; 2889 } 2890 2891 list_add_tail(&host->node, &nandc->host_list); 2892 } 2893 2894 return ret; 2895} 2896 2897/* parse custom DT properties here */ 2898static int qcom_nandc_parse_dt(struct platform_device *pdev) 2899{ 2900 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev); 2901 struct device_node *np = nandc->dev->of_node; 2902 int ret; 2903 2904 if (!nandc->props->is_bam) { 2905 ret = of_property_read_u32(np, "qcom,cmd-crci", 2906 &nandc->cmd_crci); 2907 if (ret) { 2908 dev_err(nandc->dev, "command CRCI unspecified\n"); 2909 return ret; 2910 } 2911 2912 ret = of_property_read_u32(np, "qcom,data-crci", 2913 &nandc->data_crci); 2914 if (ret) { 2915 dev_err(nandc->dev, "data CRCI unspecified\n"); 2916 return ret; 2917 } 2918 } 2919 2920 return 0; 2921} 2922 2923static int qcom_nandc_probe(struct platform_device *pdev) 2924{ 2925 struct qcom_nand_controller *nandc; 2926 const void *dev_data; 2927 struct device *dev = &pdev->dev; 2928 struct resource *res; 2929 int ret; 2930 2931 nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL); 2932 if (!nandc) 2933 return -ENOMEM; 2934 2935 platform_set_drvdata(pdev, nandc); 2936 nandc->dev = dev; 2937 2938 dev_data = of_device_get_match_data(dev); 2939 if (!dev_data) { 2940 dev_err(&pdev->dev, "failed to get device data\n"); 2941 return -ENODEV; 2942 } 2943 2944 nandc->props = dev_data; 2945 2946 nandc->core_clk = devm_clk_get(dev, "core"); 2947 if (IS_ERR(nandc->core_clk)) 2948 return PTR_ERR(nandc->core_clk); 2949 2950 nandc->aon_clk = devm_clk_get(dev, "aon"); 2951 if (IS_ERR(nandc->aon_clk)) 2952 return PTR_ERR(nandc->aon_clk); 2953 2954 ret = qcom_nandc_parse_dt(pdev); 2955 if (ret) 2956 return ret; 2957 2958 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2959 nandc->base = devm_ioremap_resource(dev, res); 2960 if (IS_ERR(nandc->base)) 2961 return PTR_ERR(nandc->base); 2962 2963 nandc->base_phys = res->start; 2964 nandc->base_dma = dma_map_resource(dev, res->start, 2965 resource_size(res), 2966 DMA_BIDIRECTIONAL, 0); 2967 if (!nandc->base_dma) 2968 return -ENXIO; 2969 2970 ret = clk_prepare_enable(nandc->core_clk); 2971 if (ret) 2972 goto err_core_clk; 2973 2974 ret = clk_prepare_enable(nandc->aon_clk); 2975 if (ret) 2976 goto err_aon_clk; 2977 2978 ret = qcom_nandc_alloc(nandc); 2979 if (ret) 2980 goto err_nandc_alloc; 2981 2982 ret = qcom_nandc_setup(nandc); 2983 if (ret) 2984 goto err_setup; 2985 2986 ret = qcom_probe_nand_devices(nandc); 2987 if (ret) 2988 goto err_setup; 2989 2990 return 0; 2991 2992err_setup: 2993 qcom_nandc_unalloc(nandc); 2994err_nandc_alloc: 2995 clk_disable_unprepare(nandc->aon_clk); 2996err_aon_clk: 2997 clk_disable_unprepare(nandc->core_clk); 2998err_core_clk: 2999 dma_unmap_resource(dev, nandc->base_dma, resource_size(res), 3000 DMA_BIDIRECTIONAL, 0); 3001 return ret; 3002} 3003 3004static int qcom_nandc_remove(struct platform_device *pdev) 3005{ 3006 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev); 3007 struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 3008 struct qcom_nand_host *host; 3009 struct nand_chip *chip; 3010 int ret; 3011 3012 list_for_each_entry(host, &nandc->host_list, node) { 3013 chip = &host->chip; 3014 ret = mtd_device_unregister(nand_to_mtd(chip)); 3015 WARN_ON(ret); 3016 nand_cleanup(chip); 3017 } 3018 3019 qcom_nandc_unalloc(nandc); 3020 3021 clk_disable_unprepare(nandc->aon_clk); 3022 clk_disable_unprepare(nandc->core_clk); 3023 3024 dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res), 3025 DMA_BIDIRECTIONAL, 0); 3026 3027 return 0; 3028} 3029 3030static const struct qcom_nandc_props ipq806x_nandc_props = { 3031 .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT), 3032 .is_bam = false, 3033 .dev_cmd_reg_start = 0x0, 3034}; 3035 3036static const struct qcom_nandc_props ipq4019_nandc_props = { 3037 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 3038 .is_bam = true, 3039 .is_qpic = true, 3040 .dev_cmd_reg_start = 0x0, 3041}; 3042 3043static const struct qcom_nandc_props ipq8074_nandc_props = { 3044 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT), 3045 .is_bam = true, 3046 .is_qpic = true, 3047 .dev_cmd_reg_start = 0x7000, 3048}; 3049 3050/* 3051 * data will hold a struct pointer containing more differences once we support 3052 * more controller variants 3053 */ 3054static const struct of_device_id qcom_nandc_of_match[] = { 3055 { 3056 .compatible = "qcom,ipq806x-nand", 3057 .data = &ipq806x_nandc_props, 3058 }, 3059 { 3060 .compatible = "qcom,ipq4019-nand", 3061 .data = &ipq4019_nandc_props, 3062 }, 3063 { 3064 .compatible = "qcom,ipq8074-nand", 3065 .data = &ipq8074_nandc_props, 3066 }, 3067 {} 3068}; 3069MODULE_DEVICE_TABLE(of, qcom_nandc_of_match); 3070 3071static struct platform_driver qcom_nandc_driver = { 3072 .driver = { 3073 .name = "qcom-nandc", 3074 .of_match_table = qcom_nandc_of_match, 3075 }, 3076 .probe = qcom_nandc_probe, 3077 .remove = qcom_nandc_remove, 3078}; 3079module_platform_driver(qcom_nandc_driver); 3080 3081MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>"); 3082MODULE_DESCRIPTION("Qualcomm NAND Controller driver"); 3083MODULE_LICENSE("GPL v2"); 3084