drivers/mtd/nand/atmel_nand.c

   1 /*
   2  *  Copyright © 2003 Rick Bronson
   3  *
   4  *  Derived from drivers/mtd/nand/autcpu12.c
   5  *       Copyright © 2001 Thomas Gleixner (gleixner@autronix.de)
   6  *
   7  *  Derived from drivers/mtd/spia.c
   8  *       Copyright © 2000 Steven J. Hill (sjhill@cotw.com)
   9  *
  10  *
  11  *  Add Hardware ECC support for AT91SAM9260 / AT91SAM9263
  12  *     Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright © 2007
  13  *
  14  *     Derived from Das U-Boot source code
  15  *              (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)
  16  *     © Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
  17  *
  18  *  Add Programmable Multibit ECC support for various AT91 SoC
  19  *     © Copyright 2012 ATMEL, Hong Xu
  20  *
  21  * This program is free software; you can redistribute it and/or modify
  22  * it under the terms of the GNU General Public License version 2 as
  23  * published by the Free Software Foundation.
  24  *
  25  */
  26
  27 #include <linux/dma-mapping.h>
  28 #include <linux/slab.h>
  29 #include <linux/module.h>
  30 #include <linux/moduleparam.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/of.h>
  33 #include <linux/of_device.h>
  34 #include <linux/of_gpio.h>
  35 #include <linux/of_mtd.h>
  36 #include <linux/mtd/mtd.h>
  37 #include <linux/mtd/nand.h>
  38 #include <linux/mtd/partitions.h>
  39
  40 #include <linux/dmaengine.h>
  41 #include <linux/gpio.h>
  42 #include <linux/io.h>
  43 #include <linux/platform_data/atmel.h>
  44 #include <linux/pinctrl/consumer.h>
  45
  46 #include <mach/cpu.h>
  47
  48 static int use_dma = 1;
  49 module_param(use_dma, int, 0);
  50
  51 static int on_flash_bbt = 0;
  52 module_param(on_flash_bbt, int, 0);
  53
  54 /* Register access macros */
  55 #define ecc_readl(add, reg)                             \
  56         __raw_readl(add + ATMEL_ECC_##reg)
  57 #define ecc_writel(add, reg, value)                     \
  58         __raw_writel((value), add + ATMEL_ECC_##reg)
  59
  60 #include "atmel_nand_ecc.h"     /* Hardware ECC registers */
  61
  62 /* oob layout for large page size
  63  * bad block info is on bytes 0 and 1
  64  * the bytes have to be consecutives to avoid
  65  * several NAND_CMD_RNDOUT during read
  66  */
  67 static struct nand_ecclayout atmel_oobinfo_large = {
  68         .eccbytes = 4,
  69         .eccpos = {60, 61, 62, 63},
  70         .oobfree = {
  71                 {2, 58}
  72         },
  73 };
  74
  75 /* oob layout for small page size
  76  * bad block info is on bytes 4 and 5
  77  * the bytes have to be consecutives to avoid
  78  * several NAND_CMD_RNDOUT during read
  79  */
  80 static struct nand_ecclayout atmel_oobinfo_small = {
  81         .eccbytes = 4,
  82         .eccpos = {0, 1, 2, 3},
  83         .oobfree = {
  84                 {6, 10}
  85         },
  86 };
  87
  88 struct atmel_nand_host {
  89         struct nand_chip        nand_chip;
  90         struct mtd_info         mtd;
  91         void __iomem            *io_base;
  92         dma_addr_t              io_phys;
  93         struct atmel_nand_data  board;
  94         struct device           *dev;
  95         void __iomem            *ecc;
  96
  97         struct completion       comp;
  98         struct dma_chan         *dma_chan;
  99
 100         bool                    has_pmecc;
 101         u8                      pmecc_corr_cap;
 102         u16                     pmecc_sector_size;
 103         u32                     pmecc_lookup_table_offset;
 104
 105         int                     pmecc_bytes_per_sector;
 106         int                     pmecc_sector_number;
 107         int                     pmecc_degree;   /* Degree of remainders */
 108         int                     pmecc_cw_len;   /* Length of codeword */
 109
 110         void __iomem            *pmerrloc_base;
 111         void __iomem            *pmecc_rom_base;
 112
 113         /* lookup table for alpha_to and index_of */
 114         void __iomem            *pmecc_alpha_to;
 115         void __iomem            *pmecc_index_of;
 116
 117         /* data for pmecc computation */
 118         int16_t                 *pmecc_partial_syn;
 119         int16_t                 *pmecc_si;
 120         int16_t                 *pmecc_smu;     /* Sigma table */
 121         int16_t                 *pmecc_lmu;     /* polynomal order */
 122         int                     *pmecc_mu;
 123         int                     *pmecc_dmu;
 124         int                     *pmecc_delta;
 125 };
 126
 127 static struct nand_ecclayout atmel_pmecc_oobinfo;
 128
 129 static int cpu_has_dma(void)
 130 {
 131         return cpu_is_at91sam9rl() || cpu_is_at91sam9g45();
 132 }
 133
 134 /*
 135  * Enable NAND.
 136  */
 137 static void atmel_nand_enable(struct atmel_nand_host *host)
 138 {
 139         if (gpio_is_valid(host->board.enable_pin))
 140                 gpio_set_value(host->board.enable_pin, 0);
 141 }
 142
 143 /*
 144  * Disable NAND.
 145  */
 146 static void atmel_nand_disable(struct atmel_nand_host *host)
 147 {
 148         if (gpio_is_valid(host->board.enable_pin))
 149                 gpio_set_value(host->board.enable_pin, 1);
 150 }
 151
 152 /*
 153  * Hardware specific access to control-lines
 154  */
 155 static void atmel_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl)
 156 {
 157         struct nand_chip *nand_chip = mtd->priv;
 158         struct atmel_nand_host *host = nand_chip->priv;
 159
 160         if (ctrl & NAND_CTRL_CHANGE) {
 161                 if (ctrl & NAND_NCE)
 162                         atmel_nand_enable(host);
 163                 else
 164                         atmel_nand_disable(host);
 165         }
 166         if (cmd == NAND_CMD_NONE)
 167                 return;
 168
 169         if (ctrl & NAND_CLE)
 170                 writeb(cmd, host->io_base + (1 << host->board.cle));
 171         else
 172                 writeb(cmd, host->io_base + (1 << host->board.ale));
 173 }
 174
 175 /*
 176  * Read the Device Ready pin.
 177  */
 178 static int atmel_nand_device_ready(struct mtd_info *mtd)
 179 {
 180         struct nand_chip *nand_chip = mtd->priv;
 181         struct atmel_nand_host *host = nand_chip->priv;
 182
 183         return gpio_get_value(host->board.rdy_pin) ^
 184                 !!host->board.rdy_pin_active_low;
 185 }
 186
 187 /*
 188  * Minimal-overhead PIO for data access.
 189  */
 190 static void atmel_read_buf8(struct mtd_info *mtd, u8 *buf, int len)
 191 {
 192         struct nand_chip        *nand_chip = mtd->priv;
 193
 194         __raw_readsb(nand_chip->IO_ADDR_R, buf, len);
 195 }
 196
 197 static void atmel_read_buf16(struct mtd_info *mtd, u8 *buf, int len)
 198 {
 199         struct nand_chip        *nand_chip = mtd->priv;
 200
 201         __raw_readsw(nand_chip->IO_ADDR_R, buf, len / 2);
 202 }
 203
 204 static void atmel_write_buf8(struct mtd_info *mtd, const u8 *buf, int len)
 205 {
 206         struct nand_chip        *nand_chip = mtd->priv;
 207
 208         __raw_writesb(nand_chip->IO_ADDR_W, buf, len);
 209 }
 210
 211 static void atmel_write_buf16(struct mtd_info *mtd, const u8 *buf, int len)
 212 {
 213         struct nand_chip        *nand_chip = mtd->priv;
 214
 215         __raw_writesw(nand_chip->IO_ADDR_W, buf, len / 2);
 216 }
 217
 218 static void dma_complete_func(void *completion)
 219 {
 220         complete(completion);
 221 }
 222
 223 static int atmel_nand_dma_op(struct mtd_info *mtd, void *buf, int len,
 224                                int is_read)
 225 {
 226         struct dma_device *dma_dev;
 227         enum dma_ctrl_flags flags;
 228         dma_addr_t dma_src_addr, dma_dst_addr, phys_addr;
 229         struct dma_async_tx_descriptor *tx = NULL;
 230         dma_cookie_t cookie;
 231         struct nand_chip *chip = mtd->priv;
 232         struct atmel_nand_host *host = chip->priv;
 233         void *p = buf;
 234         int err = -EIO;
 235         enum dma_data_direction dir = is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
 236
 237         if (buf >= high_memory)
 238                 goto err_buf;
 239
 240         dma_dev = host->dma_chan->device;
 241
 242         flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT | DMA_COMPL_SKIP_SRC_UNMAP |
 243                 DMA_COMPL_SKIP_DEST_UNMAP;
 244
 245         phys_addr = dma_map_single(dma_dev->dev, p, len, dir);
 246         if (dma_mapping_error(dma_dev->dev, phys_addr)) {
 247                 dev_err(host->dev, "Failed to dma_map_single\n");
 248                 goto err_buf;
 249         }
 250
 251         if (is_read) {
 252                 dma_src_addr = host->io_phys;
 253                 dma_dst_addr = phys_addr;
 254         } else {
 255                 dma_src_addr = phys_addr;
 256                 dma_dst_addr = host->io_phys;
 257         }
 258
 259         tx = dma_dev->device_prep_dma_memcpy(host->dma_chan, dma_dst_addr,
 260                                              dma_src_addr, len, flags);
 261         if (!tx) {
 262                 dev_err(host->dev, "Failed to prepare DMA memcpy\n");
 263                 goto err_dma;
 264         }
 265
 266         init_completion(&host->comp);
 267         tx->callback = dma_complete_func;
 268         tx->callback_param = &host->comp;
 269
 270         cookie = tx->tx_submit(tx);
 271         if (dma_submit_error(cookie)) {
 272                 dev_err(host->dev, "Failed to do DMA tx_submit\n");
 273                 goto err_dma;
 274         }
 275
 276         dma_async_issue_pending(host->dma_chan);
 277         wait_for_completion(&host->comp);
 278
 279         err = 0;
 280
 281 err_dma:
 282         dma_unmap_single(dma_dev->dev, phys_addr, len, dir);
 283 err_buf:
 284         if (err != 0)
 285                 dev_warn(host->dev, "Fall back to CPU I/O\n");
 286         return err;
 287 }
 288
 289 static void atmel_read_buf(struct mtd_info *mtd, u8 *buf, int len)
 290 {
 291         struct nand_chip *chip = mtd->priv;
 292         struct atmel_nand_host *host = chip->priv;
 293
 294         if (use_dma && len > mtd->oobsize)
 295                 /* only use DMA for bigger than oob size: better performances */
 296                 if (atmel_nand_dma_op(mtd, buf, len, 1) == 0)
 297                         return;
 298
 299         if (host->board.bus_width_16)
 300                 atmel_read_buf16(mtd, buf, len);
 301         else
 302                 atmel_read_buf8(mtd, buf, len);
 303 }
 304
 305 static void atmel_write_buf(struct mtd_info *mtd, const u8 *buf, int len)
 306 {
 307         struct nand_chip *chip = mtd->priv;
 308         struct atmel_nand_host *host = chip->priv;
 309
 310         if (use_dma && len > mtd->oobsize)
 311                 /* only use DMA for bigger than oob size: better performances */
 312                 if (atmel_nand_dma_op(mtd, (void *)buf, len, 0) == 0)
 313                         return;
 314
 315         if (host->board.bus_width_16)
 316                 atmel_write_buf16(mtd, buf, len);
 317         else
 318                 atmel_write_buf8(mtd, buf, len);
 319 }
 320
 321 /*
 322  * Return number of ecc bytes per sector according to sector size and
 323  * correction capability
 324  *
 325  * Following table shows what at91 PMECC supported:
 326  * Correction Capability        Sector_512_bytes        Sector_1024_bytes
 327  * =====================        ================        =================
 328  *                2-bits                 4-bytes                  4-bytes
 329  *                4-bits                 7-bytes                  7-bytes
 330  *                8-bits                13-bytes                 14-bytes
 331  *               12-bits                20-bytes                 21-bytes
 332  *               24-bits                39-bytes                 42-bytes
 333  */
 334 static int pmecc_get_ecc_bytes(int cap, int sector_size)
 335 {
 336         int m = 12 + sector_size / 512;
 337         return (m * cap + 7) / 8;
 338 }
 339
 340 static void pmecc_config_ecc_layout(struct nand_ecclayout *layout,
 341                                     int oobsize, int ecc_len)
 342 {
 343         int i;
 344
 345         layout->eccbytes = ecc_len;
 346
 347         /* ECC will occupy the last ecc_len bytes continuously */
 348         for (i = 0; i < ecc_len; i++)
 349                 layout->eccpos[i] = oobsize - ecc_len + i;
 350
 351         layout->oobfree[0].offset = 2;
 352         layout->oobfree[0].length =
 353                 oobsize - ecc_len - layout->oobfree[0].offset;
 354 }
 355
 356 static void __iomem *pmecc_get_alpha_to(struct atmel_nand_host *host)
 357 {
 358         int table_size;
 359
 360         table_size = host->pmecc_sector_size == 512 ?
 361                 PMECC_LOOKUP_TABLE_SIZE_512 : PMECC_LOOKUP_TABLE_SIZE_1024;
 362
 363         return host->pmecc_rom_base + host->pmecc_lookup_table_offset +
 364                         table_size * sizeof(int16_t);
 365 }
 366
 367 static void pmecc_data_free(struct atmel_nand_host *host)
 368 {
 369         kfree(host->pmecc_partial_syn);
 370         kfree(host->pmecc_si);
 371         kfree(host->pmecc_lmu);
 372         kfree(host->pmecc_smu);
 373         kfree(host->pmecc_mu);
 374         kfree(host->pmecc_dmu);
 375         kfree(host->pmecc_delta);
 376 }
 377
 378 static int pmecc_data_alloc(struct atmel_nand_host *host)
 379 {
 380         const int cap = host->pmecc_corr_cap;
 381
 382         host->pmecc_partial_syn = kzalloc((2 * cap + 1) * sizeof(int16_t),
 383                                         GFP_KERNEL);
 384         host->pmecc_si = kzalloc((2 * cap + 1) * sizeof(int16_t), GFP_KERNEL);
 385         host->pmecc_lmu = kzalloc((cap + 1) * sizeof(int16_t), GFP_KERNEL);
 386         host->pmecc_smu = kzalloc((cap + 2) * (2 * cap + 1) * sizeof(int16_t),
 387                                         GFP_KERNEL);
 388         host->pmecc_mu = kzalloc((cap + 1) * sizeof(int), GFP_KERNEL);
 389         host->pmecc_dmu = kzalloc((cap + 1) * sizeof(int), GFP_KERNEL);
 390         host->pmecc_delta = kzalloc((cap + 1) * sizeof(int), GFP_KERNEL);
 391
 392         if (host->pmecc_partial_syn &&
 393                         host->pmecc_si &&
 394                         host->pmecc_lmu &&
 395                         host->pmecc_smu &&
 396                         host->pmecc_mu &&
 397                         host->pmecc_dmu &&
 398                         host->pmecc_delta)
 399                 return 0;
 400
 401         /* error happened */
 402         pmecc_data_free(host);
 403         return -ENOMEM;
 404 }
 405
 406 static void pmecc_gen_syndrome(struct mtd_info *mtd, int sector)
 407 {
 408         struct nand_chip *nand_chip = mtd->priv;
 409         struct atmel_nand_host *host = nand_chip->priv;
 410         int i;
 411         uint32_t value;
 412
 413         /* Fill odd syndromes */
 414         for (i = 0; i < host->pmecc_corr_cap; i++) {
 415                 value = pmecc_readl_rem_relaxed(host->ecc, sector, i / 2);
 416                 if (i & 1)
 417                         value >>= 16;
 418                 value &= 0xffff;
 419                 host->pmecc_partial_syn[(2 * i) + 1] = (int16_t)value;
 420         }
 421 }
 422
 423 static void pmecc_substitute(struct mtd_info *mtd)
 424 {
 425         struct nand_chip *nand_chip = mtd->priv;
 426         struct atmel_nand_host *host = nand_chip->priv;
 427         int16_t __iomem *alpha_to = host->pmecc_alpha_to;
 428         int16_t __iomem *index_of = host->pmecc_index_of;
 429         int16_t *partial_syn = host->pmecc_partial_syn;
 430         const int cap = host->pmecc_corr_cap;
 431         int16_t *si;
 432         int i, j;
 433
 434         /* si[] is a table that holds the current syndrome value,
 435          * an element of that table belongs to the field
 436          */
 437         si = host->pmecc_si;
 438
 439         memset(&si[1], 0, sizeof(int16_t) * (2 * cap - 1));
 440
 441         /* Computation 2t syndromes based on S(x) */
 442         /* Odd syndromes */
 443         for (i = 1; i < 2 * cap; i += 2) {
 444                 for (j = 0; j < host->pmecc_degree; j++) {
 445                         if (partial_syn[i] & ((unsigned short)0x1 << j))
 446                                 si[i] = readw_relaxed(alpha_to + i * j) ^ si[i];
 447                 }
 448         }
 449         /* Even syndrome = (Odd syndrome) ** 2 */
 450         for (i = 2, j = 1; j <= cap; i = ++j << 1) {
 451                 if (si[j] == 0) {
 452                         si[i] = 0;
 453                 } else {
 454                         int16_t tmp;
 455
 456                         tmp = readw_relaxed(index_of + si[j]);
 457                         tmp = (tmp * 2) % host->pmecc_cw_len;
 458                         si[i] = readw_relaxed(alpha_to + tmp);
 459                 }
 460         }
 461
 462         return;
 463 }
 464
 465 static void pmecc_get_sigma(struct mtd_info *mtd)
 466 {
 467         struct nand_chip *nand_chip = mtd->priv;
 468         struct atmel_nand_host *host = nand_chip->priv;
 469
 470         int16_t *lmu = host->pmecc_lmu;
 471         int16_t *si = host->pmecc_si;
 472         int *mu = host->pmecc_mu;
 473         int *dmu = host->pmecc_dmu;     /* Discrepancy */
 474         int *delta = host->pmecc_delta; /* Delta order */
 475         int cw_len = host->pmecc_cw_len;
 476         const int16_t cap = host->pmecc_corr_cap;
 477         const int num = 2 * cap + 1;
 478         int16_t __iomem *index_of = host->pmecc_index_of;
 479         int16_t __iomem *alpha_to = host->pmecc_alpha_to;
 480         int i, j, k;
 481         uint32_t dmu_0_count, tmp;
 482         int16_t *smu = host->pmecc_smu;
 483
 484         /* index of largest delta */
 485         int ro;
 486         int largest;
 487         int diff;
 488
 489         dmu_0_count = 0;
 490
 491         /* First Row */
 492
 493         /* Mu */
 494         mu[0] = -1;
 495
 496         memset(smu, 0, sizeof(int16_t) * num);
 497         smu[0] = 1;
 498
 499         /* discrepancy set to 1 */
 500         dmu[0] = 1;
 501         /* polynom order set to 0 */
 502         lmu[0] = 0;
 503         delta[0] = (mu[0] * 2 - lmu[0]) >> 1;
 504
 505         /* Second Row */
 506
 507         /* Mu */
 508         mu[1] = 0;
 509         /* Sigma(x) set to 1 */
 510         memset(&smu[num], 0, sizeof(int16_t) * num);
 511         smu[num] = 1;
 512
 513         /* discrepancy set to S1 */
 514         dmu[1] = si[1];
 515
 516         /* polynom order set to 0 */
 517         lmu[1] = 0;
 518
 519         delta[1] = (mu[1] * 2 - lmu[1]) >> 1;
 520
 521         /* Init the Sigma(x) last row */
 522         memset(&smu[(cap + 1) * num], 0, sizeof(int16_t) * num);
 523
 524         for (i = 1; i <= cap; i++) {
 525                 mu[i + 1] = i << 1;
 526                 /* Begin Computing Sigma (Mu+1) and L(mu) */
 527                 /* check if discrepancy is set to 0 */
 528                 if (dmu[i] == 0) {
 529                         dmu_0_count++;
 530
 531                         tmp = ((cap - (lmu[i] >> 1) - 1) / 2);
 532                         if ((cap - (lmu[i] >> 1) - 1) & 0x1)
 533                                 tmp += 2;
 534                         else
 535                                 tmp += 1;
 536
 537                         if (dmu_0_count == tmp) {
 538                                 for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
 539                                         smu[(cap + 1) * num + j] =
 540                                                         smu[i * num + j];
 541
 542                                 lmu[cap + 1] = lmu[i];
 543                                 return;
 544                         }
 545
 546                         /* copy polynom */
 547                         for (j = 0; j <= lmu[i] >> 1; j++)
 548                                 smu[(i + 1) * num + j] = smu[i * num + j];
 549
 550                         /* copy previous polynom order to the next */
 551                         lmu[i + 1] = lmu[i];
 552                 } else {
 553                         ro = 0;
 554                         largest = -1;
 555                         /* find largest delta with dmu != 0 */
 556                         for (j = 0; j < i; j++) {
 557                                 if ((dmu[j]) && (delta[j] > largest)) {
 558                                         largest = delta[j];
 559                                         ro = j;
 560                                 }
 561                         }
 562
 563                         /* compute difference */
 564                         diff = (mu[i] - mu[ro]);
 565
 566                         /* Compute degree of the new smu polynomial */
 567                         if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
 568                                 lmu[i + 1] = lmu[i];
 569                         else
 570                                 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
 571
 572                         /* Init smu[i+1] with 0 */
 573                         for (k = 0; k < num; k++)
 574                                 smu[(i + 1) * num + k] = 0;
 575
 576                         /* Compute smu[i+1] */
 577                         for (k = 0; k <= lmu[ro] >> 1; k++) {
 578                                 int16_t a, b, c;
 579
 580                                 if (!(smu[ro * num + k] && dmu[i]))
 581                                         continue;
 582                                 a = readw_relaxed(index_of + dmu[i]);
 583                                 b = readw_relaxed(index_of + dmu[ro]);
 584                                 c = readw_relaxed(index_of + smu[ro * num + k]);
 585                                 tmp = a + (cw_len - b) + c;
 586                                 a = readw_relaxed(alpha_to + tmp % cw_len);
 587                                 smu[(i + 1) * num + (k + diff)] = a;
 588                         }
 589
 590                         for (k = 0; k <= lmu[i] >> 1; k++)
 591                                 smu[(i + 1) * num + k] ^= smu[i * num + k];
 592                 }
 593
 594                 /* End Computing Sigma (Mu+1) and L(mu) */
 595                 /* In either case compute delta */
 596                 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
 597
 598                 /* Do not compute discrepancy for the last iteration */
 599                 if (i >= cap)
 600                         continue;
 601
 602                 for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
 603                         tmp = 2 * (i - 1);
 604                         if (k == 0) {
 605                                 dmu[i + 1] = si[tmp + 3];
 606                         } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
 607                                 int16_t a, b, c;
 608                                 a = readw_relaxed(index_of +
 609                                                 smu[(i + 1) * num + k]);
 610                                 b = si[2 * (i - 1) + 3 - k];
 611                                 c = readw_relaxed(index_of + b);
 612                                 tmp = a + c;
 613                                 tmp %= cw_len;
 614                                 dmu[i + 1] = readw_relaxed(alpha_to + tmp) ^
 615                                         dmu[i + 1];
 616                         }
 617                 }
 618         }
 619
 620         return;
 621 }
 622
 623 static int pmecc_err_location(struct mtd_info *mtd)
 624 {
 625         struct nand_chip *nand_chip = mtd->priv;
 626         struct atmel_nand_host *host = nand_chip->priv;
 627         unsigned long end_time;
 628         const int cap = host->pmecc_corr_cap;
 629         const int num = 2 * cap + 1;
 630         int sector_size = host->pmecc_sector_size;
 631         int err_nbr = 0;        /* number of error */
 632         int roots_nbr;          /* number of roots */
 633         int i;
 634         uint32_t val;
 635         int16_t *smu = host->pmecc_smu;
 636
 637         pmerrloc_writel(host->pmerrloc_base, ELDIS, PMERRLOC_DISABLE);
 638
 639         for (i = 0; i <= host->pmecc_lmu[cap + 1] >> 1; i++) {
 640                 pmerrloc_writel_sigma_relaxed(host->pmerrloc_base, i,
 641                                       smu[(cap + 1) * num + i]);
 642                 err_nbr++;
 643         }
 644
 645         val = (err_nbr - 1) << 16;
 646         if (sector_size == 1024)
 647                 val |= 1;
 648
 649         pmerrloc_writel(host->pmerrloc_base, ELCFG, val);
 650         pmerrloc_writel(host->pmerrloc_base, ELEN,
 651                         sector_size * 8 + host->pmecc_degree * cap);
 652
 653         end_time = jiffies + msecs_to_jiffies(PMECC_MAX_TIMEOUT_MS);
 654         while (!(pmerrloc_readl_relaxed(host->pmerrloc_base, ELISR)
 655                  & PMERRLOC_CALC_DONE)) {
 656                 if (unlikely(time_after(jiffies, end_time))) {
 657                         dev_err(host->dev, "PMECC: Timeout to calculate error location.\n");
 658                         return -1;
 659                 }
 660                 cpu_relax();
 661         }
 662
 663         roots_nbr = (pmerrloc_readl_relaxed(host->pmerrloc_base, ELISR)
 664                 & PMERRLOC_ERR_NUM_MASK) >> 8;
 665         /* Number of roots == degree of smu hence <= cap */
 666         if (roots_nbr == host->pmecc_lmu[cap + 1] >> 1)
 667                 return err_nbr - 1;
 668
 669         /* Number of roots does not match the degree of smu
 670          * unable to correct error */
 671         return -1;
 672 }
 673
 674 static void pmecc_correct_data(struct mtd_info *mtd, uint8_t *buf, uint8_t *ecc,
 675                 int sector_num, int extra_bytes, int err_nbr)
 676 {
 677         struct nand_chip *nand_chip = mtd->priv;
 678         struct atmel_nand_host *host = nand_chip->priv;
 679         int i = 0;
 680         int byte_pos, bit_pos, sector_size, pos;
 681         uint32_t tmp;
 682         uint8_t err_byte;
 683
 684         sector_size = host->pmecc_sector_size;
 685
 686         while (err_nbr) {
 687                 tmp = pmerrloc_readl_el_relaxed(host->pmerrloc_base, i) - 1;
 688                 byte_pos = tmp / 8;
 689                 bit_pos  = tmp % 8;
 690
 691                 if (byte_pos >= (sector_size + extra_bytes))
 692                         BUG();  /* should never happen */
 693
 694                 if (byte_pos < sector_size) {
 695                         err_byte = *(buf + byte_pos);
 696                         *(buf + byte_pos) ^= (1 << bit_pos);
 697
 698                         pos = sector_num * host->pmecc_sector_size + byte_pos;
 699                         dev_info(host->dev, "Bit flip in data area, byte_pos: %d, bit_pos: %d, 0x%02x -> 0x%02x\n",
 700                                 pos, bit_pos, err_byte, *(buf + byte_pos));
 701                 } else {
 702                         /* Bit flip in OOB area */
 703                         tmp = sector_num * host->pmecc_bytes_per_sector
 704                                         + (byte_pos - sector_size);
 705                         err_byte = ecc[tmp];
 706                         ecc[tmp] ^= (1 << bit_pos);
 707
 708                         pos = tmp + nand_chip->ecc.layout->eccpos[0];
 709                         dev_info(host->dev, "Bit flip in OOB, oob_byte_pos: %d, bit_pos: %d, 0x%02x -> 0x%02x\n",
 710                                 pos, bit_pos, err_byte, ecc[tmp]);
 711                 }
 712
 713                 i++;
 714                 err_nbr--;
 715         }
 716
 717         return;
 718 }
 719
 720 static int pmecc_correction(struct mtd_info *mtd, u32 pmecc_stat, uint8_t *buf,
 721         u8 *ecc)
 722 {
 723         struct nand_chip *nand_chip = mtd->priv;
 724         struct atmel_nand_host *host = nand_chip->priv;
 725         int i, err_nbr, eccbytes;
 726         uint8_t *buf_pos;
 727         int total_err = 0;
 728
 729         eccbytes = nand_chip->ecc.bytes;
 730         for (i = 0; i < eccbytes; i++)
 731                 if (ecc[i] != 0xff)
 732                         goto normal_check;
 733         /* Erased page, return OK */
 734         return 0;
 735
 736 normal_check:
 737         for (i = 0; i < host->pmecc_sector_number; i++) {
 738                 err_nbr = 0;
 739                 if (pmecc_stat & 0x1) {
 740                         buf_pos = buf + i * host->pmecc_sector_size;
 741
 742                         pmecc_gen_syndrome(mtd, i);
 743                         pmecc_substitute(mtd);
 744                         pmecc_get_sigma(mtd);
 745
 746                         err_nbr = pmecc_err_location(mtd);
 747                         if (err_nbr == -1) {
 748                                 dev_err(host->dev, "PMECC: Too many errors\n");
 749                                 mtd->ecc_stats.failed++;
 750                                 return -EIO;
 751                         } else {
 752                                 pmecc_correct_data(mtd, buf_pos, ecc, i,
 753                                         host->pmecc_bytes_per_sector, err_nbr);
 754                                 mtd->ecc_stats.corrected += err_nbr;
 755                                 total_err += err_nbr;
 756                         }
 757                 }
 758                 pmecc_stat >>= 1;
 759         }
 760
 761         return total_err;
 762 }
 763
 764 static int atmel_nand_pmecc_read_page(struct mtd_info *mtd,
 765         struct nand_chip *chip, uint8_t *buf, int oob_required, int page)
 766 {
 767         struct atmel_nand_host *host = chip->priv;
 768         int eccsize = chip->ecc.size;
 769         uint8_t *oob = chip->oob_poi;
 770         uint32_t *eccpos = chip->ecc.layout->eccpos;
 771         uint32_t stat;
 772         unsigned long end_time;
 773         int bitflips = 0;
 774
 775         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_RST);
 776         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DISABLE);
 777         pmecc_writel(host->ecc, CFG, (pmecc_readl_relaxed(host->ecc, CFG)
 778                 & ~PMECC_CFG_WRITE_OP) | PMECC_CFG_AUTO_ENABLE);
 779
 780         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_ENABLE);
 781         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DATA);
 782
 783         chip->read_buf(mtd, buf, eccsize);
 784         chip->read_buf(mtd, oob, mtd->oobsize);
 785
 786         end_time = jiffies + msecs_to_jiffies(PMECC_MAX_TIMEOUT_MS);
 787         while ((pmecc_readl_relaxed(host->ecc, SR) & PMECC_SR_BUSY)) {
 788                 if (unlikely(time_after(jiffies, end_time))) {
 789                         dev_err(host->dev, "PMECC: Timeout to get error status.\n");
 790                         return -EIO;
 791                 }
 792                 cpu_relax();
 793         }
 794
 795         stat = pmecc_readl_relaxed(host->ecc, ISR);
 796         if (stat != 0) {
 797                 bitflips = pmecc_correction(mtd, stat, buf, &oob[eccpos[0]]);
 798                 if (bitflips < 0)
 799                         /* uncorrectable errors */
 800                         return 0;
 801         }
 802
 803         return bitflips;
 804 }
 805
 806 static int atmel_nand_pmecc_write_page(struct mtd_info *mtd,
 807                 struct nand_chip *chip, const uint8_t *buf, int oob_required)
 808 {
 809         struct atmel_nand_host *host = chip->priv;
 810         uint32_t *eccpos = chip->ecc.layout->eccpos;
 811         int i, j;
 812         unsigned long end_time;
 813
 814         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_RST);
 815         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DISABLE);
 816
 817         pmecc_writel(host->ecc, CFG, (pmecc_readl_relaxed(host->ecc, CFG) |
 818                 PMECC_CFG_WRITE_OP) & ~PMECC_CFG_AUTO_ENABLE);
 819
 820         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_ENABLE);
 821         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DATA);
 822
 823         chip->write_buf(mtd, (u8 *)buf, mtd->writesize);
 824
 825         end_time = jiffies + msecs_to_jiffies(PMECC_MAX_TIMEOUT_MS);
 826         while ((pmecc_readl_relaxed(host->ecc, SR) & PMECC_SR_BUSY)) {
 827                 if (unlikely(time_after(jiffies, end_time))) {
 828                         dev_err(host->dev, "PMECC: Timeout to get ECC value.\n");
 829                         return -EIO;
 830                 }
 831                 cpu_relax();
 832         }
 833
 834         for (i = 0; i < host->pmecc_sector_number; i++) {
 835                 for (j = 0; j < host->pmecc_bytes_per_sector; j++) {
 836                         int pos;
 837
 838                         pos = i * host->pmecc_bytes_per_sector + j;
 839                         chip->oob_poi[eccpos[pos]] =
 840                                 pmecc_readb_ecc_relaxed(host->ecc, i, j);
 841                 }
 842         }
 843         chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
 844
 845         return 0;
 846 }
 847
 848 static void atmel_pmecc_core_init(struct mtd_info *mtd)
 849 {
 850         struct nand_chip *nand_chip = mtd->priv;
 851         struct atmel_nand_host *host = nand_chip->priv;
 852         uint32_t val = 0;
 853         struct nand_ecclayout *ecc_layout;
 854
 855         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_RST);
 856         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DISABLE);
 857
 858         switch (host->pmecc_corr_cap) {
 859         case 2:
 860                 val = PMECC_CFG_BCH_ERR2;
 861                 break;
 862         case 4:
 863                 val = PMECC_CFG_BCH_ERR4;
 864                 break;
 865         case 8:
 866                 val = PMECC_CFG_BCH_ERR8;
 867                 break;
 868         case 12:
 869                 val = PMECC_CFG_BCH_ERR12;
 870                 break;
 871         case 24:
 872                 val = PMECC_CFG_BCH_ERR24;
 873                 break;
 874         }
 875
 876         if (host->pmecc_sector_size == 512)
 877                 val |= PMECC_CFG_SECTOR512;
 878         else if (host->pmecc_sector_size == 1024)
 879                 val |= PMECC_CFG_SECTOR1024;
 880
 881         switch (host->pmecc_sector_number) {
 882         case 1:
 883                 val |= PMECC_CFG_PAGE_1SECTOR;
 884                 break;
 885         case 2:
 886                 val |= PMECC_CFG_PAGE_2SECTORS;
 887                 break;
 888         case 4:
 889                 val |= PMECC_CFG_PAGE_4SECTORS;
 890                 break;
 891         case 8:
 892                 val |= PMECC_CFG_PAGE_8SECTORS;
 893                 break;
 894         }
 895
 896         val |= (PMECC_CFG_READ_OP | PMECC_CFG_SPARE_DISABLE
 897                 | PMECC_CFG_AUTO_DISABLE);
 898         pmecc_writel(host->ecc, CFG, val);
 899
 900         ecc_layout = nand_chip->ecc.layout;
 901         pmecc_writel(host->ecc, SAREA, mtd->oobsize - 1);
 902         pmecc_writel(host->ecc, SADDR, ecc_layout->eccpos[0]);
 903         pmecc_writel(host->ecc, EADDR,
 904                         ecc_layout->eccpos[ecc_layout->eccbytes - 1]);
 905         /* See datasheet about PMECC Clock Control Register */
 906         pmecc_writel(host->ecc, CLK, 2);
 907         pmecc_writel(host->ecc, IDR, 0xff);
 908         pmecc_writel(host->ecc, CTRL, PMECC_CTRL_ENABLE);
 909 }
 910
 911 static int __init atmel_pmecc_nand_init_params(struct platform_device *pdev,
 912                                          struct atmel_nand_host *host)
 913 {
 914         struct mtd_info *mtd = &host->mtd;
 915         struct nand_chip *nand_chip = &host->nand_chip;
 916         struct resource *regs, *regs_pmerr, *regs_rom;
 917         int cap, sector_size, err_no;
 918
 919         cap = host->pmecc_corr_cap;
 920         sector_size = host->pmecc_sector_size;
 921         dev_info(host->dev, "Initialize PMECC params, cap: %d, sector: %d\n",
 922                  cap, sector_size);
 923
 924         regs = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 925         if (!regs) {
 926                 dev_warn(host->dev,
 927                         "Can't get I/O resource regs for PMECC controller, rolling back on software ECC\n");
 928                 nand_chip->ecc.mode = NAND_ECC_SOFT;
 929                 return 0;
 930         }
 931
 932         host->ecc = ioremap(regs->start, resource_size(regs));
 933         if (host->ecc == NULL) {
 934                 dev_err(host->dev, "ioremap failed\n");
 935                 err_no = -EIO;
 936                 goto err_pmecc_ioremap;
 937         }
 938
 939         regs_pmerr = platform_get_resource(pdev, IORESOURCE_MEM, 2);
 940         regs_rom = platform_get_resource(pdev, IORESOURCE_MEM, 3);
 941         if (regs_pmerr && regs_rom) {
 942                 host->pmerrloc_base = ioremap(regs_pmerr->start,
 943                         resource_size(regs_pmerr));
 944                 host->pmecc_rom_base = ioremap(regs_rom->start,
 945                         resource_size(regs_rom));
 946         }
 947
 948         if (!host->pmerrloc_base || !host->pmecc_rom_base) {
 949                 dev_err(host->dev,
 950                         "Can not get I/O resource for PMECC ERRLOC controller or ROM!\n");
 951                 err_no = -EIO;
 952                 goto err_pmloc_ioremap;
 953         }
 954
 955         /* ECC is calculated for the whole page (1 step) */
 956         nand_chip->ecc.size = mtd->writesize;
 957
 958         /* set ECC page size and oob layout */
 959         switch (mtd->writesize) {
 960         case 2048:
 961                 host->pmecc_degree = PMECC_GF_DIMENSION_13;
 962                 host->pmecc_cw_len = (1 << host->pmecc_degree) - 1;
 963                 host->pmecc_sector_number = mtd->writesize / sector_size;
 964                 host->pmecc_bytes_per_sector = pmecc_get_ecc_bytes(
 965                         cap, sector_size);
 966                 host->pmecc_alpha_to = pmecc_get_alpha_to(host);
 967                 host->pmecc_index_of = host->pmecc_rom_base +
 968                         host->pmecc_lookup_table_offset;
 969
 970                 nand_chip->ecc.steps = 1;
 971                 nand_chip->ecc.strength = cap;
 972                 nand_chip->ecc.bytes = host->pmecc_bytes_per_sector *
 973                                        host->pmecc_sector_number;
 974                 if (nand_chip->ecc.bytes > mtd->oobsize - 2) {
 975                         dev_err(host->dev, "No room for ECC bytes\n");
 976                         err_no = -EINVAL;
 977                         goto err_no_ecc_room;
 978                 }
 979                 pmecc_config_ecc_layout(&atmel_pmecc_oobinfo,
 980                                         mtd->oobsize,
 981                                         nand_chip->ecc.bytes);
 982                 nand_chip->ecc.layout = &atmel_pmecc_oobinfo;
 983                 break;
 984         case 512:
 985         case 1024:
 986         case 4096:
 987                 /* TODO */
 988                 dev_warn(host->dev,
 989                         "Unsupported page size for PMECC, use Software ECC\n");
 990         default:
 991                 /* page size not handled by HW ECC */
 992                 /* switching back to soft ECC */
 993                 nand_chip->ecc.mode = NAND_ECC_SOFT;
 994                 return 0;
 995         }
 996
 997         /* Allocate data for PMECC computation */
 998         err_no = pmecc_data_alloc(host);
 999         if (err_no) {
1000                 dev_err(host->dev,
1001                                 "Cannot allocate memory for PMECC computation!\n");
1002                 goto err_pmecc_data_alloc;
1003         }
1004
1005         nand_chip->ecc.read_page = atmel_nand_pmecc_read_page;
1006         nand_chip->ecc.write_page = atmel_nand_pmecc_write_page;
1007
1008         atmel_pmecc_core_init(mtd);
1009
1010         return 0;
1011
1012 err_pmecc_data_alloc:
1013 err_no_ecc_room:
1014 err_pmloc_ioremap:
1015         iounmap(host->ecc);
1016         if (host->pmerrloc_base)
1017                 iounmap(host->pmerrloc_base);
1018         if (host->pmecc_rom_base)
1019                 iounmap(host->pmecc_rom_base);
1020 err_pmecc_ioremap:
1021         return err_no;
1022 }
1023
1024 /*
1025  * Calculate HW ECC
1026  *
1027  * function called after a write
1028  *
1029  * mtd:        MTD block structure
1030  * dat:        raw data (unused)
1031  * ecc_code:   buffer for ECC
1032  */
1033 static int atmel_nand_calculate(struct mtd_info *mtd,
1034                 const u_char *dat, unsigned char *ecc_code)
1035 {
1036         struct nand_chip *nand_chip = mtd->priv;
1037         struct atmel_nand_host *host = nand_chip->priv;
1038         unsigned int ecc_value;
1039
1040         /* get the first 2 ECC bytes */
1041         ecc_value = ecc_readl(host->ecc, PR);
1042
1043         ecc_code[0] = ecc_value & 0xFF;
1044         ecc_code[1] = (ecc_value >> 8) & 0xFF;
1045
1046         /* get the last 2 ECC bytes */
1047         ecc_value = ecc_readl(host->ecc, NPR) & ATMEL_ECC_NPARITY;
1048
1049         ecc_code[2] = ecc_value & 0xFF;
1050         ecc_code[3] = (ecc_value >> 8) & 0xFF;
1051
1052         return 0;
1053 }
1054
1055 /*
1056  * HW ECC read page function
1057  *
1058  * mtd:        mtd info structure
1059  * chip:       nand chip info structure
1060  * buf:        buffer to store read data
1061  * oob_required:    caller expects OOB data read to chip->oob_poi
1062  */
1063 static int atmel_nand_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1064                                 uint8_t *buf, int oob_required, int page)
1065 {
1066         int eccsize = chip->ecc.size;
1067         int eccbytes = chip->ecc.bytes;
1068         uint32_t *eccpos = chip->ecc.layout->eccpos;
1069         uint8_t *p = buf;
1070         uint8_t *oob = chip->oob_poi;
1071         uint8_t *ecc_pos;
1072         int stat;
1073         unsigned int max_bitflips = 0;
1074
1075         /*
1076          * Errata: ALE is incorrectly wired up to the ECC controller
1077          * on the AP7000, so it will include the address cycles in the
1078          * ECC calculation.
1079          *
1080          * Workaround: Reset the parity registers before reading the
1081          * actual data.
1082          */
1083         if (cpu_is_at32ap7000()) {
1084                 struct atmel_nand_host *host = chip->priv;
1085                 ecc_writel(host->ecc, CR, ATMEL_ECC_RST);
1086         }
1087
1088         /* read the page */
1089         chip->read_buf(mtd, p, eccsize);
1090
1091         /* move to ECC position if needed */
1092         if (eccpos[0] != 0) {
1093                 /* This only works on large pages
1094                  * because the ECC controller waits for
1095                  * NAND_CMD_RNDOUTSTART after the
1096                  * NAND_CMD_RNDOUT.
1097                  * anyway, for small pages, the eccpos[0] == 0
1098                  */
1099                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT,
1100                                 mtd->writesize + eccpos[0], -1);
1101         }
1102
1103         /* the ECC controller needs to read the ECC just after the data */
1104         ecc_pos = oob + eccpos[0];
1105         chip->read_buf(mtd, ecc_pos, eccbytes);
1106
1107         /* check if there's an error */
1108         stat = chip->ecc.correct(mtd, p, oob, NULL);
1109
1110         if (stat < 0) {
1111                 mtd->ecc_stats.failed++;
1112         } else {
1113                 mtd->ecc_stats.corrected += stat;
1114                 max_bitflips = max_t(unsigned int, max_bitflips, stat);
1115         }
1116
1117         /* get back to oob start (end of page) */
1118         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, mtd->writesize, -1);
1119
1120         /* read the oob */
1121         chip->read_buf(mtd, oob, mtd->oobsize);
1122
1123         return max_bitflips;
1124 }
1125
1126 /*
1127  * HW ECC Correction
1128  *
1129  * function called after a read
1130  *
1131  * mtd:        MTD block structure
1132  * dat:        raw data read from the chip
1133  * read_ecc:   ECC from the chip (unused)
1134  * isnull:     unused
1135  *
1136  * Detect and correct a 1 bit error for a page
1137  */
1138 static int atmel_nand_correct(struct mtd_info *mtd, u_char *dat,
1139                 u_char *read_ecc, u_char *isnull)
1140 {
1141         struct nand_chip *nand_chip = mtd->priv;
1142         struct atmel_nand_host *host = nand_chip->priv;
1143         unsigned int ecc_status;
1144         unsigned int ecc_word, ecc_bit;
1145
1146         /* get the status from the Status Register */
1147         ecc_status = ecc_readl(host->ecc, SR);
1148
1149         /* if there's no error */
1150         if (likely(!(ecc_status & ATMEL_ECC_RECERR)))
1151                 return 0;
1152
1153         /* get error bit offset (4 bits) */
1154         ecc_bit = ecc_readl(host->ecc, PR) & ATMEL_ECC_BITADDR;
1155         /* get word address (12 bits) */
1156         ecc_word = ecc_readl(host->ecc, PR) & ATMEL_ECC_WORDADDR;
1157         ecc_word >>= 4;
1158
1159         /* if there are multiple errors */
1160         if (ecc_status & ATMEL_ECC_MULERR) {
1161                 /* check if it is a freshly erased block
1162                  * (filled with 0xff) */
1163                 if ((ecc_bit == ATMEL_ECC_BITADDR)
1164                                 && (ecc_word == (ATMEL_ECC_WORDADDR >> 4))) {
1165                         /* the block has just been erased, return OK */
1166                         return 0;
1167                 }
1168                 /* it doesn't seems to be a freshly
1169                  * erased block.
1170                  * We can't correct so many errors */
1171                 dev_dbg(host->dev, "atmel_nand : multiple errors detected."
1172                                 " Unable to correct.\n");
1173                 return -EIO;
1174         }
1175
1176         /* if there's a single bit error : we can correct it */
1177         if (ecc_status & ATMEL_ECC_ECCERR) {
1178                 /* there's nothing much to do here.
1179                  * the bit error is on the ECC itself.
1180                  */
1181                 dev_dbg(host->dev, "atmel_nand : one bit error on ECC code."
1182                                 " Nothing to correct\n");
1183                 return 0;
1184         }
1185
1186         dev_dbg(host->dev, "atmel_nand : one bit error on data."
1187                         " (word offset in the page :"
1188                         " 0x%x bit offset : 0x%x)\n",
1189                         ecc_word, ecc_bit);
1190         /* correct the error */
1191         if (nand_chip->options & NAND_BUSWIDTH_16) {
1192                 /* 16 bits words */
1193                 ((unsigned short *) dat)[ecc_word] ^= (1 << ecc_bit);
1194         } else {
1195                 /* 8 bits words */
1196                 dat[ecc_word] ^= (1 << ecc_bit);
1197         }
1198         dev_dbg(host->dev, "atmel_nand : error corrected\n");
1199         return 1;
1200 }
1201
1202 /*
1203  * Enable HW ECC : unused on most chips
1204  */
1205 static void atmel_nand_hwctl(struct mtd_info *mtd, int mode)
1206 {
1207         if (cpu_is_at32ap7000()) {
1208                 struct nand_chip *nand_chip = mtd->priv;
1209                 struct atmel_nand_host *host = nand_chip->priv;
1210                 ecc_writel(host->ecc, CR, ATMEL_ECC_RST);
1211         }
1212 }
1213
1214 #if defined(CONFIG_OF)
1215 static int atmel_of_init_port(struct atmel_nand_host *host,
1216                               struct device_node *np)
1217 {
1218         u32 val, table_offset;
1219         u32 offset[2];
1220         int ecc_mode;
1221         struct atmel_nand_data *board = &host->board;
1222         enum of_gpio_flags flags;
1223
1224         if (of_property_read_u32(np, "atmel,nand-addr-offset", &val) == 0) {
1225                 if (val >= 32) {
1226                         dev_err(host->dev, "invalid addr-offset %u\n", val);
1227                         return -EINVAL;
1228                 }
1229                 board->ale = val;
1230         }
1231
1232         if (of_property_read_u32(np, "atmel,nand-cmd-offset", &val) == 0) {
1233                 if (val >= 32) {
1234                         dev_err(host->dev, "invalid cmd-offset %u\n", val);
1235                         return -EINVAL;
1236                 }
1237                 board->cle = val;
1238         }
1239
1240         ecc_mode = of_get_nand_ecc_mode(np);
1241
1242         board->ecc_mode = ecc_mode < 0 ? NAND_ECC_SOFT : ecc_mode;
1243
1244         board->on_flash_bbt = of_get_nand_on_flash_bbt(np);
1245
1246         if (of_get_nand_bus_width(np) == 16)
1247                 board->bus_width_16 = 1;
1248
1249         board->rdy_pin = of_get_gpio_flags(np, 0, &flags);
1250         board->rdy_pin_active_low = (flags == OF_GPIO_ACTIVE_LOW);
1251
1252         board->enable_pin = of_get_gpio(np, 1);
1253         board->det_pin = of_get_gpio(np, 2);
1254
1255         host->has_pmecc = of_property_read_bool(np, "atmel,has-pmecc");
1256
1257         if (!(board->ecc_mode == NAND_ECC_HW) || !host->has_pmecc)
1258                 return 0;       /* Not using PMECC */
1259
1260         /* use PMECC, get correction capability, sector size and lookup
1261          * table offset.
1262          */
1263         if (of_property_read_u32(np, "atmel,pmecc-cap", &val) != 0) {
1264                 dev_err(host->dev, "Cannot decide PMECC Capability\n");
1265                 return -EINVAL;
1266         } else if ((val != 2) && (val != 4) && (val != 8) && (val != 12) &&
1267             (val != 24)) {
1268                 dev_err(host->dev,
1269                         "Unsupported PMECC correction capability: %d; should be 2, 4, 8, 12 or 24\n",
1270                         val);
1271                 return -EINVAL;
1272         }
1273         host->pmecc_corr_cap = (u8)val;
1274
1275         if (of_property_read_u32(np, "atmel,pmecc-sector-size", &val) != 0) {
1276                 dev_err(host->dev, "Cannot decide PMECC Sector Size\n");
1277                 return -EINVAL;
1278         } else if ((val != 512) && (val != 1024)) {
1279                 dev_err(host->dev,
1280                         "Unsupported PMECC sector size: %d; should be 512 or 1024 bytes\n",
1281                         val);
1282                 return -EINVAL;
1283         }
1284         host->pmecc_sector_size = (u16)val;
1285
1286         if (of_property_read_u32_array(np, "atmel,pmecc-lookup-table-offset",
1287                         offset, 2) != 0) {
1288                 dev_err(host->dev, "Cannot get PMECC lookup table offset\n");
1289                 return -EINVAL;
1290         }
1291         table_offset = host->pmecc_sector_size == 512 ? offset[0] : offset[1];
1292
1293         if (!table_offset) {
1294                 dev_err(host->dev, "Invalid PMECC lookup table offset\n");
1295                 return -EINVAL;
1296         }
1297         host->pmecc_lookup_table_offset = table_offset;
1298
1299         return 0;
1300 }
1301 #else
1302 static int atmel_of_init_port(struct atmel_nand_host *host,
1303                               struct device_node *np)
1304 {
1305         return -EINVAL;
1306 }
1307 #endif
1308
1309 static int __init atmel_hw_nand_init_params(struct platform_device *pdev,
1310                                          struct atmel_nand_host *host)
1311 {
1312         struct mtd_info *mtd = &host->mtd;
1313         struct nand_chip *nand_chip = &host->nand_chip;
1314         struct resource         *regs;
1315
1316         regs = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1317         if (!regs) {
1318                 dev_err(host->dev,
1319                         "Can't get I/O resource regs, use software ECC\n");
1320                 nand_chip->ecc.mode = NAND_ECC_SOFT;
1321                 return 0;
1322         }
1323
1324         host->ecc = ioremap(regs->start, resource_size(regs));
1325         if (host->ecc == NULL) {
1326                 dev_err(host->dev, "ioremap failed\n");
1327                 return -EIO;
1328         }
1329
1330         /* ECC is calculated for the whole page (1 step) */
1331         nand_chip->ecc.size = mtd->writesize;
1332
1333         /* set ECC page size and oob layout */
1334         switch (mtd->writesize) {
1335         case 512:
1336                 nand_chip->ecc.layout = &atmel_oobinfo_small;
1337                 ecc_writel(host->ecc, MR, ATMEL_ECC_PAGESIZE_528);
1338                 break;
1339         case 1024:
1340                 nand_chip->ecc.layout = &atmel_oobinfo_large;
1341                 ecc_writel(host->ecc, MR, ATMEL_ECC_PAGESIZE_1056);
1342                 break;
1343         case 2048:
1344                 nand_chip->ecc.layout = &atmel_oobinfo_large;
1345                 ecc_writel(host->ecc, MR, ATMEL_ECC_PAGESIZE_2112);
1346                 break;
1347         case 4096:
1348                 nand_chip->ecc.layout = &atmel_oobinfo_large;
1349                 ecc_writel(host->ecc, MR, ATMEL_ECC_PAGESIZE_4224);
1350                 break;
1351         default:
1352                 /* page size not handled by HW ECC */
1353                 /* switching back to soft ECC */
1354                 nand_chip->ecc.mode = NAND_ECC_SOFT;
1355                 return 0;
1356         }
1357
1358         /* set up for HW ECC */
1359         nand_chip->ecc.calculate = atmel_nand_calculate;
1360         nand_chip->ecc.correct = atmel_nand_correct;
1361         nand_chip->ecc.hwctl = atmel_nand_hwctl;
1362         nand_chip->ecc.read_page = atmel_nand_read_page;
1363         nand_chip->ecc.bytes = 4;
1364         nand_chip->ecc.strength = 1;
1365
1366         return 0;
1367 }
1368
1369 /*
1370  * Probe for the NAND device.
1371  */
1372 static int __init atmel_nand_probe(struct platform_device *pdev)
1373 {
1374         struct atmel_nand_host *host;
1375         struct mtd_info *mtd;
1376         struct nand_chip *nand_chip;
1377         struct resource *mem;
1378         struct mtd_part_parser_data ppdata = {};
1379         int res;
1380         struct pinctrl *pinctrl;
1381
1382         mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1383         if (!mem) {
1384                 printk(KERN_ERR "atmel_nand: can't get I/O resource mem\n");
1385                 return -ENXIO;
1386         }
1387
1388         /* Allocate memory for the device structure (and zero it) */
1389         host = kzalloc(sizeof(struct atmel_nand_host), GFP_KERNEL);
1390         if (!host) {
1391                 printk(KERN_ERR "atmel_nand: failed to allocate device structure.\n");
1392                 return -ENOMEM;
1393         }
1394
1395         host->io_phys = (dma_addr_t)mem->start;
1396
1397         host->io_base = ioremap(mem->start, resource_size(mem));
1398         if (host->io_base == NULL) {
1399                 printk(KERN_ERR "atmel_nand: ioremap failed\n");
1400                 res = -EIO;
1401                 goto err_nand_ioremap;
1402         }
1403
1404         mtd = &host->mtd;
1405         nand_chip = &host->nand_chip;
1406         host->dev = &pdev->dev;
1407         if (pdev->dev.of_node) {
1408                 res = atmel_of_init_port(host, pdev->dev.of_node);
1409                 if (res)
1410                         goto err_ecc_ioremap;
1411         } else {
1412                 memcpy(&host->board, pdev->dev.platform_data,
1413                        sizeof(struct atmel_nand_data));
1414         }
1415
1416         nand_chip->priv = host;         /* link the private data structures */
1417         mtd->priv = nand_chip;
1418         mtd->owner = THIS_MODULE;
1419
1420         /* Set address of NAND IO lines */
1421         nand_chip->IO_ADDR_R = host->io_base;
1422         nand_chip->IO_ADDR_W = host->io_base;
1423         nand_chip->cmd_ctrl = atmel_nand_cmd_ctrl;
1424
1425         pinctrl = devm_pinctrl_get_select_default(&pdev->dev);
1426         if (IS_ERR(pinctrl)) {
1427                 dev_err(host->dev, "Failed to request pinctrl\n");
1428                 res = PTR_ERR(pinctrl);
1429                 goto err_ecc_ioremap;
1430         }
1431
1432         if (gpio_is_valid(host->board.rdy_pin)) {
1433                 res = gpio_request(host->board.rdy_pin, "nand_rdy");
1434                 if (res < 0) {
1435                         dev_err(&pdev->dev,
1436                                 "can't request rdy gpio %d\n",
1437                                 host->board.rdy_pin);
1438                         goto err_ecc_ioremap;
1439                 }
1440
1441                 res = gpio_direction_input(host->board.rdy_pin);
1442                 if (res < 0) {
1443                         dev_err(&pdev->dev,
1444                                 "can't request input direction rdy gpio %d\n",
1445                                 host->board.rdy_pin);
1446                         goto err_ecc_ioremap;
1447                 }
1448
1449                 nand_chip->dev_ready = atmel_nand_device_ready;
1450         }
1451
1452         if (gpio_is_valid(host->board.enable_pin)) {
1453                 res = gpio_request(host->board.enable_pin, "nand_enable");
1454                 if (res < 0) {
1455                         dev_err(&pdev->dev,
1456                                 "can't request enable gpio %d\n",
1457                                 host->board.enable_pin);
1458                         goto err_ecc_ioremap;
1459                 }
1460
1461                 res = gpio_direction_output(host->board.enable_pin, 1);
1462                 if (res < 0) {
1463                         dev_err(&pdev->dev,
1464                                 "can't request output direction enable gpio %d\n",
1465                                 host->board.enable_pin);
1466                         goto err_ecc_ioremap;
1467                 }
1468         }
1469
1470         nand_chip->ecc.mode = host->board.ecc_mode;
1471         nand_chip->chip_delay = 20;             /* 20us command delay time */
1472
1473         if (host->board.bus_width_16)   /* 16-bit bus width */
1474                 nand_chip->options |= NAND_BUSWIDTH_16;
1475
1476         nand_chip->read_buf = atmel_read_buf;
1477         nand_chip->write_buf = atmel_write_buf;
1478
1479         platform_set_drvdata(pdev, host);
1480         atmel_nand_enable(host);
1481
1482         if (gpio_is_valid(host->board.det_pin)) {
1483                 res = gpio_request(host->board.det_pin, "nand_det");
1484                 if (res < 0) {
1485                         dev_err(&pdev->dev,
1486                                 "can't request det gpio %d\n",
1487                                 host->board.det_pin);
1488                         goto err_no_card;
1489                 }
1490
1491                 res = gpio_direction_input(host->board.det_pin);
1492                 if (res < 0) {
1493                         dev_err(&pdev->dev,
1494                                 "can't request input direction det gpio %d\n",
1495                                 host->board.det_pin);
1496                         goto err_no_card;
1497                 }
1498
1499                 if (gpio_get_value(host->board.det_pin)) {
1500                         printk(KERN_INFO "No SmartMedia card inserted.\n");
1501                         res = -ENXIO;
1502                         goto err_no_card;
1503                 }
1504         }
1505
1506         if (host->board.on_flash_bbt || on_flash_bbt) {
1507                 printk(KERN_INFO "atmel_nand: Use On Flash BBT\n");
1508                 nand_chip->bbt_options |= NAND_BBT_USE_FLASH;
1509         }
1510
1511         if (!cpu_has_dma())
1512                 use_dma = 0;
1513
1514         if (use_dma) {
1515                 dma_cap_mask_t mask;
1516
1517                 dma_cap_zero(mask);
1518                 dma_cap_set(DMA_MEMCPY, mask);
1519                 host->dma_chan = dma_request_channel(mask, NULL, NULL);
1520                 if (!host->dma_chan) {
1521                         dev_err(host->dev, "Failed to request DMA channel\n");
1522                         use_dma = 0;
1523                 }
1524         }
1525         if (use_dma)
1526                 dev_info(host->dev, "Using %s for DMA transfers.\n",
1527                                         dma_chan_name(host->dma_chan));
1528         else
1529                 dev_info(host->dev, "No DMA support for NAND access.\n");
1530
1531         /* first scan to find the device and get the page size */
1532         if (nand_scan_ident(mtd, 1, NULL)) {
1533                 res = -ENXIO;
1534                 goto err_scan_ident;
1535         }
1536
1537         if (nand_chip->ecc.mode == NAND_ECC_HW) {
1538                 if (host->has_pmecc)
1539                         res = atmel_pmecc_nand_init_params(pdev, host);
1540                 else
1541                         res = atmel_hw_nand_init_params(pdev, host);
1542
1543                 if (res != 0)
1544                         goto err_hw_ecc;
1545         }
1546
1547         /* second phase scan */
1548         if (nand_scan_tail(mtd)) {
1549                 res = -ENXIO;
1550                 goto err_scan_tail;
1551         }
1552
1553         mtd->name = "atmel_nand";
1554         ppdata.of_node = pdev->dev.of_node;
1555         res = mtd_device_parse_register(mtd, NULL, &ppdata,
1556                         host->board.parts, host->board.num_parts);
1557         if (!res)
1558                 return res;
1559
1560 err_scan_tail:
1561         if (host->has_pmecc && host->nand_chip.ecc.mode == NAND_ECC_HW) {
1562                 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DISABLE);
1563                 pmecc_data_free(host);
1564         }
1565         if (host->ecc)
1566                 iounmap(host->ecc);
1567         if (host->pmerrloc_base)
1568                 iounmap(host->pmerrloc_base);
1569         if (host->pmecc_rom_base)
1570                 iounmap(host->pmecc_rom_base);
1571 err_hw_ecc:
1572 err_scan_ident:
1573 err_no_card:
1574         atmel_nand_disable(host);
1575         platform_set_drvdata(pdev, NULL);
1576         if (host->dma_chan)
1577                 dma_release_channel(host->dma_chan);
1578 err_ecc_ioremap:
1579         iounmap(host->io_base);
1580 err_nand_ioremap:
1581         kfree(host);
1582         return res;
1583 }
1584
1585 /*
1586  * Remove a NAND device.
1587  */
1588 static int __exit atmel_nand_remove(struct platform_device *pdev)
1589 {
1590         struct atmel_nand_host *host = platform_get_drvdata(pdev);
1591         struct mtd_info *mtd = &host->mtd;
1592
1593         nand_release(mtd);
1594
1595         atmel_nand_disable(host);
1596
1597         if (host->has_pmecc && host->nand_chip.ecc.mode == NAND_ECC_HW) {
1598                 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DISABLE);
1599                 pmerrloc_writel(host->pmerrloc_base, ELDIS,
1600                                 PMERRLOC_DISABLE);
1601                 pmecc_data_free(host);
1602         }
1603
1604         if (gpio_is_valid(host->board.det_pin))
1605                 gpio_free(host->board.det_pin);
1606
1607         if (gpio_is_valid(host->board.enable_pin))
1608                 gpio_free(host->board.enable_pin);
1609
1610         if (gpio_is_valid(host->board.rdy_pin))
1611                 gpio_free(host->board.rdy_pin);
1612
1613         if (host->ecc)
1614                 iounmap(host->ecc);
1615         if (host->pmecc_rom_base)
1616                 iounmap(host->pmecc_rom_base);
1617         if (host->pmerrloc_base)
1618                 iounmap(host->pmerrloc_base);
1619
1620         if (host->dma_chan)
1621                 dma_release_channel(host->dma_chan);
1622
1623         iounmap(host->io_base);
1624         kfree(host);
1625
1626         return 0;
1627 }
1628
1629 #if defined(CONFIG_OF)
1630 static const struct of_device_id atmel_nand_dt_ids[] = {
1631         { .compatible = "atmel,at91rm9200-nand" },
1632         { /* sentinel */ }
1633 };
1634
1635 MODULE_DEVICE_TABLE(of, atmel_nand_dt_ids);
1636 #endif
1637
1638 static struct platform_driver atmel_nand_driver = {
1639         .remove         = __exit_p(atmel_nand_remove),
1640         .driver         = {
1641                 .name   = "atmel_nand",
1642                 .owner  = THIS_MODULE,
1643                 .of_match_table = of_match_ptr(atmel_nand_dt_ids),
1644         },
1645 };
1646
1647 static int __init atmel_nand_init(void)
1648 {
1649         return platform_driver_probe(&atmel_nand_driver, atmel_nand_probe);
1650 }
1651
1652
1653 static void __exit atmel_nand_exit(void)
1654 {
1655         platform_driver_unregister(&atmel_nand_driver);
1656 }
1657
1658
1659 module_init(atmel_nand_init);
1660 module_exit(atmel_nand_exit);
1661
1662 MODULE_LICENSE("GPL");
1663 MODULE_AUTHOR("Rick Bronson");
1664 MODULE_DESCRIPTION("NAND/SmartMedia driver for AT91 / AVR32");
1665 MODULE_ALIAS("platform:atmel_nand");