drivers/mtd/nand/denali.c

   1 /*
   2  * NAND Flash Controller Device Driver
   3  * Copyright © 2009-2010, Intel Corporation and its suppliers.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  */
  19
  20 #include <linux/interrupt.h>
  21 #include <linux/delay.h>
  22 #include <linux/wait.h>
  23 #include <linux/mutex.h>
  24 #include <linux/pci.h>
  25 #include <linux/mtd/mtd.h>
  26 #include <linux/module.h>
  27
  28 #include "denali.h"
  29
  30 MODULE_LICENSE("GPL");
  31
  32 /* We define a module parameter that allows the user to override
  33  * the hardware and decide what timing mode should be used.
  34  */
  35 #define NAND_DEFAULT_TIMINGS    -1
  36
  37 static int onfi_timing_mode = NAND_DEFAULT_TIMINGS;
  38 module_param(onfi_timing_mode, int, S_IRUGO);
  39 MODULE_PARM_DESC(onfi_timing_mode, "Overrides default ONFI setting. -1 indicates"
  40                                         " use default timings");
  41
  42 #define DENALI_NAND_NAME    "denali-nand"
  43
  44 /* We define a macro here that combines all interrupts this driver uses into
  45  * a single constant value, for convenience. */
  46 #define DENALI_IRQ_ALL  (INTR_STATUS0__DMA_CMD_COMP | \
  47                         INTR_STATUS0__ECC_TRANSACTION_DONE | \
  48                         INTR_STATUS0__ECC_ERR | \
  49                         INTR_STATUS0__PROGRAM_FAIL | \
  50                         INTR_STATUS0__LOAD_COMP | \
  51                         INTR_STATUS0__PROGRAM_COMP | \
  52                         INTR_STATUS0__TIME_OUT | \
  53                         INTR_STATUS0__ERASE_FAIL | \
  54                         INTR_STATUS0__RST_COMP | \
  55                         INTR_STATUS0__ERASE_COMP)
  56
  57 /* indicates whether or not the internal value for the flash bank is
  58    valid or not */
  59 #define CHIP_SELECT_INVALID     -1
  60
  61 #define SUPPORT_8BITECC         1
  62
  63 /* This macro divides two integers and rounds fractional values up
  64  * to the nearest integer value. */
  65 #define CEIL_DIV(X, Y) (((X)%(Y)) ? ((X)/(Y)+1) : ((X)/(Y)))
  66
  67 /* this macro allows us to convert from an MTD structure to our own
  68  * device context (denali) structure.
  69  */
  70 #define mtd_to_denali(m) container_of(m, struct denali_nand_info, mtd)
  71
  72 /* These constants are defined by the driver to enable common driver
  73    configuration options. */
  74 #define SPARE_ACCESS            0x41
  75 #define MAIN_ACCESS             0x42
  76 #define MAIN_SPARE_ACCESS       0x43
  77
  78 #define DENALI_READ     0
  79 #define DENALI_WRITE    0x100
  80
  81 /* types of device accesses. We can issue commands and get status */
  82 #define COMMAND_CYCLE   0
  83 #define ADDR_CYCLE      1
  84 #define STATUS_CYCLE    2
  85
  86 /* this is a helper macro that allows us to
  87  * format the bank into the proper bits for the controller */
  88 #define BANK(x) ((x) << 24)
  89
  90 /* List of platforms this NAND controller has be integrated into */
  91 static const struct pci_device_id denali_pci_ids[] = {
  92         { PCI_VDEVICE(INTEL, 0x0701), INTEL_CE4100 },
  93         { PCI_VDEVICE(INTEL, 0x0809), INTEL_MRST },
  94         { /* end: all zeroes */ }
  95 };
  96
  97
  98 /* these are static lookup tables that give us easy access to
  99    registers in the NAND controller.
 100  */
 101 static const uint32_t intr_status_addresses[4] = {INTR_STATUS0,
 102                                                   INTR_STATUS1,
 103                                                   INTR_STATUS2,
 104                                                   INTR_STATUS3};
 105
 106 static const uint32_t device_reset_banks[4] = {DEVICE_RESET__BANK0,
 107                                                DEVICE_RESET__BANK1,
 108                                                DEVICE_RESET__BANK2,
 109                                                DEVICE_RESET__BANK3};
 110
 111 static const uint32_t operation_timeout[4] = {INTR_STATUS0__TIME_OUT,
 112                                               INTR_STATUS1__TIME_OUT,
 113                                               INTR_STATUS2__TIME_OUT,
 114                                               INTR_STATUS3__TIME_OUT};
 115
 116 static const uint32_t reset_complete[4] = {INTR_STATUS0__RST_COMP,
 117                                            INTR_STATUS1__RST_COMP,
 118                                            INTR_STATUS2__RST_COMP,
 119                                            INTR_STATUS3__RST_COMP};
 120
 121 /* specifies the debug level of the driver */
 122 static int nand_debug_level = 0;
 123
 124 /* forward declarations */
 125 static void clear_interrupts(struct denali_nand_info *denali);
 126 static uint32_t wait_for_irq(struct denali_nand_info *denali, uint32_t irq_mask);
 127 static void denali_irq_enable(struct denali_nand_info *denali, uint32_t int_mask);
 128 static uint32_t read_interrupt_status(struct denali_nand_info *denali);
 129
 130 #define DEBUG_DENALI 0
 131
 132 /* This is a wrapper for writing to the denali registers.
 133  * this allows us to create debug information so we can
 134  * observe how the driver is programming the device.
 135  * it uses standard linux convention for (val, addr) */
 136 static void denali_write32(uint32_t value, void *addr)
 137 {
 138         iowrite32(value, addr);
 139
 140 #if DEBUG_DENALI
 141         printk(KERN_ERR "wrote: 0x%x -> 0x%x\n", value, (uint32_t)((uint32_t)addr & 0x1fff));
 142 #endif
 143 }
 144
 145 /* Certain operations for the denali NAND controller use an indexed mode to read/write
 146    data. The operation is performed by writing the address value of the command to
 147    the device memory followed by the data. This function abstracts this common
 148    operation.
 149 */
 150 static void index_addr(struct denali_nand_info *denali, uint32_t address, uint32_t data)
 151 {
 152         denali_write32(address, denali->flash_mem);
 153         denali_write32(data, denali->flash_mem + 0x10);
 154 }
 155
 156 /* Perform an indexed read of the device */
 157 static void index_addr_read_data(struct denali_nand_info *denali,
 158                                  uint32_t address, uint32_t *pdata)
 159 {
 160         denali_write32(address, denali->flash_mem);
 161         *pdata = ioread32(denali->flash_mem + 0x10);
 162 }
 163
 164 /* We need to buffer some data for some of the NAND core routines.
 165  * The operations manage buffering that data. */
 166 static void reset_buf(struct denali_nand_info *denali)
 167 {
 168         denali->buf.head = denali->buf.tail = 0;
 169 }
 170
 171 static void write_byte_to_buf(struct denali_nand_info *denali, uint8_t byte)
 172 {
 173         BUG_ON(denali->buf.tail >= sizeof(denali->buf.buf));
 174         denali->buf.buf[denali->buf.tail++] = byte;
 175 }
 176
 177 /* reads the status of the device */
 178 static void read_status(struct denali_nand_info *denali)
 179 {
 180         uint32_t cmd = 0x0;
 181
 182         /* initialize the data buffer to store status */
 183         reset_buf(denali);
 184
 185         /* initiate a device status read */
 186         cmd = MODE_11 | BANK(denali->flash_bank);
 187         index_addr(denali, cmd | COMMAND_CYCLE, 0x70);
 188         denali_write32(cmd | STATUS_CYCLE, denali->flash_mem);
 189
 190         /* update buffer with status value */
 191         write_byte_to_buf(denali, ioread32(denali->flash_mem + 0x10));
 192
 193 #if DEBUG_DENALI
 194         printk("device reporting status value of 0x%2x\n", denali->buf.buf[0]);
 195 #endif
 196 }
 197
 198 /* resets a specific device connected to the core */
 199 static void reset_bank(struct denali_nand_info *denali)
 200 {
 201         uint32_t irq_status = 0;
 202         uint32_t irq_mask = reset_complete[denali->flash_bank] |
 203                             operation_timeout[denali->flash_bank];
 204         int bank = 0;
 205
 206         clear_interrupts(denali);
 207
 208         bank = device_reset_banks[denali->flash_bank];
 209         denali_write32(bank, denali->flash_reg + DEVICE_RESET);
 210
 211         irq_status = wait_for_irq(denali, irq_mask);
 212
 213         if (irq_status & operation_timeout[denali->flash_bank])
 214         {
 215                 printk(KERN_ERR "reset bank failed.\n");
 216         }
 217 }
 218
 219 /* Reset the flash controller */
 220 static uint16_t NAND_Flash_Reset(struct denali_nand_info *denali)
 221 {
 222         uint32_t i;
 223
 224         nand_dbg_print(NAND_DBG_TRACE, "%s, Line %d, Function: %s\n",
 225                        __FILE__, __LINE__, __func__);
 226
 227         for (i = 0 ; i < LLD_MAX_FLASH_BANKS; i++)
 228                 denali_write32(reset_complete[i] | operation_timeout[i],
 229                 denali->flash_reg + intr_status_addresses[i]);
 230
 231         for (i = 0 ; i < LLD_MAX_FLASH_BANKS; i++) {
 232                 denali_write32(device_reset_banks[i], denali->flash_reg + DEVICE_RESET);
 233                 while (!(ioread32(denali->flash_reg + intr_status_addresses[i]) &
 234                         (reset_complete[i] | operation_timeout[i])))
 235                         ;
 236                 if (ioread32(denali->flash_reg + intr_status_addresses[i]) &
 237                         operation_timeout[i])
 238                         nand_dbg_print(NAND_DBG_WARN,
 239                         "NAND Reset operation timed out on bank %d\n", i);
 240         }
 241
 242         for (i = 0; i < LLD_MAX_FLASH_BANKS; i++)
 243                 denali_write32(reset_complete[i] | operation_timeout[i],
 244                         denali->flash_reg + intr_status_addresses[i]);
 245
 246         return PASS;
 247 }
 248
 249 /* this routine calculates the ONFI timing values for a given mode and programs
 250  * the clocking register accordingly. The mode is determined by the get_onfi_nand_para
 251    routine.
 252  */
 253 static void NAND_ONFi_Timing_Mode(struct denali_nand_info *denali, uint16_t mode)
 254 {
 255         uint16_t Trea[6] = {40, 30, 25, 20, 20, 16};
 256         uint16_t Trp[6] = {50, 25, 17, 15, 12, 10};
 257         uint16_t Treh[6] = {30, 15, 15, 10, 10, 7};
 258         uint16_t Trc[6] = {100, 50, 35, 30, 25, 20};
 259         uint16_t Trhoh[6] = {0, 15, 15, 15, 15, 15};
 260         uint16_t Trloh[6] = {0, 0, 0, 0, 5, 5};
 261         uint16_t Tcea[6] = {100, 45, 30, 25, 25, 25};
 262         uint16_t Tadl[6] = {200, 100, 100, 100, 70, 70};
 263         uint16_t Trhw[6] = {200, 100, 100, 100, 100, 100};
 264         uint16_t Trhz[6] = {200, 100, 100, 100, 100, 100};
 265         uint16_t Twhr[6] = {120, 80, 80, 60, 60, 60};
 266         uint16_t Tcs[6] = {70, 35, 25, 25, 20, 15};
 267
 268         uint16_t TclsRising = 1;
 269         uint16_t data_invalid_rhoh, data_invalid_rloh, data_invalid;
 270         uint16_t dv_window = 0;
 271         uint16_t en_lo, en_hi;
 272         uint16_t acc_clks;
 273         uint16_t addr_2_data, re_2_we, re_2_re, we_2_re, cs_cnt;
 274
 275         nand_dbg_print(NAND_DBG_TRACE, "%s, Line %d, Function: %s\n",
 276                        __FILE__, __LINE__, __func__);
 277
 278         en_lo = CEIL_DIV(Trp[mode], CLK_X);
 279         en_hi = CEIL_DIV(Treh[mode], CLK_X);
 280 #if ONFI_BLOOM_TIME
 281         if ((en_hi * CLK_X) < (Treh[mode] + 2))
 282                 en_hi++;
 283 #endif
 284
 285         if ((en_lo + en_hi) * CLK_X < Trc[mode])
 286                 en_lo += CEIL_DIV((Trc[mode] - (en_lo + en_hi) * CLK_X), CLK_X);
 287
 288         if ((en_lo + en_hi) < CLK_MULTI)
 289                 en_lo += CLK_MULTI - en_lo - en_hi;
 290
 291         while (dv_window < 8) {
 292                 data_invalid_rhoh = en_lo * CLK_X + Trhoh[mode];
 293
 294                 data_invalid_rloh = (en_lo + en_hi) * CLK_X + Trloh[mode];
 295
 296                 data_invalid =
 297                     data_invalid_rhoh <
 298                     data_invalid_rloh ? data_invalid_rhoh : data_invalid_rloh;
 299
 300                 dv_window = data_invalid - Trea[mode];
 301
 302                 if (dv_window < 8)
 303                         en_lo++;
 304         }
 305
 306         acc_clks = CEIL_DIV(Trea[mode], CLK_X);
 307
 308         while (((acc_clks * CLK_X) - Trea[mode]) < 3)
 309                 acc_clks++;
 310
 311         if ((data_invalid - acc_clks * CLK_X) < 2)
 312                 nand_dbg_print(NAND_DBG_WARN, "%s, Line %d: Warning!\n",
 313                         __FILE__, __LINE__);
 314
 315         addr_2_data = CEIL_DIV(Tadl[mode], CLK_X);
 316         re_2_we = CEIL_DIV(Trhw[mode], CLK_X);
 317         re_2_re = CEIL_DIV(Trhz[mode], CLK_X);
 318         we_2_re = CEIL_DIV(Twhr[mode], CLK_X);
 319         cs_cnt = CEIL_DIV((Tcs[mode] - Trp[mode]), CLK_X);
 320         if (!TclsRising)
 321                 cs_cnt = CEIL_DIV(Tcs[mode], CLK_X);
 322         if (cs_cnt == 0)
 323                 cs_cnt = 1;
 324
 325         if (Tcea[mode]) {
 326                 while (((cs_cnt * CLK_X) + Trea[mode]) < Tcea[mode])
 327                         cs_cnt++;
 328         }
 329
 330 #if MODE5_WORKAROUND
 331         if (mode == 5)
 332                 acc_clks = 5;
 333 #endif
 334
 335         /* Sighting 3462430: Temporary hack for MT29F128G08CJABAWP:B */
 336         if ((ioread32(denali->flash_reg + MANUFACTURER_ID) == 0) &&
 337                 (ioread32(denali->flash_reg + DEVICE_ID) == 0x88))
 338                 acc_clks = 6;
 339
 340         denali_write32(acc_clks, denali->flash_reg + ACC_CLKS);
 341         denali_write32(re_2_we, denali->flash_reg + RE_2_WE);
 342         denali_write32(re_2_re, denali->flash_reg + RE_2_RE);
 343         denali_write32(we_2_re, denali->flash_reg + WE_2_RE);
 344         denali_write32(addr_2_data, denali->flash_reg + ADDR_2_DATA);
 345         denali_write32(en_lo, denali->flash_reg + RDWR_EN_LO_CNT);
 346         denali_write32(en_hi, denali->flash_reg + RDWR_EN_HI_CNT);
 347         denali_write32(cs_cnt, denali->flash_reg + CS_SETUP_CNT);
 348 }
 349
 350 /* configures the initial ECC settings for the controller */
 351 static void set_ecc_config(struct denali_nand_info *denali)
 352 {
 353 #if SUPPORT_8BITECC
 354         if ((ioread32(denali->flash_reg + DEVICE_MAIN_AREA_SIZE) < 4096) ||
 355                 (ioread32(denali->flash_reg + DEVICE_SPARE_AREA_SIZE) <= 128))
 356                 denali_write32(8, denali->flash_reg + ECC_CORRECTION);
 357 #endif
 358
 359         if ((ioread32(denali->flash_reg + ECC_CORRECTION) & ECC_CORRECTION__VALUE)
 360                 == 1) {
 361                 denali->dev_info.wECCBytesPerSector = 4;
 362                 denali->dev_info.wECCBytesPerSector *= denali->dev_info.wDevicesConnected;
 363                 denali->dev_info.wNumPageSpareFlag =
 364                         denali->dev_info.wPageSpareSize -
 365                         denali->dev_info.wPageDataSize /
 366                         (ECC_SECTOR_SIZE * denali->dev_info.wDevicesConnected) *
 367                         denali->dev_info.wECCBytesPerSector
 368                         - denali->dev_info.wSpareSkipBytes;
 369         } else {
 370                 denali->dev_info.wECCBytesPerSector =
 371                         (ioread32(denali->flash_reg + ECC_CORRECTION) &
 372                         ECC_CORRECTION__VALUE) * 13 / 8;
 373                 if ((denali->dev_info.wECCBytesPerSector) % 2 == 0)
 374                         denali->dev_info.wECCBytesPerSector += 2;
 375                 else
 376                         denali->dev_info.wECCBytesPerSector += 1;
 377
 378                 denali->dev_info.wECCBytesPerSector *= denali->dev_info.wDevicesConnected;
 379                 denali->dev_info.wNumPageSpareFlag = denali->dev_info.wPageSpareSize -
 380                         denali->dev_info.wPageDataSize /
 381                         (ECC_SECTOR_SIZE * denali->dev_info.wDevicesConnected) *
 382                         denali->dev_info.wECCBytesPerSector
 383                         - denali->dev_info.wSpareSkipBytes;
 384         }
 385 }
 386
 387 /* queries the NAND device to see what ONFI modes it supports. */
 388 static uint16_t get_onfi_nand_para(struct denali_nand_info *denali)
 389 {
 390         int i;
 391         uint16_t blks_lun_l, blks_lun_h, n_of_luns;
 392         uint32_t blockperlun, id;
 393
 394         denali_write32(DEVICE_RESET__BANK0, denali->flash_reg + DEVICE_RESET);
 395
 396         while (!((ioread32(denali->flash_reg + INTR_STATUS0) &
 397                 INTR_STATUS0__RST_COMP) |
 398                 (ioread32(denali->flash_reg + INTR_STATUS0) &
 399                 INTR_STATUS0__TIME_OUT)))
 400                 ;
 401
 402         if (ioread32(denali->flash_reg + INTR_STATUS0) & INTR_STATUS0__RST_COMP) {
 403                 denali_write32(DEVICE_RESET__BANK1, denali->flash_reg + DEVICE_RESET);
 404                 while (!((ioread32(denali->flash_reg + INTR_STATUS1) &
 405                         INTR_STATUS1__RST_COMP) |
 406                         (ioread32(denali->flash_reg + INTR_STATUS1) &
 407                         INTR_STATUS1__TIME_OUT)))
 408                         ;
 409
 410                 if (ioread32(denali->flash_reg + INTR_STATUS1) &
 411                         INTR_STATUS1__RST_COMP) {
 412                         denali_write32(DEVICE_RESET__BANK2,
 413                                 denali->flash_reg + DEVICE_RESET);
 414                         while (!((ioread32(denali->flash_reg + INTR_STATUS2) &
 415                                 INTR_STATUS2__RST_COMP) |
 416                                 (ioread32(denali->flash_reg + INTR_STATUS2) &
 417                                 INTR_STATUS2__TIME_OUT)))
 418                                 ;
 419
 420                         if (ioread32(denali->flash_reg + INTR_STATUS2) &
 421                                 INTR_STATUS2__RST_COMP) {
 422                                 denali_write32(DEVICE_RESET__BANK3,
 423                                         denali->flash_reg + DEVICE_RESET);
 424                                 while (!((ioread32(denali->flash_reg + INTR_STATUS3) &
 425                                         INTR_STATUS3__RST_COMP) |
 426                                         (ioread32(denali->flash_reg + INTR_STATUS3) &
 427                                         INTR_STATUS3__TIME_OUT)))
 428                                         ;
 429                         } else {
 430                                 printk(KERN_ERR "Getting a time out for bank 2!\n");
 431                         }
 432                 } else {
 433                         printk(KERN_ERR "Getting a time out for bank 1!\n");
 434                 }
 435         }
 436
 437         denali_write32(INTR_STATUS0__TIME_OUT, denali->flash_reg + INTR_STATUS0);
 438         denali_write32(INTR_STATUS1__TIME_OUT, denali->flash_reg + INTR_STATUS1);
 439         denali_write32(INTR_STATUS2__TIME_OUT, denali->flash_reg + INTR_STATUS2);
 440         denali_write32(INTR_STATUS3__TIME_OUT, denali->flash_reg + INTR_STATUS3);
 441
 442         denali->dev_info.wONFIDevFeatures =
 443                 ioread32(denali->flash_reg + ONFI_DEVICE_FEATURES);
 444         denali->dev_info.wONFIOptCommands =
 445                 ioread32(denali->flash_reg + ONFI_OPTIONAL_COMMANDS);
 446         denali->dev_info.wONFITimingMode =
 447                 ioread32(denali->flash_reg + ONFI_TIMING_MODE);
 448         denali->dev_info.wONFIPgmCacheTimingMode =
 449                 ioread32(denali->flash_reg + ONFI_PGM_CACHE_TIMING_MODE);
 450
 451         n_of_luns = ioread32(denali->flash_reg + ONFI_DEVICE_NO_OF_LUNS) &
 452                 ONFI_DEVICE_NO_OF_LUNS__NO_OF_LUNS;
 453         blks_lun_l = ioread32(denali->flash_reg + ONFI_DEVICE_NO_OF_BLOCKS_PER_LUN_L);
 454         blks_lun_h = ioread32(denali->flash_reg + ONFI_DEVICE_NO_OF_BLOCKS_PER_LUN_U);
 455
 456         blockperlun = (blks_lun_h << 16) | blks_lun_l;
 457
 458         denali->dev_info.wTotalBlocks = n_of_luns * blockperlun;
 459
 460         if (!(ioread32(denali->flash_reg + ONFI_TIMING_MODE) &
 461                 ONFI_TIMING_MODE__VALUE))
 462                 return FAIL;
 463
 464         for (i = 5; i > 0; i--) {
 465                 if (ioread32(denali->flash_reg + ONFI_TIMING_MODE) & (0x01 << i))
 466                         break;
 467         }
 468
 469         NAND_ONFi_Timing_Mode(denali, i);
 470
 471         index_addr(denali, MODE_11 | 0, 0x90);
 472         index_addr(denali, MODE_11 | 1, 0);
 473
 474         for (i = 0; i < 3; i++)
 475                 index_addr_read_data(denali, MODE_11 | 2, &id);
 476
 477         nand_dbg_print(NAND_DBG_DEBUG, "3rd ID: 0x%x\n", id);
 478
 479         denali->dev_info.MLCDevice = id & 0x0C;
 480
 481         /* By now, all the ONFI devices we know support the page cache */
 482         /* rw feature. So here we enable the pipeline_rw_ahead feature */
 483         /* iowrite32(1, denali->flash_reg + CACHE_WRITE_ENABLE); */
 484         /* iowrite32(1, denali->flash_reg + CACHE_READ_ENABLE);  */
 485
 486         return PASS;
 487 }
 488
 489 static void get_samsung_nand_para(struct denali_nand_info *denali)
 490 {
 491         uint8_t no_of_planes;
 492         uint32_t blk_size;
 493         uint64_t plane_size, capacity;
 494         uint32_t id_bytes[5];
 495         int i;
 496
 497         index_addr(denali, (uint32_t)(MODE_11 | 0), 0x90);
 498         index_addr(denali, (uint32_t)(MODE_11 | 1), 0);
 499         for (i = 0; i < 5; i++)
 500                 index_addr_read_data(denali, (uint32_t)(MODE_11 | 2), &id_bytes[i]);
 501
 502         nand_dbg_print(NAND_DBG_DEBUG,
 503                 "ID bytes: 0x%x, 0x%x, 0x%x, 0x%x, 0x%x\n",
 504                 id_bytes[0], id_bytes[1], id_bytes[2],
 505                 id_bytes[3], id_bytes[4]);
 506
 507         if ((id_bytes[1] & 0xff) == 0xd3) { /* Samsung K9WAG08U1A */
 508                 /* Set timing register values according to datasheet */
 509                 denali_write32(5, denali->flash_reg + ACC_CLKS);
 510                 denali_write32(20, denali->flash_reg + RE_2_WE);
 511                 denali_write32(12, denali->flash_reg + WE_2_RE);
 512                 denali_write32(14, denali->flash_reg + ADDR_2_DATA);
 513                 denali_write32(3, denali->flash_reg + RDWR_EN_LO_CNT);
 514                 denali_write32(2, denali->flash_reg + RDWR_EN_HI_CNT);
 515                 denali_write32(2, denali->flash_reg + CS_SETUP_CNT);
 516         }
 517
 518         no_of_planes = 1 << ((id_bytes[4] & 0x0c) >> 2);
 519         plane_size  = (uint64_t)64 << ((id_bytes[4] & 0x70) >> 4);
 520         blk_size = 64 << ((ioread32(denali->flash_reg + DEVICE_PARAM_1) & 0x30) >> 4);
 521         capacity = (uint64_t)128 * plane_size * no_of_planes;
 522
 523         do_div(capacity, blk_size);
 524         denali->dev_info.wTotalBlocks = capacity;
 525 }
 526
 527 static void get_toshiba_nand_para(struct denali_nand_info *denali)
 528 {
 529         void __iomem *scratch_reg;
 530         uint32_t tmp;
 531
 532         /* Workaround to fix a controller bug which reports a wrong */
 533         /* spare area size for some kind of Toshiba NAND device */
 534         if ((ioread32(denali->flash_reg + DEVICE_MAIN_AREA_SIZE) == 4096) &&
 535                 (ioread32(denali->flash_reg + DEVICE_SPARE_AREA_SIZE) == 64)) {
 536                 denali_write32(216, denali->flash_reg + DEVICE_SPARE_AREA_SIZE);
 537                 tmp = ioread32(denali->flash_reg + DEVICES_CONNECTED) *
 538                         ioread32(denali->flash_reg + DEVICE_SPARE_AREA_SIZE);
 539                 denali_write32(tmp, denali->flash_reg + LOGICAL_PAGE_SPARE_SIZE);
 540 #if SUPPORT_15BITECC
 541                 denali_write32(15, denali->flash_reg + ECC_CORRECTION);
 542 #elif SUPPORT_8BITECC
 543                 denali_write32(8, denali->flash_reg + ECC_CORRECTION);
 544 #endif
 545         }
 546
 547         /* As Toshiba NAND can not provide it's block number, */
 548         /* so here we need user to provide the correct block */
 549         /* number in a scratch register before the Linux NAND */
 550         /* driver is loaded. If no valid value found in the scratch */
 551         /* register, then we use default block number value */
 552         scratch_reg = ioremap_nocache(SCRATCH_REG_ADDR, SCRATCH_REG_SIZE);
 553         if (!scratch_reg) {
 554                 printk(KERN_ERR "Spectra: ioremap failed in %s, Line %d",
 555                         __FILE__, __LINE__);
 556                 denali->dev_info.wTotalBlocks = GLOB_HWCTL_DEFAULT_BLKS;
 557         } else {
 558                 nand_dbg_print(NAND_DBG_WARN,
 559                         "Spectra: ioremap reg address: 0x%p\n", scratch_reg);
 560                 denali->dev_info.wTotalBlocks = 1 << ioread8(scratch_reg);
 561                 if (denali->dev_info.wTotalBlocks < 512)
 562                         denali->dev_info.wTotalBlocks = GLOB_HWCTL_DEFAULT_BLKS;
 563                 iounmap(scratch_reg);
 564         }
 565 }
 566
 567 static void get_hynix_nand_para(struct denali_nand_info *denali)
 568 {
 569         void __iomem *scratch_reg;
 570         uint32_t main_size, spare_size;
 571
 572         switch (denali->dev_info.wDeviceID) {
 573         case 0xD5: /* Hynix H27UAG8T2A, H27UBG8U5A or H27UCG8VFA */
 574         case 0xD7: /* Hynix H27UDG8VEM, H27UCG8UDM or H27UCG8V5A */
 575                 denali_write32(128, denali->flash_reg + PAGES_PER_BLOCK);
 576                 denali_write32(4096, denali->flash_reg + DEVICE_MAIN_AREA_SIZE);
 577                 denali_write32(224, denali->flash_reg + DEVICE_SPARE_AREA_SIZE);
 578                 main_size = 4096 * ioread32(denali->flash_reg + DEVICES_CONNECTED);
 579                 spare_size = 224 * ioread32(denali->flash_reg + DEVICES_CONNECTED);
 580                 denali_write32(main_size, denali->flash_reg + LOGICAL_PAGE_DATA_SIZE);
 581                 denali_write32(spare_size, denali->flash_reg + LOGICAL_PAGE_SPARE_SIZE);
 582                 denali_write32(0, denali->flash_reg + DEVICE_WIDTH);
 583 #if SUPPORT_15BITECC
 584                 denali_write32(15, denali->flash_reg + ECC_CORRECTION);
 585 #elif SUPPORT_8BITECC
 586                 denali_write32(8, denali->flash_reg + ECC_CORRECTION);
 587 #endif
 588                 denali->dev_info.MLCDevice  = 1;
 589                 break;
 590         default:
 591                 nand_dbg_print(NAND_DBG_WARN,
 592                         "Spectra: Unknown Hynix NAND (Device ID: 0x%x)."
 593                         "Will use default parameter values instead.\n",
 594                         denali->dev_info.wDeviceID);
 595         }
 596
 597         scratch_reg = ioremap_nocache(SCRATCH_REG_ADDR, SCRATCH_REG_SIZE);
 598         if (!scratch_reg) {
 599                 printk(KERN_ERR "Spectra: ioremap failed in %s, Line %d",
 600                         __FILE__, __LINE__);
 601                 denali->dev_info.wTotalBlocks = GLOB_HWCTL_DEFAULT_BLKS;
 602         } else {
 603                 nand_dbg_print(NAND_DBG_WARN,
 604                         "Spectra: ioremap reg address: 0x%p\n", scratch_reg);
 605                 denali->dev_info.wTotalBlocks = 1 << ioread8(scratch_reg);
 606                 if (denali->dev_info.wTotalBlocks < 512)
 607                         denali->dev_info.wTotalBlocks = GLOB_HWCTL_DEFAULT_BLKS;
 608                 iounmap(scratch_reg);
 609         }
 610 }
 611
 612 /* determines how many NAND chips are connected to the controller. Note for
 613    Intel CE4100 devices we don't support more than one device.
 614  */
 615 static void find_valid_banks(struct denali_nand_info *denali)
 616 {
 617         uint32_t id[LLD_MAX_FLASH_BANKS];
 618         int i;
 619
 620         denali->total_used_banks = 1;
 621         for (i = 0; i < LLD_MAX_FLASH_BANKS; i++) {
 622                 index_addr(denali, (uint32_t)(MODE_11 | (i << 24) | 0), 0x90);
 623                 index_addr(denali, (uint32_t)(MODE_11 | (i << 24) | 1), 0);
 624                 index_addr_read_data(denali, (uint32_t)(MODE_11 | (i << 24) | 2), &id[i]);
 625
 626                 nand_dbg_print(NAND_DBG_DEBUG,
 627                         "Return 1st ID for bank[%d]: %x\n", i, id[i]);
 628
 629                 if (i == 0) {
 630                         if (!(id[i] & 0x0ff))
 631                                 break; /* WTF? */
 632                 } else {
 633                         if ((id[i] & 0x0ff) == (id[0] & 0x0ff))
 634                                 denali->total_used_banks++;
 635                         else
 636                                 break;
 637                 }
 638         }
 639
 640         if (denali->platform == INTEL_CE4100)
 641         {
 642                 /* Platform limitations of the CE4100 device limit
 643                  * users to a single chip solution for NAND.
 644                  * Multichip support is not enabled.
 645                  */
 646                 if (denali->total_used_banks != 1)
 647                 {
 648                         printk(KERN_ERR "Sorry, Intel CE4100 only supports "
 649                                         "a single NAND device.\n");
 650                         BUG();
 651                 }
 652         }
 653         nand_dbg_print(NAND_DBG_DEBUG,
 654                 "denali->total_used_banks: %d\n", denali->total_used_banks);
 655 }
 656
 657 static void detect_partition_feature(struct denali_nand_info *denali)
 658 {
 659         if (ioread32(denali->flash_reg + FEATURES) & FEATURES__PARTITION) {
 660                 if ((ioread32(denali->flash_reg + PERM_SRC_ID_1) &
 661                         PERM_SRC_ID_1__SRCID) == SPECTRA_PARTITION_ID) {
 662                         denali->dev_info.wSpectraStartBlock =
 663                             ((ioread32(denali->flash_reg + MIN_MAX_BANK_1) &
 664                               MIN_MAX_BANK_1__MIN_VALUE) *
 665                              denali->dev_info.wTotalBlocks)
 666                             +
 667                             (ioread32(denali->flash_reg + MIN_BLK_ADDR_1) &
 668                             MIN_BLK_ADDR_1__VALUE);
 669
 670                         denali->dev_info.wSpectraEndBlock =
 671                             (((ioread32(denali->flash_reg + MIN_MAX_BANK_1) &
 672                                MIN_MAX_BANK_1__MAX_VALUE) >> 2) *
 673                              denali->dev_info.wTotalBlocks)
 674                             +
 675                             (ioread32(denali->flash_reg + MAX_BLK_ADDR_1) &
 676                             MAX_BLK_ADDR_1__VALUE);
 677
 678                         denali->dev_info.wTotalBlocks *= denali->total_used_banks;
 679
 680                         if (denali->dev_info.wSpectraEndBlock >=
 681                             denali->dev_info.wTotalBlocks) {
 682                                 denali->dev_info.wSpectraEndBlock =
 683                                     denali->dev_info.wTotalBlocks - 1;
 684                         }
 685
 686                         denali->dev_info.wDataBlockNum =
 687                                 denali->dev_info.wSpectraEndBlock -
 688                                 denali->dev_info.wSpectraStartBlock + 1;
 689                 } else {
 690                         denali->dev_info.wTotalBlocks *= denali->total_used_banks;
 691                         denali->dev_info.wSpectraStartBlock = SPECTRA_START_BLOCK;
 692                         denali->dev_info.wSpectraEndBlock =
 693                                 denali->dev_info.wTotalBlocks - 1;
 694                         denali->dev_info.wDataBlockNum =
 695                                 denali->dev_info.wSpectraEndBlock -
 696                                 denali->dev_info.wSpectraStartBlock + 1;
 697                 }
 698         } else {
 699                 denali->dev_info.wTotalBlocks *= denali->total_used_banks;
 700                 denali->dev_info.wSpectraStartBlock = SPECTRA_START_BLOCK;
 701                 denali->dev_info.wSpectraEndBlock = denali->dev_info.wTotalBlocks - 1;
 702                 denali->dev_info.wDataBlockNum =
 703                         denali->dev_info.wSpectraEndBlock -
 704                         denali->dev_info.wSpectraStartBlock + 1;
 705         }
 706 }
 707
 708 static void dump_device_info(struct denali_nand_info *denali)
 709 {
 710         nand_dbg_print(NAND_DBG_DEBUG, "denali->dev_info:\n");
 711         nand_dbg_print(NAND_DBG_DEBUG, "DeviceMaker: 0x%x\n",
 712                 denali->dev_info.wDeviceMaker);
 713         nand_dbg_print(NAND_DBG_DEBUG, "DeviceID: 0x%x\n",
 714                 denali->dev_info.wDeviceID);
 715         nand_dbg_print(NAND_DBG_DEBUG, "DeviceType: 0x%x\n",
 716                 denali->dev_info.wDeviceType);
 717         nand_dbg_print(NAND_DBG_DEBUG, "SpectraStartBlock: %d\n",
 718                 denali->dev_info.wSpectraStartBlock);
 719         nand_dbg_print(NAND_DBG_DEBUG, "SpectraEndBlock: %d\n",
 720                 denali->dev_info.wSpectraEndBlock);
 721         nand_dbg_print(NAND_DBG_DEBUG, "TotalBlocks: %d\n",
 722                 denali->dev_info.wTotalBlocks);
 723         nand_dbg_print(NAND_DBG_DEBUG, "PagesPerBlock: %d\n",
 724                 denali->dev_info.wPagesPerBlock);
 725         nand_dbg_print(NAND_DBG_DEBUG, "PageSize: %d\n",
 726                 denali->dev_info.wPageSize);
 727         nand_dbg_print(NAND_DBG_DEBUG, "PageDataSize: %d\n",
 728                 denali->dev_info.wPageDataSize);
 729         nand_dbg_print(NAND_DBG_DEBUG, "PageSpareSize: %d\n",
 730                 denali->dev_info.wPageSpareSize);
 731         nand_dbg_print(NAND_DBG_DEBUG, "NumPageSpareFlag: %d\n",
 732                 denali->dev_info.wNumPageSpareFlag);
 733         nand_dbg_print(NAND_DBG_DEBUG, "ECCBytesPerSector: %d\n",
 734                 denali->dev_info.wECCBytesPerSector);
 735         nand_dbg_print(NAND_DBG_DEBUG, "BlockSize: %d\n",
 736                 denali->dev_info.wBlockSize);
 737         nand_dbg_print(NAND_DBG_DEBUG, "BlockDataSize: %d\n",
 738                 denali->dev_info.wBlockDataSize);
 739         nand_dbg_print(NAND_DBG_DEBUG, "DataBlockNum: %d\n",
 740                 denali->dev_info.wDataBlockNum);
 741         nand_dbg_print(NAND_DBG_DEBUG, "PlaneNum: %d\n",
 742                 denali->dev_info.bPlaneNum);
 743         nand_dbg_print(NAND_DBG_DEBUG, "DeviceMainAreaSize: %d\n",
 744                 denali->dev_info.wDeviceMainAreaSize);
 745         nand_dbg_print(NAND_DBG_DEBUG, "DeviceSpareAreaSize: %d\n",
 746                 denali->dev_info.wDeviceSpareAreaSize);
 747         nand_dbg_print(NAND_DBG_DEBUG, "DevicesConnected: %d\n",
 748                 denali->dev_info.wDevicesConnected);
 749         nand_dbg_print(NAND_DBG_DEBUG, "DeviceWidth: %d\n",
 750                 denali->dev_info.wDeviceWidth);
 751         nand_dbg_print(NAND_DBG_DEBUG, "HWRevision: 0x%x\n",
 752                 denali->dev_info.wHWRevision);
 753         nand_dbg_print(NAND_DBG_DEBUG, "HWFeatures: 0x%x\n",
 754                 denali->dev_info.wHWFeatures);
 755         nand_dbg_print(NAND_DBG_DEBUG, "ONFIDevFeatures: 0x%x\n",
 756                 denali->dev_info.wONFIDevFeatures);
 757         nand_dbg_print(NAND_DBG_DEBUG, "ONFIOptCommands: 0x%x\n",
 758                 denali->dev_info.wONFIOptCommands);
 759         nand_dbg_print(NAND_DBG_DEBUG, "ONFITimingMode: 0x%x\n",
 760                 denali->dev_info.wONFITimingMode);
 761         nand_dbg_print(NAND_DBG_DEBUG, "ONFIPgmCacheTimingMode: 0x%x\n",
 762                 denali->dev_info.wONFIPgmCacheTimingMode);
 763         nand_dbg_print(NAND_DBG_DEBUG, "MLCDevice: %s\n",
 764                 denali->dev_info.MLCDevice ? "Yes" : "No");
 765         nand_dbg_print(NAND_DBG_DEBUG, "SpareSkipBytes: %d\n",
 766                 denali->dev_info.wSpareSkipBytes);
 767         nand_dbg_print(NAND_DBG_DEBUG, "BitsInPageNumber: %d\n",
 768                 denali->dev_info.nBitsInPageNumber);
 769         nand_dbg_print(NAND_DBG_DEBUG, "BitsInPageDataSize: %d\n",
 770                 denali->dev_info.nBitsInPageDataSize);
 771         nand_dbg_print(NAND_DBG_DEBUG, "BitsInBlockDataSize: %d\n",
 772                 denali->dev_info.nBitsInBlockDataSize);
 773 }
 774
 775 static uint16_t NAND_Read_Device_ID(struct denali_nand_info *denali)
 776 {
 777         uint16_t status = PASS;
 778         uint8_t no_of_planes;
 779
 780         nand_dbg_print(NAND_DBG_TRACE, "%s, Line %d, Function: %s\n",
 781                        __FILE__, __LINE__, __func__);
 782
 783         denali->dev_info.wDeviceMaker = ioread32(denali->flash_reg + MANUFACTURER_ID);
 784         denali->dev_info.wDeviceID = ioread32(denali->flash_reg + DEVICE_ID);
 785         denali->dev_info.bDeviceParam0 = ioread32(denali->flash_reg + DEVICE_PARAM_0);
 786         denali->dev_info.bDeviceParam1 = ioread32(denali->flash_reg + DEVICE_PARAM_1);
 787         denali->dev_info.bDeviceParam2 = ioread32(denali->flash_reg + DEVICE_PARAM_2);
 788
 789         denali->dev_info.MLCDevice = ioread32(denali->flash_reg + DEVICE_PARAM_0) & 0x0c;
 790
 791         if (ioread32(denali->flash_reg + ONFI_DEVICE_NO_OF_LUNS) &
 792                 ONFI_DEVICE_NO_OF_LUNS__ONFI_DEVICE) { /* ONFI 1.0 NAND */
 793                 if (FAIL == get_onfi_nand_para(denali))
 794                         return FAIL;
 795         } else if (denali->dev_info.wDeviceMaker == 0xEC) { /* Samsung NAND */
 796                 get_samsung_nand_para(denali);
 797         } else if (denali->dev_info.wDeviceMaker == 0x98) { /* Toshiba NAND */
 798                 get_toshiba_nand_para(denali);
 799         } else if (denali->dev_info.wDeviceMaker == 0xAD) { /* Hynix NAND */
 800                 get_hynix_nand_para(denali);
 801         } else {
 802                 denali->dev_info.wTotalBlocks = GLOB_HWCTL_DEFAULT_BLKS;
 803         }
 804
 805         nand_dbg_print(NAND_DBG_DEBUG, "Dump timing register values:"
 806                         "acc_clks: %d, re_2_we: %d, we_2_re: %d,"
 807                         "addr_2_data: %d, rdwr_en_lo_cnt: %d, "
 808                         "rdwr_en_hi_cnt: %d, cs_setup_cnt: %d\n",
 809                         ioread32(denali->flash_reg + ACC_CLKS),
 810                         ioread32(denali->flash_reg + RE_2_WE),
 811                         ioread32(denali->flash_reg + WE_2_RE),
 812                         ioread32(denali->flash_reg + ADDR_2_DATA),
 813                         ioread32(denali->flash_reg + RDWR_EN_LO_CNT),
 814                         ioread32(denali->flash_reg + RDWR_EN_HI_CNT),
 815                         ioread32(denali->flash_reg + CS_SETUP_CNT));
 816
 817         denali->dev_info.wHWRevision = ioread32(denali->flash_reg + REVISION);
 818         denali->dev_info.wHWFeatures = ioread32(denali->flash_reg + FEATURES);
 819
 820         denali->dev_info.wDeviceMainAreaSize =
 821                 ioread32(denali->flash_reg + DEVICE_MAIN_AREA_SIZE);
 822         denali->dev_info.wDeviceSpareAreaSize =
 823                 ioread32(denali->flash_reg + DEVICE_SPARE_AREA_SIZE);
 824
 825         denali->dev_info.wPageDataSize =
 826                 ioread32(denali->flash_reg + LOGICAL_PAGE_DATA_SIZE);
 827
 828         /* Note: When using the Micon 4K NAND device, the controller will report
 829          * Page Spare Size as 216 bytes. But Micron's Spec say it's 218 bytes.
 830          * And if force set it to 218 bytes, the controller can not work
 831          * correctly. So just let it be. But keep in mind that this bug may
 832          * cause
 833          * other problems in future.       - Yunpeng  2008-10-10
 834          */
 835         denali->dev_info.wPageSpareSize =
 836                 ioread32(denali->flash_reg + LOGICAL_PAGE_SPARE_SIZE);
 837
 838         denali->dev_info.wPagesPerBlock = ioread32(denali->flash_reg + PAGES_PER_BLOCK);
 839
 840         denali->dev_info.wPageSize =
 841             denali->dev_info.wPageDataSize + denali->dev_info.wPageSpareSize;
 842         denali->dev_info.wBlockSize =
 843             denali->dev_info.wPageSize * denali->dev_info.wPagesPerBlock;
 844         denali->dev_info.wBlockDataSize =
 845             denali->dev_info.wPagesPerBlock * denali->dev_info.wPageDataSize;
 846
 847         denali->dev_info.wDeviceWidth = ioread32(denali->flash_reg + DEVICE_WIDTH);
 848         denali->dev_info.wDeviceType =
 849                 ((ioread32(denali->flash_reg + DEVICE_WIDTH) > 0) ? 16 : 8);
 850
 851         denali->dev_info.wDevicesConnected = ioread32(denali->flash_reg + DEVICES_CONNECTED);
 852
 853         denali->dev_info.wSpareSkipBytes =
 854                 ioread32(denali->flash_reg + SPARE_AREA_SKIP_BYTES) *
 855                 denali->dev_info.wDevicesConnected;
 856
 857         denali->dev_info.nBitsInPageNumber =
 858                 ilog2(denali->dev_info.wPagesPerBlock);
 859         denali->dev_info.nBitsInPageDataSize =
 860                 ilog2(denali->dev_info.wPageDataSize);
 861         denali->dev_info.nBitsInBlockDataSize =
 862                 ilog2(denali->dev_info.wBlockDataSize);
 863
 864         set_ecc_config(denali);
 865
 866         no_of_planes = ioread32(denali->flash_reg + NUMBER_OF_PLANES) &
 867                 NUMBER_OF_PLANES__VALUE;
 868
 869         switch (no_of_planes) {
 870         case 0:
 871         case 1:
 872         case 3:
 873         case 7:
 874                 denali->dev_info.bPlaneNum = no_of_planes + 1;
 875                 break;
 876         default:
 877                 status = FAIL;
 878                 break;
 879         }
 880
 881         find_valid_banks(denali);
 882
 883         detect_partition_feature(denali);
 884
 885         dump_device_info(denali);
 886
 887         /* If the user specified to override the default timings
 888          * with a specific ONFI mode, we apply those changes here.
 889          */
 890         if (onfi_timing_mode != NAND_DEFAULT_TIMINGS)
 891         {
 892                 NAND_ONFi_Timing_Mode(denali, onfi_timing_mode);
 893         }
 894
 895         return status;
 896 }
 897
 898 static void NAND_LLD_Enable_Disable_Interrupts(struct denali_nand_info *denali,
 899                                         uint16_t INT_ENABLE)
 900 {
 901         nand_dbg_print(NAND_DBG_TRACE, "%s, Line %d, Function: %s\n",
 902                        __FILE__, __LINE__, __func__);
 903
 904         if (INT_ENABLE)
 905                 denali_write32(1, denali->flash_reg + GLOBAL_INT_ENABLE);
 906         else
 907                 denali_write32(0, denali->flash_reg + GLOBAL_INT_ENABLE);
 908 }
 909
 910 /* validation function to verify that the controlling software is making
 911    a valid request
 912  */
 913 static inline bool is_flash_bank_valid(int flash_bank)
 914 {
 915         return (flash_bank >= 0 && flash_bank < 4);
 916 }
 917
 918 static void denali_irq_init(struct denali_nand_info *denali)
 919 {
 920         uint32_t int_mask = 0;
 921
 922         /* Disable global interrupts */
 923         NAND_LLD_Enable_Disable_Interrupts(denali, false);
 924
 925         int_mask = DENALI_IRQ_ALL;
 926
 927         /* Clear all status bits */
 928         denali_write32(0xFFFF, denali->flash_reg + INTR_STATUS0);
 929         denali_write32(0xFFFF, denali->flash_reg + INTR_STATUS1);
 930         denali_write32(0xFFFF, denali->flash_reg + INTR_STATUS2);
 931         denali_write32(0xFFFF, denali->flash_reg + INTR_STATUS3);
 932
 933         denali_irq_enable(denali, int_mask);
 934 }
 935
 936 static void denali_irq_cleanup(int irqnum, struct denali_nand_info *denali)
 937 {
 938         NAND_LLD_Enable_Disable_Interrupts(denali, false);
 939         free_irq(irqnum, denali);
 940 }
 941
 942 static void denali_irq_enable(struct denali_nand_info *denali, uint32_t int_mask)
 943 {
 944         denali_write32(int_mask, denali->flash_reg + INTR_EN0);
 945         denali_write32(int_mask, denali->flash_reg + INTR_EN1);
 946         denali_write32(int_mask, denali->flash_reg + INTR_EN2);
 947         denali_write32(int_mask, denali->flash_reg + INTR_EN3);
 948 }
 949
 950 /* This function only returns when an interrupt that this driver cares about
 951  * occurs. This is to reduce the overhead of servicing interrupts
 952  */
 953 static inline uint32_t denali_irq_detected(struct denali_nand_info *denali)
 954 {
 955         return (read_interrupt_status(denali) & DENALI_IRQ_ALL);
 956 }
 957
 958 /* Interrupts are cleared by writing a 1 to the appropriate status bit */
 959 static inline void clear_interrupt(struct denali_nand_info *denali, uint32_t irq_mask)
 960 {
 961         uint32_t intr_status_reg = 0;
 962
 963         intr_status_reg = intr_status_addresses[denali->flash_bank];
 964
 965         denali_write32(irq_mask, denali->flash_reg + intr_status_reg);
 966 }
 967
 968 static void clear_interrupts(struct denali_nand_info *denali)
 969 {
 970         uint32_t status = 0x0;
 971         spin_lock_irq(&denali->irq_lock);
 972
 973         status = read_interrupt_status(denali);
 974
 975 #if DEBUG_DENALI
 976         denali->irq_debug_array[denali->idx++] = 0x30000000 | status;
 977         denali->idx %= 32;
 978 #endif
 979
 980         denali->irq_status = 0x0;
 981         spin_unlock_irq(&denali->irq_lock);
 982 }
 983
 984 static uint32_t read_interrupt_status(struct denali_nand_info *denali)
 985 {
 986         uint32_t intr_status_reg = 0;
 987
 988         intr_status_reg = intr_status_addresses[denali->flash_bank];
 989
 990         return ioread32(denali->flash_reg + intr_status_reg);
 991 }
 992
 993 #if DEBUG_DENALI
 994 static void print_irq_log(struct denali_nand_info *denali)
 995 {
 996         int i = 0;
 997
 998         printk("ISR debug log index = %X\n", denali->idx);
 999         for (i = 0; i < 32; i++)
1000         {
1001                 printk("%08X: %08X\n", i, denali->irq_debug_array[i]);
1002         }
1003 }
1004 #endif
1005
1006 /* This is the interrupt service routine. It handles all interrupts
1007  * sent to this device. Note that on CE4100, this is a shared
1008  * interrupt.
1009  */
1010 static irqreturn_t denali_isr(int irq, void *dev_id)
1011 {
1012         struct denali_nand_info *denali = dev_id;
1013         uint32_t irq_status = 0x0;
1014         irqreturn_t result = IRQ_NONE;
1015
1016         spin_lock(&denali->irq_lock);
1017
1018         /* check to see if a valid NAND chip has
1019          * been selected.
1020          */
1021         if (is_flash_bank_valid(denali->flash_bank))
1022         {
1023                 /* check to see if controller generated
1024                  * the interrupt, since this is a shared interrupt */
1025                 if ((irq_status = denali_irq_detected(denali)) != 0)
1026                 {
1027 #if DEBUG_DENALI
1028                         denali->irq_debug_array[denali->idx++] = 0x10000000 | irq_status;
1029                         denali->idx %= 32;
1030
1031                         printk("IRQ status = 0x%04x\n", irq_status);
1032 #endif
1033                         /* handle interrupt */
1034                         /* first acknowledge it */
1035                         clear_interrupt(denali, irq_status);
1036                         /* store the status in the device context for someone
1037                            to read */
1038                         denali->irq_status |= irq_status;
1039                         /* notify anyone who cares that it happened */
1040                         complete(&denali->complete);
1041                         /* tell the OS that we've handled this */
1042                         result = IRQ_HANDLED;
1043                 }
1044         }
1045         spin_unlock(&denali->irq_lock);
1046         return result;
1047 }
1048 #define BANK(x) ((x) << 24)
1049
1050 static uint32_t wait_for_irq(struct denali_nand_info *denali, uint32_t irq_mask)
1051 {
1052         unsigned long comp_res = 0;
1053         uint32_t intr_status = 0;
1054         bool retry = false;
1055         unsigned long timeout = msecs_to_jiffies(1000);
1056
1057         do
1058         {
1059 #if DEBUG_DENALI
1060                 printk("waiting for 0x%x\n", irq_mask);
1061 #endif
1062                 comp_res = wait_for_completion_timeout(&denali->complete, timeout);
1063                 spin_lock_irq(&denali->irq_lock);
1064                 intr_status = denali->irq_status;
1065
1066 #if DEBUG_DENALI
1067                 denali->irq_debug_array[denali->idx++] = 0x20000000 | (irq_mask << 16) | intr_status;
1068                 denali->idx %= 32;
1069 #endif
1070
1071                 if (intr_status & irq_mask)
1072                 {
1073                         denali->irq_status &= ~irq_mask;
1074                         spin_unlock_irq(&denali->irq_lock);
1075 #if DEBUG_DENALI
1076                         if (retry) printk("status on retry = 0x%x\n", intr_status);
1077 #endif
1078                         /* our interrupt was detected */
1079                         break;
1080                 }
1081                 else
1082                 {
1083                         /* these are not the interrupts you are looking for -
1084                            need to wait again */
1085                         spin_unlock_irq(&denali->irq_lock);
1086 #if DEBUG_DENALI
1087                         print_irq_log(denali);
1088                         printk("received irq nobody cared: irq_status = 0x%x,"
1089                                 " irq_mask = 0x%x, timeout = %ld\n", intr_status, irq_mask, comp_res);
1090 #endif
1091                         retry = true;
1092                 }
1093         } while (comp_res != 0);
1094
1095         if (comp_res == 0)
1096         {
1097                 /* timeout */
1098                 printk(KERN_ERR "timeout occurred, status = 0x%x, mask = 0x%x\n",
1099                                 intr_status, irq_mask);
1100
1101                 intr_status = 0;
1102         }
1103         return intr_status;
1104 }
1105
1106 /* This helper function setups the registers for ECC and whether or not
1107    the spare area will be transfered. */
1108 static void setup_ecc_for_xfer(struct denali_nand_info *denali, bool ecc_en,
1109                                 bool transfer_spare)
1110 {
1111         int ecc_en_flag = 0, transfer_spare_flag = 0;
1112
1113         /* set ECC, transfer spare bits if needed */
1114         ecc_en_flag = ecc_en ? ECC_ENABLE__FLAG : 0;
1115         transfer_spare_flag = transfer_spare ? TRANSFER_SPARE_REG__FLAG : 0;
1116
1117         /* Enable spare area/ECC per user's request. */
1118         denali_write32(ecc_en_flag, denali->flash_reg + ECC_ENABLE);
1119         denali_write32(transfer_spare_flag, denali->flash_reg + TRANSFER_SPARE_REG);
1120 }
1121
1122 /* sends a pipeline command operation to the controller. See the Denali NAND
1123    controller's user guide for more information (section 4.2.3.6).
1124  */
1125 static int denali_send_pipeline_cmd(struct denali_nand_info *denali, bool ecc_en,
1126                                         bool transfer_spare, int access_type,
1127                                         int op)
1128 {
1129         int status = PASS;
1130         uint32_t addr = 0x0, cmd = 0x0, page_count = 1, irq_status = 0,
1131                  irq_mask = 0;
1132
1133         if (op == DENALI_READ) irq_mask = INTR_STATUS0__LOAD_COMP;
1134         else if (op == DENALI_WRITE) irq_mask = 0;
1135         else BUG();
1136
1137         setup_ecc_for_xfer(denali, ecc_en, transfer_spare);
1138
1139 #if DEBUG_DENALI
1140         spin_lock_irq(&denali->irq_lock);
1141         denali->irq_debug_array[denali->idx++] = 0x40000000 | ioread32(denali->flash_reg + ECC_ENABLE) | (access_type << 4);
1142         denali->idx %= 32;
1143         spin_unlock_irq(&denali->irq_lock);
1144 #endif
1145
1146
1147         /* clear interrupts */
1148         clear_interrupts(denali);
1149
1150         addr = BANK(denali->flash_bank) | denali->page;
1151
1152         if (op == DENALI_WRITE && access_type != SPARE_ACCESS)
1153         {
1154                 cmd = MODE_01 | addr;
1155                 denali_write32(cmd, denali->flash_mem);
1156         }
1157         else if (op == DENALI_WRITE && access_type == SPARE_ACCESS)
1158         {
1159                 /* read spare area */
1160                 cmd = MODE_10 | addr;
1161                 index_addr(denali, (uint32_t)cmd, access_type);
1162
1163                 cmd = MODE_01 | addr;
1164                 denali_write32(cmd, denali->flash_mem);
1165         }
1166         else if (op == DENALI_READ)
1167         {
1168                 /* setup page read request for access type */
1169                 cmd = MODE_10 | addr;
1170                 index_addr(denali, (uint32_t)cmd, access_type);
1171
1172                 /* page 33 of the NAND controller spec indicates we should not
1173                    use the pipeline commands in Spare area only mode. So we
1174                    don't.
1175                  */
1176                 if (access_type == SPARE_ACCESS)
1177                 {
1178                         cmd = MODE_01 | addr;
1179                         denali_write32(cmd, denali->flash_mem);
1180                 }
1181                 else
1182                 {
1183                         index_addr(denali, (uint32_t)cmd, 0x2000 | op | page_count);
1184
1185                         /* wait for command to be accepted
1186                          * can always use status0 bit as the mask is identical for each
1187                          * bank. */
1188                         irq_status = wait_for_irq(denali, irq_mask);
1189
1190                         if (irq_status == 0)
1191                         {
1192                                 printk(KERN_ERR "cmd, page, addr on timeout "
1193                                         "(0x%x, 0x%x, 0x%x)\n", cmd, denali->page, addr);
1194                                 status = FAIL;
1195                         }
1196                         else
1197                         {
1198                                 cmd = MODE_01 | addr;
1199                                 denali_write32(cmd, denali->flash_mem);
1200                         }
1201                 }
1202         }
1203         return status;
1204 }
1205
1206 /* helper function that simply writes a buffer to the flash */
1207 static int write_data_to_flash_mem(struct denali_nand_info *denali, const uint8_t *buf,
1208                                         int len)
1209 {
1210         uint32_t i = 0, *buf32;
1211
1212         /* verify that the len is a multiple of 4. see comment in
1213          * read_data_from_flash_mem() */
1214         BUG_ON((len % 4) != 0);
1215
1216         /* write the data to the flash memory */
1217         buf32 = (uint32_t *)buf;
1218         for (i = 0; i < len / 4; i++)
1219         {
1220                 denali_write32(*buf32++, denali->flash_mem + 0x10);
1221         }
1222         return i*4; /* intent is to return the number of bytes read */
1223 }
1224
1225 /* helper function that simply reads a buffer from the flash */
1226 static int read_data_from_flash_mem(struct denali_nand_info *denali, uint8_t *buf,
1227                                         int len)
1228 {
1229         uint32_t i = 0, *buf32;
1230
1231         /* we assume that len will be a multiple of 4, if not
1232          * it would be nice to know about it ASAP rather than
1233          * have random failures...
1234          *
1235          * This assumption is based on the fact that this
1236          * function is designed to be used to read flash pages,
1237          * which are typically multiples of 4...
1238          */
1239
1240         BUG_ON((len % 4) != 0);
1241
1242         /* transfer the data from the flash */
1243         buf32 = (uint32_t *)buf;
1244         for (i = 0; i < len / 4; i++)
1245         {
1246                 *buf32++ = ioread32(denali->flash_mem + 0x10);
1247         }
1248         return i*4; /* intent is to return the number of bytes read */
1249 }
1250
1251 /* writes OOB data to the device */
1252 static int write_oob_data(struct mtd_info *mtd, uint8_t *buf, int page)
1253 {
1254         struct denali_nand_info *denali = mtd_to_denali(mtd);
1255         uint32_t irq_status = 0;
1256         uint32_t irq_mask = INTR_STATUS0__PROGRAM_COMP |
1257                                                 INTR_STATUS0__PROGRAM_FAIL;
1258         int status = 0;
1259
1260         denali->page = page;
1261
1262         if (denali_send_pipeline_cmd(denali, false, false, SPARE_ACCESS,
1263                                                         DENALI_WRITE) == PASS)
1264         {
1265                 write_data_to_flash_mem(denali, buf, mtd->oobsize);
1266
1267 #if DEBUG_DENALI
1268                 spin_lock_irq(&denali->irq_lock);
1269                 denali->irq_debug_array[denali->idx++] = 0x80000000 | mtd->oobsize;
1270                 denali->idx %= 32;
1271                 spin_unlock_irq(&denali->irq_lock);
1272 #endif
1273
1274
1275                 /* wait for operation to complete */
1276                 irq_status = wait_for_irq(denali, irq_mask);
1277
1278                 if (irq_status == 0)
1279                 {
1280                         printk(KERN_ERR "OOB write failed\n");
1281                         status = -EIO;
1282                 }
1283         }
1284         else
1285         {
1286                 printk(KERN_ERR "unable to send pipeline command\n");
1287                 status = -EIO;
1288         }
1289         return status;
1290 }
1291
1292 /* reads OOB data from the device */
1293 static void read_oob_data(struct mtd_info *mtd, uint8_t *buf, int page)
1294 {
1295         struct denali_nand_info *denali = mtd_to_denali(mtd);
1296         uint32_t irq_mask = INTR_STATUS0__LOAD_COMP, irq_status = 0, addr = 0x0, cmd = 0x0;
1297
1298         denali->page = page;
1299
1300 #if DEBUG_DENALI
1301         printk("read_oob %d\n", page);
1302 #endif
1303         if (denali_send_pipeline_cmd(denali, false, true, SPARE_ACCESS,
1304                                                         DENALI_READ) == PASS)
1305         {
1306                 read_data_from_flash_mem(denali, buf, mtd->oobsize);
1307
1308                 /* wait for command to be accepted
1309                  * can always use status0 bit as the mask is identical for each
1310                  * bank. */
1311                 irq_status = wait_for_irq(denali, irq_mask);
1312
1313                 if (irq_status == 0)
1314                 {
1315                         printk(KERN_ERR "page on OOB timeout %d\n", denali->page);
1316                 }
1317
1318                 /* We set the device back to MAIN_ACCESS here as I observed
1319                  * instability with the controller if you do a block erase
1320                  * and the last transaction was a SPARE_ACCESS. Block erase
1321                  * is reliable (according to the MTD test infrastructure)
1322                  * if you are in MAIN_ACCESS.
1323                  */
1324                 addr = BANK(denali->flash_bank) | denali->page;
1325                 cmd = MODE_10 | addr;
1326                 index_addr(denali, (uint32_t)cmd, MAIN_ACCESS);
1327
1328 #if DEBUG_DENALI
1329                 spin_lock_irq(&denali->irq_lock);
1330                 denali->irq_debug_array[denali->idx++] = 0x60000000 | mtd->oobsize;
1331                 denali->idx %= 32;
1332                 spin_unlock_irq(&denali->irq_lock);
1333 #endif
1334         }
1335 }
1336
1337 /* this function examines buffers to see if they contain data that
1338  * indicate that the buffer is part of an erased region of flash.
1339  */
1340 bool is_erased(uint8_t *buf, int len)
1341 {
1342         int i = 0;
1343         for (i = 0; i < len; i++)
1344         {
1345                 if (buf[i] != 0xFF)
1346                 {
1347                         return false;
1348                 }
1349         }
1350         return true;
1351 }
1352 #define ECC_SECTOR_SIZE 512
1353
1354 #define ECC_SECTOR(x)   (((x) & ECC_ERROR_ADDRESS__SECTOR_NR) >> 12)
1355 #define ECC_BYTE(x)     (((x) & ECC_ERROR_ADDRESS__OFFSET))
1356 #define ECC_CORRECTION_VALUE(x) ((x) & ERR_CORRECTION_INFO__BYTEMASK)
1357 #define ECC_ERROR_CORRECTABLE(x) (!((x) & ERR_CORRECTION_INFO))
1358 #define ECC_ERR_DEVICE(x)       ((x) & ERR_CORRECTION_INFO__DEVICE_NR >> 8)
1359 #define ECC_LAST_ERR(x)         ((x) & ERR_CORRECTION_INFO__LAST_ERR_INFO)
1360
1361 static bool handle_ecc(struct denali_nand_info *denali, uint8_t *buf,
1362                         uint8_t *oobbuf, uint32_t irq_status)
1363 {
1364         bool check_erased_page = false;
1365
1366         if (irq_status & INTR_STATUS0__ECC_ERR)
1367         {
1368                 /* read the ECC errors. we'll ignore them for now */
1369                 uint32_t err_address = 0, err_correction_info = 0;
1370                 uint32_t err_byte = 0, err_sector = 0, err_device = 0;
1371                 uint32_t err_correction_value = 0;
1372
1373                 do
1374                 {
1375                         err_address = ioread32(denali->flash_reg +
1376                                                 ECC_ERROR_ADDRESS);
1377                         err_sector = ECC_SECTOR(err_address);
1378                         err_byte = ECC_BYTE(err_address);
1379
1380
1381                         err_correction_info = ioread32(denali->flash_reg +
1382                                                 ERR_CORRECTION_INFO);
1383                         err_correction_value =
1384                                 ECC_CORRECTION_VALUE(err_correction_info);
1385                         err_device = ECC_ERR_DEVICE(err_correction_info);
1386
1387                         if (ECC_ERROR_CORRECTABLE(err_correction_info))
1388                         {
1389                                 /* offset in our buffer is computed as:
1390                                    sector number * sector size + offset in
1391                                    sector
1392                                  */
1393                                 int offset = err_sector * ECC_SECTOR_SIZE +
1394                                                                 err_byte;
1395                                 if (offset < denali->mtd.writesize)
1396                                 {
1397                                         /* correct the ECC error */
1398                                         buf[offset] ^= err_correction_value;
1399                                         denali->mtd.ecc_stats.corrected++;
1400                                 }
1401                                 else
1402                                 {
1403                                         /* bummer, couldn't correct the error */
1404                                         printk(KERN_ERR "ECC offset invalid\n");
1405                                         denali->mtd.ecc_stats.failed++;
1406                                 }
1407                         }
1408                         else
1409                         {
1410                                 /* if the error is not correctable, need to
1411                                  * look at the page to see if it is an erased page.
1412                                  * if so, then it's not a real ECC error */
1413                                 check_erased_page = true;
1414                         }
1415
1416 #if DEBUG_DENALI
1417                         printk("Detected ECC error in page %d: err_addr = 0x%08x,"
1418                                 " info to fix is 0x%08x\n", denali->page, err_address,
1419                                 err_correction_info);
1420 #endif
1421                 } while (!ECC_LAST_ERR(err_correction_info));
1422         }
1423         return check_erased_page;
1424 }
1425
1426 /* programs the controller to either enable/disable DMA transfers */
1427 static void denali_enable_dma(struct denali_nand_info *denali, bool en)
1428 {
1429         uint32_t reg_val = 0x0;
1430
1431         if (en) reg_val = DMA_ENABLE__FLAG;
1432
1433         denali_write32(reg_val, denali->flash_reg + DMA_ENABLE);
1434         ioread32(denali->flash_reg + DMA_ENABLE);
1435 }
1436
1437 /* setups the HW to perform the data DMA */
1438 static void denali_setup_dma(struct denali_nand_info *denali, int op)
1439 {
1440         uint32_t mode = 0x0;
1441         const int page_count = 1;
1442         dma_addr_t addr = denali->buf.dma_buf;
1443
1444         mode = MODE_10 | BANK(denali->flash_bank);
1445
1446         /* DMA is a four step process */
1447
1448         /* 1. setup transfer type and # of pages */
1449         index_addr(denali, mode | denali->page, 0x2000 | op | page_count);
1450
1451         /* 2. set memory high address bits 23:8 */
1452         index_addr(denali, mode | ((uint16_t)(addr >> 16) << 8), 0x2200);
1453
1454         /* 3. set memory low address bits 23:8 */
1455         index_addr(denali, mode | ((uint16_t)addr << 8), 0x2300);
1456
1457         /* 4.  interrupt when complete, burst len = 64 bytes*/
1458         index_addr(denali, mode | 0x14000, 0x2400);
1459 }
1460
1461 /* writes a page. user specifies type, and this function handles the
1462    configuration details. */
1463 static void write_page(struct mtd_info *mtd, struct nand_chip *chip,
1464                         const uint8_t *buf, bool raw_xfer)
1465 {
1466         struct denali_nand_info *denali = mtd_to_denali(mtd);
1467         struct pci_dev *pci_dev = denali->dev;
1468
1469         dma_addr_t addr = denali->buf.dma_buf;
1470         size_t size = denali->mtd.writesize + denali->mtd.oobsize;
1471
1472         uint32_t irq_status = 0;
1473         uint32_t irq_mask = INTR_STATUS0__DMA_CMD_COMP |
1474                                                 INTR_STATUS0__PROGRAM_FAIL;
1475
1476         /* if it is a raw xfer, we want to disable ecc, and send
1477          * the spare area.
1478          * !raw_xfer - enable ecc
1479          * raw_xfer - transfer spare
1480          */
1481         setup_ecc_for_xfer(denali, !raw_xfer, raw_xfer);
1482
1483         /* copy buffer into DMA buffer */
1484         memcpy(denali->buf.buf, buf, mtd->writesize);
1485
1486         if (raw_xfer)
1487         {
1488                 /* transfer the data to the spare area */
1489                 memcpy(denali->buf.buf + mtd->writesize,
1490                         chip->oob_poi,
1491                         mtd->oobsize);
1492         }
1493
1494         pci_dma_sync_single_for_device(pci_dev, addr, size, PCI_DMA_TODEVICE);
1495
1496         clear_interrupts(denali);
1497         denali_enable_dma(denali, true);
1498
1499         denali_setup_dma(denali, DENALI_WRITE);
1500
1501         /* wait for operation to complete */
1502         irq_status = wait_for_irq(denali, irq_mask);
1503
1504         if (irq_status == 0)
1505         {
1506                 printk(KERN_ERR "timeout on write_page (type = %d)\n", raw_xfer);
1507                 denali->status =
1508                    (irq_status & INTR_STATUS0__PROGRAM_FAIL) ? NAND_STATUS_FAIL :
1509                                                              PASS;
1510         }
1511
1512         denali_enable_dma(denali, false);
1513         pci_dma_sync_single_for_cpu(pci_dev, addr, size, PCI_DMA_TODEVICE);
1514 }
1515
1516 /* NAND core entry points */
1517
1518 /* this is the callback that the NAND core calls to write a page. Since
1519    writing a page with ECC or without is similar, all the work is done
1520    by write_page above.   */
1521 static void denali_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1522                                 const uint8_t *buf)
1523 {
1524         /* for regular page writes, we let HW handle all the ECC
1525          * data written to the device. */
1526         write_page(mtd, chip, buf, false);
1527 }
1528
1529 /* This is the callback that the NAND core calls to write a page without ECC.
1530    raw access is similiar to ECC page writes, so all the work is done in the
1531    write_page() function above.
1532  */
1533 static void denali_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
1534                                         const uint8_t *buf)
1535 {
1536         /* for raw page writes, we want to disable ECC and simply write
1537            whatever data is in the buffer. */
1538         write_page(mtd, chip, buf, true);
1539 }
1540
1541 static int denali_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
1542                             int page)
1543 {
1544         return write_oob_data(mtd, chip->oob_poi, page);
1545 }
1546
1547 static int denali_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1548                            int page, int sndcmd)
1549 {
1550         read_oob_data(mtd, chip->oob_poi, page);
1551
1552         return 0; /* notify NAND core to send command to
1553                    * NAND device. */
1554 }
1555
1556 static int denali_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1557                             uint8_t *buf, int page)
1558 {
1559         struct denali_nand_info *denali = mtd_to_denali(mtd);
1560         struct pci_dev *pci_dev = denali->dev;
1561
1562         dma_addr_t addr = denali->buf.dma_buf;
1563         size_t size = denali->mtd.writesize + denali->mtd.oobsize;
1564
1565         uint32_t irq_status = 0;
1566         uint32_t irq_mask = INTR_STATUS0__ECC_TRANSACTION_DONE |
1567                             INTR_STATUS0__ECC_ERR;
1568         bool check_erased_page = false;
1569
1570         setup_ecc_for_xfer(denali, true, false);
1571
1572         denali_enable_dma(denali, true);
1573         pci_dma_sync_single_for_device(pci_dev, addr, size, PCI_DMA_FROMDEVICE);
1574
1575         clear_interrupts(denali);
1576         denali_setup_dma(denali, DENALI_READ);
1577
1578         /* wait for operation to complete */
1579         irq_status = wait_for_irq(denali, irq_mask);
1580
1581         pci_dma_sync_single_for_cpu(pci_dev, addr, size, PCI_DMA_FROMDEVICE);
1582
1583         memcpy(buf, denali->buf.buf, mtd->writesize);
1584
1585         check_erased_page = handle_ecc(denali, buf, chip->oob_poi, irq_status);
1586         denali_enable_dma(denali, false);
1587
1588         if (check_erased_page)
1589         {
1590                 read_oob_data(&denali->mtd, chip->oob_poi, denali->page);
1591
1592                 /* check ECC failures that may have occurred on erased pages */
1593                 if (check_erased_page)
1594                 {
1595                         if (!is_erased(buf, denali->mtd.writesize))
1596                         {
1597                                 denali->mtd.ecc_stats.failed++;
1598                         }
1599                         if (!is_erased(buf, denali->mtd.oobsize))
1600                         {
1601                                 denali->mtd.ecc_stats.failed++;
1602                         }
1603                 }
1604         }
1605         return 0;
1606 }
1607
1608 static int denali_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
1609                                 uint8_t *buf, int page)
1610 {
1611         struct denali_nand_info *denali = mtd_to_denali(mtd);
1612         struct pci_dev *pci_dev = denali->dev;
1613
1614         dma_addr_t addr = denali->buf.dma_buf;
1615         size_t size = denali->mtd.writesize + denali->mtd.oobsize;
1616
1617         uint32_t irq_status = 0;
1618         uint32_t irq_mask = INTR_STATUS0__DMA_CMD_COMP;
1619
1620         setup_ecc_for_xfer(denali, false, true);
1621         denali_enable_dma(denali, true);
1622
1623         pci_dma_sync_single_for_device(pci_dev, addr, size, PCI_DMA_FROMDEVICE);
1624
1625         clear_interrupts(denali);
1626         denali_setup_dma(denali, DENALI_READ);
1627
1628         /* wait for operation to complete */
1629         irq_status = wait_for_irq(denali, irq_mask);
1630
1631         pci_dma_sync_single_for_cpu(pci_dev, addr, size, PCI_DMA_FROMDEVICE);
1632
1633         denali_enable_dma(denali, false);
1634
1635         memcpy(buf, denali->buf.buf, mtd->writesize);
1636         memcpy(chip->oob_poi, denali->buf.buf + mtd->writesize, mtd->oobsize);
1637
1638         return 0;
1639 }
1640
1641 static uint8_t denali_read_byte(struct mtd_info *mtd)
1642 {
1643         struct denali_nand_info *denali = mtd_to_denali(mtd);
1644         uint8_t result = 0xff;
1645
1646         if (denali->buf.head < denali->buf.tail)
1647         {
1648                 result = denali->buf.buf[denali->buf.head++];
1649         }
1650
1651 #if DEBUG_DENALI
1652         printk("read byte -> 0x%02x\n", result);
1653 #endif
1654         return result;
1655 }
1656
1657 static void denali_select_chip(struct mtd_info *mtd, int chip)
1658 {
1659         struct denali_nand_info *denali = mtd_to_denali(mtd);
1660 #if DEBUG_DENALI
1661         printk("denali select chip %d\n", chip);
1662 #endif
1663         spin_lock_irq(&denali->irq_lock);
1664         denali->flash_bank = chip;
1665         spin_unlock_irq(&denali->irq_lock);
1666 }
1667
1668 static int denali_waitfunc(struct mtd_info *mtd, struct nand_chip *chip)
1669 {
1670         struct denali_nand_info *denali = mtd_to_denali(mtd);
1671         int status = denali->status;
1672         denali->status = 0;
1673
1674 #if DEBUG_DENALI
1675         printk("waitfunc %d\n", status);
1676 #endif
1677         return status;
1678 }
1679
1680 static void denali_erase(struct mtd_info *mtd, int page)
1681 {
1682         struct denali_nand_info *denali = mtd_to_denali(mtd);
1683
1684         uint32_t cmd = 0x0, irq_status = 0;
1685
1686 #if DEBUG_DENALI
1687         printk("erase page: %d\n", page);
1688 #endif
1689         /* clear interrupts */
1690         clear_interrupts(denali);
1691
1692         /* setup page read request for access type */
1693         cmd = MODE_10 | BANK(denali->flash_bank) | page;
1694         index_addr(denali, (uint32_t)cmd, 0x1);
1695
1696         /* wait for erase to complete or failure to occur */
1697         irq_status = wait_for_irq(denali, INTR_STATUS0__ERASE_COMP |
1698                                         INTR_STATUS0__ERASE_FAIL);
1699
1700         denali->status = (irq_status & INTR_STATUS0__ERASE_FAIL) ? NAND_STATUS_FAIL :
1701                                                                  PASS;
1702 }
1703
1704 static void denali_cmdfunc(struct mtd_info *mtd, unsigned int cmd, int col,
1705                            int page)
1706 {
1707         struct denali_nand_info *denali = mtd_to_denali(mtd);
1708
1709 #if DEBUG_DENALI
1710         printk("cmdfunc: 0x%x %d %d\n", cmd, col, page);
1711 #endif
1712         switch (cmd)
1713         {
1714                 case NAND_CMD_PAGEPROG:
1715                         break;
1716                 case NAND_CMD_STATUS:
1717                         read_status(denali);
1718                         break;
1719                 case NAND_CMD_READID:
1720                         reset_buf(denali);
1721                         if (denali->flash_bank < denali->total_used_banks)
1722                         {
1723                                 /* write manufacturer information into nand
1724                                    buffer for NAND subsystem to fetch.
1725                                  */
1726                                 write_byte_to_buf(denali, denali->dev_info.wDeviceMaker);
1727                                 write_byte_to_buf(denali, denali->dev_info.wDeviceID);
1728                                 write_byte_to_buf(denali, denali->dev_info.bDeviceParam0);
1729                                 write_byte_to_buf(denali, denali->dev_info.bDeviceParam1);
1730                                 write_byte_to_buf(denali, denali->dev_info.bDeviceParam2);
1731                         }
1732                         else
1733                         {
1734                                 int i;
1735                                 for (i = 0; i < 5; i++)
1736                                         write_byte_to_buf(denali, 0xff);
1737                         }
1738                         break;
1739                 case NAND_CMD_READ0:
1740                 case NAND_CMD_SEQIN:
1741                         denali->page = page;
1742                         break;
1743                 case NAND_CMD_RESET:
1744                         reset_bank(denali);
1745                         break;
1746                 case NAND_CMD_READOOB:
1747                         /* TODO: Read OOB data */
1748                         break;
1749                 default:
1750                         printk(KERN_ERR ": unsupported command received 0x%x\n", cmd);
1751                         break;
1752         }
1753 }
1754
1755 /* stubs for ECC functions not used by the NAND core */
1756 static int denali_ecc_calculate(struct mtd_info *mtd, const uint8_t *data,
1757                                 uint8_t *ecc_code)
1758 {
1759         printk(KERN_ERR "denali_ecc_calculate called unexpectedly\n");
1760         BUG();
1761         return -EIO;
1762 }
1763
1764 static int denali_ecc_correct(struct mtd_info *mtd, uint8_t *data,
1765                                 uint8_t *read_ecc, uint8_t *calc_ecc)
1766 {
1767         printk(KERN_ERR "denali_ecc_correct called unexpectedly\n");
1768         BUG();
1769         return -EIO;
1770 }
1771
1772 static void denali_ecc_hwctl(struct mtd_info *mtd, int mode)
1773 {
1774         printk(KERN_ERR "denali_ecc_hwctl called unexpectedly\n");
1775         BUG();
1776 }
1777 /* end NAND core entry points */
1778
1779 /* Initialization code to bring the device up to a known good state */
1780 static void denali_hw_init(struct denali_nand_info *denali)
1781 {
1782         denali_irq_init(denali);
1783         NAND_Flash_Reset(denali);
1784         denali_write32(0x0F, denali->flash_reg + RB_PIN_ENABLED);
1785         denali_write32(CHIP_EN_DONT_CARE__FLAG, denali->flash_reg + CHIP_ENABLE_DONT_CARE);
1786
1787         denali_write32(0x0, denali->flash_reg + SPARE_AREA_SKIP_BYTES);
1788         denali_write32(0xffff, denali->flash_reg + SPARE_AREA_MARKER);
1789
1790         /* Should set value for these registers when init */
1791         denali_write32(0, denali->flash_reg + TWO_ROW_ADDR_CYCLES);
1792         denali_write32(1, denali->flash_reg + ECC_ENABLE);
1793 }
1794
1795 /* ECC layout for SLC devices. Denali spec indicates SLC fixed at 4 bytes */
1796 #define ECC_BYTES_SLC   4 * (2048 / ECC_SECTOR_SIZE)
1797 static struct nand_ecclayout nand_oob_slc = {
1798         .eccbytes = 4,
1799         .eccpos = { 0, 1, 2, 3 }, /* not used */
1800         .oobfree = {{
1801                         .offset = ECC_BYTES_SLC,
1802                         .length = 64 - ECC_BYTES_SLC
1803                    }}
1804 };
1805
1806 #define ECC_BYTES_MLC   14 * (2048 / ECC_SECTOR_SIZE)
1807 static struct nand_ecclayout nand_oob_mlc_14bit = {
1808         .eccbytes = 14,
1809         .eccpos = { 0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13 }, /* not used */
1810         .oobfree = {{
1811                         .offset = ECC_BYTES_MLC,
1812                         .length = 64 - ECC_BYTES_MLC
1813                    }}
1814 };
1815
1816 static uint8_t bbt_pattern[] = {'B', 'b', 't', '0' };
1817 static uint8_t mirror_pattern[] = {'1', 't', 'b', 'B' };
1818
1819 static struct nand_bbt_descr bbt_main_descr = {
1820         .options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE
1821                 | NAND_BBT_2BIT | NAND_BBT_VERSION | NAND_BBT_PERCHIP,
1822         .offs = 8,
1823         .len = 4,
1824         .veroffs = 12,
1825         .maxblocks = 4,
1826         .pattern = bbt_pattern,
1827 };
1828
1829 static struct nand_bbt_descr bbt_mirror_descr = {
1830         .options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE
1831                 | NAND_BBT_2BIT | NAND_BBT_VERSION | NAND_BBT_PERCHIP,
1832         .offs = 8,
1833         .len = 4,
1834         .veroffs = 12,
1835         .maxblocks = 4,
1836         .pattern = mirror_pattern,
1837 };
1838
1839 /* initalize driver data structures */
1840 void denali_drv_init(struct denali_nand_info *denali)
1841 {
1842         denali->idx = 0;
1843
1844         /* setup interrupt handler */
1845         /* the completion object will be used to notify
1846          * the callee that the interrupt is done */
1847         init_completion(&denali->complete);
1848
1849         /* the spinlock will be used to synchronize the ISR
1850          * with any element that might be access shared
1851          * data (interrupt status) */
1852         spin_lock_init(&denali->irq_lock);
1853
1854         /* indicate that MTD has not selected a valid bank yet */
1855         denali->flash_bank = CHIP_SELECT_INVALID;
1856
1857         /* initialize our irq_status variable to indicate no interrupts */
1858         denali->irq_status = 0;
1859 }
1860
1861 /* driver entry point */
1862 static int denali_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
1863 {
1864         int ret = -ENODEV;
1865         resource_size_t csr_base, mem_base;
1866         unsigned long csr_len, mem_len;
1867         struct denali_nand_info *denali;
1868
1869         nand_dbg_print(NAND_DBG_TRACE, "%s, Line %d, Function: %s\n",
1870                        __FILE__, __LINE__, __func__);
1871
1872         denali = kzalloc(sizeof(*denali), GFP_KERNEL);
1873         if (!denali)
1874                 return -ENOMEM;
1875
1876         ret = pci_enable_device(dev);
1877         if (ret) {
1878                 printk(KERN_ERR "Spectra: pci_enable_device failed.\n");
1879                 goto failed_enable;
1880         }
1881
1882         if (id->driver_data == INTEL_CE4100) {
1883                 /* Due to a silicon limitation, we can only support
1884                  * ONFI timing mode 1 and below.
1885                  */
1886                 if (onfi_timing_mode < -1 || onfi_timing_mode > 1)
1887                 {
1888                         printk("Intel CE4100 only supports ONFI timing mode 1 "
1889                                 "or below\n");
1890                         ret = -EINVAL;
1891                         goto failed_enable;
1892                 }
1893                 denali->platform = INTEL_CE4100;
1894                 mem_base = pci_resource_start(dev, 0);
1895                 mem_len = pci_resource_len(dev, 1);
1896                 csr_base = pci_resource_start(dev, 1);
1897                 csr_len = pci_resource_len(dev, 1);
1898         } else {
1899                 denali->platform = INTEL_MRST;
1900                 csr_base = pci_resource_start(dev, 0);
1901                 csr_len = pci_resource_start(dev, 0);
1902                 mem_base = pci_resource_start(dev, 1);
1903                 mem_len = pci_resource_len(dev, 1);
1904                 if (!mem_len) {
1905                         mem_base = csr_base + csr_len;
1906                         mem_len = csr_len;
1907                         nand_dbg_print(NAND_DBG_WARN,
1908                                        "Spectra: No second BAR for PCI device; assuming %08Lx\n",
1909                                        (uint64_t)csr_base);
1910                 }
1911         }
1912
1913         /* Is 32-bit DMA supported? */
1914         ret = pci_set_dma_mask(dev, DMA_BIT_MASK(32));
1915
1916         if (ret)
1917         {
1918                 printk(KERN_ERR "Spectra: no usable DMA configuration\n");
1919                 goto failed_enable;
1920         }
1921         denali->buf.dma_buf = pci_map_single(dev, denali->buf.buf, DENALI_BUF_SIZE,
1922                                          PCI_DMA_BIDIRECTIONAL);
1923
1924         if (pci_dma_mapping_error(dev, denali->buf.dma_buf))
1925         {
1926                 printk(KERN_ERR "Spectra: failed to map DMA buffer\n");
1927                 goto failed_enable;
1928         }
1929
1930         pci_set_master(dev);
1931         denali->dev = dev;
1932
1933         ret = pci_request_regions(dev, DENALI_NAND_NAME);
1934         if (ret) {
1935                 printk(KERN_ERR "Spectra: Unable to request memory regions\n");
1936                 goto failed_req_csr;
1937         }
1938
1939         denali->flash_reg = ioremap_nocache(csr_base, csr_len);
1940         if (!denali->flash_reg) {
1941                 printk(KERN_ERR "Spectra: Unable to remap memory region\n");
1942                 ret = -ENOMEM;
1943                 goto failed_remap_csr;
1944         }
1945         nand_dbg_print(NAND_DBG_DEBUG, "Spectra: CSR 0x%08Lx -> 0x%p (0x%lx)\n",
1946                        (uint64_t)csr_base, denali->flash_reg, csr_len);
1947
1948         denali->flash_mem = ioremap_nocache(mem_base, mem_len);
1949         if (!denali->flash_mem) {
1950                 printk(KERN_ERR "Spectra: ioremap_nocache failed!");
1951                 iounmap(denali->flash_reg);
1952                 ret = -ENOMEM;
1953                 goto failed_remap_csr;
1954         }
1955
1956         nand_dbg_print(NAND_DBG_WARN,
1957                 "Spectra: Remapped flash base address: "
1958                 "0x%p, len: %ld\n",
1959                 denali->flash_mem, csr_len);
1960
1961         denali_hw_init(denali);
1962         denali_drv_init(denali);
1963
1964         nand_dbg_print(NAND_DBG_DEBUG, "Spectra: IRQ %d\n", dev->irq);
1965         if (request_irq(dev->irq, denali_isr, IRQF_SHARED,
1966                         DENALI_NAND_NAME, denali)) {
1967                 printk(KERN_ERR "Spectra: Unable to allocate IRQ\n");
1968                 ret = -ENODEV;
1969                 goto failed_request_irq;
1970         }
1971
1972         /* now that our ISR is registered, we can enable interrupts */
1973         NAND_LLD_Enable_Disable_Interrupts(denali, true);
1974
1975         pci_set_drvdata(dev, denali);
1976
1977         NAND_Read_Device_ID(denali);
1978
1979         /* MTD supported page sizes vary by kernel. We validate our
1980            kernel supports the device here.
1981          */
1982         if (denali->dev_info.wPageSize > NAND_MAX_PAGESIZE + NAND_MAX_OOBSIZE)
1983         {
1984                 ret = -ENODEV;
1985                 printk(KERN_ERR "Spectra: device size not supported by this "
1986                         "version of MTD.");
1987                 goto failed_nand;
1988         }
1989
1990         nand_dbg_print(NAND_DBG_DEBUG, "Dump timing register values:"
1991                         "acc_clks: %d, re_2_we: %d, we_2_re: %d,"
1992                         "addr_2_data: %d, rdwr_en_lo_cnt: %d, "
1993                         "rdwr_en_hi_cnt: %d, cs_setup_cnt: %d\n",
1994                         ioread32(denali->flash_reg + ACC_CLKS),
1995                         ioread32(denali->flash_reg + RE_2_WE),
1996                         ioread32(denali->flash_reg + WE_2_RE),
1997                         ioread32(denali->flash_reg + ADDR_2_DATA),
1998                         ioread32(denali->flash_reg + RDWR_EN_LO_CNT),
1999                         ioread32(denali->flash_reg + RDWR_EN_HI_CNT),
2000                         ioread32(denali->flash_reg + CS_SETUP_CNT));
2001
2002         denali->mtd.name = "Denali NAND";
2003         denali->mtd.owner = THIS_MODULE;
2004         denali->mtd.priv = &denali->nand;
2005
2006         /* register the driver with the NAND core subsystem */
2007         denali->nand.select_chip = denali_select_chip;
2008         denali->nand.cmdfunc = denali_cmdfunc;
2009         denali->nand.read_byte = denali_read_byte;
2010         denali->nand.waitfunc = denali_waitfunc;
2011
2012         /* scan for NAND devices attached to the controller
2013          * this is the first stage in a two step process to register
2014          * with the nand subsystem */
2015         if (nand_scan_ident(&denali->mtd, LLD_MAX_FLASH_BANKS, NULL))
2016         {
2017                 ret = -ENXIO;
2018                 goto failed_nand;
2019         }
2020
2021         /* second stage of the NAND scan
2022          * this stage requires information regarding ECC and
2023          * bad block management. */
2024
2025         /* Bad block management */
2026         denali->nand.bbt_td = &bbt_main_descr;
2027         denali->nand.bbt_md = &bbt_mirror_descr;
2028
2029         /* skip the scan for now until we have OOB read and write support */
2030         denali->nand.options |= NAND_USE_FLASH_BBT | NAND_SKIP_BBTSCAN;
2031         denali->nand.ecc.mode = NAND_ECC_HW_SYNDROME;
2032
2033         if (denali->dev_info.MLCDevice)
2034         {
2035                 denali->nand.ecc.layout = &nand_oob_mlc_14bit;
2036                 denali->nand.ecc.bytes = ECC_BYTES_MLC;
2037         }
2038         else /* SLC */
2039         {
2040                 denali->nand.ecc.layout = &nand_oob_slc;
2041                 denali->nand.ecc.bytes = ECC_BYTES_SLC;
2042         }
2043
2044         /* These functions are required by the NAND core framework, otherwise,
2045            the NAND core will assert. However, we don't need them, so we'll stub
2046            them out. */
2047         denali->nand.ecc.calculate = denali_ecc_calculate;
2048         denali->nand.ecc.correct = denali_ecc_correct;
2049         denali->nand.ecc.hwctl = denali_ecc_hwctl;
2050
2051         /* override the default read operations */
2052         denali->nand.ecc.size = denali->mtd.writesize;
2053         denali->nand.ecc.read_page = denali_read_page;
2054         denali->nand.ecc.read_page_raw = denali_read_page_raw;
2055         denali->nand.ecc.write_page = denali_write_page;
2056         denali->nand.ecc.write_page_raw = denali_write_page_raw;
2057         denali->nand.ecc.read_oob = denali_read_oob;
2058         denali->nand.ecc.write_oob = denali_write_oob;
2059         denali->nand.erase_cmd = denali_erase;
2060
2061         if (nand_scan_tail(&denali->mtd))
2062         {
2063                 ret = -ENXIO;
2064                 goto failed_nand;
2065         }
2066
2067         ret = add_mtd_device(&denali->mtd);
2068         if (ret) {
2069                 printk(KERN_ERR "Spectra: Failed to register MTD device: %d\n", ret);
2070                 goto failed_nand;
2071         }
2072         return 0;
2073
2074  failed_nand:
2075         denali_irq_cleanup(dev->irq, denali);
2076  failed_request_irq:
2077         iounmap(denali->flash_reg);
2078         iounmap(denali->flash_mem);
2079  failed_remap_csr:
2080         pci_release_regions(dev);
2081  failed_req_csr:
2082         pci_unmap_single(dev, denali->buf.dma_buf, DENALI_BUF_SIZE,
2083                                                         PCI_DMA_BIDIRECTIONAL);
2084  failed_enable:
2085         kfree(denali);
2086         return ret;
2087 }
2088
2089 /* driver exit point */
2090 static void denali_pci_remove(struct pci_dev *dev)
2091 {
2092         struct denali_nand_info *denali = pci_get_drvdata(dev);
2093
2094         nand_dbg_print(NAND_DBG_WARN, "%s, Line %d, Function: %s\n",
2095                        __FILE__, __LINE__, __func__);
2096
2097         nand_release(&denali->mtd);
2098         del_mtd_device(&denali->mtd);
2099
2100         denali_irq_cleanup(dev->irq, denali);
2101
2102         iounmap(denali->flash_reg);
2103         iounmap(denali->flash_mem);
2104         pci_release_regions(dev);
2105         pci_disable_device(dev);
2106         pci_unmap_single(dev, denali->buf.dma_buf, DENALI_BUF_SIZE,
2107                                                         PCI_DMA_BIDIRECTIONAL);
2108         pci_set_drvdata(dev, NULL);
2109         kfree(denali);
2110 }
2111
2112 MODULE_DEVICE_TABLE(pci, denali_pci_ids);
2113
2114 static struct pci_driver denali_pci_driver = {
2115         .name = DENALI_NAND_NAME,
2116         .id_table = denali_pci_ids,
2117         .probe = denali_pci_probe,
2118         .remove = denali_pci_remove,
2119 };
2120
2121 static int __devinit denali_init(void)
2122 {
2123         printk(KERN_INFO "Spectra MTD driver built on %s @ %s\n", __DATE__, __TIME__);
2124         return pci_register_driver(&denali_pci_driver);
2125 }
2126
2127 /* Free memory */
2128 static void __devexit denali_exit(void)
2129 {
2130         pci_unregister_driver(&denali_pci_driver);
2131 }
2132
2133 module_init(denali_init);
2134 module_exit(denali_exit);