drivers/ata/sata_nv.c

   1 /*
   2  *  sata_nv.c - NVIDIA nForce SATA
   3  *
   4  *  Copyright 2004 NVIDIA Corp.  All rights reserved.
   5  *  Copyright 2004 Andrew Chew
   6  *
   7  *
   8  *  This program is free software; you can redistribute it and/or modify
   9  *  it under the terms of the GNU General Public License as published by
  10  *  the Free Software Foundation; either version 2, or (at your option)
  11  *  any later version.
  12  *
  13  *  This program is distributed in the hope that it will be useful,
  14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  *  GNU General Public License for more details.
  17  *
  18  *  You should have received a copy of the GNU General Public License
  19  *  along with this program; see the file COPYING.  If not, write to
  20  *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  21  *
  22  *
  23  *  libata documentation is available via 'make {ps|pdf}docs',
  24  *  as Documentation/DocBook/libata.*
  25  *
  26  *  No hardware documentation available outside of NVIDIA.
  27  *  This driver programs the NVIDIA SATA controller in a similar
  28  *  fashion as with other PCI IDE BMDMA controllers, with a few
  29  *  NV-specific details such as register offsets, SATA phy location,
  30  *  hotplug info, etc.
  31  *
  32  *  CK804/MCP04 controllers support an alternate programming interface
  33  *  similar to the ADMA specification (with some modifications).
  34  *  This allows the use of NCQ. Non-DMA-mapped ATA commands are still
  35  *  sent through the legacy interface.
  36  *
  37  */
  38
  39 #include <linux/kernel.h>
  40 #include <linux/module.h>
  41 #include <linux/pci.h>
  42 #include <linux/init.h>
  43 #include <linux/blkdev.h>
  44 #include <linux/delay.h>
  45 #include <linux/interrupt.h>
  46 #include <linux/device.h>
  47 #include <scsi/scsi_host.h>
  48 #include <scsi/scsi_device.h>
  49 #include <linux/libata.h>
  50
  51 #define DRV_NAME                        "sata_nv"
  52 #define DRV_VERSION                     "3.5"
  53
  54 #define NV_ADMA_DMA_BOUNDARY            0xffffffffUL
  55
  56 enum {
  57         NV_MMIO_BAR                     = 5,
  58
  59         NV_PORTS                        = 2,
  60         NV_PIO_MASK                     = 0x1f,
  61         NV_MWDMA_MASK                   = 0x07,
  62         NV_UDMA_MASK                    = 0x7f,
  63         NV_PORT0_SCR_REG_OFFSET         = 0x00,
  64         NV_PORT1_SCR_REG_OFFSET         = 0x40,
  65
  66         /* INT_STATUS/ENABLE */
  67         NV_INT_STATUS                   = 0x10,
  68         NV_INT_ENABLE                   = 0x11,
  69         NV_INT_STATUS_CK804             = 0x440,
  70         NV_INT_ENABLE_CK804             = 0x441,
  71
  72         /* INT_STATUS/ENABLE bits */
  73         NV_INT_DEV                      = 0x01,
  74         NV_INT_PM                       = 0x02,
  75         NV_INT_ADDED                    = 0x04,
  76         NV_INT_REMOVED                  = 0x08,
  77
  78         NV_INT_PORT_SHIFT               = 4,    /* each port occupies 4 bits */
  79
  80         NV_INT_ALL                      = 0x0f,
  81         NV_INT_MASK                     = NV_INT_DEV |
  82                                           NV_INT_ADDED | NV_INT_REMOVED,
  83
  84         /* INT_CONFIG */
  85         NV_INT_CONFIG                   = 0x12,
  86         NV_INT_CONFIG_METHD             = 0x01, // 0 = INT, 1 = SMI
  87
  88         // For PCI config register 20
  89         NV_MCP_SATA_CFG_20              = 0x50,
  90         NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
  91         NV_MCP_SATA_CFG_20_PORT0_EN     = (1 << 17),
  92         NV_MCP_SATA_CFG_20_PORT1_EN     = (1 << 16),
  93         NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
  94         NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
  95
  96         NV_ADMA_MAX_CPBS                = 32,
  97         NV_ADMA_CPB_SZ                  = 128,
  98         NV_ADMA_APRD_SZ                 = 16,
  99         NV_ADMA_SGTBL_LEN               = (1024 - NV_ADMA_CPB_SZ) /
 100                                            NV_ADMA_APRD_SZ,
 101         NV_ADMA_SGTBL_TOTAL_LEN         = NV_ADMA_SGTBL_LEN + 5,
 102         NV_ADMA_SGTBL_SZ                = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
 103         NV_ADMA_PORT_PRIV_DMA_SZ        = NV_ADMA_MAX_CPBS *
 104                                            (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
 105
 106         /* BAR5 offset to ADMA general registers */
 107         NV_ADMA_GEN                     = 0x400,
 108         NV_ADMA_GEN_CTL                 = 0x00,
 109         NV_ADMA_NOTIFIER_CLEAR          = 0x30,
 110
 111         /* BAR5 offset to ADMA ports */
 112         NV_ADMA_PORT                    = 0x480,
 113
 114         /* size of ADMA port register space  */
 115         NV_ADMA_PORT_SIZE               = 0x100,
 116
 117         /* ADMA port registers */
 118         NV_ADMA_CTL                     = 0x40,
 119         NV_ADMA_CPB_COUNT               = 0x42,
 120         NV_ADMA_NEXT_CPB_IDX            = 0x43,
 121         NV_ADMA_STAT                    = 0x44,
 122         NV_ADMA_CPB_BASE_LOW            = 0x48,
 123         NV_ADMA_CPB_BASE_HIGH           = 0x4C,
 124         NV_ADMA_APPEND                  = 0x50,
 125         NV_ADMA_NOTIFIER                = 0x68,
 126         NV_ADMA_NOTIFIER_ERROR          = 0x6C,
 127
 128         /* NV_ADMA_CTL register bits */
 129         NV_ADMA_CTL_HOTPLUG_IEN         = (1 << 0),
 130         NV_ADMA_CTL_CHANNEL_RESET       = (1 << 5),
 131         NV_ADMA_CTL_GO                  = (1 << 7),
 132         NV_ADMA_CTL_AIEN                = (1 << 8),
 133         NV_ADMA_CTL_READ_NON_COHERENT   = (1 << 11),
 134         NV_ADMA_CTL_WRITE_NON_COHERENT  = (1 << 12),
 135
 136         /* CPB response flag bits */
 137         NV_CPB_RESP_DONE                = (1 << 0),
 138         NV_CPB_RESP_ATA_ERR             = (1 << 3),
 139         NV_CPB_RESP_CMD_ERR             = (1 << 4),
 140         NV_CPB_RESP_CPB_ERR             = (1 << 7),
 141
 142         /* CPB control flag bits */
 143         NV_CPB_CTL_CPB_VALID            = (1 << 0),
 144         NV_CPB_CTL_QUEUE                = (1 << 1),
 145         NV_CPB_CTL_APRD_VALID           = (1 << 2),
 146         NV_CPB_CTL_IEN                  = (1 << 3),
 147         NV_CPB_CTL_FPDMA                = (1 << 4),
 148
 149         /* APRD flags */
 150         NV_APRD_WRITE                   = (1 << 1),
 151         NV_APRD_END                     = (1 << 2),
 152         NV_APRD_CONT                    = (1 << 3),
 153
 154         /* NV_ADMA_STAT flags */
 155         NV_ADMA_STAT_TIMEOUT            = (1 << 0),
 156         NV_ADMA_STAT_HOTUNPLUG          = (1 << 1),
 157         NV_ADMA_STAT_HOTPLUG            = (1 << 2),
 158         NV_ADMA_STAT_CPBERR             = (1 << 4),
 159         NV_ADMA_STAT_SERROR             = (1 << 5),
 160         NV_ADMA_STAT_CMD_COMPLETE       = (1 << 6),
 161         NV_ADMA_STAT_IDLE               = (1 << 8),
 162         NV_ADMA_STAT_LEGACY             = (1 << 9),
 163         NV_ADMA_STAT_STOPPED            = (1 << 10),
 164         NV_ADMA_STAT_DONE               = (1 << 12),
 165         NV_ADMA_STAT_ERR                = NV_ADMA_STAT_CPBERR |
 166                                           NV_ADMA_STAT_TIMEOUT,
 167
 168         /* port flags */
 169         NV_ADMA_PORT_REGISTER_MODE      = (1 << 0),
 170         NV_ADMA_ATAPI_SETUP_COMPLETE    = (1 << 1),
 171
 172         /* MCP55 reg offset */
 173         NV_CTL_MCP55                    = 0x400,
 174         NV_INT_STATUS_MCP55             = 0x440,
 175         NV_INT_ENABLE_MCP55             = 0x444,
 176         NV_NCQ_REG_MCP55                = 0x448,
 177
 178         /* MCP55 */
 179         NV_INT_ALL_MCP55                = 0xffff,
 180         NV_INT_PORT_SHIFT_MCP55         = 16,   /* each port occupies 16 bits */
 181         NV_INT_MASK_MCP55               = NV_INT_ALL_MCP55 & 0xfffd,
 182
 183         /* SWNCQ ENABLE BITS*/
 184         NV_CTL_PRI_SWNCQ                = 0x02,
 185         NV_CTL_SEC_SWNCQ                = 0x04,
 186
 187         /* SW NCQ status bits*/
 188         NV_SWNCQ_IRQ_DEV                = (1 << 0),
 189         NV_SWNCQ_IRQ_PM                 = (1 << 1),
 190         NV_SWNCQ_IRQ_ADDED              = (1 << 2),
 191         NV_SWNCQ_IRQ_REMOVED            = (1 << 3),
 192
 193         NV_SWNCQ_IRQ_BACKOUT            = (1 << 4),
 194         NV_SWNCQ_IRQ_SDBFIS             = (1 << 5),
 195         NV_SWNCQ_IRQ_DHREGFIS           = (1 << 6),
 196         NV_SWNCQ_IRQ_DMASETUP           = (1 << 7),
 197
 198         NV_SWNCQ_IRQ_HOTPLUG            = NV_SWNCQ_IRQ_ADDED |
 199                                           NV_SWNCQ_IRQ_REMOVED,
 200
 201 };
 202
 203 /* ADMA Physical Region Descriptor - one SG segment */
 204 struct nv_adma_prd {
 205         __le64                  addr;
 206         __le32                  len;
 207         u8                      flags;
 208         u8                      packet_len;
 209         __le16                  reserved;
 210 };
 211
 212 enum nv_adma_regbits {
 213         CMDEND  = (1 << 15),            /* end of command list */
 214         WNB     = (1 << 14),            /* wait-not-BSY */
 215         IGN     = (1 << 13),            /* ignore this entry */
 216         CS1n    = (1 << (4 + 8)),       /* std. PATA signals follow... */
 217         DA2     = (1 << (2 + 8)),
 218         DA1     = (1 << (1 + 8)),
 219         DA0     = (1 << (0 + 8)),
 220 };
 221
 222 /* ADMA Command Parameter Block
 223    The first 5 SG segments are stored inside the Command Parameter Block itself.
 224    If there are more than 5 segments the remainder are stored in a separate
 225    memory area indicated by next_aprd. */
 226 struct nv_adma_cpb {
 227         u8                      resp_flags;    /* 0 */
 228         u8                      reserved1;     /* 1 */
 229         u8                      ctl_flags;     /* 2 */
 230         /* len is length of taskfile in 64 bit words */
 231         u8                      len;            /* 3  */
 232         u8                      tag;           /* 4 */
 233         u8                      next_cpb_idx;  /* 5 */
 234         __le16                  reserved2;     /* 6-7 */
 235         __le16                  tf[12];        /* 8-31 */
 236         struct nv_adma_prd      aprd[5];       /* 32-111 */
 237         __le64                  next_aprd;     /* 112-119 */
 238         __le64                  reserved3;     /* 120-127 */
 239 };
 240
 241
 242 struct nv_adma_port_priv {
 243         struct nv_adma_cpb      *cpb;
 244         dma_addr_t              cpb_dma;
 245         struct nv_adma_prd      *aprd;
 246         dma_addr_t              aprd_dma;
 247         void __iomem            *ctl_block;
 248         void __iomem            *gen_block;
 249         void __iomem            *notifier_clear_block;
 250         u8                      flags;
 251         int                     last_issue_ncq;
 252 };
 253
 254 struct nv_host_priv {
 255         unsigned long           type;
 256 };
 257
 258 struct defer_queue {
 259         u32             defer_bits;
 260         unsigned int    head;
 261         unsigned int    tail;
 262         unsigned int    tag[ATA_MAX_QUEUE];
 263 };
 264
 265 enum ncq_saw_flag_list {
 266         ncq_saw_d2h     = (1U << 0),
 267         ncq_saw_dmas    = (1U << 1),
 268         ncq_saw_sdb     = (1U << 2),
 269         ncq_saw_backout = (1U << 3),
 270 };
 271
 272 struct nv_swncq_port_priv {
 273         struct ata_prd  *prd;    /* our SG list */
 274         dma_addr_t      prd_dma; /* and its DMA mapping */
 275         void __iomem    *sactive_block;
 276         void __iomem    *irq_block;
 277         void __iomem    *tag_block;
 278         u32             qc_active;
 279
 280         unsigned int    last_issue_tag;
 281
 282         /* fifo circular queue to store deferral command */
 283         struct defer_queue defer_queue;
 284
 285         /* for NCQ interrupt analysis */
 286         u32             dhfis_bits;
 287         u32             dmafis_bits;
 288         u32             sdbfis_bits;
 289
 290         unsigned int    ncq_flags;
 291 };
 292
 293
 294 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
 295
 296 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
 297 #ifdef CONFIG_PM
 298 static int nv_pci_device_resume(struct pci_dev *pdev);
 299 #endif
 300 static void nv_ck804_host_stop(struct ata_host *host);
 301 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
 302 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
 303 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
 304 static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
 305 static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
 306
 307 static void nv_nf2_freeze(struct ata_port *ap);
 308 static void nv_nf2_thaw(struct ata_port *ap);
 309 static void nv_ck804_freeze(struct ata_port *ap);
 310 static void nv_ck804_thaw(struct ata_port *ap);
 311 static void nv_error_handler(struct ata_port *ap);
 312 static int nv_adma_slave_config(struct scsi_device *sdev);
 313 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
 314 static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
 315 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
 316 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
 317 static void nv_adma_irq_clear(struct ata_port *ap);
 318 static int nv_adma_port_start(struct ata_port *ap);
 319 static void nv_adma_port_stop(struct ata_port *ap);
 320 #ifdef CONFIG_PM
 321 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
 322 static int nv_adma_port_resume(struct ata_port *ap);
 323 #endif
 324 static void nv_adma_freeze(struct ata_port *ap);
 325 static void nv_adma_thaw(struct ata_port *ap);
 326 static void nv_adma_error_handler(struct ata_port *ap);
 327 static void nv_adma_host_stop(struct ata_host *host);
 328 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
 329 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
 330
 331 static void nv_mcp55_thaw(struct ata_port *ap);
 332 static void nv_mcp55_freeze(struct ata_port *ap);
 333 static void nv_swncq_error_handler(struct ata_port *ap);
 334 static int nv_swncq_slave_config(struct scsi_device *sdev);
 335 static int nv_swncq_port_start(struct ata_port *ap);
 336 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
 337 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
 338 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
 339 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
 340 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
 341 #ifdef CONFIG_PM
 342 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
 343 static int nv_swncq_port_resume(struct ata_port *ap);
 344 #endif
 345
 346 enum nv_host_type
 347 {
 348         GENERIC,
 349         NFORCE2,
 350         NFORCE3 = NFORCE2,      /* NF2 == NF3 as far as sata_nv is concerned */
 351         CK804,
 352         ADMA,
 353         SWNCQ,
 354 };
 355
 356 static const struct pci_device_id nv_pci_tbl[] = {
 357         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
 358         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
 359         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
 360         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
 361         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
 362         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
 363         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
 364         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), SWNCQ },
 365         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), SWNCQ },
 366         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), SWNCQ },
 367         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), SWNCQ },
 368         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
 369         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
 370         { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
 371
 372         { } /* terminate list */
 373 };
 374
 375 static struct pci_driver nv_pci_driver = {
 376         .name                   = DRV_NAME,
 377         .id_table               = nv_pci_tbl,
 378         .probe                  = nv_init_one,
 379 #ifdef CONFIG_PM
 380         .suspend                = ata_pci_device_suspend,
 381         .resume                 = nv_pci_device_resume,
 382 #endif
 383         .remove                 = ata_pci_remove_one,
 384 };
 385
 386 static struct scsi_host_template nv_sht = {
 387         .module                 = THIS_MODULE,
 388         .name                   = DRV_NAME,
 389         .ioctl                  = ata_scsi_ioctl,
 390         .queuecommand           = ata_scsi_queuecmd,
 391         .can_queue              = ATA_DEF_QUEUE,
 392         .this_id                = ATA_SHT_THIS_ID,
 393         .sg_tablesize           = LIBATA_MAX_PRD,
 394         .cmd_per_lun            = ATA_SHT_CMD_PER_LUN,
 395         .emulated               = ATA_SHT_EMULATED,
 396         .use_clustering         = ATA_SHT_USE_CLUSTERING,
 397         .proc_name              = DRV_NAME,
 398         .dma_boundary           = ATA_DMA_BOUNDARY,
 399         .slave_configure        = ata_scsi_slave_config,
 400         .slave_destroy          = ata_scsi_slave_destroy,
 401         .bios_param             = ata_std_bios_param,
 402 };
 403
 404 static struct scsi_host_template nv_adma_sht = {
 405         .module                 = THIS_MODULE,
 406         .name                   = DRV_NAME,
 407         .ioctl                  = ata_scsi_ioctl,
 408         .queuecommand           = ata_scsi_queuecmd,
 409         .change_queue_depth     = ata_scsi_change_queue_depth,
 410         .can_queue              = NV_ADMA_MAX_CPBS,
 411         .this_id                = ATA_SHT_THIS_ID,
 412         .sg_tablesize           = NV_ADMA_SGTBL_TOTAL_LEN,
 413         .cmd_per_lun            = ATA_SHT_CMD_PER_LUN,
 414         .emulated               = ATA_SHT_EMULATED,
 415         .use_clustering         = ATA_SHT_USE_CLUSTERING,
 416         .proc_name              = DRV_NAME,
 417         .dma_boundary           = NV_ADMA_DMA_BOUNDARY,
 418         .slave_configure        = nv_adma_slave_config,
 419         .slave_destroy          = ata_scsi_slave_destroy,
 420         .bios_param             = ata_std_bios_param,
 421 };
 422
 423 static struct scsi_host_template nv_swncq_sht = {
 424         .module                 = THIS_MODULE,
 425         .name                   = DRV_NAME,
 426         .ioctl                  = ata_scsi_ioctl,
 427         .queuecommand           = ata_scsi_queuecmd,
 428         .change_queue_depth     = ata_scsi_change_queue_depth,
 429         .can_queue              = ATA_MAX_QUEUE,
 430         .this_id                = ATA_SHT_THIS_ID,
 431         .sg_tablesize           = LIBATA_MAX_PRD,
 432         .cmd_per_lun            = ATA_SHT_CMD_PER_LUN,
 433         .emulated               = ATA_SHT_EMULATED,
 434         .use_clustering         = ATA_SHT_USE_CLUSTERING,
 435         .proc_name              = DRV_NAME,
 436         .dma_boundary           = ATA_DMA_BOUNDARY,
 437         .slave_configure        = nv_swncq_slave_config,
 438         .slave_destroy          = ata_scsi_slave_destroy,
 439         .bios_param             = ata_std_bios_param,
 440 };
 441
 442 static const struct ata_port_operations nv_generic_ops = {
 443         .tf_load                = ata_tf_load,
 444         .tf_read                = ata_tf_read,
 445         .exec_command           = ata_exec_command,
 446         .check_status           = ata_check_status,
 447         .dev_select             = ata_std_dev_select,
 448         .bmdma_setup            = ata_bmdma_setup,
 449         .bmdma_start            = ata_bmdma_start,
 450         .bmdma_stop             = ata_bmdma_stop,
 451         .bmdma_status           = ata_bmdma_status,
 452         .qc_prep                = ata_qc_prep,
 453         .qc_issue               = ata_qc_issue_prot,
 454         .freeze                 = ata_bmdma_freeze,
 455         .thaw                   = ata_bmdma_thaw,
 456         .error_handler          = nv_error_handler,
 457         .post_internal_cmd      = ata_bmdma_post_internal_cmd,
 458         .data_xfer              = ata_data_xfer,
 459         .irq_clear              = ata_bmdma_irq_clear,
 460         .irq_on                 = ata_irq_on,
 461         .scr_read               = nv_scr_read,
 462         .scr_write              = nv_scr_write,
 463         .port_start             = ata_port_start,
 464 };
 465
 466 static const struct ata_port_operations nv_nf2_ops = {
 467         .tf_load                = ata_tf_load,
 468         .tf_read                = ata_tf_read,
 469         .exec_command           = ata_exec_command,
 470         .check_status           = ata_check_status,
 471         .dev_select             = ata_std_dev_select,
 472         .bmdma_setup            = ata_bmdma_setup,
 473         .bmdma_start            = ata_bmdma_start,
 474         .bmdma_stop             = ata_bmdma_stop,
 475         .bmdma_status           = ata_bmdma_status,
 476         .qc_prep                = ata_qc_prep,
 477         .qc_issue               = ata_qc_issue_prot,
 478         .freeze                 = nv_nf2_freeze,
 479         .thaw                   = nv_nf2_thaw,
 480         .error_handler          = nv_error_handler,
 481         .post_internal_cmd      = ata_bmdma_post_internal_cmd,
 482         .data_xfer              = ata_data_xfer,
 483         .irq_clear              = ata_bmdma_irq_clear,
 484         .irq_on                 = ata_irq_on,
 485         .scr_read               = nv_scr_read,
 486         .scr_write              = nv_scr_write,
 487         .port_start             = ata_port_start,
 488 };
 489
 490 static const struct ata_port_operations nv_ck804_ops = {
 491         .tf_load                = ata_tf_load,
 492         .tf_read                = ata_tf_read,
 493         .exec_command           = ata_exec_command,
 494         .check_status           = ata_check_status,
 495         .dev_select             = ata_std_dev_select,
 496         .bmdma_setup            = ata_bmdma_setup,
 497         .bmdma_start            = ata_bmdma_start,
 498         .bmdma_stop             = ata_bmdma_stop,
 499         .bmdma_status           = ata_bmdma_status,
 500         .qc_prep                = ata_qc_prep,
 501         .qc_issue               = ata_qc_issue_prot,
 502         .freeze                 = nv_ck804_freeze,
 503         .thaw                   = nv_ck804_thaw,
 504         .error_handler          = nv_error_handler,
 505         .post_internal_cmd      = ata_bmdma_post_internal_cmd,
 506         .data_xfer              = ata_data_xfer,
 507         .irq_clear              = ata_bmdma_irq_clear,
 508         .irq_on                 = ata_irq_on,
 509         .scr_read               = nv_scr_read,
 510         .scr_write              = nv_scr_write,
 511         .port_start             = ata_port_start,
 512         .host_stop              = nv_ck804_host_stop,
 513 };
 514
 515 static const struct ata_port_operations nv_adma_ops = {
 516         .tf_load                = ata_tf_load,
 517         .tf_read                = nv_adma_tf_read,
 518         .check_atapi_dma        = nv_adma_check_atapi_dma,
 519         .exec_command           = ata_exec_command,
 520         .check_status           = ata_check_status,
 521         .dev_select             = ata_std_dev_select,
 522         .bmdma_setup            = ata_bmdma_setup,
 523         .bmdma_start            = ata_bmdma_start,
 524         .bmdma_stop             = ata_bmdma_stop,
 525         .bmdma_status           = ata_bmdma_status,
 526         .qc_defer               = ata_std_qc_defer,
 527         .qc_prep                = nv_adma_qc_prep,
 528         .qc_issue               = nv_adma_qc_issue,
 529         .freeze                 = nv_adma_freeze,
 530         .thaw                   = nv_adma_thaw,
 531         .error_handler          = nv_adma_error_handler,
 532         .post_internal_cmd      = nv_adma_post_internal_cmd,
 533         .data_xfer              = ata_data_xfer,
 534         .irq_clear              = nv_adma_irq_clear,
 535         .irq_on                 = ata_irq_on,
 536         .scr_read               = nv_scr_read,
 537         .scr_write              = nv_scr_write,
 538         .port_start             = nv_adma_port_start,
 539         .port_stop              = nv_adma_port_stop,
 540 #ifdef CONFIG_PM
 541         .port_suspend           = nv_adma_port_suspend,
 542         .port_resume            = nv_adma_port_resume,
 543 #endif
 544         .host_stop              = nv_adma_host_stop,
 545 };
 546
 547 static const struct ata_port_operations nv_swncq_ops = {
 548         .tf_load                = ata_tf_load,
 549         .tf_read                = ata_tf_read,
 550         .exec_command           = ata_exec_command,
 551         .check_status           = ata_check_status,
 552         .dev_select             = ata_std_dev_select,
 553         .bmdma_setup            = ata_bmdma_setup,
 554         .bmdma_start            = ata_bmdma_start,
 555         .bmdma_stop             = ata_bmdma_stop,
 556         .bmdma_status           = ata_bmdma_status,
 557         .qc_defer               = ata_std_qc_defer,
 558         .qc_prep                = nv_swncq_qc_prep,
 559         .qc_issue               = nv_swncq_qc_issue,
 560         .freeze                 = nv_mcp55_freeze,
 561         .thaw                   = nv_mcp55_thaw,
 562         .error_handler          = nv_swncq_error_handler,
 563         .post_internal_cmd      = ata_bmdma_post_internal_cmd,
 564         .data_xfer              = ata_data_xfer,
 565         .irq_clear              = ata_bmdma_irq_clear,
 566         .irq_on                 = ata_irq_on,
 567         .scr_read               = nv_scr_read,
 568         .scr_write              = nv_scr_write,
 569 #ifdef CONFIG_PM
 570         .port_suspend           = nv_swncq_port_suspend,
 571         .port_resume            = nv_swncq_port_resume,
 572 #endif
 573         .port_start             = nv_swncq_port_start,
 574 };
 575
 576 static const struct ata_port_info nv_port_info[] = {
 577         /* generic */
 578         {
 579                 .sht            = &nv_sht,
 580                 .flags          = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
 581                 .link_flags     = ATA_LFLAG_HRST_TO_RESUME,
 582                 .pio_mask       = NV_PIO_MASK,
 583                 .mwdma_mask     = NV_MWDMA_MASK,
 584                 .udma_mask      = NV_UDMA_MASK,
 585                 .port_ops       = &nv_generic_ops,
 586                 .irq_handler    = nv_generic_interrupt,
 587         },
 588         /* nforce2/3 */
 589         {
 590                 .sht            = &nv_sht,
 591                 .flags          = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
 592                 .link_flags     = ATA_LFLAG_HRST_TO_RESUME,
 593                 .pio_mask       = NV_PIO_MASK,
 594                 .mwdma_mask     = NV_MWDMA_MASK,
 595                 .udma_mask      = NV_UDMA_MASK,
 596                 .port_ops       = &nv_nf2_ops,
 597                 .irq_handler    = nv_nf2_interrupt,
 598         },
 599         /* ck804 */
 600         {
 601                 .sht            = &nv_sht,
 602                 .flags          = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
 603                 .link_flags     = ATA_LFLAG_HRST_TO_RESUME,
 604                 .pio_mask       = NV_PIO_MASK,
 605                 .mwdma_mask     = NV_MWDMA_MASK,
 606                 .udma_mask      = NV_UDMA_MASK,
 607                 .port_ops       = &nv_ck804_ops,
 608                 .irq_handler    = nv_ck804_interrupt,
 609         },
 610         /* ADMA */
 611         {
 612                 .sht            = &nv_adma_sht,
 613                 .flags          = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
 614                                   ATA_FLAG_MMIO | ATA_FLAG_NCQ,
 615                 .link_flags     = ATA_LFLAG_HRST_TO_RESUME,
 616                 .pio_mask       = NV_PIO_MASK,
 617                 .mwdma_mask     = NV_MWDMA_MASK,
 618                 .udma_mask      = NV_UDMA_MASK,
 619                 .port_ops       = &nv_adma_ops,
 620                 .irq_handler    = nv_adma_interrupt,
 621         },
 622         /* SWNCQ */
 623         {
 624                 .sht            = &nv_swncq_sht,
 625                 .flags          = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
 626                                   ATA_FLAG_NCQ,
 627                 .link_flags     = ATA_LFLAG_HRST_TO_RESUME,
 628                 .pio_mask       = NV_PIO_MASK,
 629                 .mwdma_mask     = NV_MWDMA_MASK,
 630                 .udma_mask      = NV_UDMA_MASK,
 631                 .port_ops       = &nv_swncq_ops,
 632                 .irq_handler    = nv_swncq_interrupt,
 633         },
 634 };
 635
 636 MODULE_AUTHOR("NVIDIA");
 637 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
 638 MODULE_LICENSE("GPL");
 639 MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
 640 MODULE_VERSION(DRV_VERSION);
 641
 642 static int adma_enabled = 1;
 643 static int swncq_enabled;
 644
 645 static void nv_adma_register_mode(struct ata_port *ap)
 646 {
 647         struct nv_adma_port_priv *pp = ap->private_data;
 648         void __iomem *mmio = pp->ctl_block;
 649         u16 tmp, status;
 650         int count = 0;
 651
 652         if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
 653                 return;
 654
 655         status = readw(mmio + NV_ADMA_STAT);
 656         while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
 657                 ndelay(50);
 658                 status = readw(mmio + NV_ADMA_STAT);
 659                 count++;
 660         }
 661         if (count == 20)
 662                 ata_port_printk(ap, KERN_WARNING,
 663                         "timeout waiting for ADMA IDLE, stat=0x%hx\n",
 664                         status);
 665
 666         tmp = readw(mmio + NV_ADMA_CTL);
 667         writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
 668
 669         count = 0;
 670         status = readw(mmio + NV_ADMA_STAT);
 671         while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
 672                 ndelay(50);
 673                 status = readw(mmio + NV_ADMA_STAT);
 674                 count++;
 675         }
 676         if (count == 20)
 677                 ata_port_printk(ap, KERN_WARNING,
 678                          "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
 679                          status);
 680
 681         pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
 682 }
 683
 684 static void nv_adma_mode(struct ata_port *ap)
 685 {
 686         struct nv_adma_port_priv *pp = ap->private_data;
 687         void __iomem *mmio = pp->ctl_block;
 688         u16 tmp, status;
 689         int count = 0;
 690
 691         if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
 692                 return;
 693
 694         WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
 695
 696         tmp = readw(mmio + NV_ADMA_CTL);
 697         writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
 698
 699         status = readw(mmio + NV_ADMA_STAT);
 700         while (((status & NV_ADMA_STAT_LEGACY) ||
 701               !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
 702                 ndelay(50);
 703                 status = readw(mmio + NV_ADMA_STAT);
 704                 count++;
 705         }
 706         if (count == 20)
 707                 ata_port_printk(ap, KERN_WARNING,
 708                         "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
 709                         status);
 710
 711         pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
 712 }
 713
 714 static int nv_adma_slave_config(struct scsi_device *sdev)
 715 {
 716         struct ata_port *ap = ata_shost_to_port(sdev->host);
 717         struct nv_adma_port_priv *pp = ap->private_data;
 718         struct pci_dev *pdev = to_pci_dev(ap->host->dev);
 719         u64 bounce_limit;
 720         unsigned long segment_boundary;
 721         unsigned short sg_tablesize;
 722         int rc;
 723         int adma_enable;
 724         u32 current_reg, new_reg, config_mask;
 725
 726         rc = ata_scsi_slave_config(sdev);
 727
 728         if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
 729                 /* Not a proper libata device, ignore */
 730                 return rc;
 731
 732         if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
 733                 /*
 734                  * NVIDIA reports that ADMA mode does not support ATAPI commands.
 735                  * Therefore ATAPI commands are sent through the legacy interface.
 736                  * However, the legacy interface only supports 32-bit DMA.
 737                  * Restrict DMA parameters as required by the legacy interface
 738                  * when an ATAPI device is connected.
 739                  */
 740                 bounce_limit = ATA_DMA_MASK;
 741                 segment_boundary = ATA_DMA_BOUNDARY;
 742                 /* Subtract 1 since an extra entry may be needed for padding, see
 743                    libata-scsi.c */
 744                 sg_tablesize = LIBATA_MAX_PRD - 1;
 745
 746                 /* Since the legacy DMA engine is in use, we need to disable ADMA
 747                    on the port. */
 748                 adma_enable = 0;
 749                 nv_adma_register_mode(ap);
 750         } else {
 751                 bounce_limit = *ap->dev->dma_mask;
 752                 segment_boundary = NV_ADMA_DMA_BOUNDARY;
 753                 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
 754                 adma_enable = 1;
 755         }
 756
 757         pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
 758
 759         if (ap->port_no == 1)
 760                 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
 761                               NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
 762         else
 763                 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
 764                               NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
 765
 766         if (adma_enable) {
 767                 new_reg = current_reg | config_mask;
 768                 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
 769         } else {
 770                 new_reg = current_reg & ~config_mask;
 771                 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
 772         }
 773
 774         if (current_reg != new_reg)
 775                 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
 776
 777         blk_queue_bounce_limit(sdev->request_queue, bounce_limit);
 778         blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
 779         blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
 780         ata_port_printk(ap, KERN_INFO,
 781                 "bounce limit 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
 782                 (unsigned long long)bounce_limit, segment_boundary, sg_tablesize);
 783         return rc;
 784 }
 785
 786 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
 787 {
 788         struct nv_adma_port_priv *pp = qc->ap->private_data;
 789         return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
 790 }
 791
 792 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
 793 {
 794         /* Other than when internal or pass-through commands are executed,
 795            the only time this function will be called in ADMA mode will be
 796            if a command fails. In the failure case we don't care about going
 797            into register mode with ADMA commands pending, as the commands will
 798            all shortly be aborted anyway. We assume that NCQ commands are not
 799            issued via passthrough, which is the only way that switching into
 800            ADMA mode could abort outstanding commands. */
 801         nv_adma_register_mode(ap);
 802
 803         ata_tf_read(ap, tf);
 804 }
 805
 806 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
 807 {
 808         unsigned int idx = 0;
 809
 810         if (tf->flags & ATA_TFLAG_ISADDR) {
 811                 if (tf->flags & ATA_TFLAG_LBA48) {
 812                         cpb[idx++] = cpu_to_le16((ATA_REG_ERR   << 8) | tf->hob_feature | WNB);
 813                         cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
 814                         cpb[idx++] = cpu_to_le16((ATA_REG_LBAL  << 8) | tf->hob_lbal);
 815                         cpb[idx++] = cpu_to_le16((ATA_REG_LBAM  << 8) | tf->hob_lbam);
 816                         cpb[idx++] = cpu_to_le16((ATA_REG_LBAH  << 8) | tf->hob_lbah);
 817                         cpb[idx++] = cpu_to_le16((ATA_REG_ERR    << 8) | tf->feature);
 818                 } else
 819                         cpb[idx++] = cpu_to_le16((ATA_REG_ERR    << 8) | tf->feature | WNB);
 820
 821                 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT  << 8) | tf->nsect);
 822                 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL   << 8) | tf->lbal);
 823                 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM   << 8) | tf->lbam);
 824                 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH   << 8) | tf->lbah);
 825         }
 826
 827         if (tf->flags & ATA_TFLAG_DEVICE)
 828                 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
 829
 830         cpb[idx++] = cpu_to_le16((ATA_REG_CMD    << 8) | tf->command | CMDEND);
 831
 832         while (idx < 12)
 833                 cpb[idx++] = cpu_to_le16(IGN);
 834
 835         return idx;
 836 }
 837
 838 static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
 839 {
 840         struct nv_adma_port_priv *pp = ap->private_data;
 841         u8 flags = pp->cpb[cpb_num].resp_flags;
 842
 843         VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
 844
 845         if (unlikely((force_err ||
 846                      flags & (NV_CPB_RESP_ATA_ERR |
 847                               NV_CPB_RESP_CMD_ERR |
 848                               NV_CPB_RESP_CPB_ERR)))) {
 849                 struct ata_eh_info *ehi = &ap->link.eh_info;
 850                 int freeze = 0;
 851
 852                 ata_ehi_clear_desc(ehi);
 853                 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
 854                 if (flags & NV_CPB_RESP_ATA_ERR) {
 855                         ata_ehi_push_desc(ehi, "ATA error");
 856                         ehi->err_mask |= AC_ERR_DEV;
 857                 } else if (flags & NV_CPB_RESP_CMD_ERR) {
 858                         ata_ehi_push_desc(ehi, "CMD error");
 859                         ehi->err_mask |= AC_ERR_DEV;
 860                 } else if (flags & NV_CPB_RESP_CPB_ERR) {
 861                         ata_ehi_push_desc(ehi, "CPB error");
 862                         ehi->err_mask |= AC_ERR_SYSTEM;
 863                         freeze = 1;
 864                 } else {
 865                         /* notifier error, but no error in CPB flags? */
 866                         ata_ehi_push_desc(ehi, "unknown");
 867                         ehi->err_mask |= AC_ERR_OTHER;
 868                         freeze = 1;
 869                 }
 870                 /* Kill all commands. EH will determine what actually failed. */
 871                 if (freeze)
 872                         ata_port_freeze(ap);
 873                 else
 874                         ata_port_abort(ap);
 875                 return 1;
 876         }
 877
 878         if (likely(flags & NV_CPB_RESP_DONE)) {
 879                 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
 880                 VPRINTK("CPB flags done, flags=0x%x\n", flags);
 881                 if (likely(qc)) {
 882                         DPRINTK("Completing qc from tag %d\n", cpb_num);
 883                         ata_qc_complete(qc);
 884                 } else {
 885                         struct ata_eh_info *ehi = &ap->link.eh_info;
 886                         /* Notifier bits set without a command may indicate the drive
 887                            is misbehaving. Raise host state machine violation on this
 888                            condition. */
 889                         ata_port_printk(ap, KERN_ERR,
 890                                         "notifier for tag %d with no cmd?\n",
 891                                         cpb_num);
 892                         ehi->err_mask |= AC_ERR_HSM;
 893                         ehi->action |= ATA_EH_SOFTRESET;
 894                         ata_port_freeze(ap);
 895                         return 1;
 896                 }
 897         }
 898         return 0;
 899 }
 900
 901 static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
 902 {
 903         struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
 904
 905         /* freeze if hotplugged */
 906         if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
 907                 ata_port_freeze(ap);
 908                 return 1;
 909         }
 910
 911         /* bail out if not our interrupt */
 912         if (!(irq_stat & NV_INT_DEV))
 913                 return 0;
 914
 915         /* DEV interrupt w/ no active qc? */
 916         if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
 917                 ata_check_status(ap);
 918                 return 1;
 919         }
 920
 921         /* handle interrupt */
 922         return ata_host_intr(ap, qc);
 923 }
 924
 925 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
 926 {
 927         struct ata_host *host = dev_instance;
 928         int i, handled = 0;
 929         u32 notifier_clears[2];
 930
 931         spin_lock(&host->lock);
 932
 933         for (i = 0; i < host->n_ports; i++) {
 934                 struct ata_port *ap = host->ports[i];
 935                 notifier_clears[i] = 0;
 936
 937                 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
 938                         struct nv_adma_port_priv *pp = ap->private_data;
 939                         void __iomem *mmio = pp->ctl_block;
 940                         u16 status;
 941                         u32 gen_ctl;
 942                         u32 notifier, notifier_error;
 943
 944                         /* if ADMA is disabled, use standard ata interrupt handler */
 945                         if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
 946                                 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
 947                                         >> (NV_INT_PORT_SHIFT * i);
 948                                 handled += nv_host_intr(ap, irq_stat);
 949                                 continue;
 950                         }
 951
 952                         /* if in ATA register mode, check for standard interrupts */
 953                         if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
 954                                 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
 955                                         >> (NV_INT_PORT_SHIFT * i);
 956                                 if (ata_tag_valid(ap->link.active_tag))
 957                                         /** NV_INT_DEV indication seems unreliable at times
 958                                             at least in ADMA mode. Force it on always when a
 959                                             command is active, to prevent losing interrupts. */
 960                                         irq_stat |= NV_INT_DEV;
 961                                 handled += nv_host_intr(ap, irq_stat);
 962                         }
 963
 964                         notifier = readl(mmio + NV_ADMA_NOTIFIER);
 965                         notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
 966                         notifier_clears[i] = notifier | notifier_error;
 967
 968                         gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
 969
 970                         if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
 971                             !notifier_error)
 972                                 /* Nothing to do */
 973                                 continue;
 974
 975                         status = readw(mmio + NV_ADMA_STAT);
 976
 977                         /* Clear status. Ensure the controller sees the clearing before we start
 978                            looking at any of the CPB statuses, so that any CPB completions after
 979                            this point in the handler will raise another interrupt. */
 980                         writew(status, mmio + NV_ADMA_STAT);
 981                         readw(mmio + NV_ADMA_STAT); /* flush posted write */
 982                         rmb();
 983
 984                         handled++; /* irq handled if we got here */
 985
 986                         /* freeze if hotplugged or controller error */
 987                         if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
 988                                                NV_ADMA_STAT_HOTUNPLUG |
 989                                                NV_ADMA_STAT_TIMEOUT |
 990                                                NV_ADMA_STAT_SERROR))) {
 991                                 struct ata_eh_info *ehi = &ap->link.eh_info;
 992
 993                                 ata_ehi_clear_desc(ehi);
 994                                 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
 995                                 if (status & NV_ADMA_STAT_TIMEOUT) {
 996                                         ehi->err_mask |= AC_ERR_SYSTEM;
 997                                         ata_ehi_push_desc(ehi, "timeout");
 998                                 } else if (status & NV_ADMA_STAT_HOTPLUG) {
 999                                         ata_ehi_hotplugged(ehi);
1000                                         ata_ehi_push_desc(ehi, "hotplug");
1001                                 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
1002                                         ata_ehi_hotplugged(ehi);
1003                                         ata_ehi_push_desc(ehi, "hot unplug");
1004                                 } else if (status & NV_ADMA_STAT_SERROR) {
1005                                         /* let libata analyze SError and figure out the cause */
1006                                         ata_ehi_push_desc(ehi, "SError");
1007                                 } else
1008                                         ata_ehi_push_desc(ehi, "unknown");
1009                                 ata_port_freeze(ap);
1010                                 continue;
1011                         }
1012
1013                         if (status & (NV_ADMA_STAT_DONE |
1014                                       NV_ADMA_STAT_CPBERR)) {
1015                                 u32 check_commands;
1016                                 int pos, error = 0;
1017
1018                                 if (ata_tag_valid(ap->link.active_tag))
1019                                         check_commands = 1 << ap->link.active_tag;
1020                                 else
1021                                         check_commands = ap->link.sactive;
1022
1023                                 /** Check CPBs for completed commands */
1024                                 while ((pos = ffs(check_commands)) && !error) {
1025                                         pos--;
1026                                         error = nv_adma_check_cpb(ap, pos,
1027                                                 notifier_error & (1 << pos));
1028                                         check_commands &= ~(1 << pos);
1029                                 }
1030                         }
1031                 }
1032         }
1033
1034         if (notifier_clears[0] || notifier_clears[1]) {
1035                 /* Note: Both notifier clear registers must be written
1036                    if either is set, even if one is zero, according to NVIDIA. */
1037                 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1038                 writel(notifier_clears[0], pp->notifier_clear_block);
1039                 pp = host->ports[1]->private_data;
1040                 writel(notifier_clears[1], pp->notifier_clear_block);
1041         }
1042
1043         spin_unlock(&host->lock);
1044
1045         return IRQ_RETVAL(handled);
1046 }
1047
1048 static void nv_adma_freeze(struct ata_port *ap)
1049 {
1050         struct nv_adma_port_priv *pp = ap->private_data;
1051         void __iomem *mmio = pp->ctl_block;
1052         u16 tmp;
1053
1054         nv_ck804_freeze(ap);
1055
1056         if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1057                 return;
1058
1059         /* clear any outstanding CK804 notifications */
1060         writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1061                 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1062
1063         /* Disable interrupt */
1064         tmp = readw(mmio + NV_ADMA_CTL);
1065         writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1066                 mmio + NV_ADMA_CTL);
1067         readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1068 }
1069
1070 static void nv_adma_thaw(struct ata_port *ap)
1071 {
1072         struct nv_adma_port_priv *pp = ap->private_data;
1073         void __iomem *mmio = pp->ctl_block;
1074         u16 tmp;
1075
1076         nv_ck804_thaw(ap);
1077
1078         if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1079                 return;
1080
1081         /* Enable interrupt */
1082         tmp = readw(mmio + NV_ADMA_CTL);
1083         writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1084                 mmio + NV_ADMA_CTL);
1085         readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1086 }
1087
1088 static void nv_adma_irq_clear(struct ata_port *ap)
1089 {
1090         struct nv_adma_port_priv *pp = ap->private_data;
1091         void __iomem *mmio = pp->ctl_block;
1092         u32 notifier_clears[2];
1093
1094         if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1095                 ata_bmdma_irq_clear(ap);
1096                 return;
1097         }
1098
1099         /* clear any outstanding CK804 notifications */
1100         writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1101                 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1102
1103         /* clear ADMA status */
1104         writew(0xffff, mmio + NV_ADMA_STAT);
1105
1106         /* clear notifiers - note both ports need to be written with
1107            something even though we are only clearing on one */
1108         if (ap->port_no == 0) {
1109                 notifier_clears[0] = 0xFFFFFFFF;
1110                 notifier_clears[1] = 0;
1111         } else {
1112                 notifier_clears[0] = 0;
1113                 notifier_clears[1] = 0xFFFFFFFF;
1114         }
1115         pp = ap->host->ports[0]->private_data;
1116         writel(notifier_clears[0], pp->notifier_clear_block);
1117         pp = ap->host->ports[1]->private_data;
1118         writel(notifier_clears[1], pp->notifier_clear_block);
1119 }
1120
1121 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
1122 {
1123         struct nv_adma_port_priv *pp = qc->ap->private_data;
1124
1125         if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
1126                 ata_bmdma_post_internal_cmd(qc);
1127 }
1128
1129 static int nv_adma_port_start(struct ata_port *ap)
1130 {
1131         struct device *dev = ap->host->dev;
1132         struct nv_adma_port_priv *pp;
1133         int rc;
1134         void *mem;
1135         dma_addr_t mem_dma;
1136         void __iomem *mmio;
1137         u16 tmp;
1138
1139         VPRINTK("ENTER\n");
1140
1141         rc = ata_port_start(ap);
1142         if (rc)
1143                 return rc;
1144
1145         pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1146         if (!pp)
1147                 return -ENOMEM;
1148
1149         mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
1150                ap->port_no * NV_ADMA_PORT_SIZE;
1151         pp->ctl_block = mmio;
1152         pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
1153         pp->notifier_clear_block = pp->gen_block +
1154                NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1155
1156         mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1157                                   &mem_dma, GFP_KERNEL);
1158         if (!mem)
1159                 return -ENOMEM;
1160         memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1161
1162         /*
1163          * First item in chunk of DMA memory:
1164          * 128-byte command parameter block (CPB)
1165          * one for each command tag
1166          */
1167         pp->cpb     = mem;
1168         pp->cpb_dma = mem_dma;
1169
1170         writel(mem_dma & 0xFFFFFFFF,    mmio + NV_ADMA_CPB_BASE_LOW);
1171         writel((mem_dma >> 16) >> 16,   mmio + NV_ADMA_CPB_BASE_HIGH);
1172
1173         mem     += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1174         mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1175
1176         /*
1177          * Second item: block of ADMA_SGTBL_LEN s/g entries
1178          */
1179         pp->aprd = mem;
1180         pp->aprd_dma = mem_dma;
1181
1182         ap->private_data = pp;
1183
1184         /* clear any outstanding interrupt conditions */
1185         writew(0xffff, mmio + NV_ADMA_STAT);
1186
1187         /* initialize port variables */
1188         pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1189
1190         /* clear CPB fetch count */
1191         writew(0, mmio + NV_ADMA_CPB_COUNT);
1192
1193         /* clear GO for register mode, enable interrupt */
1194         tmp = readw(mmio + NV_ADMA_CTL);
1195         writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1196                 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1197
1198         tmp = readw(mmio + NV_ADMA_CTL);
1199         writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1200         readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1201         udelay(1);
1202         writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1203         readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1204
1205         return 0;
1206 }
1207
1208 static void nv_adma_port_stop(struct ata_port *ap)
1209 {
1210         struct nv_adma_port_priv *pp = ap->private_data;
1211         void __iomem *mmio = pp->ctl_block;
1212
1213         VPRINTK("ENTER\n");
1214         writew(0, mmio + NV_ADMA_CTL);
1215 }
1216
1217 #ifdef CONFIG_PM
1218 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1219 {
1220         struct nv_adma_port_priv *pp = ap->private_data;
1221         void __iomem *mmio = pp->ctl_block;
1222
1223         /* Go to register mode - clears GO */
1224         nv_adma_register_mode(ap);
1225
1226         /* clear CPB fetch count */
1227         writew(0, mmio + NV_ADMA_CPB_COUNT);
1228
1229         /* disable interrupt, shut down port */
1230         writew(0, mmio + NV_ADMA_CTL);
1231
1232         return 0;
1233 }
1234
1235 static int nv_adma_port_resume(struct ata_port *ap)
1236 {
1237         struct nv_adma_port_priv *pp = ap->private_data;
1238         void __iomem *mmio = pp->ctl_block;
1239         u16 tmp;
1240
1241         /* set CPB block location */
1242         writel(pp->cpb_dma & 0xFFFFFFFF,        mmio + NV_ADMA_CPB_BASE_LOW);
1243         writel((pp->cpb_dma >> 16) >> 16,       mmio + NV_ADMA_CPB_BASE_HIGH);
1244
1245         /* clear any outstanding interrupt conditions */
1246         writew(0xffff, mmio + NV_ADMA_STAT);
1247
1248         /* initialize port variables */
1249         pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1250
1251         /* clear CPB fetch count */
1252         writew(0, mmio + NV_ADMA_CPB_COUNT);
1253
1254         /* clear GO for register mode, enable interrupt */
1255         tmp = readw(mmio + NV_ADMA_CTL);
1256         writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1257                 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1258
1259         tmp = readw(mmio + NV_ADMA_CTL);
1260         writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1261         readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1262         udelay(1);
1263         writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1264         readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1265
1266         return 0;
1267 }
1268 #endif
1269
1270 static void nv_adma_setup_port(struct ata_port *ap)
1271 {
1272         void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1273         struct ata_ioports *ioport = &ap->ioaddr;
1274
1275         VPRINTK("ENTER\n");
1276
1277         mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
1278
1279         ioport->cmd_addr        = mmio;
1280         ioport->data_addr       = mmio + (ATA_REG_DATA * 4);
1281         ioport->error_addr      =
1282         ioport->feature_addr    = mmio + (ATA_REG_ERR * 4);
1283         ioport->nsect_addr      = mmio + (ATA_REG_NSECT * 4);
1284         ioport->lbal_addr       = mmio + (ATA_REG_LBAL * 4);
1285         ioport->lbam_addr       = mmio + (ATA_REG_LBAM * 4);
1286         ioport->lbah_addr       = mmio + (ATA_REG_LBAH * 4);
1287         ioport->device_addr     = mmio + (ATA_REG_DEVICE * 4);
1288         ioport->status_addr     =
1289         ioport->command_addr    = mmio + (ATA_REG_STATUS * 4);
1290         ioport->altstatus_addr  =
1291         ioport->ctl_addr        = mmio + 0x20;
1292 }
1293
1294 static int nv_adma_host_init(struct ata_host *host)
1295 {
1296         struct pci_dev *pdev = to_pci_dev(host->dev);
1297         unsigned int i;
1298         u32 tmp32;
1299
1300         VPRINTK("ENTER\n");
1301
1302         /* enable ADMA on the ports */
1303         pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1304         tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1305                  NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1306                  NV_MCP_SATA_CFG_20_PORT1_EN |
1307                  NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1308
1309         pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1310
1311         for (i = 0; i < host->n_ports; i++)
1312                 nv_adma_setup_port(host->ports[i]);
1313
1314         return 0;
1315 }
1316
1317 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1318                               struct scatterlist *sg,
1319                               int idx,
1320                               struct nv_adma_prd *aprd)
1321 {
1322         u8 flags = 0;
1323         if (qc->tf.flags & ATA_TFLAG_WRITE)
1324                 flags |= NV_APRD_WRITE;
1325         if (idx == qc->n_elem - 1)
1326                 flags |= NV_APRD_END;
1327         else if (idx != 4)
1328                 flags |= NV_APRD_CONT;
1329
1330         aprd->addr  = cpu_to_le64(((u64)sg_dma_address(sg)));
1331         aprd->len   = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1332         aprd->flags = flags;
1333         aprd->packet_len = 0;
1334 }
1335
1336 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1337 {
1338         struct nv_adma_port_priv *pp = qc->ap->private_data;
1339         struct nv_adma_prd *aprd;
1340         struct scatterlist *sg;
1341         unsigned int si;
1342
1343         VPRINTK("ENTER\n");
1344
1345         for_each_sg(qc->sg, sg, qc->n_elem, si) {
1346                 aprd = (si < 5) ? &cpb->aprd[si] :
1347                                &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
1348                 nv_adma_fill_aprd(qc, sg, si, aprd);
1349         }
1350         if (si > 5)
1351                 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
1352         else
1353                 cpb->next_aprd = cpu_to_le64(0);
1354 }
1355
1356 static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1357 {
1358         struct nv_adma_port_priv *pp = qc->ap->private_data;
1359
1360         /* ADMA engine can only be used for non-ATAPI DMA commands,
1361            or interrupt-driven no-data commands. */
1362         if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
1363            (qc->tf.flags & ATA_TFLAG_POLLING))
1364                 return 1;
1365
1366         if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
1367            (qc->tf.protocol == ATA_PROT_NODATA))
1368                 return 0;
1369
1370         return 1;
1371 }
1372
1373 static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1374 {
1375         struct nv_adma_port_priv *pp = qc->ap->private_data;
1376         struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1377         u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1378                        NV_CPB_CTL_IEN;
1379
1380         if (nv_adma_use_reg_mode(qc)) {
1381                 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1382                         (qc->flags & ATA_QCFLAG_DMAMAP));
1383                 nv_adma_register_mode(qc->ap);
1384                 ata_qc_prep(qc);
1385                 return;
1386         }
1387
1388         cpb->resp_flags = NV_CPB_RESP_DONE;
1389         wmb();
1390         cpb->ctl_flags = 0;
1391         wmb();
1392
1393         cpb->len                = 3;
1394         cpb->tag                = qc->tag;
1395         cpb->next_cpb_idx       = 0;
1396
1397         /* turn on NCQ flags for NCQ commands */
1398         if (qc->tf.protocol == ATA_PROT_NCQ)
1399                 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1400
1401         VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1402
1403         nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1404
1405         if (qc->flags & ATA_QCFLAG_DMAMAP) {
1406                 nv_adma_fill_sg(qc, cpb);
1407                 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1408         } else
1409                 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
1410
1411         /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1412            until we are finished filling in all of the contents */
1413         wmb();
1414         cpb->ctl_flags = ctl_flags;
1415         wmb();
1416         cpb->resp_flags = 0;
1417 }
1418
1419 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1420 {
1421         struct nv_adma_port_priv *pp = qc->ap->private_data;
1422         void __iomem *mmio = pp->ctl_block;
1423         int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
1424
1425         VPRINTK("ENTER\n");
1426
1427         /* We can't handle result taskfile with NCQ commands, since
1428            retrieving the taskfile switches us out of ADMA mode and would abort
1429            existing commands. */
1430         if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1431                      (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1432                 ata_dev_printk(qc->dev, KERN_ERR,
1433                         "NCQ w/ RESULT_TF not allowed\n");
1434                 return AC_ERR_SYSTEM;
1435         }
1436
1437         if (nv_adma_use_reg_mode(qc)) {
1438                 /* use ATA register mode */
1439                 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
1440                 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1441                         (qc->flags & ATA_QCFLAG_DMAMAP));
1442                 nv_adma_register_mode(qc->ap);
1443                 return ata_qc_issue_prot(qc);
1444         } else
1445                 nv_adma_mode(qc->ap);
1446
1447         /* write append register, command tag in lower 8 bits
1448            and (number of cpbs to append -1) in top 8 bits */
1449         wmb();
1450
1451         if (curr_ncq != pp->last_issue_ncq) {
1452                 /* Seems to need some delay before switching between NCQ and
1453                    non-NCQ commands, else we get command timeouts and such. */
1454                 udelay(20);
1455                 pp->last_issue_ncq = curr_ncq;
1456         }
1457
1458         writew(qc->tag, mmio + NV_ADMA_APPEND);
1459
1460         DPRINTK("Issued tag %u\n", qc->tag);
1461
1462         return 0;
1463 }
1464
1465 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1466 {
1467         struct ata_host *host = dev_instance;
1468         unsigned int i;
1469         unsigned int handled = 0;
1470         unsigned long flags;
1471
1472         spin_lock_irqsave(&host->lock, flags);
1473
1474         for (i = 0; i < host->n_ports; i++) {
1475                 struct ata_port *ap;
1476
1477                 ap = host->ports[i];
1478                 if (ap &&
1479                     !(ap->flags & ATA_FLAG_DISABLED)) {
1480                         struct ata_queued_cmd *qc;
1481
1482                         qc = ata_qc_from_tag(ap, ap->link.active_tag);
1483                         if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
1484                                 handled += ata_host_intr(ap, qc);
1485                         else
1486                                 // No request pending?  Clear interrupt status
1487                                 // anyway, in case there's one pending.
1488                                 ap->ops->check_status(ap);
1489                 }
1490
1491         }
1492
1493         spin_unlock_irqrestore(&host->lock, flags);
1494
1495         return IRQ_RETVAL(handled);
1496 }
1497
1498 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1499 {
1500         int i, handled = 0;
1501
1502         for (i = 0; i < host->n_ports; i++) {
1503                 struct ata_port *ap = host->ports[i];
1504
1505                 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1506                         handled += nv_host_intr(ap, irq_stat);
1507
1508                 irq_stat >>= NV_INT_PORT_SHIFT;
1509         }
1510
1511         return IRQ_RETVAL(handled);
1512 }
1513
1514 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1515 {
1516         struct ata_host *host = dev_instance;
1517         u8 irq_stat;
1518         irqreturn_t ret;
1519
1520         spin_lock(&host->lock);
1521         irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1522         ret = nv_do_interrupt(host, irq_stat);
1523         spin_unlock(&host->lock);
1524
1525         return ret;
1526 }
1527
1528 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1529 {
1530         struct ata_host *host = dev_instance;
1531         u8 irq_stat;
1532         irqreturn_t ret;
1533
1534         spin_lock(&host->lock);
1535         irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1536         ret = nv_do_interrupt(host, irq_stat);
1537         spin_unlock(&host->lock);
1538
1539         return ret;
1540 }
1541
1542 static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
1543 {
1544         if (sc_reg > SCR_CONTROL)
1545                 return -EINVAL;
1546
1547         *val = ioread32(ap->ioaddr.scr_addr + (sc_reg * 4));
1548         return 0;
1549 }
1550
1551 static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
1552 {
1553         if (sc_reg > SCR_CONTROL)
1554                 return -EINVAL;
1555
1556         iowrite32(val, ap->ioaddr.scr_addr + (sc_reg * 4));
1557         return 0;
1558 }
1559
1560 static void nv_nf2_freeze(struct ata_port *ap)
1561 {
1562         void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1563         int shift = ap->port_no * NV_INT_PORT_SHIFT;
1564         u8 mask;
1565
1566         mask = ioread8(scr_addr + NV_INT_ENABLE);
1567         mask &= ~(NV_INT_ALL << shift);
1568         iowrite8(mask, scr_addr + NV_INT_ENABLE);
1569 }
1570
1571 static void nv_nf2_thaw(struct ata_port *ap)
1572 {
1573         void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1574         int shift = ap->port_no * NV_INT_PORT_SHIFT;
1575         u8 mask;
1576
1577         iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1578
1579         mask = ioread8(scr_addr + NV_INT_ENABLE);
1580         mask |= (NV_INT_MASK << shift);
1581         iowrite8(mask, scr_addr + NV_INT_ENABLE);
1582 }
1583
1584 static void nv_ck804_freeze(struct ata_port *ap)
1585 {
1586         void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1587         int shift = ap->port_no * NV_INT_PORT_SHIFT;
1588         u8 mask;
1589
1590         mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1591         mask &= ~(NV_INT_ALL << shift);
1592         writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1593 }
1594
1595 static void nv_ck804_thaw(struct ata_port *ap)
1596 {
1597         void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1598         int shift = ap->port_no * NV_INT_PORT_SHIFT;
1599         u8 mask;
1600
1601         writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1602
1603         mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1604         mask |= (NV_INT_MASK << shift);
1605         writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1606 }
1607
1608 static void nv_mcp55_freeze(struct ata_port *ap)
1609 {
1610         void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1611         int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1612         u32 mask;
1613
1614         writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1615
1616         mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1617         mask &= ~(NV_INT_ALL_MCP55 << shift);
1618         writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1619         ata_bmdma_freeze(ap);
1620 }
1621
1622 static void nv_mcp55_thaw(struct ata_port *ap)
1623 {
1624         void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1625         int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1626         u32 mask;
1627
1628         writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1629
1630         mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1631         mask |= (NV_INT_MASK_MCP55 << shift);
1632         writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1633         ata_bmdma_thaw(ap);
1634 }
1635
1636 static int nv_hardreset(struct ata_link *link, unsigned int *class,
1637                         unsigned long deadline)
1638 {
1639         unsigned int dummy;
1640
1641         /* SATA hardreset fails to retrieve proper device signature on
1642          * some controllers.  Don't classify on hardreset.  For more
1643          * info, see http://bugzilla.kernel.org/show_bug.cgi?id=3352
1644          */
1645         return sata_std_hardreset(link, &dummy, deadline);
1646 }
1647
1648 static void nv_error_handler(struct ata_port *ap)
1649 {
1650         ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1651                            nv_hardreset, ata_std_postreset);
1652 }
1653
1654 static void nv_adma_error_handler(struct ata_port *ap)
1655 {
1656         struct nv_adma_port_priv *pp = ap->private_data;
1657         if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1658                 void __iomem *mmio = pp->ctl_block;
1659                 int i;
1660                 u16 tmp;
1661
1662                 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
1663                         u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1664                         u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1665                         u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1666                         u32 status = readw(mmio + NV_ADMA_STAT);
1667                         u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1668                         u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
1669
1670                         ata_port_printk(ap, KERN_ERR,
1671                                 "EH in ADMA mode, notifier 0x%X "
1672                                 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1673                                 "next cpb count 0x%X next cpb idx 0x%x\n",
1674                                 notifier, notifier_error, gen_ctl, status,
1675                                 cpb_count, next_cpb_idx);
1676
1677                         for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
1678                                 struct nv_adma_cpb *cpb = &pp->cpb[i];
1679                                 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
1680                                     ap->link.sactive & (1 << i))
1681                                         ata_port_printk(ap, KERN_ERR,
1682                                                 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1683                                                 i, cpb->ctl_flags, cpb->resp_flags);
1684                         }
1685                 }
1686
1687                 /* Push us back into port register mode for error handling. */
1688                 nv_adma_register_mode(ap);
1689
1690                 /* Mark all of the CPBs as invalid to prevent them from
1691                    being executed */
1692                 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
1693                         pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1694
1695                 /* clear CPB fetch count */
1696                 writew(0, mmio + NV_ADMA_CPB_COUNT);
1697
1698                 /* Reset channel */
1699                 tmp = readw(mmio + NV_ADMA_CTL);
1700                 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1701                 readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1702                 udelay(1);
1703                 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1704                 readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1705         }
1706
1707         ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1708                            nv_hardreset, ata_std_postreset);
1709 }
1710
1711 static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1712 {
1713         struct nv_swncq_port_priv *pp = ap->private_data;
1714         struct defer_queue *dq = &pp->defer_queue;
1715
1716         /* queue is full */
1717         WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1718         dq->defer_bits |= (1 << qc->tag);
1719         dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1720 }
1721
1722 static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1723 {
1724         struct nv_swncq_port_priv *pp = ap->private_data;
1725         struct defer_queue *dq = &pp->defer_queue;
1726         unsigned int tag;
1727
1728         if (dq->head == dq->tail)       /* null queue */
1729                 return NULL;
1730
1731         tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1732         dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1733         WARN_ON(!(dq->defer_bits & (1 << tag)));
1734         dq->defer_bits &= ~(1 << tag);
1735
1736         return ata_qc_from_tag(ap, tag);
1737 }
1738
1739 static void nv_swncq_fis_reinit(struct ata_port *ap)
1740 {
1741         struct nv_swncq_port_priv *pp = ap->private_data;
1742
1743         pp->dhfis_bits = 0;
1744         pp->dmafis_bits = 0;
1745         pp->sdbfis_bits = 0;
1746         pp->ncq_flags = 0;
1747 }
1748
1749 static void nv_swncq_pp_reinit(struct ata_port *ap)
1750 {
1751         struct nv_swncq_port_priv *pp = ap->private_data;
1752         struct defer_queue *dq = &pp->defer_queue;
1753
1754         dq->head = 0;
1755         dq->tail = 0;
1756         dq->defer_bits = 0;
1757         pp->qc_active = 0;
1758         pp->last_issue_tag = ATA_TAG_POISON;
1759         nv_swncq_fis_reinit(ap);
1760 }
1761
1762 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1763 {
1764         struct nv_swncq_port_priv *pp = ap->private_data;
1765
1766         writew(fis, pp->irq_block);
1767 }
1768
1769 static void __ata_bmdma_stop(struct ata_port *ap)
1770 {
1771         struct ata_queued_cmd qc;
1772
1773         qc.ap = ap;
1774         ata_bmdma_stop(&qc);
1775 }
1776
1777 static void nv_swncq_ncq_stop(struct ata_port *ap)
1778 {
1779         struct nv_swncq_port_priv *pp = ap->private_data;
1780         unsigned int i;
1781         u32 sactive;
1782         u32 done_mask;
1783
1784         ata_port_printk(ap, KERN_ERR,
1785                         "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1786                         ap->qc_active, ap->link.sactive);
1787         ata_port_printk(ap, KERN_ERR,
1788                 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n  "
1789                 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1790                 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1791                 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1792
1793         ata_port_printk(ap, KERN_ERR, "ATA_REG 0x%X ERR_REG 0x%X\n",
1794                         ap->ops->check_status(ap),
1795                         ioread8(ap->ioaddr.error_addr));
1796
1797         sactive = readl(pp->sactive_block);
1798         done_mask = pp->qc_active ^ sactive;
1799
1800         ata_port_printk(ap, KERN_ERR, "tag : dhfis dmafis sdbfis sacitve\n");
1801         for (i = 0; i < ATA_MAX_QUEUE; i++) {
1802                 u8 err = 0;
1803                 if (pp->qc_active & (1 << i))
1804                         err = 0;
1805                 else if (done_mask & (1 << i))
1806                         err = 1;
1807                 else
1808                         continue;
1809
1810                 ata_port_printk(ap, KERN_ERR,
1811                                 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1812                                 (pp->dhfis_bits >> i) & 0x1,
1813                                 (pp->dmafis_bits >> i) & 0x1,
1814                                 (pp->sdbfis_bits >> i) & 0x1,
1815                                 (sactive >> i) & 0x1,
1816                                 (err ? "error! tag doesn't exit" : " "));
1817         }
1818
1819         nv_swncq_pp_reinit(ap);
1820         ap->ops->irq_clear(ap);
1821         __ata_bmdma_stop(ap);
1822         nv_swncq_irq_clear(ap, 0xffff);
1823 }
1824
1825 static void nv_swncq_error_handler(struct ata_port *ap)
1826 {
1827         struct ata_eh_context *ehc = &ap->link.eh_context;
1828
1829         if (ap->link.sactive) {
1830                 nv_swncq_ncq_stop(ap);
1831                 ehc->i.action |= ATA_EH_HARDRESET;
1832         }
1833
1834         ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1835                            nv_hardreset, ata_std_postreset);
1836 }
1837
1838 #ifdef CONFIG_PM
1839 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1840 {
1841         void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1842         u32 tmp;
1843
1844         /* clear irq */
1845         writel(~0, mmio + NV_INT_STATUS_MCP55);
1846
1847         /* disable irq */
1848         writel(0, mmio + NV_INT_ENABLE_MCP55);
1849
1850         /* disable swncq */
1851         tmp = readl(mmio + NV_CTL_MCP55);
1852         tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1853         writel(tmp, mmio + NV_CTL_MCP55);
1854
1855         return 0;
1856 }
1857
1858 static int nv_swncq_port_resume(struct ata_port *ap)
1859 {
1860         void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1861         u32 tmp;
1862
1863         /* clear irq */
1864         writel(~0, mmio + NV_INT_STATUS_MCP55);
1865
1866         /* enable irq */
1867         writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1868
1869         /* enable swncq */
1870         tmp = readl(mmio + NV_CTL_MCP55);
1871         writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1872
1873         return 0;
1874 }
1875 #endif
1876
1877 static void nv_swncq_host_init(struct ata_host *host)
1878 {
1879         u32 tmp;
1880         void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1881         struct pci_dev *pdev = to_pci_dev(host->dev);
1882         u8 regval;
1883
1884         /* disable  ECO 398 */
1885         pci_read_config_byte(pdev, 0x7f, &regval);
1886         regval &= ~(1 << 7);
1887         pci_write_config_byte(pdev, 0x7f, regval);
1888
1889         /* enable swncq */
1890         tmp = readl(mmio + NV_CTL_MCP55);
1891         VPRINTK("HOST_CTL:0x%X\n", tmp);
1892         writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1893
1894         /* enable irq intr */
1895         tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1896         VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1897         writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1898
1899         /*  clear port irq */
1900         writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1901 }
1902
1903 static int nv_swncq_slave_config(struct scsi_device *sdev)
1904 {
1905         struct ata_port *ap = ata_shost_to_port(sdev->host);
1906         struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1907         struct ata_device *dev;
1908         int rc;
1909         u8 rev;
1910         u8 check_maxtor = 0;
1911         unsigned char model_num[ATA_ID_PROD_LEN + 1];
1912
1913         rc = ata_scsi_slave_config(sdev);
1914         if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1915                 /* Not a proper libata device, ignore */
1916                 return rc;
1917
1918         dev = &ap->link.device[sdev->id];
1919         if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1920                 return rc;
1921
1922         /* if MCP51 and Maxtor, then disable ncq */
1923         if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1924                 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1925                 check_maxtor = 1;
1926
1927         /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1928         if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1929                 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1930                 pci_read_config_byte(pdev, 0x8, &rev);
1931                 if (rev <= 0xa2)
1932                         check_maxtor = 1;
1933         }
1934
1935         if (!check_maxtor)
1936                 return rc;
1937
1938         ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1939
1940         if (strncmp(model_num, "Maxtor", 6) == 0) {
1941                 ata_scsi_change_queue_depth(sdev, 1);
1942                 ata_dev_printk(dev, KERN_NOTICE,
1943                         "Disabling SWNCQ mode (depth %x)\n", sdev->queue_depth);
1944         }
1945
1946         return rc;
1947 }
1948
1949 static int nv_swncq_port_start(struct ata_port *ap)
1950 {
1951         struct device *dev = ap->host->dev;
1952         void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1953         struct nv_swncq_port_priv *pp;
1954         int rc;
1955
1956         rc = ata_port_start(ap);
1957         if (rc)
1958                 return rc;
1959
1960         pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1961         if (!pp)
1962                 return -ENOMEM;
1963
1964         pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1965                                       &pp->prd_dma, GFP_KERNEL);
1966         if (!pp->prd)
1967                 return -ENOMEM;
1968         memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1969
1970         ap->private_data = pp;
1971         pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1972         pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1973         pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1974
1975         return 0;
1976 }
1977
1978 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1979 {
1980         if (qc->tf.protocol != ATA_PROT_NCQ) {
1981                 ata_qc_prep(qc);
1982                 return;
1983         }
1984
1985         if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1986                 return;
1987
1988         nv_swncq_fill_sg(qc);
1989 }
1990
1991 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1992 {
1993         struct ata_port *ap = qc->ap;
1994         struct scatterlist *sg;
1995         struct nv_swncq_port_priv *pp = ap->private_data;
1996         struct ata_prd *prd;
1997         unsigned int si, idx;
1998
1999         prd = pp->prd + ATA_MAX_PRD * qc->tag;
2000
2001         idx = 0;
2002         for_each_sg(qc->sg, sg, qc->n_elem, si) {
2003                 u32 addr, offset;
2004                 u32 sg_len, len;
2005
2006                 addr = (u32)sg_dma_address(sg);
2007                 sg_len = sg_dma_len(sg);
2008
2009                 while (sg_len) {
2010                         offset = addr & 0xffff;
2011                         len = sg_len;
2012                         if ((offset + sg_len) > 0x10000)
2013                                 len = 0x10000 - offset;
2014
2015                         prd[idx].addr = cpu_to_le32(addr);
2016                         prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2017
2018                         idx++;
2019                         sg_len -= len;
2020                         addr += len;
2021                 }
2022         }
2023
2024         prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
2025 }
2026
2027 static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2028                                           struct ata_queued_cmd *qc)
2029 {
2030         struct nv_swncq_port_priv *pp = ap->private_data;
2031
2032         if (qc == NULL)
2033                 return 0;
2034
2035         DPRINTK("Enter\n");
2036
2037         writel((1 << qc->tag), pp->sactive_block);
2038         pp->last_issue_tag = qc->tag;
2039         pp->dhfis_bits &= ~(1 << qc->tag);
2040         pp->dmafis_bits &= ~(1 << qc->tag);
2041         pp->qc_active |= (0x1 << qc->tag);
2042
2043         ap->ops->tf_load(ap, &qc->tf);   /* load tf registers */
2044         ap->ops->exec_command(ap, &qc->tf);
2045
2046         DPRINTK("Issued tag %u\n", qc->tag);
2047
2048         return 0;
2049 }
2050
2051 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2052 {
2053         struct ata_port *ap = qc->ap;
2054         struct nv_swncq_port_priv *pp = ap->private_data;
2055
2056         if (qc->tf.protocol != ATA_PROT_NCQ)
2057                 return ata_qc_issue_prot(qc);
2058
2059         DPRINTK("Enter\n");
2060
2061         if (!pp->qc_active)
2062                 nv_swncq_issue_atacmd(ap, qc);
2063         else
2064                 nv_swncq_qc_to_dq(ap, qc);      /* add qc to defer queue */
2065
2066         return 0;
2067 }
2068
2069 static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2070 {
2071         u32 serror;
2072         struct ata_eh_info *ehi = &ap->link.eh_info;
2073
2074         ata_ehi_clear_desc(ehi);
2075
2076         /* AHCI needs SError cleared; otherwise, it might lock up */
2077         sata_scr_read(&ap->link, SCR_ERROR, &serror);
2078         sata_scr_write(&ap->link, SCR_ERROR, serror);
2079
2080         /* analyze @irq_stat */
2081         if (fis & NV_SWNCQ_IRQ_ADDED)
2082                 ata_ehi_push_desc(ehi, "hot plug");
2083         else if (fis & NV_SWNCQ_IRQ_REMOVED)
2084                 ata_ehi_push_desc(ehi, "hot unplug");
2085
2086         ata_ehi_hotplugged(ehi);
2087
2088         /* okay, let's hand over to EH */
2089         ehi->serror |= serror;
2090
2091         ata_port_freeze(ap);
2092 }
2093
2094 static int nv_swncq_sdbfis(struct ata_port *ap)
2095 {
2096         struct ata_queued_cmd *qc;
2097         struct nv_swncq_port_priv *pp = ap->private_data;
2098         struct ata_eh_info *ehi = &ap->link.eh_info;
2099         u32 sactive;
2100         int nr_done = 0;
2101         u32 done_mask;
2102         int i;
2103         u8 host_stat;
2104         u8 lack_dhfis = 0;
2105
2106         host_stat = ap->ops->bmdma_status(ap);
2107         if (unlikely(host_stat & ATA_DMA_ERR)) {
2108                 /* error when transfering data to/from memory */
2109                 ata_ehi_clear_desc(ehi);
2110                 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2111                 ehi->err_mask |= AC_ERR_HOST_BUS;
2112                 ehi->action |= ATA_EH_SOFTRESET;
2113                 return -EINVAL;
2114         }
2115
2116         ap->ops->irq_clear(ap);
2117         __ata_bmdma_stop(ap);
2118
2119         sactive = readl(pp->sactive_block);
2120         done_mask = pp->qc_active ^ sactive;
2121
2122         if (unlikely(done_mask & sactive)) {
2123                 ata_ehi_clear_desc(ehi);
2124                 ata_ehi_push_desc(ehi, "illegal SWNCQ:qc_active transition"
2125                                   "(%08x->%08x)", pp->qc_active, sactive);
2126                 ehi->err_mask |= AC_ERR_HSM;
2127                 ehi->action |= ATA_EH_HARDRESET;
2128                 return -EINVAL;
2129         }
2130         for (i = 0; i < ATA_MAX_QUEUE; i++) {
2131                 if (!(done_mask & (1 << i)))
2132                         continue;
2133
2134                 qc = ata_qc_from_tag(ap, i);
2135                 if (qc) {
2136                         ata_qc_complete(qc);
2137                         pp->qc_active &= ~(1 << i);
2138                         pp->dhfis_bits &= ~(1 << i);
2139                         pp->dmafis_bits &= ~(1 << i);
2140                         pp->sdbfis_bits |= (1 << i);
2141                         nr_done++;
2142                 }
2143         }
2144
2145         if (!ap->qc_active) {
2146                 DPRINTK("over\n");
2147                 nv_swncq_pp_reinit(ap);
2148                 return nr_done;
2149         }
2150
2151         if (pp->qc_active & pp->dhfis_bits)
2152                 return nr_done;
2153
2154         if ((pp->ncq_flags & ncq_saw_backout) ||
2155             (pp->qc_active ^ pp->dhfis_bits))
2156                 /* if the controller cann't get a device to host register FIS,
2157                  * The driver needs to reissue the new command.
2158                  */
2159                 lack_dhfis = 1;
2160
2161         DPRINTK("id 0x%x QC: qc_active 0x%x,"
2162                 "SWNCQ:qc_active 0x%X defer_bits %X "
2163                 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2164                 ap->print_id, ap->qc_active, pp->qc_active,
2165                 pp->defer_queue.defer_bits, pp->dhfis_bits,
2166                 pp->dmafis_bits, pp->last_issue_tag);
2167
2168         nv_swncq_fis_reinit(ap);
2169
2170         if (lack_dhfis) {
2171                 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2172                 nv_swncq_issue_atacmd(ap, qc);
2173                 return nr_done;
2174         }
2175
2176         if (pp->defer_queue.defer_bits) {
2177                 /* send deferral queue command */
2178                 qc = nv_swncq_qc_from_dq(ap);
2179                 WARN_ON(qc == NULL);
2180                 nv_swncq_issue_atacmd(ap, qc);
2181         }
2182
2183         return nr_done;
2184 }
2185
2186 static inline u32 nv_swncq_tag(struct ata_port *ap)
2187 {
2188         struct nv_swncq_port_priv *pp = ap->private_data;
2189         u32 tag;
2190
2191         tag = readb(pp->tag_block) >> 2;
2192         return (tag & 0x1f);
2193 }
2194
2195 static int nv_swncq_dmafis(struct ata_port *ap)
2196 {
2197         struct ata_queued_cmd *qc;
2198         unsigned int rw;
2199         u8 dmactl;
2200         u32 tag;
2201         struct nv_swncq_port_priv *pp = ap->private_data;
2202
2203         __ata_bmdma_stop(ap);
2204         tag = nv_swncq_tag(ap);
2205
2206         DPRINTK("dma setup tag 0x%x\n", tag);
2207         qc = ata_qc_from_tag(ap, tag);
2208
2209         if (unlikely(!qc))
2210                 return 0;
2211
2212         rw = qc->tf.flags & ATA_TFLAG_WRITE;
2213
2214         /* load PRD table addr. */
2215         iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2216                   ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2217
2218         /* specify data direction, triple-check start bit is clear */
2219         dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2220         dmactl &= ~ATA_DMA_WR;
2221         if (!rw)
2222                 dmactl |= ATA_DMA_WR;
2223
2224         iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2225
2226         return 1;
2227 }
2228
2229 static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2230 {
2231         struct nv_swncq_port_priv *pp = ap->private_data;
2232         struct ata_queued_cmd *qc;
2233         struct ata_eh_info *ehi = &ap->link.eh_info;
2234         u32 serror;
2235         u8 ata_stat;
2236         int rc = 0;
2237
2238         ata_stat = ap->ops->check_status(ap);
2239         nv_swncq_irq_clear(ap, fis);
2240         if (!fis)
2241                 return;
2242
2243         if (ap->pflags & ATA_PFLAG_FROZEN)
2244                 return;
2245
2246         if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2247                 nv_swncq_hotplug(ap, fis);
2248                 return;
2249         }
2250
2251         if (!pp->qc_active)
2252                 return;
2253
2254         if (ap->ops->scr_read(ap, SCR_ERROR, &serror))
2255                 return;
2256         ap->ops->scr_write(ap, SCR_ERROR, serror);
2257
2258         if (ata_stat & ATA_ERR) {
2259                 ata_ehi_clear_desc(ehi);
2260                 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2261                 ehi->err_mask |= AC_ERR_DEV;
2262                 ehi->serror |= serror;
2263                 ehi->action |= ATA_EH_SOFTRESET;
2264                 ata_port_freeze(ap);
2265                 return;
2266         }
2267
2268         if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2269                 /* If the IRQ is backout, driver must issue
2270                  * the new command again some time later.
2271                  */
2272                 pp->ncq_flags |= ncq_saw_backout;
2273         }
2274
2275         if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2276                 pp->ncq_flags |= ncq_saw_sdb;
2277                 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2278                         "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2279                         ap->print_id, pp->qc_active, pp->dhfis_bits,
2280                         pp->dmafis_bits, readl(pp->sactive_block));
2281                 rc = nv_swncq_sdbfis(ap);
2282                 if (rc < 0)
2283                         goto irq_error;
2284         }
2285
2286         if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2287                 /* The interrupt indicates the new command
2288                  * was transmitted correctly to the drive.
2289                  */
2290                 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2291                 pp->ncq_flags |= ncq_saw_d2h;
2292                 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2293                         ata_ehi_push_desc(ehi, "illegal fis transaction");
2294                         ehi->err_mask |= AC_ERR_HSM;
2295                         ehi->action |= ATA_EH_HARDRESET;
2296                         goto irq_error;
2297                 }
2298
2299                 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2300                     !(pp->ncq_flags & ncq_saw_dmas)) {
2301                         ata_stat = ap->ops->check_status(ap);
2302                         if (ata_stat & ATA_BUSY)
2303                                 goto irq_exit;
2304
2305                         if (pp->defer_queue.defer_bits) {
2306                                 DPRINTK("send next command\n");
2307                                 qc = nv_swncq_qc_from_dq(ap);
2308                                 nv_swncq_issue_atacmd(ap, qc);
2309                         }
2310                 }
2311         }
2312
2313         if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2314                 /* program the dma controller with appropriate PRD buffers
2315                  * and start the DMA transfer for requested command.
2316                  */
2317                 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2318                 pp->ncq_flags |= ncq_saw_dmas;
2319                 rc = nv_swncq_dmafis(ap);
2320         }
2321
2322 irq_exit:
2323         return;
2324 irq_error:
2325         ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2326         ata_port_freeze(ap);
2327         return;
2328 }
2329
2330 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2331 {
2332         struct ata_host *host = dev_instance;
2333         unsigned int i;
2334         unsigned int handled = 0;
2335         unsigned long flags;
2336         u32 irq_stat;
2337
2338         spin_lock_irqsave(&host->lock, flags);
2339
2340         irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2341
2342         for (i = 0; i < host->n_ports; i++) {
2343                 struct ata_port *ap = host->ports[i];
2344
2345                 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
2346                         if (ap->link.sactive) {
2347                                 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2348                                 handled = 1;
2349                         } else {
2350                                 if (irq_stat)   /* reserve Hotplug */
2351                                         nv_swncq_irq_clear(ap, 0xfff0);
2352
2353                                 handled += nv_host_intr(ap, (u8)irq_stat);
2354                         }
2355                 }
2356                 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2357         }
2358
2359         spin_unlock_irqrestore(&host->lock, flags);
2360
2361         return IRQ_RETVAL(handled);
2362 }
2363
2364 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
2365 {
2366         static int printed_version;
2367         const struct ata_port_info *ppi[] = { NULL, NULL };
2368         struct ata_host *host;
2369         struct nv_host_priv *hpriv;
2370         int rc;
2371         u32 bar;
2372         void __iomem *base;
2373         unsigned long type = ent->driver_data;
2374
2375         // Make sure this is a SATA controller by counting the number of bars
2376         // (NVIDIA SATA controllers will always have six bars).  Otherwise,
2377         // it's an IDE controller and we ignore it.
2378         for (bar = 0; bar < 6; bar++)
2379                 if (pci_resource_start(pdev, bar) == 0)
2380                         return -ENODEV;
2381
2382         if (!printed_version++)
2383                 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
2384
2385         rc = pcim_enable_device(pdev);
2386         if (rc)
2387                 return rc;
2388
2389         /* determine type and allocate host */
2390         if (type == CK804 && adma_enabled) {
2391                 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
2392                 type = ADMA;
2393         }
2394
2395         if (type == SWNCQ) {
2396                 if (swncq_enabled)
2397                         dev_printk(KERN_NOTICE, &pdev->dev,
2398                                    "Using SWNCQ mode\n");
2399                 else
2400                         type = GENERIC;
2401         }
2402
2403         ppi[0] = &nv_port_info[type];
2404         rc = ata_pci_prepare_sff_host(pdev, ppi, &host);
2405         if (rc)
2406                 return rc;
2407
2408         hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
2409         if (!hpriv)
2410                 return -ENOMEM;
2411         hpriv->type = type;
2412         host->private_data = hpriv;
2413
2414         /* set 64bit dma masks, may fail */
2415         if (type == ADMA) {
2416                 if (pci_set_dma_mask(pdev, DMA_64BIT_MASK) == 0)
2417                         pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2418         }
2419
2420         /* request and iomap NV_MMIO_BAR */
2421         rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2422         if (rc)
2423                 return rc;
2424
2425         /* configure SCR access */
2426         base = host->iomap[NV_MMIO_BAR];
2427         host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2428         host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
2429
2430         /* enable SATA space for CK804 */
2431         if (type >= CK804) {
2432                 u8 regval;
2433
2434                 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2435                 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2436                 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2437         }
2438
2439         /* init ADMA */
2440         if (type == ADMA) {
2441                 rc = nv_adma_host_init(host);
2442                 if (rc)
2443                         return rc;
2444         } else if (type == SWNCQ)
2445                 nv_swncq_host_init(host);
2446
2447         pci_set_master(pdev);
2448         return ata_host_activate(host, pdev->irq, ppi[0]->irq_handler,
2449                                  IRQF_SHARED, ppi[0]->sht);
2450 }
2451
2452 #ifdef CONFIG_PM
2453 static int nv_pci_device_resume(struct pci_dev *pdev)
2454 {
2455         struct ata_host *host = dev_get_drvdata(&pdev->dev);
2456         struct nv_host_priv *hpriv = host->private_data;
2457         int rc;
2458
2459         rc = ata_pci_device_do_resume(pdev);
2460         if (rc)
2461                 return rc;
2462
2463         if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
2464                 if (hpriv->type >= CK804) {
2465                         u8 regval;
2466
2467                         pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2468                         regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2469                         pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2470                 }
2471                 if (hpriv->type == ADMA) {
2472                         u32 tmp32;
2473                         struct nv_adma_port_priv *pp;
2474                         /* enable/disable ADMA on the ports appropriately */
2475                         pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2476
2477                         pp = host->ports[0]->private_data;
2478                         if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2479                                 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2480                                            NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2481                         else
2482                                 tmp32 |=  (NV_MCP_SATA_CFG_20_PORT0_EN |
2483                                            NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2484                         pp = host->ports[1]->private_data;
2485                         if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2486                                 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
2487                                            NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2488                         else
2489                                 tmp32 |=  (NV_MCP_SATA_CFG_20_PORT1_EN |
2490                                            NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2491
2492                         pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2493                 }
2494         }
2495
2496         ata_host_resume(host);
2497
2498         return 0;
2499 }
2500 #endif
2501
2502 static void nv_ck804_host_stop(struct ata_host *host)
2503 {
2504         struct pci_dev *pdev = to_pci_dev(host->dev);
2505         u8 regval;
2506
2507         /* disable SATA space for CK804 */
2508         pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2509         regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2510         pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2511 }
2512
2513 static void nv_adma_host_stop(struct ata_host *host)
2514 {
2515         struct pci_dev *pdev = to_pci_dev(host->dev);
2516         u32 tmp32;
2517
2518         /* disable ADMA on the ports */
2519         pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2520         tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2521                    NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2522                    NV_MCP_SATA_CFG_20_PORT1_EN |
2523                    NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2524
2525         pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2526
2527         nv_ck804_host_stop(host);
2528 }
2529
2530 static int __init nv_init(void)
2531 {
2532         return pci_register_driver(&nv_pci_driver);
2533 }
2534
2535 static void __exit nv_exit(void)
2536 {
2537         pci_unregister_driver(&nv_pci_driver);
2538 }
2539
2540 module_init(nv_init);
2541 module_exit(nv_exit);
2542 module_param_named(adma, adma_enabled, bool, 0444);
2543 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");
2544 module_param_named(swncq, swncq_enabled, bool, 0444);
2545 MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: false)");
2546