2 * sata_nv.c - NVIDIA nForce SATA
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
39 #include <linux/kernel.h>
40 #include <linux/module.h>
41 #include <linux/pci.h>
42 #include <linux/init.h>
43 #include <linux/blkdev.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/device.h>
47 #include <scsi/scsi_host.h>
48 #include <scsi/scsi_device.h>
49 #include <linux/libata.h>
51 #define DRV_NAME "sata_nv"
52 #define DRV_VERSION "3.3"
54 #define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
66 /* INT_STATUS/ENABLE */
69 NV_INT_STATUS_CK804 = 0x440,
70 NV_INT_ENABLE_CK804 = 0x441,
72 /* INT_STATUS/ENABLE bits */
76 NV_INT_REMOVED = 0x08,
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
81 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
88 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
91 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
96 NV_ADMA_MAX_CPBS = 32,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
106 /* BAR5 offset to ADMA general registers */
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
117 /* ADMA port registers */
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
166 NV_ADMA_STAT_TIMEOUT,
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
174 /* ADMA Physical Region Descriptor - one SG segment */
183 enum nv_adma_regbits {
184 CMDEND = (1 << 15), /* end of command list */
185 WNB = (1 << 14), /* wait-not-BSY */
186 IGN = (1 << 13), /* ignore this entry */
187 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
188 DA2 = (1 << (2 + 8)),
189 DA1 = (1 << (1 + 8)),
190 DA0 = (1 << (0 + 8)),
193 /* ADMA Command Parameter Block
194 The first 5 SG segments are stored inside the Command Parameter Block itself.
195 If there are more than 5 segments the remainder are stored in a separate
196 memory area indicated by next_aprd. */
198 u8 resp_flags; /* 0 */
199 u8 reserved1; /* 1 */
200 u8 ctl_flags; /* 2 */
201 /* len is length of taskfile in 64 bit words */
204 u8 next_cpb_idx; /* 5 */
205 __le16 reserved2; /* 6-7 */
206 __le16 tf[12]; /* 8-31 */
207 struct nv_adma_prd aprd[5]; /* 32-111 */
208 __le64 next_aprd; /* 112-119 */
209 __le64 reserved3; /* 120-127 */
213 struct nv_adma_port_priv {
214 struct nv_adma_cpb *cpb;
216 struct nv_adma_prd *aprd;
218 void __iomem * ctl_block;
219 void __iomem * gen_block;
220 void __iomem * notifier_clear_block;
225 struct nv_host_priv {
229 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & ( 1 << (19 + (12 * (PORT)))))
231 static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
232 static void nv_remove_one (struct pci_dev *pdev);
233 static int nv_pci_device_resume(struct pci_dev *pdev);
234 static void nv_ck804_host_stop(struct ata_host *host);
235 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
236 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
237 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
238 static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg);
239 static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val);
241 static void nv_nf2_freeze(struct ata_port *ap);
242 static void nv_nf2_thaw(struct ata_port *ap);
243 static void nv_ck804_freeze(struct ata_port *ap);
244 static void nv_ck804_thaw(struct ata_port *ap);
245 static void nv_error_handler(struct ata_port *ap);
246 static int nv_adma_slave_config(struct scsi_device *sdev);
247 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
248 static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
249 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
250 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
251 static void nv_adma_irq_clear(struct ata_port *ap);
252 static int nv_adma_port_start(struct ata_port *ap);
253 static void nv_adma_port_stop(struct ata_port *ap);
254 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
255 static int nv_adma_port_resume(struct ata_port *ap);
256 static void nv_adma_error_handler(struct ata_port *ap);
257 static void nv_adma_host_stop(struct ata_host *host);
258 static void nv_adma_bmdma_setup(struct ata_queued_cmd *qc);
259 static void nv_adma_bmdma_start(struct ata_queued_cmd *qc);
260 static void nv_adma_bmdma_stop(struct ata_queued_cmd *qc);
261 static u8 nv_adma_bmdma_status(struct ata_port *ap);
267 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
272 static const struct pci_device_id nv_pci_tbl[] = {
273 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
274 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
275 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
276 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
277 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
278 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
279 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
280 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), GENERIC },
281 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), GENERIC },
282 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), GENERIC },
283 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), GENERIC },
284 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
285 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
286 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
287 { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
288 PCI_ANY_ID, PCI_ANY_ID,
289 PCI_CLASS_STORAGE_IDE<<8, 0xffff00, GENERIC },
290 { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
291 PCI_ANY_ID, PCI_ANY_ID,
292 PCI_CLASS_STORAGE_RAID<<8, 0xffff00, GENERIC },
294 { } /* terminate list */
297 static struct pci_driver nv_pci_driver = {
299 .id_table = nv_pci_tbl,
300 .probe = nv_init_one,
301 .suspend = ata_pci_device_suspend,
302 .resume = nv_pci_device_resume,
303 .remove = nv_remove_one,
306 static struct scsi_host_template nv_sht = {
307 .module = THIS_MODULE,
309 .ioctl = ata_scsi_ioctl,
310 .queuecommand = ata_scsi_queuecmd,
311 .can_queue = ATA_DEF_QUEUE,
312 .this_id = ATA_SHT_THIS_ID,
313 .sg_tablesize = LIBATA_MAX_PRD,
314 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
315 .emulated = ATA_SHT_EMULATED,
316 .use_clustering = ATA_SHT_USE_CLUSTERING,
317 .proc_name = DRV_NAME,
318 .dma_boundary = ATA_DMA_BOUNDARY,
319 .slave_configure = ata_scsi_slave_config,
320 .slave_destroy = ata_scsi_slave_destroy,
321 .bios_param = ata_std_bios_param,
322 .suspend = ata_scsi_device_suspend,
323 .resume = ata_scsi_device_resume,
326 static struct scsi_host_template nv_adma_sht = {
327 .module = THIS_MODULE,
329 .ioctl = ata_scsi_ioctl,
330 .queuecommand = ata_scsi_queuecmd,
331 .can_queue = NV_ADMA_MAX_CPBS,
332 .this_id = ATA_SHT_THIS_ID,
333 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
334 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
335 .emulated = ATA_SHT_EMULATED,
336 .use_clustering = ATA_SHT_USE_CLUSTERING,
337 .proc_name = DRV_NAME,
338 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
339 .slave_configure = nv_adma_slave_config,
340 .slave_destroy = ata_scsi_slave_destroy,
341 .bios_param = ata_std_bios_param,
342 .suspend = ata_scsi_device_suspend,
343 .resume = ata_scsi_device_resume,
346 static const struct ata_port_operations nv_generic_ops = {
347 .port_disable = ata_port_disable,
348 .tf_load = ata_tf_load,
349 .tf_read = ata_tf_read,
350 .exec_command = ata_exec_command,
351 .check_status = ata_check_status,
352 .dev_select = ata_std_dev_select,
353 .bmdma_setup = ata_bmdma_setup,
354 .bmdma_start = ata_bmdma_start,
355 .bmdma_stop = ata_bmdma_stop,
356 .bmdma_status = ata_bmdma_status,
357 .qc_prep = ata_qc_prep,
358 .qc_issue = ata_qc_issue_prot,
359 .freeze = ata_bmdma_freeze,
360 .thaw = ata_bmdma_thaw,
361 .error_handler = nv_error_handler,
362 .post_internal_cmd = ata_bmdma_post_internal_cmd,
363 .data_xfer = ata_data_xfer,
364 .irq_handler = nv_generic_interrupt,
365 .irq_clear = ata_bmdma_irq_clear,
366 .irq_on = ata_irq_on,
367 .irq_ack = ata_irq_ack,
368 .scr_read = nv_scr_read,
369 .scr_write = nv_scr_write,
370 .port_start = ata_port_start,
373 static const struct ata_port_operations nv_nf2_ops = {
374 .port_disable = ata_port_disable,
375 .tf_load = ata_tf_load,
376 .tf_read = ata_tf_read,
377 .exec_command = ata_exec_command,
378 .check_status = ata_check_status,
379 .dev_select = ata_std_dev_select,
380 .bmdma_setup = ata_bmdma_setup,
381 .bmdma_start = ata_bmdma_start,
382 .bmdma_stop = ata_bmdma_stop,
383 .bmdma_status = ata_bmdma_status,
384 .qc_prep = ata_qc_prep,
385 .qc_issue = ata_qc_issue_prot,
386 .freeze = nv_nf2_freeze,
388 .error_handler = nv_error_handler,
389 .post_internal_cmd = ata_bmdma_post_internal_cmd,
390 .data_xfer = ata_data_xfer,
391 .irq_handler = nv_nf2_interrupt,
392 .irq_clear = ata_bmdma_irq_clear,
393 .irq_on = ata_irq_on,
394 .irq_ack = ata_irq_ack,
395 .scr_read = nv_scr_read,
396 .scr_write = nv_scr_write,
397 .port_start = ata_port_start,
400 static const struct ata_port_operations nv_ck804_ops = {
401 .port_disable = ata_port_disable,
402 .tf_load = ata_tf_load,
403 .tf_read = ata_tf_read,
404 .exec_command = ata_exec_command,
405 .check_status = ata_check_status,
406 .dev_select = ata_std_dev_select,
407 .bmdma_setup = ata_bmdma_setup,
408 .bmdma_start = ata_bmdma_start,
409 .bmdma_stop = ata_bmdma_stop,
410 .bmdma_status = ata_bmdma_status,
411 .qc_prep = ata_qc_prep,
412 .qc_issue = ata_qc_issue_prot,
413 .freeze = nv_ck804_freeze,
414 .thaw = nv_ck804_thaw,
415 .error_handler = nv_error_handler,
416 .post_internal_cmd = ata_bmdma_post_internal_cmd,
417 .data_xfer = ata_data_xfer,
418 .irq_handler = nv_ck804_interrupt,
419 .irq_clear = ata_bmdma_irq_clear,
420 .irq_on = ata_irq_on,
421 .irq_ack = ata_irq_ack,
422 .scr_read = nv_scr_read,
423 .scr_write = nv_scr_write,
424 .port_start = ata_port_start,
425 .host_stop = nv_ck804_host_stop,
428 static const struct ata_port_operations nv_adma_ops = {
429 .port_disable = ata_port_disable,
430 .tf_load = ata_tf_load,
431 .tf_read = ata_tf_read,
432 .check_atapi_dma = nv_adma_check_atapi_dma,
433 .exec_command = ata_exec_command,
434 .check_status = ata_check_status,
435 .dev_select = ata_std_dev_select,
436 .bmdma_setup = nv_adma_bmdma_setup,
437 .bmdma_start = nv_adma_bmdma_start,
438 .bmdma_stop = nv_adma_bmdma_stop,
439 .bmdma_status = nv_adma_bmdma_status,
440 .qc_prep = nv_adma_qc_prep,
441 .qc_issue = nv_adma_qc_issue,
442 .freeze = nv_ck804_freeze,
443 .thaw = nv_ck804_thaw,
444 .error_handler = nv_adma_error_handler,
445 .post_internal_cmd = nv_adma_bmdma_stop,
446 .data_xfer = ata_data_xfer,
447 .irq_handler = nv_adma_interrupt,
448 .irq_clear = nv_adma_irq_clear,
449 .irq_on = ata_irq_on,
450 .irq_ack = ata_irq_ack,
451 .scr_read = nv_scr_read,
452 .scr_write = nv_scr_write,
453 .port_start = nv_adma_port_start,
454 .port_stop = nv_adma_port_stop,
455 .port_suspend = nv_adma_port_suspend,
456 .port_resume = nv_adma_port_resume,
457 .host_stop = nv_adma_host_stop,
460 static struct ata_port_info nv_port_info[] = {
464 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
465 ATA_FLAG_HRST_TO_RESUME,
466 .pio_mask = NV_PIO_MASK,
467 .mwdma_mask = NV_MWDMA_MASK,
468 .udma_mask = NV_UDMA_MASK,
469 .port_ops = &nv_generic_ops,
474 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
475 ATA_FLAG_HRST_TO_RESUME,
476 .pio_mask = NV_PIO_MASK,
477 .mwdma_mask = NV_MWDMA_MASK,
478 .udma_mask = NV_UDMA_MASK,
479 .port_ops = &nv_nf2_ops,
484 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
485 ATA_FLAG_HRST_TO_RESUME,
486 .pio_mask = NV_PIO_MASK,
487 .mwdma_mask = NV_MWDMA_MASK,
488 .udma_mask = NV_UDMA_MASK,
489 .port_ops = &nv_ck804_ops,
494 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
495 ATA_FLAG_HRST_TO_RESUME |
496 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
497 .pio_mask = NV_PIO_MASK,
498 .mwdma_mask = NV_MWDMA_MASK,
499 .udma_mask = NV_UDMA_MASK,
500 .port_ops = &nv_adma_ops,
504 MODULE_AUTHOR("NVIDIA");
505 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
506 MODULE_LICENSE("GPL");
507 MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
508 MODULE_VERSION(DRV_VERSION);
510 static int adma_enabled = 1;
512 static void nv_adma_register_mode(struct ata_port *ap)
514 struct nv_adma_port_priv *pp = ap->private_data;
515 void __iomem *mmio = pp->ctl_block;
519 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
522 status = readw(mmio + NV_ADMA_STAT);
523 while(!(status & NV_ADMA_STAT_IDLE) && count < 20) {
525 status = readw(mmio + NV_ADMA_STAT);
529 ata_port_printk(ap, KERN_WARNING,
530 "timeout waiting for ADMA IDLE, stat=0x%hx\n",
533 tmp = readw(mmio + NV_ADMA_CTL);
534 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
537 status = readw(mmio + NV_ADMA_STAT);
538 while(!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
540 status = readw(mmio + NV_ADMA_STAT);
544 ata_port_printk(ap, KERN_WARNING,
545 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
548 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
551 static void nv_adma_mode(struct ata_port *ap)
553 struct nv_adma_port_priv *pp = ap->private_data;
554 void __iomem *mmio = pp->ctl_block;
558 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
561 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
563 tmp = readw(mmio + NV_ADMA_CTL);
564 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
566 status = readw(mmio + NV_ADMA_STAT);
567 while(((status & NV_ADMA_STAT_LEGACY) ||
568 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
570 status = readw(mmio + NV_ADMA_STAT);
574 ata_port_printk(ap, KERN_WARNING,
575 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
578 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
581 static int nv_adma_slave_config(struct scsi_device *sdev)
583 struct ata_port *ap = ata_shost_to_port(sdev->host);
584 struct nv_adma_port_priv *pp = ap->private_data;
585 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
587 unsigned long segment_boundary;
588 unsigned short sg_tablesize;
591 u32 current_reg, new_reg, config_mask;
593 rc = ata_scsi_slave_config(sdev);
595 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
596 /* Not a proper libata device, ignore */
599 if (ap->device[sdev->id].class == ATA_DEV_ATAPI) {
601 * NVIDIA reports that ADMA mode does not support ATAPI commands.
602 * Therefore ATAPI commands are sent through the legacy interface.
603 * However, the legacy interface only supports 32-bit DMA.
604 * Restrict DMA parameters as required by the legacy interface
605 * when an ATAPI device is connected.
607 bounce_limit = ATA_DMA_MASK;
608 segment_boundary = ATA_DMA_BOUNDARY;
609 /* Subtract 1 since an extra entry may be needed for padding, see
611 sg_tablesize = LIBATA_MAX_PRD - 1;
613 /* Since the legacy DMA engine is in use, we need to disable ADMA
616 nv_adma_register_mode(ap);
619 bounce_limit = *ap->dev->dma_mask;
620 segment_boundary = NV_ADMA_DMA_BOUNDARY;
621 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
625 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, ¤t_reg);
628 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
629 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
631 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
632 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
635 new_reg = current_reg | config_mask;
636 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
639 new_reg = current_reg & ~config_mask;
640 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
643 if(current_reg != new_reg)
644 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
646 blk_queue_bounce_limit(sdev->request_queue, bounce_limit);
647 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
648 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
649 ata_port_printk(ap, KERN_INFO,
650 "bounce limit 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
651 (unsigned long long)bounce_limit, segment_boundary, sg_tablesize);
655 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
657 struct nv_adma_port_priv *pp = qc->ap->private_data;
658 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
661 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
663 unsigned int idx = 0;
665 if(tf->flags & ATA_TFLAG_ISADDR) {
666 if (tf->flags & ATA_TFLAG_LBA48) {
667 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
668 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
669 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
670 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
671 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
672 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
674 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
676 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
677 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
678 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
679 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
682 if(tf->flags & ATA_TFLAG_DEVICE)
683 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
685 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
688 cpb[idx++] = cpu_to_le16(IGN);
693 static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
695 struct nv_adma_port_priv *pp = ap->private_data;
696 u8 flags = pp->cpb[cpb_num].resp_flags;
698 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
700 if (unlikely((force_err ||
701 flags & (NV_CPB_RESP_ATA_ERR |
702 NV_CPB_RESP_CMD_ERR |
703 NV_CPB_RESP_CPB_ERR)))) {
704 struct ata_eh_info *ehi = &ap->eh_info;
707 ata_ehi_clear_desc(ehi);
708 ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x", flags );
709 if (flags & NV_CPB_RESP_ATA_ERR) {
710 ata_ehi_push_desc(ehi, ": ATA error");
711 ehi->err_mask |= AC_ERR_DEV;
712 } else if (flags & NV_CPB_RESP_CMD_ERR) {
713 ata_ehi_push_desc(ehi, ": CMD error");
714 ehi->err_mask |= AC_ERR_DEV;
715 } else if (flags & NV_CPB_RESP_CPB_ERR) {
716 ata_ehi_push_desc(ehi, ": CPB error");
717 ehi->err_mask |= AC_ERR_SYSTEM;
720 /* notifier error, but no error in CPB flags? */
721 ehi->err_mask |= AC_ERR_OTHER;
724 /* Kill all commands. EH will determine what actually failed. */
732 if (flags & NV_CPB_RESP_DONE) {
733 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
734 VPRINTK("CPB flags done, flags=0x%x\n", flags);
736 /* Grab the ATA port status for non-NCQ commands.
737 For NCQ commands the current status may have nothing to do with
738 the command just completed. */
739 if (qc->tf.protocol != ATA_PROT_NCQ) {
740 u8 ata_status = readb(pp->ctl_block + (ATA_REG_STATUS * 4));
741 qc->err_mask |= ac_err_mask(ata_status);
743 DPRINTK("Completing qc from tag %d with err_mask %u\n",cpb_num,
751 static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
753 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->active_tag);
755 /* freeze if hotplugged */
756 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
761 /* bail out if not our interrupt */
762 if (!(irq_stat & NV_INT_DEV))
765 /* DEV interrupt w/ no active qc? */
766 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
767 ata_check_status(ap);
771 /* handle interrupt */
772 return ata_host_intr(ap, qc);
775 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
777 struct ata_host *host = dev_instance;
779 u32 notifier_clears[2];
781 spin_lock(&host->lock);
783 for (i = 0; i < host->n_ports; i++) {
784 struct ata_port *ap = host->ports[i];
785 notifier_clears[i] = 0;
787 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
788 struct nv_adma_port_priv *pp = ap->private_data;
789 void __iomem *mmio = pp->ctl_block;
792 u32 notifier, notifier_error;
794 /* if in ATA register mode, use standard ata interrupt handler */
795 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
796 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
797 >> (NV_INT_PORT_SHIFT * i);
798 if(ata_tag_valid(ap->active_tag))
799 /** NV_INT_DEV indication seems unreliable at times
800 at least in ADMA mode. Force it on always when a
801 command is active, to prevent losing interrupts. */
802 irq_stat |= NV_INT_DEV;
803 handled += nv_host_intr(ap, irq_stat);
807 notifier = readl(mmio + NV_ADMA_NOTIFIER);
808 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
809 notifier_clears[i] = notifier | notifier_error;
811 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
813 if( !NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
818 status = readw(mmio + NV_ADMA_STAT);
820 /* Clear status. Ensure the controller sees the clearing before we start
821 looking at any of the CPB statuses, so that any CPB completions after
822 this point in the handler will raise another interrupt. */
823 writew(status, mmio + NV_ADMA_STAT);
824 readw(mmio + NV_ADMA_STAT); /* flush posted write */
827 handled++; /* irq handled if we got here */
829 /* freeze if hotplugged or controller error */
830 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
831 NV_ADMA_STAT_HOTUNPLUG |
832 NV_ADMA_STAT_TIMEOUT |
833 NV_ADMA_STAT_SERROR))) {
834 struct ata_eh_info *ehi = &ap->eh_info;
836 ata_ehi_clear_desc(ehi);
837 ata_ehi_push_desc(ehi, "ADMA status 0x%08x", status );
838 if (status & NV_ADMA_STAT_TIMEOUT) {
839 ehi->err_mask |= AC_ERR_SYSTEM;
840 ata_ehi_push_desc(ehi, ": timeout");
841 } else if (status & NV_ADMA_STAT_HOTPLUG) {
842 ata_ehi_hotplugged(ehi);
843 ata_ehi_push_desc(ehi, ": hotplug");
844 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
845 ata_ehi_hotplugged(ehi);
846 ata_ehi_push_desc(ehi, ": hot unplug");
847 } else if (status & NV_ADMA_STAT_SERROR) {
848 /* let libata analyze SError and figure out the cause */
849 ata_ehi_push_desc(ehi, ": SError");
855 if (status & (NV_ADMA_STAT_DONE |
856 NV_ADMA_STAT_CPBERR)) {
857 u32 check_commands = notifier | notifier_error;
859 /** Check CPBs for completed commands */
860 while ((pos = ffs(check_commands)) && !error) {
862 error = nv_adma_check_cpb(ap, pos,
863 notifier_error & (1 << pos) );
864 check_commands &= ~(1 << pos );
870 if(notifier_clears[0] || notifier_clears[1]) {
871 /* Note: Both notifier clear registers must be written
872 if either is set, even if one is zero, according to NVIDIA. */
873 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
874 writel(notifier_clears[0], pp->notifier_clear_block);
875 pp = host->ports[1]->private_data;
876 writel(notifier_clears[1], pp->notifier_clear_block);
879 spin_unlock(&host->lock);
881 return IRQ_RETVAL(handled);
884 static void nv_adma_irq_clear(struct ata_port *ap)
886 struct nv_adma_port_priv *pp = ap->private_data;
887 void __iomem *mmio = pp->ctl_block;
888 u16 status = readw(mmio + NV_ADMA_STAT);
889 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
890 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
891 void __iomem *dma_stat_addr = ap->ioaddr.bmdma_addr + ATA_DMA_STATUS;
893 /* clear ADMA status */
894 writew(status, mmio + NV_ADMA_STAT);
895 writel(notifier | notifier_error,
896 pp->notifier_clear_block);
898 /** clear legacy status */
899 iowrite8(ioread8(dma_stat_addr), dma_stat_addr);
902 static void nv_adma_bmdma_setup(struct ata_queued_cmd *qc)
904 struct ata_port *ap = qc->ap;
905 unsigned int rw = (qc->tf.flags & ATA_TFLAG_WRITE);
906 struct nv_adma_port_priv *pp = ap->private_data;
909 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
914 /* load PRD table addr. */
915 iowrite32(ap->prd_dma, ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
917 /* specify data direction, triple-check start bit is clear */
918 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
919 dmactl &= ~(ATA_DMA_WR | ATA_DMA_START);
921 dmactl |= ATA_DMA_WR;
923 iowrite8(dmactl, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
925 /* issue r/w command */
926 ata_exec_command(ap, &qc->tf);
929 static void nv_adma_bmdma_start(struct ata_queued_cmd *qc)
931 struct ata_port *ap = qc->ap;
932 struct nv_adma_port_priv *pp = ap->private_data;
935 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
940 /* start host DMA transaction */
941 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
942 iowrite8(dmactl | ATA_DMA_START,
943 ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
946 static void nv_adma_bmdma_stop(struct ata_queued_cmd *qc)
948 struct ata_port *ap = qc->ap;
949 struct nv_adma_port_priv *pp = ap->private_data;
951 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
954 /* clear start/stop bit */
955 iowrite8(ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD) & ~ATA_DMA_START,
956 ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
958 /* one-PIO-cycle guaranteed wait, per spec, for HDMA1:0 transition */
959 ata_altstatus(ap); /* dummy read */
962 static u8 nv_adma_bmdma_status(struct ata_port *ap)
964 struct nv_adma_port_priv *pp = ap->private_data;
966 WARN_ON(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE));
968 return ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_STATUS);
971 static int nv_adma_port_start(struct ata_port *ap)
973 struct device *dev = ap->host->dev;
974 struct nv_adma_port_priv *pp;
983 rc = ata_port_start(ap);
987 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
991 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
992 ap->port_no * NV_ADMA_PORT_SIZE;
993 pp->ctl_block = mmio;
994 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
995 pp->notifier_clear_block = pp->gen_block +
996 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
998 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
999 &mem_dma, GFP_KERNEL);
1002 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1005 * First item in chunk of DMA memory:
1006 * 128-byte command parameter block (CPB)
1007 * one for each command tag
1010 pp->cpb_dma = mem_dma;
1012 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1013 writel((mem_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1015 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1016 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1019 * Second item: block of ADMA_SGTBL_LEN s/g entries
1022 pp->aprd_dma = mem_dma;
1024 ap->private_data = pp;
1026 /* clear any outstanding interrupt conditions */
1027 writew(0xffff, mmio + NV_ADMA_STAT);
1029 /* initialize port variables */
1030 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1032 /* clear CPB fetch count */
1033 writew(0, mmio + NV_ADMA_CPB_COUNT);
1035 /* clear GO for register mode, enable interrupt */
1036 tmp = readw(mmio + NV_ADMA_CTL);
1037 writew( (tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1038 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1040 tmp = readw(mmio + NV_ADMA_CTL);
1041 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1042 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
1044 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1045 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
1050 static void nv_adma_port_stop(struct ata_port *ap)
1052 struct nv_adma_port_priv *pp = ap->private_data;
1053 void __iomem *mmio = pp->ctl_block;
1056 writew(0, mmio + NV_ADMA_CTL);
1059 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1061 struct nv_adma_port_priv *pp = ap->private_data;
1062 void __iomem *mmio = pp->ctl_block;
1064 /* Go to register mode - clears GO */
1065 nv_adma_register_mode(ap);
1067 /* clear CPB fetch count */
1068 writew(0, mmio + NV_ADMA_CPB_COUNT);
1070 /* disable interrupt, shut down port */
1071 writew(0, mmio + NV_ADMA_CTL);
1076 static int nv_adma_port_resume(struct ata_port *ap)
1078 struct nv_adma_port_priv *pp = ap->private_data;
1079 void __iomem *mmio = pp->ctl_block;
1082 /* set CPB block location */
1083 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1084 writel((pp->cpb_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1086 /* clear any outstanding interrupt conditions */
1087 writew(0xffff, mmio + NV_ADMA_STAT);
1089 /* initialize port variables */
1090 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1092 /* clear CPB fetch count */
1093 writew(0, mmio + NV_ADMA_CPB_COUNT);
1095 /* clear GO for register mode, enable interrupt */
1096 tmp = readw(mmio + NV_ADMA_CTL);
1097 writew( (tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1098 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1100 tmp = readw(mmio + NV_ADMA_CTL);
1101 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1102 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
1104 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1105 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
1110 static void nv_adma_setup_port(struct ata_probe_ent *probe_ent, unsigned int port)
1112 void __iomem *mmio = probe_ent->iomap[NV_MMIO_BAR];
1113 struct ata_ioports *ioport = &probe_ent->port[port];
1117 mmio += NV_ADMA_PORT + port * NV_ADMA_PORT_SIZE;
1119 ioport->cmd_addr = mmio;
1120 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
1121 ioport->error_addr =
1122 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1123 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1124 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1125 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1126 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1127 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
1128 ioport->status_addr =
1129 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
1130 ioport->altstatus_addr =
1131 ioport->ctl_addr = mmio + 0x20;
1134 static int nv_adma_host_init(struct ata_probe_ent *probe_ent)
1136 struct pci_dev *pdev = to_pci_dev(probe_ent->dev);
1142 /* enable ADMA on the ports */
1143 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1144 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1145 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1146 NV_MCP_SATA_CFG_20_PORT1_EN |
1147 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1149 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1151 for (i = 0; i < probe_ent->n_ports; i++)
1152 nv_adma_setup_port(probe_ent, i);
1157 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1158 struct scatterlist *sg,
1160 struct nv_adma_prd *aprd)
1163 if (qc->tf.flags & ATA_TFLAG_WRITE)
1164 flags |= NV_APRD_WRITE;
1165 if (idx == qc->n_elem - 1)
1166 flags |= NV_APRD_END;
1168 flags |= NV_APRD_CONT;
1170 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1171 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1172 aprd->flags = flags;
1173 aprd->packet_len = 0;
1176 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1178 struct nv_adma_port_priv *pp = qc->ap->private_data;
1180 struct nv_adma_prd *aprd;
1181 struct scatterlist *sg;
1187 ata_for_each_sg(sg, qc) {
1188 aprd = (idx < 5) ? &cpb->aprd[idx] : &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (idx-5)];
1189 nv_adma_fill_aprd(qc, sg, idx, aprd);
1193 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
1195 cpb->next_aprd = cpu_to_le64(0);
1198 static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1200 struct nv_adma_port_priv *pp = qc->ap->private_data;
1202 /* ADMA engine can only be used for non-ATAPI DMA commands,
1203 or interrupt-driven no-data commands. */
1204 if((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
1205 (qc->tf.flags & ATA_TFLAG_POLLING))
1208 if((qc->flags & ATA_QCFLAG_DMAMAP) ||
1209 (qc->tf.protocol == ATA_PROT_NODATA))
1215 static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1217 struct nv_adma_port_priv *pp = qc->ap->private_data;
1218 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1219 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1222 if (nv_adma_use_reg_mode(qc)) {
1223 nv_adma_register_mode(qc->ap);
1228 cpb->resp_flags = NV_CPB_RESP_DONE;
1235 cpb->next_cpb_idx = 0;
1237 /* turn on NCQ flags for NCQ commands */
1238 if (qc->tf.protocol == ATA_PROT_NCQ)
1239 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1241 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1243 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1245 if(qc->flags & ATA_QCFLAG_DMAMAP) {
1246 nv_adma_fill_sg(qc, cpb);
1247 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1249 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
1251 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID until we are
1252 finished filling in all of the contents */
1254 cpb->ctl_flags = ctl_flags;
1256 cpb->resp_flags = 0;
1259 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1261 struct nv_adma_port_priv *pp = qc->ap->private_data;
1262 void __iomem *mmio = pp->ctl_block;
1263 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
1267 if (nv_adma_use_reg_mode(qc)) {
1268 /* use ATA register mode */
1269 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
1270 nv_adma_register_mode(qc->ap);
1271 return ata_qc_issue_prot(qc);
1273 nv_adma_mode(qc->ap);
1275 /* write append register, command tag in lower 8 bits
1276 and (number of cpbs to append -1) in top 8 bits */
1279 if(curr_ncq != pp->last_issue_ncq) {
1280 /* Seems to need some delay before switching between NCQ and non-NCQ
1281 commands, else we get command timeouts and such. */
1283 pp->last_issue_ncq = curr_ncq;
1286 writew(qc->tag, mmio + NV_ADMA_APPEND);
1288 DPRINTK("Issued tag %u\n",qc->tag);
1293 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1295 struct ata_host *host = dev_instance;
1297 unsigned int handled = 0;
1298 unsigned long flags;
1300 spin_lock_irqsave(&host->lock, flags);
1302 for (i = 0; i < host->n_ports; i++) {
1303 struct ata_port *ap;
1305 ap = host->ports[i];
1307 !(ap->flags & ATA_FLAG_DISABLED)) {
1308 struct ata_queued_cmd *qc;
1310 qc = ata_qc_from_tag(ap, ap->active_tag);
1311 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
1312 handled += ata_host_intr(ap, qc);
1314 // No request pending? Clear interrupt status
1315 // anyway, in case there's one pending.
1316 ap->ops->check_status(ap);
1321 spin_unlock_irqrestore(&host->lock, flags);
1323 return IRQ_RETVAL(handled);
1326 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1330 for (i = 0; i < host->n_ports; i++) {
1331 struct ata_port *ap = host->ports[i];
1333 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1334 handled += nv_host_intr(ap, irq_stat);
1336 irq_stat >>= NV_INT_PORT_SHIFT;
1339 return IRQ_RETVAL(handled);
1342 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1344 struct ata_host *host = dev_instance;
1348 spin_lock(&host->lock);
1349 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1350 ret = nv_do_interrupt(host, irq_stat);
1351 spin_unlock(&host->lock);
1356 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1358 struct ata_host *host = dev_instance;
1362 spin_lock(&host->lock);
1363 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1364 ret = nv_do_interrupt(host, irq_stat);
1365 spin_unlock(&host->lock);
1370 static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg)
1372 if (sc_reg > SCR_CONTROL)
1375 return ioread32(ap->ioaddr.scr_addr + (sc_reg * 4));
1378 static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val)
1380 if (sc_reg > SCR_CONTROL)
1383 iowrite32(val, ap->ioaddr.scr_addr + (sc_reg * 4));
1386 static void nv_nf2_freeze(struct ata_port *ap)
1388 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1389 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1392 mask = ioread8(scr_addr + NV_INT_ENABLE);
1393 mask &= ~(NV_INT_ALL << shift);
1394 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1397 static void nv_nf2_thaw(struct ata_port *ap)
1399 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1400 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1403 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1405 mask = ioread8(scr_addr + NV_INT_ENABLE);
1406 mask |= (NV_INT_MASK << shift);
1407 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1410 static void nv_ck804_freeze(struct ata_port *ap)
1412 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1413 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1416 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1417 mask &= ~(NV_INT_ALL << shift);
1418 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1421 static void nv_ck804_thaw(struct ata_port *ap)
1423 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1424 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1427 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1429 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1430 mask |= (NV_INT_MASK << shift);
1431 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1434 static int nv_hardreset(struct ata_port *ap, unsigned int *class)
1438 /* SATA hardreset fails to retrieve proper device signature on
1439 * some controllers. Don't classify on hardreset. For more
1440 * info, see http://bugme.osdl.org/show_bug.cgi?id=3352
1442 return sata_std_hardreset(ap, &dummy);
1445 static void nv_error_handler(struct ata_port *ap)
1447 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1448 nv_hardreset, ata_std_postreset);
1451 static void nv_adma_error_handler(struct ata_port *ap)
1453 struct nv_adma_port_priv *pp = ap->private_data;
1454 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1455 void __iomem *mmio = pp->ctl_block;
1459 if(ata_tag_valid(ap->active_tag) || ap->sactive) {
1460 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1461 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1462 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1463 u32 status = readw(mmio + NV_ADMA_STAT);
1464 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1465 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
1467 ata_port_printk(ap, KERN_ERR, "EH in ADMA mode, notifier 0x%X "
1468 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1469 "next cpb count 0x%X next cpb idx 0x%x\n",
1470 notifier, notifier_error, gen_ctl, status,
1471 cpb_count, next_cpb_idx);
1473 for( i=0;i<NV_ADMA_MAX_CPBS;i++) {
1474 struct nv_adma_cpb *cpb = &pp->cpb[i];
1475 if( (ata_tag_valid(ap->active_tag) && i == ap->active_tag) ||
1476 ap->sactive & (1 << i) )
1477 ata_port_printk(ap, KERN_ERR,
1478 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1479 i, cpb->ctl_flags, cpb->resp_flags);
1483 /* Push us back into port register mode for error handling. */
1484 nv_adma_register_mode(ap);
1486 /* Mark all of the CPBs as invalid to prevent them from being executed */
1487 for( i=0;i<NV_ADMA_MAX_CPBS;i++)
1488 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1490 /* clear CPB fetch count */
1491 writew(0, mmio + NV_ADMA_CPB_COUNT);
1494 tmp = readw(mmio + NV_ADMA_CTL);
1495 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1496 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
1498 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1499 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
1502 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1503 nv_hardreset, ata_std_postreset);
1506 static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
1508 static int printed_version = 0;
1509 struct ata_port_info *ppi[2];
1510 struct ata_probe_ent *probe_ent;
1511 struct nv_host_priv *hpriv;
1515 unsigned long type = ent->driver_data;
1518 // Make sure this is a SATA controller by counting the number of bars
1519 // (NVIDIA SATA controllers will always have six bars). Otherwise,
1520 // it's an IDE controller and we ignore it.
1521 for (bar=0; bar<6; bar++)
1522 if (pci_resource_start(pdev, bar) == 0)
1525 if (!printed_version++)
1526 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
1528 rc = pcim_enable_device(pdev);
1532 rc = pci_request_regions(pdev, DRV_NAME);
1534 pcim_pin_device(pdev);
1538 if(type >= CK804 && adma_enabled) {
1539 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
1541 if(!pci_set_dma_mask(pdev, DMA_64BIT_MASK) &&
1542 !pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK))
1547 rc = pci_set_dma_mask(pdev, ATA_DMA_MASK);
1550 rc = pci_set_consistent_dma_mask(pdev, ATA_DMA_MASK);
1557 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
1561 ppi[0] = ppi[1] = &nv_port_info[type];
1562 probe_ent = ata_pci_init_native_mode(pdev, ppi, ATA_PORT_PRIMARY | ATA_PORT_SECONDARY);
1566 if (!pcim_iomap(pdev, NV_MMIO_BAR, 0))
1568 probe_ent->iomap = pcim_iomap_table(pdev);
1570 probe_ent->private_data = hpriv;
1573 base = probe_ent->iomap[NV_MMIO_BAR];
1574 probe_ent->port[0].scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
1575 probe_ent->port[1].scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
1577 /* enable SATA space for CK804 */
1578 if (type >= CK804) {
1581 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
1582 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1583 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1586 pci_set_master(pdev);
1589 rc = nv_adma_host_init(probe_ent);
1594 rc = ata_device_add(probe_ent);
1598 devm_kfree(&pdev->dev, probe_ent);
1602 static void nv_remove_one (struct pci_dev *pdev)
1604 struct ata_host *host = dev_get_drvdata(&pdev->dev);
1605 struct nv_host_priv *hpriv = host->private_data;
1607 ata_pci_remove_one(pdev);
1611 static int nv_pci_device_resume(struct pci_dev *pdev)
1613 struct ata_host *host = dev_get_drvdata(&pdev->dev);
1614 struct nv_host_priv *hpriv = host->private_data;
1617 rc = ata_pci_device_do_resume(pdev);
1621 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
1622 if(hpriv->type >= CK804) {
1625 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
1626 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1627 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1629 if(hpriv->type == ADMA) {
1631 struct nv_adma_port_priv *pp;
1632 /* enable/disable ADMA on the ports appropriately */
1633 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1635 pp = host->ports[0]->private_data;
1636 if(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1637 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
1638 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
1640 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
1641 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
1642 pp = host->ports[1]->private_data;
1643 if(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1644 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
1645 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1647 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
1648 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1650 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1654 ata_host_resume(host);
1659 static void nv_ck804_host_stop(struct ata_host *host)
1661 struct pci_dev *pdev = to_pci_dev(host->dev);
1664 /* disable SATA space for CK804 */
1665 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
1666 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1667 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1670 static void nv_adma_host_stop(struct ata_host *host)
1672 struct pci_dev *pdev = to_pci_dev(host->dev);
1675 /* disable ADMA on the ports */
1676 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1677 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
1678 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1679 NV_MCP_SATA_CFG_20_PORT1_EN |
1680 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1682 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1684 nv_ck804_host_stop(host);
1687 static int __init nv_init(void)
1689 return pci_register_driver(&nv_pci_driver);
1692 static void __exit nv_exit(void)
1694 pci_unregister_driver(&nv_pci_driver);
1697 module_init(nv_init);
1698 module_exit(nv_exit);
1699 module_param_named(adma, adma_enabled, bool, 0444);
1700 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");