2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
23 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/interrupt.h>
31 #include <linux/dmaengine.h>
32 #include <linux/delay.h>
33 #include <linux/dma-mapping.h>
35 #include "ioatdma_registers.h"
36 #include "ioatdma_hw.h"
38 #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
39 #define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
40 #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
41 #define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
43 /* internal functions */
44 static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
45 static void ioat_shutdown(struct pci_dev *pdev);
46 static void __devexit ioat_remove(struct pci_dev *pdev);
48 static int enumerate_dma_channels(struct ioat_device *device)
53 struct ioat_dma_chan *ioat_chan;
55 device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
56 xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
57 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
59 for (i = 0; i < device->common.chancnt; i++) {
60 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
62 device->common.chancnt = i;
66 ioat_chan->device = device;
67 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
68 ioat_chan->xfercap = xfercap;
69 spin_lock_init(&ioat_chan->cleanup_lock);
70 spin_lock_init(&ioat_chan->desc_lock);
71 INIT_LIST_HEAD(&ioat_chan->free_desc);
72 INIT_LIST_HEAD(&ioat_chan->used_desc);
73 /* This should be made common somewhere in dmaengine.c */
74 ioat_chan->common.device = &device->common;
75 ioat_chan->common.client = NULL;
76 list_add_tail(&ioat_chan->common.device_node,
77 &device->common.channels);
79 return device->common.chancnt;
83 ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
85 struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
86 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
88 pci_unmap_addr_set(desc, src, addr);
90 list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
91 iter->hw->src_addr = addr;
92 addr += ioat_chan->xfercap;
98 ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
100 struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
101 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
103 pci_unmap_addr_set(desc, dst, addr);
105 list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
106 iter->hw->dst_addr = addr;
107 addr += ioat_chan->xfercap;
112 ioat_tx_submit(struct dma_async_tx_descriptor *tx)
114 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
115 struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
118 struct ioat_desc_sw *group_start;
120 group_start = list_entry(desc->async_tx.tx_list.next,
121 struct ioat_desc_sw, node);
122 spin_lock_bh(&ioat_chan->desc_lock);
123 /* cookie incr and addition to used_list must be atomic */
124 cookie = ioat_chan->common.cookie;
128 ioat_chan->common.cookie = desc->async_tx.cookie = cookie;
130 /* write address into NextDescriptor field of last desc in chain */
131 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
132 group_start->async_tx.phys;
133 list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev);
135 ioat_chan->pending += desc->tx_cnt;
136 if (ioat_chan->pending >= 4) {
138 ioat_chan->pending = 0;
140 spin_unlock_bh(&ioat_chan->desc_lock);
143 writeb(IOAT_CHANCMD_APPEND,
144 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
149 static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
150 struct ioat_dma_chan *ioat_chan,
153 struct ioat_dma_descriptor *desc;
154 struct ioat_desc_sw *desc_sw;
155 struct ioat_device *ioat_device;
158 ioat_device = to_ioat_device(ioat_chan->common.device);
159 desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys);
163 desc_sw = kzalloc(sizeof(*desc_sw), flags);
164 if (unlikely(!desc_sw)) {
165 pci_pool_free(ioat_device->dma_pool, desc, phys);
169 memset(desc, 0, sizeof(*desc));
170 dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
171 desc_sw->async_tx.tx_set_src = ioat_set_src;
172 desc_sw->async_tx.tx_set_dest = ioat_set_dest;
173 desc_sw->async_tx.tx_submit = ioat_tx_submit;
174 INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);
176 desc_sw->async_tx.phys = phys;
181 #define INITIAL_IOAT_DESC_COUNT 128
183 static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan);
185 /* returns the actual number of allocated descriptors */
186 static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
188 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
189 struct ioat_desc_sw *desc = NULL;
196 * In-use bit automatically set by reading chanctrl
197 * If 0, we got it, if 1, someone else did
199 chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
200 if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE)
203 /* Setup register to interrupt and write completion status on error */
204 chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE |
205 IOAT_CHANCTRL_ERR_INT_EN |
206 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
207 IOAT_CHANCTRL_ERR_COMPLETION_EN;
208 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
210 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
212 printk("IOAT: CHANERR = %x, clearing\n", chanerr);
213 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
216 /* Allocate descriptors */
217 for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) {
218 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
220 printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i);
223 list_add_tail(&desc->node, &tmp_list);
225 spin_lock_bh(&ioat_chan->desc_lock);
226 list_splice(&tmp_list, &ioat_chan->free_desc);
227 spin_unlock_bh(&ioat_chan->desc_lock);
229 /* allocate a completion writeback area */
230 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
231 ioat_chan->completion_virt =
232 pci_pool_alloc(ioat_chan->device->completion_pool,
234 &ioat_chan->completion_addr);
235 memset(ioat_chan->completion_virt, 0,
236 sizeof(*ioat_chan->completion_virt));
237 writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
238 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
239 writel(((u64) ioat_chan->completion_addr) >> 32,
240 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
242 ioat_start_null_desc(ioat_chan);
246 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
248 static void ioat_dma_free_chan_resources(struct dma_chan *chan)
250 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
251 struct ioat_device *ioat_device = to_ioat_device(chan->device);
252 struct ioat_desc_sw *desc, *_desc;
254 int in_use_descs = 0;
256 ioat_dma_memcpy_cleanup(ioat_chan);
258 writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
260 spin_lock_bh(&ioat_chan->desc_lock);
261 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
263 list_del(&desc->node);
264 pci_pool_free(ioat_device->dma_pool, desc->hw,
265 desc->async_tx.phys);
268 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
269 list_del(&desc->node);
270 pci_pool_free(ioat_device->dma_pool, desc->hw,
271 desc->async_tx.phys);
274 spin_unlock_bh(&ioat_chan->desc_lock);
276 pci_pool_free(ioat_device->completion_pool,
277 ioat_chan->completion_virt,
278 ioat_chan->completion_addr);
280 /* one is ok since we left it on there on purpose */
281 if (in_use_descs > 1)
282 printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n",
285 ioat_chan->last_completion = ioat_chan->completion_addr = 0;
287 /* Tell hw the chan is free */
288 chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
289 chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE;
290 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
293 static struct dma_async_tx_descriptor *
294 ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en)
296 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
297 struct ioat_desc_sw *first, *prev, *new;
298 LIST_HEAD(new_chain);
311 spin_lock_bh(&ioat_chan->desc_lock);
313 if (!list_empty(&ioat_chan->free_desc)) {
314 new = to_ioat_desc(ioat_chan->free_desc.next);
315 list_del(&new->node);
317 /* try to get another desc */
318 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
319 /* will this ever happen? */
320 /* TODO add upper limit on these */
324 copy = min((u32) len, ioat_chan->xfercap);
326 new->hw->size = copy;
328 new->async_tx.cookie = 0;
329 new->async_tx.ack = 1;
331 /* chain together the physical address list for the HW */
335 prev->hw->next = (u64) new->async_tx.phys;
339 list_add_tail(&new->node, &new_chain);
343 list_splice(&new_chain, &new->async_tx.tx_list);
345 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
347 new->tx_cnt = desc_count;
348 new->async_tx.ack = 0; /* client is in control of this ack */
349 new->async_tx.cookie = -EBUSY;
351 pci_unmap_len_set(new, src_len, orig_len);
352 pci_unmap_len_set(new, dst_len, orig_len);
353 spin_unlock_bh(&ioat_chan->desc_lock);
355 return new ? &new->async_tx : NULL;
360 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw
361 * @chan: DMA channel handle
364 static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
366 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
368 if (ioat_chan->pending != 0) {
369 ioat_chan->pending = 0;
370 writeb(IOAT_CHANCMD_APPEND,
371 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
375 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
377 unsigned long phys_complete;
378 struct ioat_desc_sw *desc, *_desc;
379 dma_cookie_t cookie = 0;
381 prefetch(chan->completion_virt);
383 if (!spin_trylock(&chan->cleanup_lock))
386 /* The completion writeback can happen at any time,
387 so reads by the driver need to be atomic operations
388 The descriptor physical addresses are limited to 32-bits
389 when the CPU can only do a 32-bit mov */
391 #if (BITS_PER_LONG == 64)
393 chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
395 phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
398 if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
399 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
400 printk("IOAT: Channel halted, chanerr = %x\n",
401 readl(chan->reg_base + IOAT_CHANERR_OFFSET));
403 /* TODO do something to salvage the situation */
406 if (phys_complete == chan->last_completion) {
407 spin_unlock(&chan->cleanup_lock);
411 spin_lock_bh(&chan->desc_lock);
412 list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) {
415 * Incoming DMA requests may use multiple descriptors, due to
416 * exceeding xfercap, perhaps. If so, only the last one will
417 * have a cookie, and require unmapping.
419 if (desc->async_tx.cookie) {
420 cookie = desc->async_tx.cookie;
422 /* yes we are unmapping both _page and _single alloc'd
423 regions with unmap_page. Is this *really* that bad?
425 pci_unmap_page(chan->device->pdev,
426 pci_unmap_addr(desc, dst),
427 pci_unmap_len(desc, dst_len),
429 pci_unmap_page(chan->device->pdev,
430 pci_unmap_addr(desc, src),
431 pci_unmap_len(desc, src_len),
435 if (desc->async_tx.phys != phys_complete) {
436 /* a completed entry, but not the last, so cleanup
437 * if the client is done with the descriptor
439 if (desc->async_tx.ack) {
440 list_del(&desc->node);
441 list_add_tail(&desc->node, &chan->free_desc);
443 desc->async_tx.cookie = 0;
445 /* last used desc. Do not remove, so we can append from
446 it, but don't look at it next time, either */
447 desc->async_tx.cookie = 0;
449 /* TODO check status bits? */
454 spin_unlock_bh(&chan->desc_lock);
456 chan->last_completion = phys_complete;
458 chan->completed_cookie = cookie;
460 spin_unlock(&chan->cleanup_lock);
463 static void ioat_dma_dependency_added(struct dma_chan *chan)
465 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
466 spin_lock_bh(&ioat_chan->desc_lock);
467 if (ioat_chan->pending == 0) {
468 spin_unlock_bh(&ioat_chan->desc_lock);
469 ioat_dma_memcpy_cleanup(ioat_chan);
471 spin_unlock_bh(&ioat_chan->desc_lock);
475 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
476 * @chan: IOAT DMA channel handle
477 * @cookie: DMA transaction identifier
478 * @done: if not %NULL, updated with last completed transaction
479 * @used: if not %NULL, updated with last used transaction
482 static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
487 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
488 dma_cookie_t last_used;
489 dma_cookie_t last_complete;
492 last_used = chan->cookie;
493 last_complete = ioat_chan->completed_cookie;
496 *done= last_complete;
500 ret = dma_async_is_complete(cookie, last_complete, last_used);
501 if (ret == DMA_SUCCESS)
504 ioat_dma_memcpy_cleanup(ioat_chan);
506 last_used = chan->cookie;
507 last_complete = ioat_chan->completed_cookie;
510 *done= last_complete;
514 return dma_async_is_complete(cookie, last_complete, last_used);
519 static struct pci_device_id ioat_pci_tbl[] = {
520 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
524 static struct pci_driver ioat_pci_driver = {
526 .id_table = ioat_pci_tbl,
528 .shutdown = ioat_shutdown,
529 .remove = __devexit_p(ioat_remove),
532 static irqreturn_t ioat_do_interrupt(int irq, void *data)
534 struct ioat_device *instance = data;
535 unsigned long attnstatus;
538 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
540 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
543 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
544 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
548 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
550 printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus);
552 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
556 static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan)
558 struct ioat_desc_sw *desc;
560 spin_lock_bh(&ioat_chan->desc_lock);
562 if (!list_empty(&ioat_chan->free_desc)) {
563 desc = to_ioat_desc(ioat_chan->free_desc.next);
564 list_del(&desc->node);
566 /* try to get another desc */
567 spin_unlock_bh(&ioat_chan->desc_lock);
568 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
569 spin_lock_bh(&ioat_chan->desc_lock);
570 /* will this ever happen? */
574 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
576 desc->async_tx.ack = 1;
578 list_add_tail(&desc->node, &ioat_chan->used_desc);
579 spin_unlock_bh(&ioat_chan->desc_lock);
581 writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
582 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW);
583 writel(((u64) desc->async_tx.phys) >> 32,
584 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH);
586 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
590 * Perform a IOAT transaction to verify the HW works.
592 #define IOAT_TEST_SIZE 2000
594 static int ioat_self_test(struct ioat_device *device)
599 struct dma_chan *dma_chan;
600 struct dma_async_tx_descriptor *tx;
605 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
608 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
614 /* Fill in src buffer */
615 for (i = 0; i < IOAT_TEST_SIZE; i++)
618 /* Start copy, using first DMA channel */
619 dma_chan = container_of(device->common.channels.next,
622 if (ioat_dma_alloc_chan_resources(dma_chan) < 1) {
627 tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0);
629 addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
631 ioat_set_src(addr, tx, 0);
632 addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
634 ioat_set_dest(addr, tx, 0);
635 cookie = ioat_tx_submit(tx);
636 ioat_dma_memcpy_issue_pending(dma_chan);
639 if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
640 printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n");
644 if (memcmp(src, dest, IOAT_TEST_SIZE)) {
645 printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n");
651 ioat_dma_free_chan_resources(dma_chan);
658 static int __devinit ioat_probe(struct pci_dev *pdev,
659 const struct pci_device_id *ent)
662 unsigned long mmio_start, mmio_len;
663 void __iomem *reg_base;
664 struct ioat_device *device;
666 err = pci_enable_device(pdev);
668 goto err_enable_device;
670 err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
672 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
674 goto err_set_dma_mask;
676 err = pci_request_regions(pdev, ioat_pci_driver.name);
678 goto err_request_regions;
680 mmio_start = pci_resource_start(pdev, 0);
681 mmio_len = pci_resource_len(pdev, 0);
683 reg_base = ioremap(mmio_start, mmio_len);
689 device = kzalloc(sizeof(*device), GFP_KERNEL);
695 /* DMA coherent memory pool for DMA descriptor allocations */
696 device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
697 sizeof(struct ioat_dma_descriptor), 64, 0);
698 if (!device->dma_pool) {
703 device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES);
704 if (!device->completion_pool) {
706 goto err_completion_pool;
710 pci_set_drvdata(pdev, device);
711 #ifdef CONFIG_PCI_MSI
712 if (pci_enable_msi(pdev) == 0) {
718 err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat",
723 device->reg_base = reg_base;
725 writeb(IOAT_INTRCTRL_MASTER_INT_EN, device->reg_base + IOAT_INTRCTRL_OFFSET);
726 pci_set_master(pdev);
728 INIT_LIST_HEAD(&device->common.channels);
729 enumerate_dma_channels(device);
731 dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
732 device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
733 device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
734 device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy;
735 device->common.device_is_tx_complete = ioat_dma_is_complete;
736 device->common.device_issue_pending = ioat_dma_memcpy_issue_pending;
737 device->common.device_dependency_added = ioat_dma_dependency_added;
738 device->common.dev = &pdev->dev;
739 printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
740 device->common.chancnt);
742 err = ioat_self_test(device);
746 dma_async_device_register(&device->common);
752 pci_pool_destroy(device->completion_pool);
754 pci_pool_destroy(device->dma_pool);
760 pci_release_regions(pdev);
763 pci_disable_device(pdev);
766 printk(KERN_ERR "Intel(R) I/OAT DMA Engine initialization failed\n");
771 static void ioat_shutdown(struct pci_dev *pdev)
773 struct ioat_device *device;
774 device = pci_get_drvdata(pdev);
776 dma_async_device_unregister(&device->common);
779 static void __devexit ioat_remove(struct pci_dev *pdev)
781 struct ioat_device *device;
782 struct dma_chan *chan, *_chan;
783 struct ioat_dma_chan *ioat_chan;
785 device = pci_get_drvdata(pdev);
786 dma_async_device_unregister(&device->common);
788 free_irq(device->pdev->irq, device);
789 #ifdef CONFIG_PCI_MSI
791 pci_disable_msi(device->pdev);
793 pci_pool_destroy(device->dma_pool);
794 pci_pool_destroy(device->completion_pool);
795 iounmap(device->reg_base);
796 pci_release_regions(pdev);
797 pci_disable_device(pdev);
798 list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) {
799 ioat_chan = to_ioat_chan(chan);
800 list_del(&chan->device_node);
807 MODULE_VERSION("1.9");
808 MODULE_LICENSE("GPL");
809 MODULE_AUTHOR("Intel Corporation");
811 static int __init ioat_init_module(void)
813 /* it's currently unsafe to unload this module */
814 /* if forced, worst case is that rmmod hangs */
815 __unsafe(THIS_MODULE);
817 return pci_register_driver(&ioat_pci_driver);
820 module_init(ioat_init_module);
822 static void __exit ioat_exit_module(void)
824 pci_unregister_driver(&ioat_pci_driver);
827 module_exit(ioat_exit_module);