2 * RapidIO mport driver for Tsi721 PCIExpress-to-SRIO bridge
4 * Copyright 2011 Integrated Device Technology, Inc.
5 * Alexandre Bounine <alexandre.bounine@idt.com>
6 * Chul Kim <chul.kim@idt.com>
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc., 59
20 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
24 #include <linux/errno.h>
25 #include <linux/init.h>
26 #include <linux/ioport.h>
27 #include <linux/kernel.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/rio.h>
31 #include <linux/rio_drv.h>
32 #include <linux/dma-mapping.h>
33 #include <linux/interrupt.h>
34 #include <linux/kfifo.h>
35 #include <linux/delay.h>
39 #define DEBUG_PW /* Inbound Port-Write debugging */
41 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch);
42 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch);
45 * tsi721_lcread - read from local SREP config space
46 * @mport: RapidIO master port info
47 * @index: ID of RapdiIO interface
48 * @offset: Offset into configuration space
49 * @len: Length (in bytes) of the maintenance transaction
50 * @data: Value to be read into
52 * Generates a local SREP space read. Returns %0 on
53 * success or %-EINVAL on failure.
55 static int tsi721_lcread(struct rio_mport *mport, int index, u32 offset,
58 struct tsi721_device *priv = mport->priv;
60 if (len != sizeof(u32))
61 return -EINVAL; /* only 32-bit access is supported */
63 *data = ioread32(priv->regs + offset);
69 * tsi721_lcwrite - write into local SREP config space
70 * @mport: RapidIO master port info
71 * @index: ID of RapdiIO interface
72 * @offset: Offset into configuration space
73 * @len: Length (in bytes) of the maintenance transaction
74 * @data: Value to be written
76 * Generates a local write into SREP configuration space. Returns %0 on
77 * success or %-EINVAL on failure.
79 static int tsi721_lcwrite(struct rio_mport *mport, int index, u32 offset,
82 struct tsi721_device *priv = mport->priv;
84 if (len != sizeof(u32))
85 return -EINVAL; /* only 32-bit access is supported */
87 iowrite32(data, priv->regs + offset);
93 * tsi721_maint_dma - Helper function to generate RapidIO maintenance
94 * transactions using designated Tsi721 DMA channel.
95 * @priv: pointer to tsi721 private data
96 * @sys_size: RapdiIO transport system size
97 * @destid: Destination ID of transaction
98 * @hopcount: Number of hops to target device
99 * @offset: Offset into configuration space
100 * @len: Length (in bytes) of the maintenance transaction
101 * @data: Location to be read from or write into
102 * @do_wr: Operation flag (1 == MAINT_WR)
104 * Generates a RapidIO maintenance transaction (Read or Write).
105 * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
107 static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size,
108 u16 destid, u8 hopcount, u32 offset, int len,
109 u32 *data, int do_wr)
111 struct tsi721_dma_desc *bd_ptr;
112 u32 rd_count, swr_ptr, ch_stat;
114 u32 op = do_wr ? MAINT_WR : MAINT_RD;
116 if (offset > (RIO_MAINT_SPACE_SZ - len) || (len != sizeof(u32)))
119 bd_ptr = priv->bdma[TSI721_DMACH_MAINT].bd_base;
122 priv->regs + TSI721_DMAC_DRDCNT(TSI721_DMACH_MAINT));
124 /* Initialize DMA descriptor */
125 bd_ptr[0].type_id = cpu_to_le32((DTYPE2 << 29) | (op << 19) | destid);
126 bd_ptr[0].bcount = cpu_to_le32((sys_size << 26) | 0x04);
127 bd_ptr[0].raddr_lo = cpu_to_le32((hopcount << 24) | offset);
128 bd_ptr[0].raddr_hi = 0;
130 bd_ptr[0].data[0] = cpu_to_be32p(data);
132 bd_ptr[0].data[0] = 0xffffffff;
136 /* Start DMA operation */
137 iowrite32(rd_count + 2,
138 priv->regs + TSI721_DMAC_DWRCNT(TSI721_DMACH_MAINT));
139 ioread32(priv->regs + TSI721_DMAC_DWRCNT(TSI721_DMACH_MAINT));
142 /* Wait until DMA transfer is finished */
143 while ((ch_stat = ioread32(priv->regs +
144 TSI721_DMAC_STS(TSI721_DMACH_MAINT))) & TSI721_DMAC_STS_RUN) {
146 if (++i >= 5000000) {
147 dev_dbg(&priv->pdev->dev,
148 "%s : DMA[%d] read timeout ch_status=%x\n",
149 __func__, TSI721_DMACH_MAINT, ch_stat);
157 if (ch_stat & TSI721_DMAC_STS_ABORT) {
158 /* If DMA operation aborted due to error,
159 * reinitialize DMA channel
161 dev_dbg(&priv->pdev->dev, "%s : DMA ABORT ch_stat=%x\n",
163 dev_dbg(&priv->pdev->dev, "OP=%d : destid=%x hc=%x off=%x\n",
164 do_wr ? MAINT_WR : MAINT_RD, destid, hopcount, offset);
165 iowrite32(TSI721_DMAC_INT_ALL,
166 priv->regs + TSI721_DMAC_INT(TSI721_DMACH_MAINT));
167 iowrite32(TSI721_DMAC_CTL_INIT,
168 priv->regs + TSI721_DMAC_CTL(TSI721_DMACH_MAINT));
170 iowrite32(0, priv->regs +
171 TSI721_DMAC_DWRCNT(TSI721_DMACH_MAINT));
180 *data = be32_to_cpu(bd_ptr[0].data[0]);
183 * Update descriptor status FIFO RD pointer.
184 * NOTE: Skipping check and clear FIFO entries because we are waiting
185 * for transfer to be completed.
187 swr_ptr = ioread32(priv->regs + TSI721_DMAC_DSWP(TSI721_DMACH_MAINT));
188 iowrite32(swr_ptr, priv->regs + TSI721_DMAC_DSRP(TSI721_DMACH_MAINT));
195 * tsi721_cread_dma - Generate a RapidIO maintenance read transaction
196 * using Tsi721 BDMA engine.
197 * @mport: RapidIO master port control structure
198 * @index: ID of RapdiIO interface
199 * @destid: Destination ID of transaction
200 * @hopcount: Number of hops to target device
201 * @offset: Offset into configuration space
202 * @len: Length (in bytes) of the maintenance transaction
203 * @val: Location to be read into
205 * Generates a RapidIO maintenance read transaction.
206 * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
208 static int tsi721_cread_dma(struct rio_mport *mport, int index, u16 destid,
209 u8 hopcount, u32 offset, int len, u32 *data)
211 struct tsi721_device *priv = mport->priv;
213 return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
214 offset, len, data, 0);
218 * tsi721_cwrite_dma - Generate a RapidIO maintenance write transaction
219 * using Tsi721 BDMA engine
220 * @mport: RapidIO master port control structure
221 * @index: ID of RapdiIO interface
222 * @destid: Destination ID of transaction
223 * @hopcount: Number of hops to target device
224 * @offset: Offset into configuration space
225 * @len: Length (in bytes) of the maintenance transaction
226 * @val: Value to be written
228 * Generates a RapidIO maintenance write transaction.
229 * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
231 static int tsi721_cwrite_dma(struct rio_mport *mport, int index, u16 destid,
232 u8 hopcount, u32 offset, int len, u32 data)
234 struct tsi721_device *priv = mport->priv;
237 return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
238 offset, len, &temp, 1);
242 * tsi721_pw_handler - Tsi721 inbound port-write interrupt handler
243 * @mport: RapidIO master port structure
245 * Handles inbound port-write interrupts. Copies PW message from an internal
246 * buffer into PW message FIFO and schedules deferred routine to process
250 tsi721_pw_handler(struct rio_mport *mport)
252 struct tsi721_device *priv = mport->priv;
254 u32 pw_buf[TSI721_RIO_PW_MSG_SIZE/sizeof(u32)];
257 pw_stat = ioread32(priv->regs + TSI721_RIO_PW_RX_STAT);
259 if (pw_stat & TSI721_RIO_PW_RX_STAT_PW_VAL) {
260 pw_buf[0] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(0));
261 pw_buf[1] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(1));
262 pw_buf[2] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(2));
263 pw_buf[3] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(3));
265 /* Queue PW message (if there is room in FIFO),
266 * otherwise discard it.
268 spin_lock(&priv->pw_fifo_lock);
269 if (kfifo_avail(&priv->pw_fifo) >= TSI721_RIO_PW_MSG_SIZE)
270 kfifo_in(&priv->pw_fifo, pw_buf,
271 TSI721_RIO_PW_MSG_SIZE);
273 priv->pw_discard_count++;
274 spin_unlock(&priv->pw_fifo_lock);
277 /* Clear pending PW interrupts */
278 iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
279 priv->regs + TSI721_RIO_PW_RX_STAT);
281 schedule_work(&priv->pw_work);
286 static void tsi721_pw_dpc(struct work_struct *work)
288 struct tsi721_device *priv = container_of(work, struct tsi721_device,
290 u32 msg_buffer[RIO_PW_MSG_SIZE/sizeof(u32)]; /* Use full size PW message
291 buffer for RIO layer */
294 * Process port-write messages
296 while (kfifo_out_spinlocked(&priv->pw_fifo, (unsigned char *)msg_buffer,
297 TSI721_RIO_PW_MSG_SIZE, &priv->pw_fifo_lock)) {
298 /* Process one message */
302 pr_debug("%s : Port-Write Message:", __func__);
303 for (i = 0; i < RIO_PW_MSG_SIZE/sizeof(u32); ) {
304 pr_debug("0x%02x: %08x %08x %08x %08x", i*4,
305 msg_buffer[i], msg_buffer[i + 1],
306 msg_buffer[i + 2], msg_buffer[i + 3]);
312 /* Pass the port-write message to RIO core for processing */
313 rio_inb_pwrite_handler((union rio_pw_msg *)msg_buffer);
318 * tsi721_pw_enable - enable/disable port-write interface init
319 * @mport: Master port implementing the port write unit
320 * @enable: 1=enable; 0=disable port-write message handling
322 static int tsi721_pw_enable(struct rio_mport *mport, int enable)
324 struct tsi721_device *priv = mport->priv;
327 rval = ioread32(priv->regs + TSI721_RIO_EM_INT_ENABLE);
330 rval |= TSI721_RIO_EM_INT_ENABLE_PW_RX;
332 rval &= ~TSI721_RIO_EM_INT_ENABLE_PW_RX;
334 /* Clear pending PW interrupts */
335 iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
336 priv->regs + TSI721_RIO_PW_RX_STAT);
337 /* Update enable bits */
338 iowrite32(rval, priv->regs + TSI721_RIO_EM_INT_ENABLE);
344 * tsi721_dsend - Send a RapidIO doorbell
345 * @mport: RapidIO master port info
346 * @index: ID of RapidIO interface
347 * @destid: Destination ID of target device
348 * @data: 16-bit info field of RapidIO doorbell
350 * Sends a RapidIO doorbell message. Always returns %0.
352 static int tsi721_dsend(struct rio_mport *mport, int index,
353 u16 destid, u16 data)
355 struct tsi721_device *priv = mport->priv;
358 offset = (((mport->sys_size) ? RIO_TT_CODE_16 : RIO_TT_CODE_8) << 18) |
361 dev_dbg(&priv->pdev->dev,
362 "Send Doorbell 0x%04x to destID 0x%x\n", data, destid);
363 iowrite16be(data, priv->odb_base + offset);
369 * tsi721_dbell_handler - Tsi721 doorbell interrupt handler
370 * @mport: RapidIO master port structure
372 * Handles inbound doorbell interrupts. Copies doorbell entry from an internal
373 * buffer into DB message FIFO and schedules deferred routine to process
377 tsi721_dbell_handler(struct rio_mport *mport)
379 struct tsi721_device *priv = mport->priv;
382 /* Disable IDB interrupts */
383 regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
384 regval &= ~TSI721_SR_CHINT_IDBQRCV;
386 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
388 schedule_work(&priv->idb_work);
393 static void tsi721_db_dpc(struct work_struct *work)
395 struct tsi721_device *priv = container_of(work, struct tsi721_device,
397 struct rio_mport *mport;
398 struct rio_dbell *dbell;
409 * Process queued inbound doorbells
413 wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
414 rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE;
416 while (wr_ptr != rd_ptr) {
417 idb_entry = (u64 *)(priv->idb_base +
418 (TSI721_IDB_ENTRY_SIZE * rd_ptr));
421 idb.msg = *idb_entry;
424 /* Process one doorbell */
425 list_for_each_entry(dbell, &mport->dbells, node) {
426 if ((dbell->res->start <= DBELL_INF(idb.bytes)) &&
427 (dbell->res->end >= DBELL_INF(idb.bytes))) {
434 dbell->dinb(mport, dbell->dev_id, DBELL_SID(idb.bytes),
435 DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
437 dev_dbg(&priv->pdev->dev,
438 "spurious inb doorbell, sid %2.2x tid %2.2x"
439 " info %4.4x\n", DBELL_SID(idb.bytes),
440 DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
443 wr_ptr = ioread32(priv->regs +
444 TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
447 iowrite32(rd_ptr & (IDB_QSIZE - 1),
448 priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
450 /* Re-enable IDB interrupts */
451 regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
452 regval |= TSI721_SR_CHINT_IDBQRCV;
454 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
456 wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
457 if (wr_ptr != rd_ptr)
458 schedule_work(&priv->idb_work);
462 * tsi721_irqhandler - Tsi721 interrupt handler
463 * @irq: Linux interrupt number
464 * @ptr: Pointer to interrupt-specific data (mport structure)
466 * Handles Tsi721 interrupts signaled using MSI and INTA. Checks reported
467 * interrupt events and calls an event-specific handler(s).
469 static irqreturn_t tsi721_irqhandler(int irq, void *ptr)
471 struct rio_mport *mport = (struct rio_mport *)ptr;
472 struct tsi721_device *priv = mport->priv;
478 /* For MSI mode disable all device-level interrupts */
479 if (priv->flags & TSI721_USING_MSI)
480 iowrite32(0, priv->regs + TSI721_DEV_INTE);
482 dev_int = ioread32(priv->regs + TSI721_DEV_INT);
486 dev_ch_int = ioread32(priv->regs + TSI721_DEV_CHAN_INT);
488 if (dev_int & TSI721_DEV_INT_SR2PC_CH) {
489 /* Service SR2PC Channel interrupts */
490 if (dev_ch_int & TSI721_INT_SR2PC_CHAN(IDB_QUEUE)) {
491 /* Service Inbound Doorbell interrupt */
492 intval = ioread32(priv->regs +
493 TSI721_SR_CHINT(IDB_QUEUE));
494 if (intval & TSI721_SR_CHINT_IDBQRCV)
495 tsi721_dbell_handler(mport);
497 dev_info(&priv->pdev->dev,
498 "Unsupported SR_CH_INT %x\n", intval);
500 /* Clear interrupts */
502 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
503 ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
507 if (dev_int & TSI721_DEV_INT_SMSG_CH) {
511 * Service channel interrupts from Messaging Engine
514 if (dev_ch_int & TSI721_INT_IMSG_CHAN_M) { /* Inbound Msg */
515 /* Disable signaled OB MSG Channel interrupts */
516 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
517 ch_inte &= ~(dev_ch_int & TSI721_INT_IMSG_CHAN_M);
518 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
521 * Process Inbound Message interrupt for each MBOX
523 for (ch = 4; ch < RIO_MAX_MBOX + 4; ch++) {
524 if (!(dev_ch_int & TSI721_INT_IMSG_CHAN(ch)))
526 tsi721_imsg_handler(priv, ch);
530 if (dev_ch_int & TSI721_INT_OMSG_CHAN_M) { /* Outbound Msg */
531 /* Disable signaled OB MSG Channel interrupts */
532 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
533 ch_inte &= ~(dev_ch_int & TSI721_INT_OMSG_CHAN_M);
534 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
537 * Process Outbound Message interrupts for each MBOX
540 for (ch = 0; ch < RIO_MAX_MBOX; ch++) {
541 if (!(dev_ch_int & TSI721_INT_OMSG_CHAN(ch)))
543 tsi721_omsg_handler(priv, ch);
548 if (dev_int & TSI721_DEV_INT_SRIO) {
549 /* Service SRIO MAC interrupts */
550 intval = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
551 if (intval & TSI721_RIO_EM_INT_STAT_PW_RX)
552 tsi721_pw_handler(mport);
555 /* For MSI mode re-enable device-level interrupts */
556 if (priv->flags & TSI721_USING_MSI) {
557 dev_int = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
558 TSI721_DEV_INT_SMSG_CH;
559 iowrite32(dev_int, priv->regs + TSI721_DEV_INTE);
565 static void tsi721_interrupts_init(struct tsi721_device *priv)
569 /* Enable IDB interrupts */
570 iowrite32(TSI721_SR_CHINT_ALL,
571 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
572 iowrite32(TSI721_SR_CHINT_IDBQRCV,
573 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
574 iowrite32(TSI721_INT_SR2PC_CHAN(IDB_QUEUE),
575 priv->regs + TSI721_DEV_CHAN_INTE);
577 /* Enable SRIO MAC interrupts */
578 iowrite32(TSI721_RIO_EM_DEV_INT_EN_INT,
579 priv->regs + TSI721_RIO_EM_DEV_INT_EN);
581 if (priv->flags & TSI721_USING_MSIX)
582 intr = TSI721_DEV_INT_SRIO;
584 intr = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
585 TSI721_DEV_INT_SMSG_CH;
587 iowrite32(intr, priv->regs + TSI721_DEV_INTE);
588 ioread32(priv->regs + TSI721_DEV_INTE);
591 #ifdef CONFIG_PCI_MSI
593 * tsi721_omsg_msix - MSI-X interrupt handler for outbound messaging
594 * @irq: Linux interrupt number
595 * @ptr: Pointer to interrupt-specific data (mport structure)
597 * Handles outbound messaging interrupts signaled using MSI-X.
599 static irqreturn_t tsi721_omsg_msix(int irq, void *ptr)
601 struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
604 mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX;
605 tsi721_omsg_handler(priv, mbox);
610 * tsi721_imsg_msix - MSI-X interrupt handler for inbound messaging
611 * @irq: Linux interrupt number
612 * @ptr: Pointer to interrupt-specific data (mport structure)
614 * Handles inbound messaging interrupts signaled using MSI-X.
616 static irqreturn_t tsi721_imsg_msix(int irq, void *ptr)
618 struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
621 mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX;
622 tsi721_imsg_handler(priv, mbox + 4);
627 * tsi721_srio_msix - Tsi721 MSI-X SRIO MAC interrupt handler
628 * @irq: Linux interrupt number
629 * @ptr: Pointer to interrupt-specific data (mport structure)
631 * Handles Tsi721 interrupts from SRIO MAC.
633 static irqreturn_t tsi721_srio_msix(int irq, void *ptr)
635 struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
638 /* Service SRIO MAC interrupts */
639 srio_int = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
640 if (srio_int & TSI721_RIO_EM_INT_STAT_PW_RX)
641 tsi721_pw_handler((struct rio_mport *)ptr);
647 * tsi721_sr2pc_ch_msix - Tsi721 MSI-X SR2PC Channel interrupt handler
648 * @irq: Linux interrupt number
649 * @ptr: Pointer to interrupt-specific data (mport structure)
651 * Handles Tsi721 interrupts from SR2PC Channel.
652 * NOTE: At this moment services only one SR2PC channel associated with inbound
655 static irqreturn_t tsi721_sr2pc_ch_msix(int irq, void *ptr)
657 struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
660 /* Service Inbound DB interrupt from SR2PC channel */
661 sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
662 if (sr_ch_int & TSI721_SR_CHINT_IDBQRCV)
663 tsi721_dbell_handler((struct rio_mport *)ptr);
665 /* Clear interrupts */
666 iowrite32(sr_ch_int, priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
667 /* Read back to ensure that interrupt was cleared */
668 sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
674 * tsi721_request_msix - register interrupt service for MSI-X mode.
675 * @mport: RapidIO master port structure
677 * Registers MSI-X interrupt service routines for interrupts that are active
678 * immediately after mport initialization. Messaging interrupt service routines
679 * should be registered during corresponding open requests.
681 static int tsi721_request_msix(struct rio_mport *mport)
683 struct tsi721_device *priv = mport->priv;
686 err = request_irq(priv->msix[TSI721_VECT_IDB].vector,
687 tsi721_sr2pc_ch_msix, 0,
688 priv->msix[TSI721_VECT_IDB].irq_name, (void *)mport);
692 err = request_irq(priv->msix[TSI721_VECT_PWRX].vector,
694 priv->msix[TSI721_VECT_PWRX].irq_name, (void *)mport);
697 priv->msix[TSI721_VECT_IDB].vector,
704 * tsi721_enable_msix - Attempts to enable MSI-X support for Tsi721.
705 * @priv: pointer to tsi721 private data
707 * Configures MSI-X support for Tsi721. Supports only an exact number
708 * of requested vectors.
710 static int tsi721_enable_msix(struct tsi721_device *priv)
712 struct msix_entry entries[TSI721_VECT_MAX];
716 entries[TSI721_VECT_IDB].entry = TSI721_MSIX_SR2PC_IDBQ_RCV(IDB_QUEUE);
717 entries[TSI721_VECT_PWRX].entry = TSI721_MSIX_SRIO_MAC_INT;
720 * Initialize MSI-X entries for Messaging Engine:
721 * this driver supports four RIO mailboxes (inbound and outbound)
722 * NOTE: Inbound message MBOX 0...4 use IB channels 4...7. Therefore
723 * offset +4 is added to IB MBOX number.
725 for (i = 0; i < RIO_MAX_MBOX; i++) {
726 entries[TSI721_VECT_IMB0_RCV + i].entry =
727 TSI721_MSIX_IMSG_DQ_RCV(i + 4);
728 entries[TSI721_VECT_IMB0_INT + i].entry =
729 TSI721_MSIX_IMSG_INT(i + 4);
730 entries[TSI721_VECT_OMB0_DONE + i].entry =
731 TSI721_MSIX_OMSG_DONE(i);
732 entries[TSI721_VECT_OMB0_INT + i].entry =
733 TSI721_MSIX_OMSG_INT(i);
736 err = pci_enable_msix(priv->pdev, entries, ARRAY_SIZE(entries));
739 dev_info(&priv->pdev->dev,
740 "Only %d MSI-X vectors available, "
741 "not using MSI-X\n", err);
746 * Copy MSI-X vector information into tsi721 private structure
748 priv->msix[TSI721_VECT_IDB].vector = entries[TSI721_VECT_IDB].vector;
749 snprintf(priv->msix[TSI721_VECT_IDB].irq_name, IRQ_DEVICE_NAME_MAX,
750 DRV_NAME "-idb@pci:%s", pci_name(priv->pdev));
751 priv->msix[TSI721_VECT_PWRX].vector = entries[TSI721_VECT_PWRX].vector;
752 snprintf(priv->msix[TSI721_VECT_PWRX].irq_name, IRQ_DEVICE_NAME_MAX,
753 DRV_NAME "-pwrx@pci:%s", pci_name(priv->pdev));
755 for (i = 0; i < RIO_MAX_MBOX; i++) {
756 priv->msix[TSI721_VECT_IMB0_RCV + i].vector =
757 entries[TSI721_VECT_IMB0_RCV + i].vector;
758 snprintf(priv->msix[TSI721_VECT_IMB0_RCV + i].irq_name,
759 IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbr%d@pci:%s",
760 i, pci_name(priv->pdev));
762 priv->msix[TSI721_VECT_IMB0_INT + i].vector =
763 entries[TSI721_VECT_IMB0_INT + i].vector;
764 snprintf(priv->msix[TSI721_VECT_IMB0_INT + i].irq_name,
765 IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbi%d@pci:%s",
766 i, pci_name(priv->pdev));
768 priv->msix[TSI721_VECT_OMB0_DONE + i].vector =
769 entries[TSI721_VECT_OMB0_DONE + i].vector;
770 snprintf(priv->msix[TSI721_VECT_OMB0_DONE + i].irq_name,
771 IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombd%d@pci:%s",
772 i, pci_name(priv->pdev));
774 priv->msix[TSI721_VECT_OMB0_INT + i].vector =
775 entries[TSI721_VECT_OMB0_INT + i].vector;
776 snprintf(priv->msix[TSI721_VECT_OMB0_INT + i].irq_name,
777 IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombi%d@pci:%s",
778 i, pci_name(priv->pdev));
783 #endif /* CONFIG_PCI_MSI */
785 static int tsi721_request_irq(struct rio_mport *mport)
787 struct tsi721_device *priv = mport->priv;
790 #ifdef CONFIG_PCI_MSI
791 if (priv->flags & TSI721_USING_MSIX)
792 err = tsi721_request_msix(mport);
795 err = request_irq(priv->pdev->irq, tsi721_irqhandler,
796 (priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,
797 DRV_NAME, (void *)mport);
800 dev_err(&priv->pdev->dev,
801 "Unable to allocate interrupt, Error: %d\n", err);
807 * tsi721_init_pc2sr_mapping - initializes outbound (PCIe->SRIO)
808 * translation regions.
809 * @priv: pointer to tsi721 private data
811 * Disables SREP translation regions.
813 static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv)
817 /* Disable all PC2SR translation windows */
818 for (i = 0; i < TSI721_OBWIN_NUM; i++)
819 iowrite32(0, priv->regs + TSI721_OBWINLB(i));
823 * tsi721_init_sr2pc_mapping - initializes inbound (SRIO->PCIe)
824 * translation regions.
825 * @priv: pointer to tsi721 private data
827 * Disables inbound windows.
829 static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv)
833 /* Disable all SR2PC inbound windows */
834 for (i = 0; i < TSI721_IBWIN_NUM; i++)
835 iowrite32(0, priv->regs + TSI721_IBWINLB(i));
839 * tsi721_port_write_init - Inbound port write interface init
840 * @priv: pointer to tsi721 private data
842 * Initializes inbound port write handler.
843 * Returns %0 on success or %-ENOMEM on failure.
845 static int tsi721_port_write_init(struct tsi721_device *priv)
847 priv->pw_discard_count = 0;
848 INIT_WORK(&priv->pw_work, tsi721_pw_dpc);
849 spin_lock_init(&priv->pw_fifo_lock);
850 if (kfifo_alloc(&priv->pw_fifo,
851 TSI721_RIO_PW_MSG_SIZE * 32, GFP_KERNEL)) {
852 dev_err(&priv->pdev->dev, "PW FIFO allocation failed\n");
856 /* Use reliable port-write capture mode */
857 iowrite32(TSI721_RIO_PW_CTL_PWC_REL, priv->regs + TSI721_RIO_PW_CTL);
861 static int tsi721_doorbell_init(struct tsi721_device *priv)
863 /* Outbound Doorbells do not require any setup.
864 * Tsi721 uses dedicated PCI BAR1 to generate doorbells.
865 * That BAR1 was mapped during the probe routine.
868 /* Initialize Inbound Doorbell processing DPC and queue */
869 priv->db_discard_count = 0;
870 INIT_WORK(&priv->idb_work, tsi721_db_dpc);
872 /* Allocate buffer for inbound doorbells queue */
873 priv->idb_base = dma_zalloc_coherent(&priv->pdev->dev,
874 IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
875 &priv->idb_dma, GFP_KERNEL);
879 dev_dbg(&priv->pdev->dev, "Allocated IDB buffer @ %p (phys = %llx)\n",
880 priv->idb_base, (unsigned long long)priv->idb_dma);
882 iowrite32(TSI721_IDQ_SIZE_VAL(IDB_QSIZE),
883 priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE));
884 iowrite32(((u64)priv->idb_dma >> 32),
885 priv->regs + TSI721_IDQ_BASEU(IDB_QUEUE));
886 iowrite32(((u64)priv->idb_dma & TSI721_IDQ_BASEL_ADDR),
887 priv->regs + TSI721_IDQ_BASEL(IDB_QUEUE));
888 /* Enable accepting all inbound doorbells */
889 iowrite32(0, priv->regs + TSI721_IDQ_MASK(IDB_QUEUE));
891 iowrite32(TSI721_IDQ_INIT, priv->regs + TSI721_IDQ_CTL(IDB_QUEUE));
893 iowrite32(0, priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
898 static void tsi721_doorbell_free(struct tsi721_device *priv)
900 if (priv->idb_base == NULL)
903 /* Free buffer allocated for inbound doorbell queue */
904 dma_free_coherent(&priv->pdev->dev, IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
905 priv->idb_base, priv->idb_dma);
906 priv->idb_base = NULL;
909 static int tsi721_bdma_ch_init(struct tsi721_device *priv, int chnum)
911 struct tsi721_dma_desc *bd_ptr;
913 dma_addr_t bd_phys, sts_phys;
915 int bd_num = priv->bdma[chnum].bd_num;
917 dev_dbg(&priv->pdev->dev, "Init Block DMA Engine, CH%d\n", chnum);
920 * Initialize DMA channel for maintenance requests
923 /* Allocate space for DMA descriptors */
924 bd_ptr = dma_zalloc_coherent(&priv->pdev->dev,
925 bd_num * sizeof(struct tsi721_dma_desc),
926 &bd_phys, GFP_KERNEL);
930 priv->bdma[chnum].bd_phys = bd_phys;
931 priv->bdma[chnum].bd_base = bd_ptr;
933 dev_dbg(&priv->pdev->dev, "DMA descriptors @ %p (phys = %llx)\n",
934 bd_ptr, (unsigned long long)bd_phys);
936 /* Allocate space for descriptor status FIFO */
937 sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ?
938 bd_num : TSI721_DMA_MINSTSSZ;
939 sts_size = roundup_pow_of_two(sts_size);
940 sts_ptr = dma_zalloc_coherent(&priv->pdev->dev,
941 sts_size * sizeof(struct tsi721_dma_sts),
942 &sts_phys, GFP_KERNEL);
944 /* Free space allocated for DMA descriptors */
945 dma_free_coherent(&priv->pdev->dev,
946 bd_num * sizeof(struct tsi721_dma_desc),
948 priv->bdma[chnum].bd_base = NULL;
952 priv->bdma[chnum].sts_phys = sts_phys;
953 priv->bdma[chnum].sts_base = sts_ptr;
954 priv->bdma[chnum].sts_size = sts_size;
956 dev_dbg(&priv->pdev->dev,
957 "desc status FIFO @ %p (phys = %llx) size=0x%x\n",
958 sts_ptr, (unsigned long long)sts_phys, sts_size);
960 /* Initialize DMA descriptors ring */
961 bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29);
962 bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys &
963 TSI721_DMAC_DPTRL_MASK);
964 bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32);
966 /* Setup DMA descriptor pointers */
967 iowrite32(((u64)bd_phys >> 32),
968 priv->regs + TSI721_DMAC_DPTRH(chnum));
969 iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
970 priv->regs + TSI721_DMAC_DPTRL(chnum));
972 /* Setup descriptor status FIFO */
973 iowrite32(((u64)sts_phys >> 32),
974 priv->regs + TSI721_DMAC_DSBH(chnum));
975 iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
976 priv->regs + TSI721_DMAC_DSBL(chnum));
977 iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
978 priv->regs + TSI721_DMAC_DSSZ(chnum));
980 /* Clear interrupt bits */
981 iowrite32(TSI721_DMAC_INT_ALL,
982 priv->regs + TSI721_DMAC_INT(chnum));
984 ioread32(priv->regs + TSI721_DMAC_INT(chnum));
986 /* Toggle DMA channel initialization */
987 iowrite32(TSI721_DMAC_CTL_INIT, priv->regs + TSI721_DMAC_CTL(chnum));
988 ioread32(priv->regs + TSI721_DMAC_CTL(chnum));
994 static int tsi721_bdma_ch_free(struct tsi721_device *priv, int chnum)
998 if (priv->bdma[chnum].bd_base == NULL)
1001 /* Check if DMA channel still running */
1002 ch_stat = ioread32(priv->regs + TSI721_DMAC_STS(chnum));
1003 if (ch_stat & TSI721_DMAC_STS_RUN)
1006 /* Put DMA channel into init state */
1007 iowrite32(TSI721_DMAC_CTL_INIT,
1008 priv->regs + TSI721_DMAC_CTL(chnum));
1010 /* Free space allocated for DMA descriptors */
1011 dma_free_coherent(&priv->pdev->dev,
1012 priv->bdma[chnum].bd_num * sizeof(struct tsi721_dma_desc),
1013 priv->bdma[chnum].bd_base, priv->bdma[chnum].bd_phys);
1014 priv->bdma[chnum].bd_base = NULL;
1016 /* Free space allocated for status FIFO */
1017 dma_free_coherent(&priv->pdev->dev,
1018 priv->bdma[chnum].sts_size * sizeof(struct tsi721_dma_sts),
1019 priv->bdma[chnum].sts_base, priv->bdma[chnum].sts_phys);
1020 priv->bdma[chnum].sts_base = NULL;
1024 static int tsi721_bdma_init(struct tsi721_device *priv)
1026 /* Initialize BDMA channel allocated for RapidIO maintenance read/write
1027 * request generation
1029 priv->bdma[TSI721_DMACH_MAINT].bd_num = 2;
1030 if (tsi721_bdma_ch_init(priv, TSI721_DMACH_MAINT)) {
1031 dev_err(&priv->pdev->dev, "Unable to initialize maintenance DMA"
1032 " channel %d, aborting\n", TSI721_DMACH_MAINT);
1039 static void tsi721_bdma_free(struct tsi721_device *priv)
1041 tsi721_bdma_ch_free(priv, TSI721_DMACH_MAINT);
1044 /* Enable Inbound Messaging Interrupts */
1046 tsi721_imsg_interrupt_enable(struct tsi721_device *priv, int ch,
1054 /* Clear pending Inbound Messaging interrupts */
1055 iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1057 /* Enable Inbound Messaging interrupts */
1058 rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1059 iowrite32(rval | inte_mask, priv->regs + TSI721_IBDMAC_INTE(ch));
1061 if (priv->flags & TSI721_USING_MSIX)
1062 return; /* Finished if we are in MSI-X mode */
1065 * For MSI and INTA interrupt signalling we need to enable next levels
1068 /* Enable Device Channel Interrupt */
1069 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1070 iowrite32(rval | TSI721_INT_IMSG_CHAN(ch),
1071 priv->regs + TSI721_DEV_CHAN_INTE);
1074 /* Disable Inbound Messaging Interrupts */
1076 tsi721_imsg_interrupt_disable(struct tsi721_device *priv, int ch,
1084 /* Clear pending Inbound Messaging interrupts */
1085 iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1087 /* Disable Inbound Messaging interrupts */
1088 rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1090 iowrite32(rval, priv->regs + TSI721_IBDMAC_INTE(ch));
1092 if (priv->flags & TSI721_USING_MSIX)
1093 return; /* Finished if we are in MSI-X mode */
1096 * For MSI and INTA interrupt signalling we need to disable next levels
1099 /* Disable Device Channel Interrupt */
1100 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1101 rval &= ~TSI721_INT_IMSG_CHAN(ch);
1102 iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1105 /* Enable Outbound Messaging interrupts */
1107 tsi721_omsg_interrupt_enable(struct tsi721_device *priv, int ch,
1115 /* Clear pending Outbound Messaging interrupts */
1116 iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1118 /* Enable Outbound Messaging channel interrupts */
1119 rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1120 iowrite32(rval | inte_mask, priv->regs + TSI721_OBDMAC_INTE(ch));
1122 if (priv->flags & TSI721_USING_MSIX)
1123 return; /* Finished if we are in MSI-X mode */
1126 * For MSI and INTA interrupt signalling we need to enable next levels
1129 /* Enable Device Channel Interrupt */
1130 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1131 iowrite32(rval | TSI721_INT_OMSG_CHAN(ch),
1132 priv->regs + TSI721_DEV_CHAN_INTE);
1135 /* Disable Outbound Messaging interrupts */
1137 tsi721_omsg_interrupt_disable(struct tsi721_device *priv, int ch,
1145 /* Clear pending Outbound Messaging interrupts */
1146 iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1148 /* Disable Outbound Messaging interrupts */
1149 rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1151 iowrite32(rval, priv->regs + TSI721_OBDMAC_INTE(ch));
1153 if (priv->flags & TSI721_USING_MSIX)
1154 return; /* Finished if we are in MSI-X mode */
1157 * For MSI and INTA interrupt signalling we need to disable next levels
1160 /* Disable Device Channel Interrupt */
1161 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1162 rval &= ~TSI721_INT_OMSG_CHAN(ch);
1163 iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1167 * tsi721_add_outb_message - Add message to the Tsi721 outbound message queue
1168 * @mport: Master port with outbound message queue
1169 * @rdev: Target of outbound message
1170 * @mbox: Outbound mailbox
1171 * @buffer: Message to add to outbound queue
1172 * @len: Length of message
1175 tsi721_add_outb_message(struct rio_mport *mport, struct rio_dev *rdev, int mbox,
1176 void *buffer, size_t len)
1178 struct tsi721_device *priv = mport->priv;
1179 struct tsi721_omsg_desc *desc;
1182 if (!priv->omsg_init[mbox] ||
1183 len > TSI721_MSG_MAX_SIZE || len < 8)
1186 tx_slot = priv->omsg_ring[mbox].tx_slot;
1188 /* Copy copy message into transfer buffer */
1189 memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len);
1194 /* Build descriptor associated with buffer */
1195 desc = priv->omsg_ring[mbox].omd_base;
1196 desc[tx_slot].type_id = cpu_to_le32((DTYPE4 << 29) | rdev->destid);
1197 if (tx_slot % 4 == 0)
1198 desc[tx_slot].type_id |= cpu_to_le32(TSI721_OMD_IOF);
1200 desc[tx_slot].msg_info =
1201 cpu_to_le32((mport->sys_size << 26) | (mbox << 22) |
1202 (0xe << 12) | (len & 0xff8));
1203 desc[tx_slot].bufptr_lo =
1204 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] &
1206 desc[tx_slot].bufptr_hi =
1207 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32);
1209 priv->omsg_ring[mbox].wr_count++;
1211 /* Go to next descriptor */
1212 if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) {
1213 priv->omsg_ring[mbox].tx_slot = 0;
1214 /* Move through the ring link descriptor at the end */
1215 priv->omsg_ring[mbox].wr_count++;
1220 /* Set new write count value */
1221 iowrite32(priv->omsg_ring[mbox].wr_count,
1222 priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1223 ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1229 * tsi721_omsg_handler - Outbound Message Interrupt Handler
1230 * @priv: pointer to tsi721 private data
1231 * @ch: number of OB MSG channel to service
1233 * Services channel interrupts from outbound messaging engine.
1235 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch)
1239 spin_lock(&priv->omsg_ring[ch].lock);
1241 omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch));
1243 if (omsg_int & TSI721_OBDMAC_INT_ST_FULL)
1244 dev_info(&priv->pdev->dev,
1245 "OB MBOX%d: Status FIFO is full\n", ch);
1247 if (omsg_int & (TSI721_OBDMAC_INT_DONE | TSI721_OBDMAC_INT_IOF_DONE)) {
1249 u64 *sts_ptr, last_ptr = 0, prev_ptr = 0;
1254 * Find last successfully processed descriptor
1257 /* Check and clear descriptor status FIFO entries */
1258 srd_ptr = priv->omsg_ring[ch].sts_rdptr;
1259 sts_ptr = priv->omsg_ring[ch].sts_base;
1261 while (sts_ptr[j]) {
1262 for (i = 0; i < 8 && sts_ptr[j]; i++, j++) {
1263 prev_ptr = last_ptr;
1264 last_ptr = le64_to_cpu(sts_ptr[j]);
1269 srd_ptr %= priv->omsg_ring[ch].sts_size;
1276 priv->omsg_ring[ch].sts_rdptr = srd_ptr;
1277 iowrite32(srd_ptr, priv->regs + TSI721_OBDMAC_DSRP(ch));
1279 if (!priv->mport->outb_msg[ch].mcback)
1282 /* Inform upper layer about transfer completion */
1284 tx_slot = (last_ptr - (u64)priv->omsg_ring[ch].omd_phys)/
1285 sizeof(struct tsi721_omsg_desc);
1288 * Check if this is a Link Descriptor (LD).
1289 * If yes, ignore LD and use descriptor processed
1292 if (tx_slot == priv->omsg_ring[ch].size) {
1294 tx_slot = (prev_ptr -
1295 (u64)priv->omsg_ring[ch].omd_phys)/
1296 sizeof(struct tsi721_omsg_desc);
1301 /* Move slot index to the next message to be sent */
1303 if (tx_slot == priv->omsg_ring[ch].size)
1305 BUG_ON(tx_slot >= priv->omsg_ring[ch].size);
1306 priv->mport->outb_msg[ch].mcback(priv->mport,
1307 priv->omsg_ring[ch].dev_id, ch,
1313 if (omsg_int & TSI721_OBDMAC_INT_ERROR) {
1315 * Outbound message operation aborted due to error,
1316 * reinitialize OB MSG channel
1319 dev_dbg(&priv->pdev->dev, "OB MSG ABORT ch_stat=%x\n",
1320 ioread32(priv->regs + TSI721_OBDMAC_STS(ch)));
1322 iowrite32(TSI721_OBDMAC_INT_ERROR,
1323 priv->regs + TSI721_OBDMAC_INT(ch));
1324 iowrite32(TSI721_OBDMAC_CTL_INIT,
1325 priv->regs + TSI721_OBDMAC_CTL(ch));
1326 ioread32(priv->regs + TSI721_OBDMAC_CTL(ch));
1328 /* Inform upper level to clear all pending tx slots */
1329 if (priv->mport->outb_msg[ch].mcback)
1330 priv->mport->outb_msg[ch].mcback(priv->mport,
1331 priv->omsg_ring[ch].dev_id, ch,
1332 priv->omsg_ring[ch].tx_slot);
1333 /* Synch tx_slot tracking */
1334 iowrite32(priv->omsg_ring[ch].tx_slot,
1335 priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1336 ioread32(priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1337 priv->omsg_ring[ch].wr_count = priv->omsg_ring[ch].tx_slot;
1338 priv->omsg_ring[ch].sts_rdptr = 0;
1341 /* Clear channel interrupts */
1342 iowrite32(omsg_int, priv->regs + TSI721_OBDMAC_INT(ch));
1344 if (!(priv->flags & TSI721_USING_MSIX)) {
1347 /* Re-enable channel interrupts */
1348 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1349 ch_inte |= TSI721_INT_OMSG_CHAN(ch);
1350 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1353 spin_unlock(&priv->omsg_ring[ch].lock);
1357 * tsi721_open_outb_mbox - Initialize Tsi721 outbound mailbox
1358 * @mport: Master port implementing Outbound Messaging Engine
1359 * @dev_id: Device specific pointer to pass on event
1360 * @mbox: Mailbox to open
1361 * @entries: Number of entries in the outbound mailbox ring
1363 static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id,
1364 int mbox, int entries)
1366 struct tsi721_device *priv = mport->priv;
1367 struct tsi721_omsg_desc *bd_ptr;
1370 if ((entries < TSI721_OMSGD_MIN_RING_SIZE) ||
1371 (entries > (TSI721_OMSGD_RING_SIZE)) ||
1372 (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1377 priv->omsg_ring[mbox].dev_id = dev_id;
1378 priv->omsg_ring[mbox].size = entries;
1379 priv->omsg_ring[mbox].sts_rdptr = 0;
1380 spin_lock_init(&priv->omsg_ring[mbox].lock);
1382 /* Outbound Msg Buffer allocation based on
1383 the number of maximum descriptor entries */
1384 for (i = 0; i < entries; i++) {
1385 priv->omsg_ring[mbox].omq_base[i] =
1387 &priv->pdev->dev, TSI721_MSG_BUFFER_SIZE,
1388 &priv->omsg_ring[mbox].omq_phys[i],
1390 if (priv->omsg_ring[mbox].omq_base[i] == NULL) {
1391 dev_dbg(&priv->pdev->dev,
1392 "Unable to allocate OB MSG data buffer for"
1399 /* Outbound message descriptor allocation */
1400 priv->omsg_ring[mbox].omd_base = dma_alloc_coherent(
1402 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1403 &priv->omsg_ring[mbox].omd_phys, GFP_KERNEL);
1404 if (priv->omsg_ring[mbox].omd_base == NULL) {
1405 dev_dbg(&priv->pdev->dev,
1406 "Unable to allocate OB MSG descriptor memory "
1407 "for MBOX%d\n", mbox);
1412 priv->omsg_ring[mbox].tx_slot = 0;
1414 /* Outbound message descriptor status FIFO allocation */
1415 priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1);
1416 priv->omsg_ring[mbox].sts_base = dma_zalloc_coherent(&priv->pdev->dev,
1417 priv->omsg_ring[mbox].sts_size *
1418 sizeof(struct tsi721_dma_sts),
1419 &priv->omsg_ring[mbox].sts_phys, GFP_KERNEL);
1420 if (priv->omsg_ring[mbox].sts_base == NULL) {
1421 dev_dbg(&priv->pdev->dev,
1422 "Unable to allocate OB MSG descriptor status FIFO "
1423 "for MBOX%d\n", mbox);
1429 * Configure Outbound Messaging Engine
1432 /* Setup Outbound Message descriptor pointer */
1433 iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32),
1434 priv->regs + TSI721_OBDMAC_DPTRH(mbox));
1435 iowrite32(((u64)priv->omsg_ring[mbox].omd_phys &
1436 TSI721_OBDMAC_DPTRL_MASK),
1437 priv->regs + TSI721_OBDMAC_DPTRL(mbox));
1439 /* Setup Outbound Message descriptor status FIFO */
1440 iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32),
1441 priv->regs + TSI721_OBDMAC_DSBH(mbox));
1442 iowrite32(((u64)priv->omsg_ring[mbox].sts_phys &
1443 TSI721_OBDMAC_DSBL_MASK),
1444 priv->regs + TSI721_OBDMAC_DSBL(mbox));
1445 iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size),
1446 priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox));
1448 /* Enable interrupts */
1450 #ifdef CONFIG_PCI_MSI
1451 if (priv->flags & TSI721_USING_MSIX) {
1452 /* Request interrupt service if we are in MSI-X mode */
1454 priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1455 tsi721_omsg_msix, 0,
1456 priv->msix[TSI721_VECT_OMB0_DONE + mbox].irq_name,
1460 dev_dbg(&priv->pdev->dev,
1461 "Unable to allocate MSI-X interrupt for "
1462 "OBOX%d-DONE\n", mbox);
1466 rc = request_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
1467 tsi721_omsg_msix, 0,
1468 priv->msix[TSI721_VECT_OMB0_INT + mbox].irq_name,
1472 dev_dbg(&priv->pdev->dev,
1473 "Unable to allocate MSI-X interrupt for "
1474 "MBOX%d-INT\n", mbox);
1476 priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1481 #endif /* CONFIG_PCI_MSI */
1483 tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1485 /* Initialize Outbound Message descriptors ring */
1486 bd_ptr = priv->omsg_ring[mbox].omd_base;
1487 bd_ptr[entries].type_id = cpu_to_le32(DTYPE5 << 29);
1488 bd_ptr[entries].msg_info = 0;
1489 bd_ptr[entries].next_lo =
1490 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys &
1491 TSI721_OBDMAC_DPTRL_MASK);
1492 bd_ptr[entries].next_hi =
1493 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32);
1494 priv->omsg_ring[mbox].wr_count = 0;
1497 /* Initialize Outbound Message engine */
1498 iowrite32(TSI721_OBDMAC_CTL_INIT, priv->regs + TSI721_OBDMAC_CTL(mbox));
1499 ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1502 priv->omsg_init[mbox] = 1;
1506 #ifdef CONFIG_PCI_MSI
1508 dma_free_coherent(&priv->pdev->dev,
1509 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
1510 priv->omsg_ring[mbox].sts_base,
1511 priv->omsg_ring[mbox].sts_phys);
1513 priv->omsg_ring[mbox].sts_base = NULL;
1514 #endif /* CONFIG_PCI_MSI */
1517 dma_free_coherent(&priv->pdev->dev,
1518 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1519 priv->omsg_ring[mbox].omd_base,
1520 priv->omsg_ring[mbox].omd_phys);
1522 priv->omsg_ring[mbox].omd_base = NULL;
1525 for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
1526 if (priv->omsg_ring[mbox].omq_base[i]) {
1527 dma_free_coherent(&priv->pdev->dev,
1528 TSI721_MSG_BUFFER_SIZE,
1529 priv->omsg_ring[mbox].omq_base[i],
1530 priv->omsg_ring[mbox].omq_phys[i]);
1532 priv->omsg_ring[mbox].omq_base[i] = NULL;
1541 * tsi721_close_outb_mbox - Close Tsi721 outbound mailbox
1542 * @mport: Master port implementing the outbound message unit
1543 * @mbox: Mailbox to close
1545 static void tsi721_close_outb_mbox(struct rio_mport *mport, int mbox)
1547 struct tsi721_device *priv = mport->priv;
1550 if (!priv->omsg_init[mbox])
1552 priv->omsg_init[mbox] = 0;
1554 /* Disable Interrupts */
1556 tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1558 #ifdef CONFIG_PCI_MSI
1559 if (priv->flags & TSI721_USING_MSIX) {
1560 free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1562 free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
1565 #endif /* CONFIG_PCI_MSI */
1567 /* Free OMSG Descriptor Status FIFO */
1568 dma_free_coherent(&priv->pdev->dev,
1569 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
1570 priv->omsg_ring[mbox].sts_base,
1571 priv->omsg_ring[mbox].sts_phys);
1573 priv->omsg_ring[mbox].sts_base = NULL;
1575 /* Free OMSG descriptors */
1576 dma_free_coherent(&priv->pdev->dev,
1577 (priv->omsg_ring[mbox].size + 1) *
1578 sizeof(struct tsi721_omsg_desc),
1579 priv->omsg_ring[mbox].omd_base,
1580 priv->omsg_ring[mbox].omd_phys);
1582 priv->omsg_ring[mbox].omd_base = NULL;
1584 /* Free message buffers */
1585 for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
1586 if (priv->omsg_ring[mbox].omq_base[i]) {
1587 dma_free_coherent(&priv->pdev->dev,
1588 TSI721_MSG_BUFFER_SIZE,
1589 priv->omsg_ring[mbox].omq_base[i],
1590 priv->omsg_ring[mbox].omq_phys[i]);
1592 priv->omsg_ring[mbox].omq_base[i] = NULL;
1598 * tsi721_imsg_handler - Inbound Message Interrupt Handler
1599 * @priv: pointer to tsi721 private data
1600 * @ch: inbound message channel number to service
1602 * Services channel interrupts from inbound messaging engine.
1604 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch)
1609 spin_lock(&priv->imsg_ring[mbox].lock);
1611 imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch));
1613 if (imsg_int & TSI721_IBDMAC_INT_SRTO)
1614 dev_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout\n",
1617 if (imsg_int & TSI721_IBDMAC_INT_PC_ERROR)
1618 dev_info(&priv->pdev->dev, "IB MBOX%d PCIe error\n",
1621 if (imsg_int & TSI721_IBDMAC_INT_FQ_LOW)
1622 dev_info(&priv->pdev->dev,
1623 "IB MBOX%d IB free queue low\n", mbox);
1625 /* Clear IB channel interrupts */
1626 iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch));
1628 /* If an IB Msg is received notify the upper layer */
1629 if (imsg_int & TSI721_IBDMAC_INT_DQ_RCV &&
1630 priv->mport->inb_msg[mbox].mcback)
1631 priv->mport->inb_msg[mbox].mcback(priv->mport,
1632 priv->imsg_ring[mbox].dev_id, mbox, -1);
1634 if (!(priv->flags & TSI721_USING_MSIX)) {
1637 /* Re-enable channel interrupts */
1638 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1639 ch_inte |= TSI721_INT_IMSG_CHAN(ch);
1640 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1643 spin_unlock(&priv->imsg_ring[mbox].lock);
1647 * tsi721_open_inb_mbox - Initialize Tsi721 inbound mailbox
1648 * @mport: Master port implementing the Inbound Messaging Engine
1649 * @dev_id: Device specific pointer to pass on event
1650 * @mbox: Mailbox to open
1651 * @entries: Number of entries in the inbound mailbox ring
1653 static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id,
1654 int mbox, int entries)
1656 struct tsi721_device *priv = mport->priv;
1662 if ((entries < TSI721_IMSGD_MIN_RING_SIZE) ||
1663 (entries > TSI721_IMSGD_RING_SIZE) ||
1664 (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1669 /* Initialize IB Messaging Ring */
1670 priv->imsg_ring[mbox].dev_id = dev_id;
1671 priv->imsg_ring[mbox].size = entries;
1672 priv->imsg_ring[mbox].rx_slot = 0;
1673 priv->imsg_ring[mbox].desc_rdptr = 0;
1674 priv->imsg_ring[mbox].fq_wrptr = 0;
1675 for (i = 0; i < priv->imsg_ring[mbox].size; i++)
1676 priv->imsg_ring[mbox].imq_base[i] = NULL;
1677 spin_lock_init(&priv->imsg_ring[mbox].lock);
1679 /* Allocate buffers for incoming messages */
1680 priv->imsg_ring[mbox].buf_base =
1681 dma_alloc_coherent(&priv->pdev->dev,
1682 entries * TSI721_MSG_BUFFER_SIZE,
1683 &priv->imsg_ring[mbox].buf_phys,
1686 if (priv->imsg_ring[mbox].buf_base == NULL) {
1687 dev_err(&priv->pdev->dev,
1688 "Failed to allocate buffers for IB MBOX%d\n", mbox);
1693 /* Allocate memory for circular free list */
1694 priv->imsg_ring[mbox].imfq_base =
1695 dma_alloc_coherent(&priv->pdev->dev,
1697 &priv->imsg_ring[mbox].imfq_phys,
1700 if (priv->imsg_ring[mbox].imfq_base == NULL) {
1701 dev_err(&priv->pdev->dev,
1702 "Failed to allocate free queue for IB MBOX%d\n", mbox);
1707 /* Allocate memory for Inbound message descriptors */
1708 priv->imsg_ring[mbox].imd_base =
1709 dma_alloc_coherent(&priv->pdev->dev,
1710 entries * sizeof(struct tsi721_imsg_desc),
1711 &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL);
1713 if (priv->imsg_ring[mbox].imd_base == NULL) {
1714 dev_err(&priv->pdev->dev,
1715 "Failed to allocate descriptor memory for IB MBOX%d\n",
1721 /* Fill free buffer pointer list */
1722 free_ptr = priv->imsg_ring[mbox].imfq_base;
1723 for (i = 0; i < entries; i++)
1724 free_ptr[i] = cpu_to_le64(
1725 (u64)(priv->imsg_ring[mbox].buf_phys) +
1731 * For mapping of inbound SRIO Messages into appropriate queues we need
1732 * to set Inbound Device ID register in the messaging engine. We do it
1733 * once when first inbound mailbox is requested.
1735 if (!(priv->flags & TSI721_IMSGID_SET)) {
1736 iowrite32((u32)priv->mport->host_deviceid,
1737 priv->regs + TSI721_IB_DEVID);
1738 priv->flags |= TSI721_IMSGID_SET;
1742 * Configure Inbound Messaging channel (ch = mbox + 4)
1745 /* Setup Inbound Message free queue */
1746 iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32),
1747 priv->regs + TSI721_IBDMAC_FQBH(ch));
1748 iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys &
1749 TSI721_IBDMAC_FQBL_MASK),
1750 priv->regs+TSI721_IBDMAC_FQBL(ch));
1751 iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
1752 priv->regs + TSI721_IBDMAC_FQSZ(ch));
1754 /* Setup Inbound Message descriptor queue */
1755 iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32),
1756 priv->regs + TSI721_IBDMAC_DQBH(ch));
1757 iowrite32(((u32)priv->imsg_ring[mbox].imd_phys &
1758 (u32)TSI721_IBDMAC_DQBL_MASK),
1759 priv->regs+TSI721_IBDMAC_DQBL(ch));
1760 iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
1761 priv->regs + TSI721_IBDMAC_DQSZ(ch));
1763 /* Enable interrupts */
1765 #ifdef CONFIG_PCI_MSI
1766 if (priv->flags & TSI721_USING_MSIX) {
1767 /* Request interrupt service if we are in MSI-X mode */
1768 rc = request_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1769 tsi721_imsg_msix, 0,
1770 priv->msix[TSI721_VECT_IMB0_RCV + mbox].irq_name,
1774 dev_dbg(&priv->pdev->dev,
1775 "Unable to allocate MSI-X interrupt for "
1776 "IBOX%d-DONE\n", mbox);
1780 rc = request_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
1781 tsi721_imsg_msix, 0,
1782 priv->msix[TSI721_VECT_IMB0_INT + mbox].irq_name,
1786 dev_dbg(&priv->pdev->dev,
1787 "Unable to allocate MSI-X interrupt for "
1788 "IBOX%d-INT\n", mbox);
1790 priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1795 #endif /* CONFIG_PCI_MSI */
1797 tsi721_imsg_interrupt_enable(priv, ch, TSI721_IBDMAC_INT_ALL);
1799 /* Initialize Inbound Message Engine */
1800 iowrite32(TSI721_IBDMAC_CTL_INIT, priv->regs + TSI721_IBDMAC_CTL(ch));
1801 ioread32(priv->regs + TSI721_IBDMAC_CTL(ch));
1803 priv->imsg_ring[mbox].fq_wrptr = entries - 1;
1804 iowrite32(entries - 1, priv->regs + TSI721_IBDMAC_FQWP(ch));
1806 priv->imsg_init[mbox] = 1;
1809 #ifdef CONFIG_PCI_MSI
1811 dma_free_coherent(&priv->pdev->dev,
1812 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
1813 priv->imsg_ring[mbox].imd_base,
1814 priv->imsg_ring[mbox].imd_phys);
1816 priv->imsg_ring[mbox].imd_base = NULL;
1817 #endif /* CONFIG_PCI_MSI */
1820 dma_free_coherent(&priv->pdev->dev,
1821 priv->imsg_ring[mbox].size * 8,
1822 priv->imsg_ring[mbox].imfq_base,
1823 priv->imsg_ring[mbox].imfq_phys);
1825 priv->imsg_ring[mbox].imfq_base = NULL;
1828 dma_free_coherent(&priv->pdev->dev,
1829 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
1830 priv->imsg_ring[mbox].buf_base,
1831 priv->imsg_ring[mbox].buf_phys);
1833 priv->imsg_ring[mbox].buf_base = NULL;
1840 * tsi721_close_inb_mbox - Shut down Tsi721 inbound mailbox
1841 * @mport: Master port implementing the Inbound Messaging Engine
1842 * @mbox: Mailbox to close
1844 static void tsi721_close_inb_mbox(struct rio_mport *mport, int mbox)
1846 struct tsi721_device *priv = mport->priv;
1850 if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */
1852 priv->imsg_init[mbox] = 0;
1854 /* Disable Inbound Messaging Engine */
1856 /* Disable Interrupts */
1857 tsi721_imsg_interrupt_disable(priv, ch, TSI721_OBDMAC_INT_MASK);
1859 #ifdef CONFIG_PCI_MSI
1860 if (priv->flags & TSI721_USING_MSIX) {
1861 free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1863 free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
1866 #endif /* CONFIG_PCI_MSI */
1868 /* Clear Inbound Buffer Queue */
1869 for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++)
1870 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
1872 /* Free memory allocated for message buffers */
1873 dma_free_coherent(&priv->pdev->dev,
1874 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
1875 priv->imsg_ring[mbox].buf_base,
1876 priv->imsg_ring[mbox].buf_phys);
1878 priv->imsg_ring[mbox].buf_base = NULL;
1880 /* Free memory allocated for free pointr list */
1881 dma_free_coherent(&priv->pdev->dev,
1882 priv->imsg_ring[mbox].size * 8,
1883 priv->imsg_ring[mbox].imfq_base,
1884 priv->imsg_ring[mbox].imfq_phys);
1886 priv->imsg_ring[mbox].imfq_base = NULL;
1888 /* Free memory allocated for RX descriptors */
1889 dma_free_coherent(&priv->pdev->dev,
1890 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
1891 priv->imsg_ring[mbox].imd_base,
1892 priv->imsg_ring[mbox].imd_phys);
1894 priv->imsg_ring[mbox].imd_base = NULL;
1898 * tsi721_add_inb_buffer - Add buffer to the Tsi721 inbound message queue
1899 * @mport: Master port implementing the Inbound Messaging Engine
1900 * @mbox: Inbound mailbox number
1901 * @buf: Buffer to add to inbound queue
1903 static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf)
1905 struct tsi721_device *priv = mport->priv;
1909 rx_slot = priv->imsg_ring[mbox].rx_slot;
1910 if (priv->imsg_ring[mbox].imq_base[rx_slot]) {
1911 dev_err(&priv->pdev->dev,
1912 "Error adding inbound buffer %d, buffer exists\n",
1918 priv->imsg_ring[mbox].imq_base[rx_slot] = buf;
1920 if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size)
1921 priv->imsg_ring[mbox].rx_slot = 0;
1928 * tsi721_get_inb_message - Fetch inbound message from the Tsi721 MSG Queue
1929 * @mport: Master port implementing the Inbound Messaging Engine
1930 * @mbox: Inbound mailbox number
1932 * Returns pointer to the message on success or NULL on failure.
1934 static void *tsi721_get_inb_message(struct rio_mport *mport, int mbox)
1936 struct tsi721_device *priv = mport->priv;
1937 struct tsi721_imsg_desc *desc;
1939 void *rx_virt = NULL;
1946 if (!priv->imsg_init[mbox])
1949 desc = priv->imsg_ring[mbox].imd_base;
1950 desc += priv->imsg_ring[mbox].desc_rdptr;
1952 if (!(le32_to_cpu(desc->msg_info) & TSI721_IMD_HO))
1955 rx_slot = priv->imsg_ring[mbox].rx_slot;
1956 while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) {
1957 if (++rx_slot == priv->imsg_ring[mbox].size)
1961 rx_phys = ((u64)le32_to_cpu(desc->bufptr_hi) << 32) |
1962 le32_to_cpu(desc->bufptr_lo);
1964 rx_virt = priv->imsg_ring[mbox].buf_base +
1965 (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys);
1967 buf = priv->imsg_ring[mbox].imq_base[rx_slot];
1968 msg_size = le32_to_cpu(desc->msg_info) & TSI721_IMD_BCOUNT;
1970 msg_size = RIO_MAX_MSG_SIZE;
1972 memcpy(buf, rx_virt, msg_size);
1973 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
1975 desc->msg_info &= cpu_to_le32(~TSI721_IMD_HO);
1976 if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size)
1977 priv->imsg_ring[mbox].desc_rdptr = 0;
1979 iowrite32(priv->imsg_ring[mbox].desc_rdptr,
1980 priv->regs + TSI721_IBDMAC_DQRP(ch));
1982 /* Return free buffer into the pointer list */
1983 free_ptr = priv->imsg_ring[mbox].imfq_base;
1984 free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys);
1986 if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size)
1987 priv->imsg_ring[mbox].fq_wrptr = 0;
1989 iowrite32(priv->imsg_ring[mbox].fq_wrptr,
1990 priv->regs + TSI721_IBDMAC_FQWP(ch));
1996 * tsi721_messages_init - Initialization of Messaging Engine
1997 * @priv: pointer to tsi721 private data
1999 * Configures Tsi721 messaging engine.
2001 static int tsi721_messages_init(struct tsi721_device *priv)
2005 iowrite32(0, priv->regs + TSI721_SMSG_ECC_LOG);
2006 iowrite32(0, priv->regs + TSI721_RETRY_GEN_CNT);
2007 iowrite32(0, priv->regs + TSI721_RETRY_RX_CNT);
2009 /* Set SRIO Message Request/Response Timeout */
2010 iowrite32(TSI721_RQRPTO_VAL, priv->regs + TSI721_RQRPTO);
2012 /* Initialize Inbound Messaging Engine Registers */
2013 for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++) {
2014 /* Clear interrupt bits */
2015 iowrite32(TSI721_IBDMAC_INT_MASK,
2016 priv->regs + TSI721_IBDMAC_INT(ch));
2018 iowrite32(0, priv->regs + TSI721_IBDMAC_STS(ch));
2020 iowrite32(TSI721_SMSG_ECC_COR_LOG_MASK,
2021 priv->regs + TSI721_SMSG_ECC_COR_LOG(ch));
2022 iowrite32(TSI721_SMSG_ECC_NCOR_MASK,
2023 priv->regs + TSI721_SMSG_ECC_NCOR(ch));
2030 * tsi721_disable_ints - disables all device interrupts
2031 * @priv: pointer to tsi721 private data
2033 static void tsi721_disable_ints(struct tsi721_device *priv)
2037 /* Disable all device level interrupts */
2038 iowrite32(0, priv->regs + TSI721_DEV_INTE);
2040 /* Disable all Device Channel interrupts */
2041 iowrite32(0, priv->regs + TSI721_DEV_CHAN_INTE);
2043 /* Disable all Inbound Msg Channel interrupts */
2044 for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++)
2045 iowrite32(0, priv->regs + TSI721_IBDMAC_INTE(ch));
2047 /* Disable all Outbound Msg Channel interrupts */
2048 for (ch = 0; ch < TSI721_OMSG_CHNUM; ch++)
2049 iowrite32(0, priv->regs + TSI721_OBDMAC_INTE(ch));
2051 /* Disable all general messaging interrupts */
2052 iowrite32(0, priv->regs + TSI721_SMSG_INTE);
2054 /* Disable all BDMA Channel interrupts */
2055 for (ch = 0; ch < TSI721_DMA_MAXCH; ch++)
2056 iowrite32(0, priv->regs + TSI721_DMAC_INTE(ch));
2058 /* Disable all general BDMA interrupts */
2059 iowrite32(0, priv->regs + TSI721_BDMA_INTE);
2061 /* Disable all SRIO Channel interrupts */
2062 for (ch = 0; ch < TSI721_SRIO_MAXCH; ch++)
2063 iowrite32(0, priv->regs + TSI721_SR_CHINTE(ch));
2065 /* Disable all general SR2PC interrupts */
2066 iowrite32(0, priv->regs + TSI721_SR2PC_GEN_INTE);
2068 /* Disable all PC2SR interrupts */
2069 iowrite32(0, priv->regs + TSI721_PC2SR_INTE);
2071 /* Disable all I2C interrupts */
2072 iowrite32(0, priv->regs + TSI721_I2C_INT_ENABLE);
2074 /* Disable SRIO MAC interrupts */
2075 iowrite32(0, priv->regs + TSI721_RIO_EM_INT_ENABLE);
2076 iowrite32(0, priv->regs + TSI721_RIO_EM_DEV_INT_EN);
2080 * tsi721_setup_mport - Setup Tsi721 as RapidIO subsystem master port
2081 * @priv: pointer to tsi721 private data
2083 * Configures Tsi721 as RapidIO master port.
2085 static int __devinit tsi721_setup_mport(struct tsi721_device *priv)
2087 struct pci_dev *pdev = priv->pdev;
2089 struct rio_ops *ops;
2091 struct rio_mport *mport;
2093 ops = kzalloc(sizeof(struct rio_ops), GFP_KERNEL);
2095 dev_dbg(&pdev->dev, "Unable to allocate memory for rio_ops\n");
2099 ops->lcread = tsi721_lcread;
2100 ops->lcwrite = tsi721_lcwrite;
2101 ops->cread = tsi721_cread_dma;
2102 ops->cwrite = tsi721_cwrite_dma;
2103 ops->dsend = tsi721_dsend;
2104 ops->open_inb_mbox = tsi721_open_inb_mbox;
2105 ops->close_inb_mbox = tsi721_close_inb_mbox;
2106 ops->open_outb_mbox = tsi721_open_outb_mbox;
2107 ops->close_outb_mbox = tsi721_close_outb_mbox;
2108 ops->add_outb_message = tsi721_add_outb_message;
2109 ops->add_inb_buffer = tsi721_add_inb_buffer;
2110 ops->get_inb_message = tsi721_get_inb_message;
2112 mport = kzalloc(sizeof(struct rio_mport), GFP_KERNEL);
2115 dev_dbg(&pdev->dev, "Unable to allocate memory for mport\n");
2121 mport->sys_size = 0; /* small system */
2122 mport->phy_type = RIO_PHY_SERIAL;
2123 mport->priv = (void *)priv;
2124 mport->phys_efptr = 0x100;
2126 INIT_LIST_HEAD(&mport->dbells);
2128 rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff);
2129 rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3);
2130 rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3);
2131 strcpy(mport->name, "Tsi721 mport");
2133 /* Hook up interrupt handler */
2135 #ifdef CONFIG_PCI_MSI
2136 if (!tsi721_enable_msix(priv))
2137 priv->flags |= TSI721_USING_MSIX;
2138 else if (!pci_enable_msi(pdev))
2139 priv->flags |= TSI721_USING_MSI;
2141 dev_info(&pdev->dev,
2142 "MSI/MSI-X is not available. Using legacy INTx.\n");
2143 #endif /* CONFIG_PCI_MSI */
2145 err = tsi721_request_irq(mport);
2148 tsi721_interrupts_init(priv);
2149 ops->pwenable = tsi721_pw_enable;
2151 dev_err(&pdev->dev, "Unable to get assigned PCI IRQ "
2152 "vector %02X err=0x%x\n", pdev->irq, err);
2154 /* Enable SRIO link */
2155 iowrite32(ioread32(priv->regs + TSI721_DEVCTL) |
2156 TSI721_DEVCTL_SRBOOT_CMPL,
2157 priv->regs + TSI721_DEVCTL);
2159 rio_register_mport(mport);
2160 priv->mport = mport;
2162 if (mport->host_deviceid >= 0)
2163 iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER |
2164 RIO_PORT_GEN_DISCOVERED,
2165 priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2167 iowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2172 static int __devinit tsi721_probe(struct pci_dev *pdev,
2173 const struct pci_device_id *id)
2175 struct tsi721_device *priv;
2180 priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL);
2182 dev_err(&pdev->dev, "Failed to allocate memory for device\n");
2187 err = pci_enable_device(pdev);
2189 dev_err(&pdev->dev, "Failed to enable PCI device\n");
2198 for (i = 0; i <= PCI_STD_RESOURCE_END; i++) {
2199 dev_dbg(&pdev->dev, "res[%d] @ 0x%llx (0x%lx, 0x%lx)\n",
2200 i, (unsigned long long)pci_resource_start(pdev, i),
2201 (unsigned long)pci_resource_len(pdev, i),
2202 pci_resource_flags(pdev, i));
2207 * Verify BAR configuration
2210 /* BAR_0 (registers) must be 512KB+ in 32-bit address space */
2211 if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) ||
2212 pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 ||
2213 pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) {
2215 "Missing or misconfigured CSR BAR0, aborting.\n");
2217 goto err_disable_pdev;
2220 /* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */
2221 if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) ||
2222 pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 ||
2223 pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) {
2225 "Missing or misconfigured Doorbell BAR1, aborting.\n");
2227 goto err_disable_pdev;
2231 * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address
2233 * NOTE: BAR_2 and BAR_4 are not used by this version of driver.
2234 * It may be a good idea to keep them disabled using HW configuration
2235 * to save PCI memory space.
2237 if ((pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM) &&
2238 (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64)) {
2239 dev_info(&pdev->dev, "Outbound BAR2 is not used but enabled.\n");
2242 if ((pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM) &&
2243 (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64)) {
2244 dev_info(&pdev->dev, "Outbound BAR4 is not used but enabled.\n");
2247 err = pci_request_regions(pdev, DRV_NAME);
2249 dev_err(&pdev->dev, "Cannot obtain PCI resources, "
2251 goto err_disable_pdev;
2254 pci_set_master(pdev);
2256 priv->regs = pci_ioremap_bar(pdev, BAR_0);
2259 "Unable to map device registers space, aborting\n");
2264 priv->odb_base = pci_ioremap_bar(pdev, BAR_1);
2265 if (!priv->odb_base) {
2267 "Unable to map outbound doorbells space, aborting\n");
2269 goto err_unmap_bars;
2272 /* Configure DMA attributes. */
2273 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
2274 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) {
2275 dev_info(&pdev->dev, "Unable to set DMA mask\n");
2276 goto err_unmap_bars;
2279 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
2280 dev_info(&pdev->dev, "Unable to set consistent DMA mask\n");
2282 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2284 dev_info(&pdev->dev, "Unable to set consistent DMA mask\n");
2287 cap = pci_pcie_cap(pdev);
2290 /* Clear "no snoop" and "relaxed ordering" bits, use default MRRS. */
2291 pci_read_config_dword(pdev, cap + PCI_EXP_DEVCTL, ®val);
2292 regval &= ~(PCI_EXP_DEVCTL_READRQ | PCI_EXP_DEVCTL_RELAX_EN |
2293 PCI_EXP_DEVCTL_NOSNOOP_EN);
2294 regval |= 0x2 << MAX_READ_REQUEST_SZ_SHIFT;
2295 pci_write_config_dword(pdev, cap + PCI_EXP_DEVCTL, regval);
2297 /* Adjust PCIe completion timeout. */
2298 pci_read_config_dword(pdev, cap + PCI_EXP_DEVCTL2, ®val);
2300 pci_write_config_dword(pdev, cap + PCI_EXP_DEVCTL2, regval | 0x2);
2303 * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block
2305 pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0x01);
2306 pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL,
2307 TSI721_MSIXTBL_OFFSET);
2308 pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA,
2309 TSI721_MSIXPBA_OFFSET);
2310 pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0);
2313 tsi721_disable_ints(priv);
2315 tsi721_init_pc2sr_mapping(priv);
2316 tsi721_init_sr2pc_mapping(priv);
2318 if (tsi721_bdma_init(priv)) {
2319 dev_err(&pdev->dev, "BDMA initialization failed, aborting\n");
2321 goto err_unmap_bars;
2324 err = tsi721_doorbell_init(priv);
2328 tsi721_port_write_init(priv);
2330 err = tsi721_messages_init(priv);
2332 goto err_free_consistent;
2334 err = tsi721_setup_mport(priv);
2336 goto err_free_consistent;
2340 err_free_consistent:
2341 tsi721_doorbell_free(priv);
2343 tsi721_bdma_free(priv);
2346 iounmap(priv->regs);
2348 iounmap(priv->odb_base);
2350 pci_release_regions(pdev);
2351 pci_clear_master(pdev);
2353 pci_disable_device(pdev);
2360 static DEFINE_PCI_DEVICE_TABLE(tsi721_pci_tbl) = {
2361 { PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721) },
2362 { 0, } /* terminate list */
2365 MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl);
2367 static struct pci_driver tsi721_driver = {
2369 .id_table = tsi721_pci_tbl,
2370 .probe = tsi721_probe,
2373 static int __init tsi721_init(void)
2375 return pci_register_driver(&tsi721_driver);
2378 static void __exit tsi721_exit(void)
2380 pci_unregister_driver(&tsi721_driver);
2383 device_initcall(tsi721_init);