drivers/rapidio/devices/tsi721.c

   1 /*
   2  * RapidIO mport driver for Tsi721 PCIExpress-to-SRIO bridge
   3  *
   4  * Copyright 2011 Integrated Device Technology, Inc.
   5  * Alexandre Bounine <alexandre.bounine@idt.com>
   6  * Chul Kim <chul.kim@idt.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU General Public License as published by the Free
  10  * Software Foundation; either version 2 of the License, or (at your option)
  11  * any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along with
  19  * this program; if not, write to the Free Software Foundation, Inc., 59
  20  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  21  */
  22
  23 #include <linux/io.h>
  24 #include <linux/errno.h>
  25 #include <linux/init.h>
  26 #include <linux/ioport.h>
  27 #include <linux/kernel.h>
  28 #include <linux/module.h>
  29 #include <linux/pci.h>
  30 #include <linux/rio.h>
  31 #include <linux/rio_drv.h>
  32 #include <linux/dma-mapping.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/kfifo.h>
  35 #include <linux/delay.h>
  36
  37 #include "tsi721.h"
  38
  39 #define DEBUG_PW        /* Inbound Port-Write debugging */
  40
  41 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch);
  42 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch);
  43
  44 /**
  45  * tsi721_lcread - read from local SREP config space
  46  * @mport: RapidIO master port info
  47  * @index: ID of RapdiIO interface
  48  * @offset: Offset into configuration space
  49  * @len: Length (in bytes) of the maintenance transaction
  50  * @data: Value to be read into
  51  *
  52  * Generates a local SREP space read. Returns %0 on
  53  * success or %-EINVAL on failure.
  54  */
  55 static int tsi721_lcread(struct rio_mport *mport, int index, u32 offset,
  56                          int len, u32 *data)
  57 {
  58         struct tsi721_device *priv = mport->priv;
  59
  60         if (len != sizeof(u32))
  61                 return -EINVAL; /* only 32-bit access is supported */
  62
  63         *data = ioread32(priv->regs + offset);
  64
  65         return 0;
  66 }
  67
  68 /**
  69  * tsi721_lcwrite - write into local SREP config space
  70  * @mport: RapidIO master port info
  71  * @index: ID of RapdiIO interface
  72  * @offset: Offset into configuration space
  73  * @len: Length (in bytes) of the maintenance transaction
  74  * @data: Value to be written
  75  *
  76  * Generates a local write into SREP configuration space. Returns %0 on
  77  * success or %-EINVAL on failure.
  78  */
  79 static int tsi721_lcwrite(struct rio_mport *mport, int index, u32 offset,
  80                           int len, u32 data)
  81 {
  82         struct tsi721_device *priv = mport->priv;
  83
  84         if (len != sizeof(u32))
  85                 return -EINVAL; /* only 32-bit access is supported */
  86
  87         iowrite32(data, priv->regs + offset);
  88
  89         return 0;
  90 }
  91
  92 /**
  93  * tsi721_maint_dma - Helper function to generate RapidIO maintenance
  94  *                    transactions using designated Tsi721 DMA channel.
  95  * @priv: pointer to tsi721 private data
  96  * @sys_size: RapdiIO transport system size
  97  * @destid: Destination ID of transaction
  98  * @hopcount: Number of hops to target device
  99  * @offset: Offset into configuration space
 100  * @len: Length (in bytes) of the maintenance transaction
 101  * @data: Location to be read from or write into
 102  * @do_wr: Operation flag (1 == MAINT_WR)
 103  *
 104  * Generates a RapidIO maintenance transaction (Read or Write).
 105  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
 106  */
 107 static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size,
 108                         u16 destid, u8 hopcount, u32 offset, int len,
 109                         u32 *data, int do_wr)
 110 {
 111         void __iomem *regs = priv->regs + TSI721_DMAC_BASE(priv->mdma.ch_id);
 112         struct tsi721_dma_desc *bd_ptr;
 113         u32 rd_count, swr_ptr, ch_stat;
 114         int i, err = 0;
 115         u32 op = do_wr ? MAINT_WR : MAINT_RD;
 116
 117         if (offset > (RIO_MAINT_SPACE_SZ - len) || (len != sizeof(u32)))
 118                 return -EINVAL;
 119
 120         bd_ptr = priv->mdma.bd_base;
 121
 122         rd_count = ioread32(regs + TSI721_DMAC_DRDCNT);
 123
 124         /* Initialize DMA descriptor */
 125         bd_ptr[0].type_id = cpu_to_le32((DTYPE2 << 29) | (op << 19) | destid);
 126         bd_ptr[0].bcount = cpu_to_le32((sys_size << 26) | 0x04);
 127         bd_ptr[0].raddr_lo = cpu_to_le32((hopcount << 24) | offset);
 128         bd_ptr[0].raddr_hi = 0;
 129         if (do_wr)
 130                 bd_ptr[0].data[0] = cpu_to_be32p(data);
 131         else
 132                 bd_ptr[0].data[0] = 0xffffffff;
 133
 134         mb();
 135
 136         /* Start DMA operation */
 137         iowrite32(rd_count + 2, regs + TSI721_DMAC_DWRCNT);
 138         ioread32(regs + TSI721_DMAC_DWRCNT);
 139         i = 0;
 140
 141         /* Wait until DMA transfer is finished */
 142         while ((ch_stat = ioread32(regs + TSI721_DMAC_STS))
 143                                                         & TSI721_DMAC_STS_RUN) {
 144                 udelay(1);
 145                 if (++i >= 5000000) {
 146                         dev_dbg(&priv->pdev->dev,
 147                                 "%s : DMA[%d] read timeout ch_status=%x\n",
 148                                 __func__, priv->mdma.ch_id, ch_stat);
 149                         if (!do_wr)
 150                                 *data = 0xffffffff;
 151                         err = -EIO;
 152                         goto err_out;
 153                 }
 154         }
 155
 156         if (ch_stat & TSI721_DMAC_STS_ABORT) {
 157                 /* If DMA operation aborted due to error,
 158                  * reinitialize DMA channel
 159                  */
 160                 dev_dbg(&priv->pdev->dev, "%s : DMA ABORT ch_stat=%x\n",
 161                         __func__, ch_stat);
 162                 dev_dbg(&priv->pdev->dev, "OP=%d : destid=%x hc=%x off=%x\n",
 163                         do_wr ? MAINT_WR : MAINT_RD, destid, hopcount, offset);
 164                 iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
 165                 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
 166                 udelay(10);
 167                 iowrite32(0, regs + TSI721_DMAC_DWRCNT);
 168                 udelay(1);
 169                 if (!do_wr)
 170                         *data = 0xffffffff;
 171                 err = -EIO;
 172                 goto err_out;
 173         }
 174
 175         if (!do_wr)
 176                 *data = be32_to_cpu(bd_ptr[0].data[0]);
 177
 178         /*
 179          * Update descriptor status FIFO RD pointer.
 180          * NOTE: Skipping check and clear FIFO entries because we are waiting
 181          * for transfer to be completed.
 182          */
 183         swr_ptr = ioread32(regs + TSI721_DMAC_DSWP);
 184         iowrite32(swr_ptr, regs + TSI721_DMAC_DSRP);
 185 err_out:
 186
 187         return err;
 188 }
 189
 190 /**
 191  * tsi721_cread_dma - Generate a RapidIO maintenance read transaction
 192  *                    using Tsi721 BDMA engine.
 193  * @mport: RapidIO master port control structure
 194  * @index: ID of RapdiIO interface
 195  * @destid: Destination ID of transaction
 196  * @hopcount: Number of hops to target device
 197  * @offset: Offset into configuration space
 198  * @len: Length (in bytes) of the maintenance transaction
 199  * @val: Location to be read into
 200  *
 201  * Generates a RapidIO maintenance read transaction.
 202  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
 203  */
 204 static int tsi721_cread_dma(struct rio_mport *mport, int index, u16 destid,
 205                         u8 hopcount, u32 offset, int len, u32 *data)
 206 {
 207         struct tsi721_device *priv = mport->priv;
 208
 209         return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
 210                                 offset, len, data, 0);
 211 }
 212
 213 /**
 214  * tsi721_cwrite_dma - Generate a RapidIO maintenance write transaction
 215  *                     using Tsi721 BDMA engine
 216  * @mport: RapidIO master port control structure
 217  * @index: ID of RapdiIO interface
 218  * @destid: Destination ID of transaction
 219  * @hopcount: Number of hops to target device
 220  * @offset: Offset into configuration space
 221  * @len: Length (in bytes) of the maintenance transaction
 222  * @val: Value to be written
 223  *
 224  * Generates a RapidIO maintenance write transaction.
 225  * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
 226  */
 227 static int tsi721_cwrite_dma(struct rio_mport *mport, int index, u16 destid,
 228                          u8 hopcount, u32 offset, int len, u32 data)
 229 {
 230         struct tsi721_device *priv = mport->priv;
 231         u32 temp = data;
 232
 233         return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
 234                                 offset, len, &temp, 1);
 235 }
 236
 237 /**
 238  * tsi721_pw_handler - Tsi721 inbound port-write interrupt handler
 239  * @mport: RapidIO master port structure
 240  *
 241  * Handles inbound port-write interrupts. Copies PW message from an internal
 242  * buffer into PW message FIFO and schedules deferred routine to process
 243  * queued messages.
 244  */
 245 static int
 246 tsi721_pw_handler(struct rio_mport *mport)
 247 {
 248         struct tsi721_device *priv = mport->priv;
 249         u32 pw_stat;
 250         u32 pw_buf[TSI721_RIO_PW_MSG_SIZE/sizeof(u32)];
 251
 252
 253         pw_stat = ioread32(priv->regs + TSI721_RIO_PW_RX_STAT);
 254
 255         if (pw_stat & TSI721_RIO_PW_RX_STAT_PW_VAL) {
 256                 pw_buf[0] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(0));
 257                 pw_buf[1] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(1));
 258                 pw_buf[2] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(2));
 259                 pw_buf[3] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(3));
 260
 261                 /* Queue PW message (if there is room in FIFO),
 262                  * otherwise discard it.
 263                  */
 264                 spin_lock(&priv->pw_fifo_lock);
 265                 if (kfifo_avail(&priv->pw_fifo) >= TSI721_RIO_PW_MSG_SIZE)
 266                         kfifo_in(&priv->pw_fifo, pw_buf,
 267                                                 TSI721_RIO_PW_MSG_SIZE);
 268                 else
 269                         priv->pw_discard_count++;
 270                 spin_unlock(&priv->pw_fifo_lock);
 271         }
 272
 273         /* Clear pending PW interrupts */
 274         iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
 275                   priv->regs + TSI721_RIO_PW_RX_STAT);
 276
 277         schedule_work(&priv->pw_work);
 278
 279         return 0;
 280 }
 281
 282 static void tsi721_pw_dpc(struct work_struct *work)
 283 {
 284         struct tsi721_device *priv = container_of(work, struct tsi721_device,
 285                                                     pw_work);
 286         u32 msg_buffer[RIO_PW_MSG_SIZE/sizeof(u32)]; /* Use full size PW message
 287                                                         buffer for RIO layer */
 288
 289         /*
 290          * Process port-write messages
 291          */
 292         while (kfifo_out_spinlocked(&priv->pw_fifo, (unsigned char *)msg_buffer,
 293                          TSI721_RIO_PW_MSG_SIZE, &priv->pw_fifo_lock)) {
 294                 /* Process one message */
 295 #ifdef DEBUG_PW
 296                 {
 297                 u32 i;
 298                 pr_debug("%s : Port-Write Message:", __func__);
 299                 for (i = 0; i < RIO_PW_MSG_SIZE/sizeof(u32); ) {
 300                         pr_debug("0x%02x: %08x %08x %08x %08x", i*4,
 301                                 msg_buffer[i], msg_buffer[i + 1],
 302                                 msg_buffer[i + 2], msg_buffer[i + 3]);
 303                         i += 4;
 304                 }
 305                 pr_debug("\n");
 306                 }
 307 #endif
 308                 /* Pass the port-write message to RIO core for processing */
 309                 rio_inb_pwrite_handler((union rio_pw_msg *)msg_buffer);
 310         }
 311 }
 312
 313 /**
 314  * tsi721_pw_enable - enable/disable port-write interface init
 315  * @mport: Master port implementing the port write unit
 316  * @enable:    1=enable; 0=disable port-write message handling
 317  */
 318 static int tsi721_pw_enable(struct rio_mport *mport, int enable)
 319 {
 320         struct tsi721_device *priv = mport->priv;
 321         u32 rval;
 322
 323         rval = ioread32(priv->regs + TSI721_RIO_EM_INT_ENABLE);
 324
 325         if (enable)
 326                 rval |= TSI721_RIO_EM_INT_ENABLE_PW_RX;
 327         else
 328                 rval &= ~TSI721_RIO_EM_INT_ENABLE_PW_RX;
 329
 330         /* Clear pending PW interrupts */
 331         iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
 332                   priv->regs + TSI721_RIO_PW_RX_STAT);
 333         /* Update enable bits */
 334         iowrite32(rval, priv->regs + TSI721_RIO_EM_INT_ENABLE);
 335
 336         return 0;
 337 }
 338
 339 /**
 340  * tsi721_dsend - Send a RapidIO doorbell
 341  * @mport: RapidIO master port info
 342  * @index: ID of RapidIO interface
 343  * @destid: Destination ID of target device
 344  * @data: 16-bit info field of RapidIO doorbell
 345  *
 346  * Sends a RapidIO doorbell message. Always returns %0.
 347  */
 348 static int tsi721_dsend(struct rio_mport *mport, int index,
 349                         u16 destid, u16 data)
 350 {
 351         struct tsi721_device *priv = mport->priv;
 352         u32 offset;
 353
 354         offset = (((mport->sys_size) ? RIO_TT_CODE_16 : RIO_TT_CODE_8) << 18) |
 355                  (destid << 2);
 356
 357         dev_dbg(&priv->pdev->dev,
 358                 "Send Doorbell 0x%04x to destID 0x%x\n", data, destid);
 359         iowrite16be(data, priv->odb_base + offset);
 360
 361         return 0;
 362 }
 363
 364 /**
 365  * tsi721_dbell_handler - Tsi721 doorbell interrupt handler
 366  * @mport: RapidIO master port structure
 367  *
 368  * Handles inbound doorbell interrupts. Copies doorbell entry from an internal
 369  * buffer into DB message FIFO and schedules deferred  routine to process
 370  * queued DBs.
 371  */
 372 static int
 373 tsi721_dbell_handler(struct rio_mport *mport)
 374 {
 375         struct tsi721_device *priv = mport->priv;
 376         u32 regval;
 377
 378         /* Disable IDB interrupts */
 379         regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 380         regval &= ~TSI721_SR_CHINT_IDBQRCV;
 381         iowrite32(regval,
 382                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 383
 384         schedule_work(&priv->idb_work);
 385
 386         return 0;
 387 }
 388
 389 static void tsi721_db_dpc(struct work_struct *work)
 390 {
 391         struct tsi721_device *priv = container_of(work, struct tsi721_device,
 392                                                     idb_work);
 393         struct rio_mport *mport;
 394         struct rio_dbell *dbell;
 395         int found = 0;
 396         u32 wr_ptr, rd_ptr;
 397         u64 *idb_entry;
 398         u32 regval;
 399         union {
 400                 u64 msg;
 401                 u8  bytes[8];
 402         } idb;
 403
 404         /*
 405          * Process queued inbound doorbells
 406          */
 407         mport = priv->mport;
 408
 409         wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
 410         rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE;
 411
 412         while (wr_ptr != rd_ptr) {
 413                 idb_entry = (u64 *)(priv->idb_base +
 414                                         (TSI721_IDB_ENTRY_SIZE * rd_ptr));
 415                 rd_ptr++;
 416                 rd_ptr %= IDB_QSIZE;
 417                 idb.msg = *idb_entry;
 418                 *idb_entry = 0;
 419
 420                 /* Process one doorbell */
 421                 list_for_each_entry(dbell, &mport->dbells, node) {
 422                         if ((dbell->res->start <= DBELL_INF(idb.bytes)) &&
 423                             (dbell->res->end >= DBELL_INF(idb.bytes))) {
 424                                 found = 1;
 425                                 break;
 426                         }
 427                 }
 428
 429                 if (found) {
 430                         dbell->dinb(mport, dbell->dev_id, DBELL_SID(idb.bytes),
 431                                     DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
 432                 } else {
 433                         dev_dbg(&priv->pdev->dev,
 434                                 "spurious inb doorbell, sid %2.2x tid %2.2x"
 435                                 " info %4.4x\n", DBELL_SID(idb.bytes),
 436                                 DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
 437                 }
 438
 439                 wr_ptr = ioread32(priv->regs +
 440                                   TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
 441         }
 442
 443         iowrite32(rd_ptr & (IDB_QSIZE - 1),
 444                 priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
 445
 446         /* Re-enable IDB interrupts */
 447         regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 448         regval |= TSI721_SR_CHINT_IDBQRCV;
 449         iowrite32(regval,
 450                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 451
 452         wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
 453         if (wr_ptr != rd_ptr)
 454                 schedule_work(&priv->idb_work);
 455 }
 456
 457 /**
 458  * tsi721_irqhandler - Tsi721 interrupt handler
 459  * @irq: Linux interrupt number
 460  * @ptr: Pointer to interrupt-specific data (mport structure)
 461  *
 462  * Handles Tsi721 interrupts signaled using MSI and INTA. Checks reported
 463  * interrupt events and calls an event-specific handler(s).
 464  */
 465 static irqreturn_t tsi721_irqhandler(int irq, void *ptr)
 466 {
 467         struct rio_mport *mport = (struct rio_mport *)ptr;
 468         struct tsi721_device *priv = mport->priv;
 469         u32 dev_int;
 470         u32 dev_ch_int;
 471         u32 intval;
 472         u32 ch_inte;
 473
 474         dev_int = ioread32(priv->regs + TSI721_DEV_INT);
 475         if (!dev_int)
 476                 return IRQ_NONE;
 477
 478         dev_ch_int = ioread32(priv->regs + TSI721_DEV_CHAN_INT);
 479
 480         if (dev_int & TSI721_DEV_INT_SR2PC_CH) {
 481                 /* Service SR2PC Channel interrupts */
 482                 if (dev_ch_int & TSI721_INT_SR2PC_CHAN(IDB_QUEUE)) {
 483                         /* Service Inbound Doorbell interrupt */
 484                         intval = ioread32(priv->regs +
 485                                                 TSI721_SR_CHINT(IDB_QUEUE));
 486                         if (intval & TSI721_SR_CHINT_IDBQRCV)
 487                                 tsi721_dbell_handler(mport);
 488                         else
 489                                 dev_info(&priv->pdev->dev,
 490                                         "Unsupported SR_CH_INT %x\n", intval);
 491
 492                         /* Clear interrupts */
 493                         iowrite32(intval,
 494                                 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 495                         ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 496                 }
 497         }
 498
 499         if (dev_int & TSI721_DEV_INT_SMSG_CH) {
 500                 int ch;
 501
 502                 /*
 503                  * Service channel interrupts from Messaging Engine
 504                  */
 505
 506                 if (dev_ch_int & TSI721_INT_IMSG_CHAN_M) { /* Inbound Msg */
 507                         /* Disable signaled OB MSG Channel interrupts */
 508                         ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
 509                         ch_inte &= ~(dev_ch_int & TSI721_INT_IMSG_CHAN_M);
 510                         iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
 511
 512                         /*
 513                          * Process Inbound Message interrupt for each MBOX
 514                          */
 515                         for (ch = 4; ch < RIO_MAX_MBOX + 4; ch++) {
 516                                 if (!(dev_ch_int & TSI721_INT_IMSG_CHAN(ch)))
 517                                         continue;
 518                                 tsi721_imsg_handler(priv, ch);
 519                         }
 520                 }
 521
 522                 if (dev_ch_int & TSI721_INT_OMSG_CHAN_M) { /* Outbound Msg */
 523                         /* Disable signaled OB MSG Channel interrupts */
 524                         ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
 525                         ch_inte &= ~(dev_ch_int & TSI721_INT_OMSG_CHAN_M);
 526                         iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
 527
 528                         /*
 529                          * Process Outbound Message interrupts for each MBOX
 530                          */
 531
 532                         for (ch = 0; ch < RIO_MAX_MBOX; ch++) {
 533                                 if (!(dev_ch_int & TSI721_INT_OMSG_CHAN(ch)))
 534                                         continue;
 535                                 tsi721_omsg_handler(priv, ch);
 536                         }
 537                 }
 538         }
 539
 540         if (dev_int & TSI721_DEV_INT_SRIO) {
 541                 /* Service SRIO MAC interrupts */
 542                 intval = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
 543                 if (intval & TSI721_RIO_EM_INT_STAT_PW_RX)
 544                         tsi721_pw_handler(mport);
 545         }
 546
 547 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 548         if (dev_int & TSI721_DEV_INT_BDMA_CH) {
 549                 int ch;
 550
 551                 if (dev_ch_int & TSI721_INT_BDMA_CHAN_M) {
 552                         dev_dbg(&priv->pdev->dev,
 553                                 "IRQ from DMA channel 0x%08x\n", dev_ch_int);
 554
 555                         for (ch = 0; ch < TSI721_DMA_MAXCH; ch++) {
 556                                 if (!(dev_ch_int & TSI721_INT_BDMA_CHAN(ch)))
 557                                         continue;
 558                                 tsi721_bdma_handler(&priv->bdma[ch]);
 559                         }
 560                 }
 561         }
 562 #endif
 563         return IRQ_HANDLED;
 564 }
 565
 566 static void tsi721_interrupts_init(struct tsi721_device *priv)
 567 {
 568         u32 intr;
 569
 570         /* Enable IDB interrupts */
 571         iowrite32(TSI721_SR_CHINT_ALL,
 572                 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 573         iowrite32(TSI721_SR_CHINT_IDBQRCV,
 574                 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
 575
 576         /* Enable SRIO MAC interrupts */
 577         iowrite32(TSI721_RIO_EM_DEV_INT_EN_INT,
 578                 priv->regs + TSI721_RIO_EM_DEV_INT_EN);
 579
 580         /* Enable interrupts from channels in use */
 581 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 582         intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE) |
 583                 (TSI721_INT_BDMA_CHAN_M &
 584                  ~TSI721_INT_BDMA_CHAN(TSI721_DMACH_MAINT));
 585 #else
 586         intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE);
 587 #endif
 588         iowrite32(intr, priv->regs + TSI721_DEV_CHAN_INTE);
 589
 590         if (priv->flags & TSI721_USING_MSIX)
 591                 intr = TSI721_DEV_INT_SRIO;
 592         else
 593                 intr = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
 594                         TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
 595
 596         iowrite32(intr, priv->regs + TSI721_DEV_INTE);
 597         ioread32(priv->regs + TSI721_DEV_INTE);
 598 }
 599
 600 #ifdef CONFIG_PCI_MSI
 601 /**
 602  * tsi721_omsg_msix - MSI-X interrupt handler for outbound messaging
 603  * @irq: Linux interrupt number
 604  * @ptr: Pointer to interrupt-specific data (mport structure)
 605  *
 606  * Handles outbound messaging interrupts signaled using MSI-X.
 607  */
 608 static irqreturn_t tsi721_omsg_msix(int irq, void *ptr)
 609 {
 610         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
 611         int mbox;
 612
 613         mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX;
 614         tsi721_omsg_handler(priv, mbox);
 615         return IRQ_HANDLED;
 616 }
 617
 618 /**
 619  * tsi721_imsg_msix - MSI-X interrupt handler for inbound messaging
 620  * @irq: Linux interrupt number
 621  * @ptr: Pointer to interrupt-specific data (mport structure)
 622  *
 623  * Handles inbound messaging interrupts signaled using MSI-X.
 624  */
 625 static irqreturn_t tsi721_imsg_msix(int irq, void *ptr)
 626 {
 627         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
 628         int mbox;
 629
 630         mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX;
 631         tsi721_imsg_handler(priv, mbox + 4);
 632         return IRQ_HANDLED;
 633 }
 634
 635 /**
 636  * tsi721_srio_msix - Tsi721 MSI-X SRIO MAC interrupt handler
 637  * @irq: Linux interrupt number
 638  * @ptr: Pointer to interrupt-specific data (mport structure)
 639  *
 640  * Handles Tsi721 interrupts from SRIO MAC.
 641  */
 642 static irqreturn_t tsi721_srio_msix(int irq, void *ptr)
 643 {
 644         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
 645         u32 srio_int;
 646
 647         /* Service SRIO MAC interrupts */
 648         srio_int = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
 649         if (srio_int & TSI721_RIO_EM_INT_STAT_PW_RX)
 650                 tsi721_pw_handler((struct rio_mport *)ptr);
 651
 652         return IRQ_HANDLED;
 653 }
 654
 655 /**
 656  * tsi721_sr2pc_ch_msix - Tsi721 MSI-X SR2PC Channel interrupt handler
 657  * @irq: Linux interrupt number
 658  * @ptr: Pointer to interrupt-specific data (mport structure)
 659  *
 660  * Handles Tsi721 interrupts from SR2PC Channel.
 661  * NOTE: At this moment services only one SR2PC channel associated with inbound
 662  * doorbells.
 663  */
 664 static irqreturn_t tsi721_sr2pc_ch_msix(int irq, void *ptr)
 665 {
 666         struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;
 667         u32 sr_ch_int;
 668
 669         /* Service Inbound DB interrupt from SR2PC channel */
 670         sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 671         if (sr_ch_int & TSI721_SR_CHINT_IDBQRCV)
 672                 tsi721_dbell_handler((struct rio_mport *)ptr);
 673
 674         /* Clear interrupts */
 675         iowrite32(sr_ch_int, priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 676         /* Read back to ensure that interrupt was cleared */
 677         sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
 678
 679         return IRQ_HANDLED;
 680 }
 681
 682 /**
 683  * tsi721_request_msix - register interrupt service for MSI-X mode.
 684  * @mport: RapidIO master port structure
 685  *
 686  * Registers MSI-X interrupt service routines for interrupts that are active
 687  * immediately after mport initialization. Messaging interrupt service routines
 688  * should be registered during corresponding open requests.
 689  */
 690 static int tsi721_request_msix(struct rio_mport *mport)
 691 {
 692         struct tsi721_device *priv = mport->priv;
 693         int err = 0;
 694
 695         err = request_irq(priv->msix[TSI721_VECT_IDB].vector,
 696                         tsi721_sr2pc_ch_msix, 0,
 697                         priv->msix[TSI721_VECT_IDB].irq_name, (void *)mport);
 698         if (err)
 699                 goto out;
 700
 701         err = request_irq(priv->msix[TSI721_VECT_PWRX].vector,
 702                         tsi721_srio_msix, 0,
 703                         priv->msix[TSI721_VECT_PWRX].irq_name, (void *)mport);
 704         if (err)
 705                 free_irq(
 706                         priv->msix[TSI721_VECT_IDB].vector,
 707                         (void *)mport);
 708 out:
 709         return err;
 710 }
 711
 712 /**
 713  * tsi721_enable_msix - Attempts to enable MSI-X support for Tsi721.
 714  * @priv: pointer to tsi721 private data
 715  *
 716  * Configures MSI-X support for Tsi721. Supports only an exact number
 717  * of requested vectors.
 718  */
 719 static int tsi721_enable_msix(struct tsi721_device *priv)
 720 {
 721         struct msix_entry entries[TSI721_VECT_MAX];
 722         int err;
 723         int i;
 724
 725         entries[TSI721_VECT_IDB].entry = TSI721_MSIX_SR2PC_IDBQ_RCV(IDB_QUEUE);
 726         entries[TSI721_VECT_PWRX].entry = TSI721_MSIX_SRIO_MAC_INT;
 727
 728         /*
 729          * Initialize MSI-X entries for Messaging Engine:
 730          * this driver supports four RIO mailboxes (inbound and outbound)
 731          * NOTE: Inbound message MBOX 0...4 use IB channels 4...7. Therefore
 732          * offset +4 is added to IB MBOX number.
 733          */
 734         for (i = 0; i < RIO_MAX_MBOX; i++) {
 735                 entries[TSI721_VECT_IMB0_RCV + i].entry =
 736                                         TSI721_MSIX_IMSG_DQ_RCV(i + 4);
 737                 entries[TSI721_VECT_IMB0_INT + i].entry =
 738                                         TSI721_MSIX_IMSG_INT(i + 4);
 739                 entries[TSI721_VECT_OMB0_DONE + i].entry =
 740                                         TSI721_MSIX_OMSG_DONE(i);
 741                 entries[TSI721_VECT_OMB0_INT + i].entry =
 742                                         TSI721_MSIX_OMSG_INT(i);
 743         }
 744
 745 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 746         /*
 747          * Initialize MSI-X entries for Block DMA Engine:
 748          * this driver supports XXX DMA channels
 749          * (one is reserved for SRIO maintenance transactions)
 750          */
 751         for (i = 0; i < TSI721_DMA_CHNUM; i++) {
 752                 entries[TSI721_VECT_DMA0_DONE + i].entry =
 753                                         TSI721_MSIX_DMACH_DONE(i);
 754                 entries[TSI721_VECT_DMA0_INT + i].entry =
 755                                         TSI721_MSIX_DMACH_INT(i);
 756         }
 757 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
 758
 759         err = pci_enable_msix(priv->pdev, entries, ARRAY_SIZE(entries));
 760         if (err) {
 761                 if (err > 0)
 762                         dev_info(&priv->pdev->dev,
 763                                  "Only %d MSI-X vectors available, "
 764                                  "not using MSI-X\n", err);
 765                 else
 766                         dev_err(&priv->pdev->dev,
 767                                 "Failed to enable MSI-X (err=%d)\n", err);
 768                 return err;
 769         }
 770
 771         /*
 772          * Copy MSI-X vector information into tsi721 private structure
 773          */
 774         priv->msix[TSI721_VECT_IDB].vector = entries[TSI721_VECT_IDB].vector;
 775         snprintf(priv->msix[TSI721_VECT_IDB].irq_name, IRQ_DEVICE_NAME_MAX,
 776                  DRV_NAME "-idb@pci:%s", pci_name(priv->pdev));
 777         priv->msix[TSI721_VECT_PWRX].vector = entries[TSI721_VECT_PWRX].vector;
 778         snprintf(priv->msix[TSI721_VECT_PWRX].irq_name, IRQ_DEVICE_NAME_MAX,
 779                  DRV_NAME "-pwrx@pci:%s", pci_name(priv->pdev));
 780
 781         for (i = 0; i < RIO_MAX_MBOX; i++) {
 782                 priv->msix[TSI721_VECT_IMB0_RCV + i].vector =
 783                                 entries[TSI721_VECT_IMB0_RCV + i].vector;
 784                 snprintf(priv->msix[TSI721_VECT_IMB0_RCV + i].irq_name,
 785                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbr%d@pci:%s",
 786                          i, pci_name(priv->pdev));
 787
 788                 priv->msix[TSI721_VECT_IMB0_INT + i].vector =
 789                                 entries[TSI721_VECT_IMB0_INT + i].vector;
 790                 snprintf(priv->msix[TSI721_VECT_IMB0_INT + i].irq_name,
 791                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbi%d@pci:%s",
 792                          i, pci_name(priv->pdev));
 793
 794                 priv->msix[TSI721_VECT_OMB0_DONE + i].vector =
 795                                 entries[TSI721_VECT_OMB0_DONE + i].vector;
 796                 snprintf(priv->msix[TSI721_VECT_OMB0_DONE + i].irq_name,
 797                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombd%d@pci:%s",
 798                          i, pci_name(priv->pdev));
 799
 800                 priv->msix[TSI721_VECT_OMB0_INT + i].vector =
 801                                 entries[TSI721_VECT_OMB0_INT + i].vector;
 802                 snprintf(priv->msix[TSI721_VECT_OMB0_INT + i].irq_name,
 803                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombi%d@pci:%s",
 804                          i, pci_name(priv->pdev));
 805         }
 806
 807 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 808         for (i = 0; i < TSI721_DMA_CHNUM; i++) {
 809                 priv->msix[TSI721_VECT_DMA0_DONE + i].vector =
 810                                 entries[TSI721_VECT_DMA0_DONE + i].vector;
 811                 snprintf(priv->msix[TSI721_VECT_DMA0_DONE + i].irq_name,
 812                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmad%d@pci:%s",
 813                          i, pci_name(priv->pdev));
 814
 815                 priv->msix[TSI721_VECT_DMA0_INT + i].vector =
 816                                 entries[TSI721_VECT_DMA0_INT + i].vector;
 817                 snprintf(priv->msix[TSI721_VECT_DMA0_INT + i].irq_name,
 818                          IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmai%d@pci:%s",
 819                          i, pci_name(priv->pdev));
 820         }
 821 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
 822
 823         return 0;
 824 }
 825 #endif /* CONFIG_PCI_MSI */
 826
 827 static int tsi721_request_irq(struct rio_mport *mport)
 828 {
 829         struct tsi721_device *priv = mport->priv;
 830         int err;
 831
 832 #ifdef CONFIG_PCI_MSI
 833         if (priv->flags & TSI721_USING_MSIX)
 834                 err = tsi721_request_msix(mport);
 835         else
 836 #endif
 837                 err = request_irq(priv->pdev->irq, tsi721_irqhandler,
 838                           (priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,
 839                           DRV_NAME, (void *)mport);
 840
 841         if (err)
 842                 dev_err(&priv->pdev->dev,
 843                         "Unable to allocate interrupt, Error: %d\n", err);
 844
 845         return err;
 846 }
 847
 848 /**
 849  * tsi721_init_pc2sr_mapping - initializes outbound (PCIe->SRIO)
 850  * translation regions.
 851  * @priv: pointer to tsi721 private data
 852  *
 853  * Disables SREP translation regions.
 854  */
 855 static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv)
 856 {
 857         int i;
 858
 859         /* Disable all PC2SR translation windows */
 860         for (i = 0; i < TSI721_OBWIN_NUM; i++)
 861                 iowrite32(0, priv->regs + TSI721_OBWINLB(i));
 862 }
 863
 864 /**
 865  * tsi721_rio_map_inb_mem -- Mapping inbound memory region.
 866  * @mport: RapidIO master port
 867  * @lstart: Local memory space start address.
 868  * @rstart: RapidIO space start address.
 869  * @size: The mapping region size.
 870  * @flags: Flags for mapping. 0 for using default flags.
 871  *
 872  * Return: 0 -- Success.
 873  *
 874  * This function will create the inbound mapping
 875  * from rstart to lstart.
 876  */
 877 static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart,
 878                 u64 rstart, u32 size, u32 flags)
 879 {
 880         struct tsi721_device *priv = mport->priv;
 881         int i;
 882         u32 regval;
 883
 884         if (!is_power_of_2(size) || size < 0x1000 ||
 885             ((u64)lstart & (size - 1)) || (rstart & (size - 1)))
 886                 return -EINVAL;
 887
 888         /* Search for free inbound translation window */
 889         for (i = 0; i < TSI721_IBWIN_NUM; i++) {
 890                 regval = ioread32(priv->regs + TSI721_IBWIN_LB(i));
 891                 if (!(regval & TSI721_IBWIN_LB_WEN))
 892                         break;
 893         }
 894
 895         if (i >= TSI721_IBWIN_NUM) {
 896                 dev_err(&priv->pdev->dev,
 897                         "Unable to find free inbound window\n");
 898                 return -EBUSY;
 899         }
 900
 901         iowrite32(TSI721_IBWIN_SIZE(size) << 8,
 902                         priv->regs + TSI721_IBWIN_SZ(i));
 903
 904         iowrite32(((u64)lstart >> 32), priv->regs + TSI721_IBWIN_TUA(i));
 905         iowrite32(((u64)lstart & TSI721_IBWIN_TLA_ADD),
 906                   priv->regs + TSI721_IBWIN_TLA(i));
 907
 908         iowrite32(rstart >> 32, priv->regs + TSI721_IBWIN_UB(i));
 909         iowrite32((rstart & TSI721_IBWIN_LB_BA) | TSI721_IBWIN_LB_WEN,
 910                 priv->regs + TSI721_IBWIN_LB(i));
 911         dev_dbg(&priv->pdev->dev,
 912                 "Configured IBWIN%d mapping (RIO_0x%llx -> PCIe_0x%llx)\n",
 913                 i, rstart, (unsigned long long)lstart);
 914
 915         return 0;
 916 }
 917
 918 /**
 919  * fsl_rio_unmap_inb_mem -- Unmapping inbound memory region.
 920  * @mport: RapidIO master port
 921  * @lstart: Local memory space start address.
 922  */
 923 static void tsi721_rio_unmap_inb_mem(struct rio_mport *mport,
 924                                 dma_addr_t lstart)
 925 {
 926         struct tsi721_device *priv = mport->priv;
 927         int i;
 928         u64 addr;
 929         u32 regval;
 930
 931         /* Search for matching active inbound translation window */
 932         for (i = 0; i < TSI721_IBWIN_NUM; i++) {
 933                 regval = ioread32(priv->regs + TSI721_IBWIN_LB(i));
 934                 if (regval & TSI721_IBWIN_LB_WEN) {
 935                         regval = ioread32(priv->regs + TSI721_IBWIN_TUA(i));
 936                         addr = (u64)regval << 32;
 937                         regval = ioread32(priv->regs + TSI721_IBWIN_TLA(i));
 938                         addr |= regval & TSI721_IBWIN_TLA_ADD;
 939
 940                         if (addr == (u64)lstart) {
 941                                 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
 942                                 break;
 943                         }
 944                 }
 945         }
 946 }
 947
 948 /**
 949  * tsi721_init_sr2pc_mapping - initializes inbound (SRIO->PCIe)
 950  * translation regions.
 951  * @priv: pointer to tsi721 private data
 952  *
 953  * Disables inbound windows.
 954  */
 955 static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv)
 956 {
 957         int i;
 958
 959         /* Disable all SR2PC inbound windows */
 960         for (i = 0; i < TSI721_IBWIN_NUM; i++)
 961                 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
 962 }
 963
 964 /**
 965  * tsi721_port_write_init - Inbound port write interface init
 966  * @priv: pointer to tsi721 private data
 967  *
 968  * Initializes inbound port write handler.
 969  * Returns %0 on success or %-ENOMEM on failure.
 970  */
 971 static int tsi721_port_write_init(struct tsi721_device *priv)
 972 {
 973         priv->pw_discard_count = 0;
 974         INIT_WORK(&priv->pw_work, tsi721_pw_dpc);
 975         spin_lock_init(&priv->pw_fifo_lock);
 976         if (kfifo_alloc(&priv->pw_fifo,
 977                         TSI721_RIO_PW_MSG_SIZE * 32, GFP_KERNEL)) {
 978                 dev_err(&priv->pdev->dev, "PW FIFO allocation failed\n");
 979                 return -ENOMEM;
 980         }
 981
 982         /* Use reliable port-write capture mode */
 983         iowrite32(TSI721_RIO_PW_CTL_PWC_REL, priv->regs + TSI721_RIO_PW_CTL);
 984         return 0;
 985 }
 986
 987 static int tsi721_doorbell_init(struct tsi721_device *priv)
 988 {
 989         /* Outbound Doorbells do not require any setup.
 990          * Tsi721 uses dedicated PCI BAR1 to generate doorbells.
 991          * That BAR1 was mapped during the probe routine.
 992          */
 993
 994         /* Initialize Inbound Doorbell processing DPC and queue */
 995         priv->db_discard_count = 0;
 996         INIT_WORK(&priv->idb_work, tsi721_db_dpc);
 997
 998         /* Allocate buffer for inbound doorbells queue */
 999         priv->idb_base = dma_zalloc_coherent(&priv->pdev->dev,
1000                                 IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
1001                                 &priv->idb_dma, GFP_KERNEL);
1002         if (!priv->idb_base)
1003                 return -ENOMEM;
1004
1005         dev_dbg(&priv->pdev->dev, "Allocated IDB buffer @ %p (phys = %llx)\n",
1006                 priv->idb_base, (unsigned long long)priv->idb_dma);
1007
1008         iowrite32(TSI721_IDQ_SIZE_VAL(IDB_QSIZE),
1009                 priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE));
1010         iowrite32(((u64)priv->idb_dma >> 32),
1011                 priv->regs + TSI721_IDQ_BASEU(IDB_QUEUE));
1012         iowrite32(((u64)priv->idb_dma & TSI721_IDQ_BASEL_ADDR),
1013                 priv->regs + TSI721_IDQ_BASEL(IDB_QUEUE));
1014         /* Enable accepting all inbound doorbells */
1015         iowrite32(0, priv->regs + TSI721_IDQ_MASK(IDB_QUEUE));
1016
1017         iowrite32(TSI721_IDQ_INIT, priv->regs + TSI721_IDQ_CTL(IDB_QUEUE));
1018
1019         iowrite32(0, priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
1020
1021         return 0;
1022 }
1023
1024 static void tsi721_doorbell_free(struct tsi721_device *priv)
1025 {
1026         if (priv->idb_base == NULL)
1027                 return;
1028
1029         /* Free buffer allocated for inbound doorbell queue */
1030         dma_free_coherent(&priv->pdev->dev, IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
1031                           priv->idb_base, priv->idb_dma);
1032         priv->idb_base = NULL;
1033 }
1034
1035 /**
1036  * tsi721_bdma_maint_init - Initialize maintenance request BDMA channel.
1037  * @priv: pointer to tsi721 private data
1038  *
1039  * Initialize BDMA channel allocated for RapidIO maintenance read/write
1040  * request generation
1041  * Returns %0 on success or %-ENOMEM on failure.
1042  */
1043 static int tsi721_bdma_maint_init(struct tsi721_device *priv)
1044 {
1045         struct tsi721_dma_desc *bd_ptr;
1046         u64             *sts_ptr;
1047         dma_addr_t      bd_phys, sts_phys;
1048         int             sts_size;
1049         int             bd_num = 2;
1050         void __iomem    *regs;
1051
1052         dev_dbg(&priv->pdev->dev,
1053                 "Init Block DMA Engine for Maintenance requests, CH%d\n",
1054                 TSI721_DMACH_MAINT);
1055
1056         /*
1057          * Initialize DMA channel for maintenance requests
1058          */
1059
1060         priv->mdma.ch_id = TSI721_DMACH_MAINT;
1061         regs = priv->regs + TSI721_DMAC_BASE(TSI721_DMACH_MAINT);
1062
1063         /* Allocate space for DMA descriptors */
1064         bd_ptr = dma_zalloc_coherent(&priv->pdev->dev,
1065                                         bd_num * sizeof(struct tsi721_dma_desc),
1066                                         &bd_phys, GFP_KERNEL);
1067         if (!bd_ptr)
1068                 return -ENOMEM;
1069
1070         priv->mdma.bd_num = bd_num;
1071         priv->mdma.bd_phys = bd_phys;
1072         priv->mdma.bd_base = bd_ptr;
1073
1074         dev_dbg(&priv->pdev->dev, "DMA descriptors @ %p (phys = %llx)\n",
1075                 bd_ptr, (unsigned long long)bd_phys);
1076
1077         /* Allocate space for descriptor status FIFO */
1078         sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ?
1079                                         bd_num : TSI721_DMA_MINSTSSZ;
1080         sts_size = roundup_pow_of_two(sts_size);
1081         sts_ptr = dma_zalloc_coherent(&priv->pdev->dev,
1082                                      sts_size * sizeof(struct tsi721_dma_sts),
1083                                      &sts_phys, GFP_KERNEL);
1084         if (!sts_ptr) {
1085                 /* Free space allocated for DMA descriptors */
1086                 dma_free_coherent(&priv->pdev->dev,
1087                                   bd_num * sizeof(struct tsi721_dma_desc),
1088                                   bd_ptr, bd_phys);
1089                 priv->mdma.bd_base = NULL;
1090                 return -ENOMEM;
1091         }
1092
1093         priv->mdma.sts_phys = sts_phys;
1094         priv->mdma.sts_base = sts_ptr;
1095         priv->mdma.sts_size = sts_size;
1096
1097         dev_dbg(&priv->pdev->dev,
1098                 "desc status FIFO @ %p (phys = %llx) size=0x%x\n",
1099                 sts_ptr, (unsigned long long)sts_phys, sts_size);
1100
1101         /* Initialize DMA descriptors ring */
1102         bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29);
1103         bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys &
1104                                                  TSI721_DMAC_DPTRL_MASK);
1105         bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32);
1106
1107         /* Setup DMA descriptor pointers */
1108         iowrite32(((u64)bd_phys >> 32), regs + TSI721_DMAC_DPTRH);
1109         iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
1110                 regs + TSI721_DMAC_DPTRL);
1111
1112         /* Setup descriptor status FIFO */
1113         iowrite32(((u64)sts_phys >> 32), regs + TSI721_DMAC_DSBH);
1114         iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
1115                 regs + TSI721_DMAC_DSBL);
1116         iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
1117                 regs + TSI721_DMAC_DSSZ);
1118
1119         /* Clear interrupt bits */
1120         iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
1121
1122         ioread32(regs + TSI721_DMAC_INT);
1123
1124         /* Toggle DMA channel initialization */
1125         iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1126         ioread32(regs + TSI721_DMAC_CTL);
1127         udelay(10);
1128
1129         return 0;
1130 }
1131
1132 static int tsi721_bdma_maint_free(struct tsi721_device *priv)
1133 {
1134         u32 ch_stat;
1135         struct tsi721_bdma_maint *mdma = &priv->mdma;
1136         void __iomem *regs = priv->regs + TSI721_DMAC_BASE(mdma->ch_id);
1137
1138         if (mdma->bd_base == NULL)
1139                 return 0;
1140
1141         /* Check if DMA channel still running */
1142         ch_stat = ioread32(regs + TSI721_DMAC_STS);
1143         if (ch_stat & TSI721_DMAC_STS_RUN)
1144                 return -EFAULT;
1145
1146         /* Put DMA channel into init state */
1147         iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1148
1149         /* Free space allocated for DMA descriptors */
1150         dma_free_coherent(&priv->pdev->dev,
1151                 mdma->bd_num * sizeof(struct tsi721_dma_desc),
1152                 mdma->bd_base, mdma->bd_phys);
1153         mdma->bd_base = NULL;
1154
1155         /* Free space allocated for status FIFO */
1156         dma_free_coherent(&priv->pdev->dev,
1157                 mdma->sts_size * sizeof(struct tsi721_dma_sts),
1158                 mdma->sts_base, mdma->sts_phys);
1159         mdma->sts_base = NULL;
1160         return 0;
1161 }
1162
1163 /* Enable Inbound Messaging Interrupts */
1164 static void
1165 tsi721_imsg_interrupt_enable(struct tsi721_device *priv, int ch,
1166                                   u32 inte_mask)
1167 {
1168         u32 rval;
1169
1170         if (!inte_mask)
1171                 return;
1172
1173         /* Clear pending Inbound Messaging interrupts */
1174         iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1175
1176         /* Enable Inbound Messaging interrupts */
1177         rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1178         iowrite32(rval | inte_mask, priv->regs + TSI721_IBDMAC_INTE(ch));
1179
1180         if (priv->flags & TSI721_USING_MSIX)
1181                 return; /* Finished if we are in MSI-X mode */
1182
1183         /*
1184          * For MSI and INTA interrupt signalling we need to enable next levels
1185          */
1186
1187         /* Enable Device Channel Interrupt */
1188         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1189         iowrite32(rval | TSI721_INT_IMSG_CHAN(ch),
1190                   priv->regs + TSI721_DEV_CHAN_INTE);
1191 }
1192
1193 /* Disable Inbound Messaging Interrupts */
1194 static void
1195 tsi721_imsg_interrupt_disable(struct tsi721_device *priv, int ch,
1196                                    u32 inte_mask)
1197 {
1198         u32 rval;
1199
1200         if (!inte_mask)
1201                 return;
1202
1203         /* Clear pending Inbound Messaging interrupts */
1204         iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1205
1206         /* Disable Inbound Messaging interrupts */
1207         rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1208         rval &= ~inte_mask;
1209         iowrite32(rval, priv->regs + TSI721_IBDMAC_INTE(ch));
1210
1211         if (priv->flags & TSI721_USING_MSIX)
1212                 return; /* Finished if we are in MSI-X mode */
1213
1214         /*
1215          * For MSI and INTA interrupt signalling we need to disable next levels
1216          */
1217
1218         /* Disable Device Channel Interrupt */
1219         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1220         rval &= ~TSI721_INT_IMSG_CHAN(ch);
1221         iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1222 }
1223
1224 /* Enable Outbound Messaging interrupts */
1225 static void
1226 tsi721_omsg_interrupt_enable(struct tsi721_device *priv, int ch,
1227                                   u32 inte_mask)
1228 {
1229         u32 rval;
1230
1231         if (!inte_mask)
1232                 return;
1233
1234         /* Clear pending Outbound Messaging interrupts */
1235         iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1236
1237         /* Enable Outbound Messaging channel interrupts */
1238         rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1239         iowrite32(rval | inte_mask, priv->regs + TSI721_OBDMAC_INTE(ch));
1240
1241         if (priv->flags & TSI721_USING_MSIX)
1242                 return; /* Finished if we are in MSI-X mode */
1243
1244         /*
1245          * For MSI and INTA interrupt signalling we need to enable next levels
1246          */
1247
1248         /* Enable Device Channel Interrupt */
1249         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1250         iowrite32(rval | TSI721_INT_OMSG_CHAN(ch),
1251                   priv->regs + TSI721_DEV_CHAN_INTE);
1252 }
1253
1254 /* Disable Outbound Messaging interrupts */
1255 static void
1256 tsi721_omsg_interrupt_disable(struct tsi721_device *priv, int ch,
1257                                    u32 inte_mask)
1258 {
1259         u32 rval;
1260
1261         if (!inte_mask)
1262                 return;
1263
1264         /* Clear pending Outbound Messaging interrupts */
1265         iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1266
1267         /* Disable Outbound Messaging interrupts */
1268         rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1269         rval &= ~inte_mask;
1270         iowrite32(rval, priv->regs + TSI721_OBDMAC_INTE(ch));
1271
1272         if (priv->flags & TSI721_USING_MSIX)
1273                 return; /* Finished if we are in MSI-X mode */
1274
1275         /*
1276          * For MSI and INTA interrupt signalling we need to disable next levels
1277          */
1278
1279         /* Disable Device Channel Interrupt */
1280         rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1281         rval &= ~TSI721_INT_OMSG_CHAN(ch);
1282         iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1283 }
1284
1285 /**
1286  * tsi721_add_outb_message - Add message to the Tsi721 outbound message queue
1287  * @mport: Master port with outbound message queue
1288  * @rdev: Target of outbound message
1289  * @mbox: Outbound mailbox
1290  * @buffer: Message to add to outbound queue
1291  * @len: Length of message
1292  */
1293 static int
1294 tsi721_add_outb_message(struct rio_mport *mport, struct rio_dev *rdev, int mbox,
1295                         void *buffer, size_t len)
1296 {
1297         struct tsi721_device *priv = mport->priv;
1298         struct tsi721_omsg_desc *desc;
1299         u32 tx_slot;
1300
1301         if (!priv->omsg_init[mbox] ||
1302             len > TSI721_MSG_MAX_SIZE || len < 8)
1303                 return -EINVAL;
1304
1305         tx_slot = priv->omsg_ring[mbox].tx_slot;
1306
1307         /* Copy copy message into transfer buffer */
1308         memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len);
1309
1310         if (len & 0x7)
1311                 len += 8;
1312
1313         /* Build descriptor associated with buffer */
1314         desc = priv->omsg_ring[mbox].omd_base;
1315         desc[tx_slot].type_id = cpu_to_le32((DTYPE4 << 29) | rdev->destid);
1316         if (tx_slot % 4 == 0)
1317                 desc[tx_slot].type_id |= cpu_to_le32(TSI721_OMD_IOF);
1318
1319         desc[tx_slot].msg_info =
1320                 cpu_to_le32((mport->sys_size << 26) | (mbox << 22) |
1321                             (0xe << 12) | (len & 0xff8));
1322         desc[tx_slot].bufptr_lo =
1323                 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] &
1324                             0xffffffff);
1325         desc[tx_slot].bufptr_hi =
1326                 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32);
1327
1328         priv->omsg_ring[mbox].wr_count++;
1329
1330         /* Go to next descriptor */
1331         if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) {
1332                 priv->omsg_ring[mbox].tx_slot = 0;
1333                 /* Move through the ring link descriptor at the end */
1334                 priv->omsg_ring[mbox].wr_count++;
1335         }
1336
1337         mb();
1338
1339         /* Set new write count value */
1340         iowrite32(priv->omsg_ring[mbox].wr_count,
1341                 priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1342         ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1343
1344         return 0;
1345 }
1346
1347 /**
1348  * tsi721_omsg_handler - Outbound Message Interrupt Handler
1349  * @priv: pointer to tsi721 private data
1350  * @ch:   number of OB MSG channel to service
1351  *
1352  * Services channel interrupts from outbound messaging engine.
1353  */
1354 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch)
1355 {
1356         u32 omsg_int;
1357
1358         spin_lock(&priv->omsg_ring[ch].lock);
1359
1360         omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch));
1361
1362         if (omsg_int & TSI721_OBDMAC_INT_ST_FULL)
1363                 dev_info(&priv->pdev->dev,
1364                         "OB MBOX%d: Status FIFO is full\n", ch);
1365
1366         if (omsg_int & (TSI721_OBDMAC_INT_DONE | TSI721_OBDMAC_INT_IOF_DONE)) {
1367                 u32 srd_ptr;
1368                 u64 *sts_ptr, last_ptr = 0, prev_ptr = 0;
1369                 int i, j;
1370                 u32 tx_slot;
1371
1372                 /*
1373                  * Find last successfully processed descriptor
1374                  */
1375
1376                 /* Check and clear descriptor status FIFO entries */
1377                 srd_ptr = priv->omsg_ring[ch].sts_rdptr;
1378                 sts_ptr = priv->omsg_ring[ch].sts_base;
1379                 j = srd_ptr * 8;
1380                 while (sts_ptr[j]) {
1381                         for (i = 0; i < 8 && sts_ptr[j]; i++, j++) {
1382                                 prev_ptr = last_ptr;
1383                                 last_ptr = le64_to_cpu(sts_ptr[j]);
1384                                 sts_ptr[j] = 0;
1385                         }
1386
1387                         ++srd_ptr;
1388                         srd_ptr %= priv->omsg_ring[ch].sts_size;
1389                         j = srd_ptr * 8;
1390                 }
1391
1392                 if (last_ptr == 0)
1393                         goto no_sts_update;
1394
1395                 priv->omsg_ring[ch].sts_rdptr = srd_ptr;
1396                 iowrite32(srd_ptr, priv->regs + TSI721_OBDMAC_DSRP(ch));
1397
1398                 if (!priv->mport->outb_msg[ch].mcback)
1399                         goto no_sts_update;
1400
1401                 /* Inform upper layer about transfer completion */
1402
1403                 tx_slot = (last_ptr - (u64)priv->omsg_ring[ch].omd_phys)/
1404                                                 sizeof(struct tsi721_omsg_desc);
1405
1406                 /*
1407                  * Check if this is a Link Descriptor (LD).
1408                  * If yes, ignore LD and use descriptor processed
1409                  * before LD.
1410                  */
1411                 if (tx_slot == priv->omsg_ring[ch].size) {
1412                         if (prev_ptr)
1413                                 tx_slot = (prev_ptr -
1414                                         (u64)priv->omsg_ring[ch].omd_phys)/
1415                                                 sizeof(struct tsi721_omsg_desc);
1416                         else
1417                                 goto no_sts_update;
1418                 }
1419
1420                 /* Move slot index to the next message to be sent */
1421                 ++tx_slot;
1422                 if (tx_slot == priv->omsg_ring[ch].size)
1423                         tx_slot = 0;
1424                 BUG_ON(tx_slot >= priv->omsg_ring[ch].size);
1425                 priv->mport->outb_msg[ch].mcback(priv->mport,
1426                                 priv->omsg_ring[ch].dev_id, ch,
1427                                 tx_slot);
1428         }
1429
1430 no_sts_update:
1431
1432         if (omsg_int & TSI721_OBDMAC_INT_ERROR) {
1433                 /*
1434                 * Outbound message operation aborted due to error,
1435                 * reinitialize OB MSG channel
1436                 */
1437
1438                 dev_dbg(&priv->pdev->dev, "OB MSG ABORT ch_stat=%x\n",
1439                         ioread32(priv->regs + TSI721_OBDMAC_STS(ch)));
1440
1441                 iowrite32(TSI721_OBDMAC_INT_ERROR,
1442                                 priv->regs + TSI721_OBDMAC_INT(ch));
1443                 iowrite32(TSI721_OBDMAC_CTL_INIT,
1444                                 priv->regs + TSI721_OBDMAC_CTL(ch));
1445                 ioread32(priv->regs + TSI721_OBDMAC_CTL(ch));
1446
1447                 /* Inform upper level to clear all pending tx slots */
1448                 if (priv->mport->outb_msg[ch].mcback)
1449                         priv->mport->outb_msg[ch].mcback(priv->mport,
1450                                         priv->omsg_ring[ch].dev_id, ch,
1451                                         priv->omsg_ring[ch].tx_slot);
1452                 /* Synch tx_slot tracking */
1453                 iowrite32(priv->omsg_ring[ch].tx_slot,
1454                         priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1455                 ioread32(priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1456                 priv->omsg_ring[ch].wr_count = priv->omsg_ring[ch].tx_slot;
1457                 priv->omsg_ring[ch].sts_rdptr = 0;
1458         }
1459
1460         /* Clear channel interrupts */
1461         iowrite32(omsg_int, priv->regs + TSI721_OBDMAC_INT(ch));
1462
1463         if (!(priv->flags & TSI721_USING_MSIX)) {
1464                 u32 ch_inte;
1465
1466                 /* Re-enable channel interrupts */
1467                 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1468                 ch_inte |= TSI721_INT_OMSG_CHAN(ch);
1469                 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1470         }
1471
1472         spin_unlock(&priv->omsg_ring[ch].lock);
1473 }
1474
1475 /**
1476  * tsi721_open_outb_mbox - Initialize Tsi721 outbound mailbox
1477  * @mport: Master port implementing Outbound Messaging Engine
1478  * @dev_id: Device specific pointer to pass on event
1479  * @mbox: Mailbox to open
1480  * @entries: Number of entries in the outbound mailbox ring
1481  */
1482 static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id,
1483                                  int mbox, int entries)
1484 {
1485         struct tsi721_device *priv = mport->priv;
1486         struct tsi721_omsg_desc *bd_ptr;
1487         int i, rc = 0;
1488
1489         if ((entries < TSI721_OMSGD_MIN_RING_SIZE) ||
1490             (entries > (TSI721_OMSGD_RING_SIZE)) ||
1491             (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1492                 rc = -EINVAL;
1493                 goto out;
1494         }
1495
1496         priv->omsg_ring[mbox].dev_id = dev_id;
1497         priv->omsg_ring[mbox].size = entries;
1498         priv->omsg_ring[mbox].sts_rdptr = 0;
1499         spin_lock_init(&priv->omsg_ring[mbox].lock);
1500
1501         /* Outbound Msg Buffer allocation based on
1502            the number of maximum descriptor entries */
1503         for (i = 0; i < entries; i++) {
1504                 priv->omsg_ring[mbox].omq_base[i] =
1505                         dma_alloc_coherent(
1506                                 &priv->pdev->dev, TSI721_MSG_BUFFER_SIZE,
1507                                 &priv->omsg_ring[mbox].omq_phys[i],
1508                                 GFP_KERNEL);
1509                 if (priv->omsg_ring[mbox].omq_base[i] == NULL) {
1510                         dev_dbg(&priv->pdev->dev,
1511                                 "Unable to allocate OB MSG data buffer for"
1512                                 " MBOX%d\n", mbox);
1513                         rc = -ENOMEM;
1514                         goto out_buf;
1515                 }
1516         }
1517
1518         /* Outbound message descriptor allocation */
1519         priv->omsg_ring[mbox].omd_base = dma_alloc_coherent(
1520                                 &priv->pdev->dev,
1521                                 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1522                                 &priv->omsg_ring[mbox].omd_phys, GFP_KERNEL);
1523         if (priv->omsg_ring[mbox].omd_base == NULL) {
1524                 dev_dbg(&priv->pdev->dev,
1525                         "Unable to allocate OB MSG descriptor memory "
1526                         "for MBOX%d\n", mbox);
1527                 rc = -ENOMEM;
1528                 goto out_buf;
1529         }
1530
1531         priv->omsg_ring[mbox].tx_slot = 0;
1532
1533         /* Outbound message descriptor status FIFO allocation */
1534         priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1);
1535         priv->omsg_ring[mbox].sts_base = dma_zalloc_coherent(&priv->pdev->dev,
1536                         priv->omsg_ring[mbox].sts_size *
1537                                                 sizeof(struct tsi721_dma_sts),
1538                         &priv->omsg_ring[mbox].sts_phys, GFP_KERNEL);
1539         if (priv->omsg_ring[mbox].sts_base == NULL) {
1540                 dev_dbg(&priv->pdev->dev,
1541                         "Unable to allocate OB MSG descriptor status FIFO "
1542                         "for MBOX%d\n", mbox);
1543                 rc = -ENOMEM;
1544                 goto out_desc;
1545         }
1546
1547         /*
1548          * Configure Outbound Messaging Engine
1549          */
1550
1551         /* Setup Outbound Message descriptor pointer */
1552         iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32),
1553                         priv->regs + TSI721_OBDMAC_DPTRH(mbox));
1554         iowrite32(((u64)priv->omsg_ring[mbox].omd_phys &
1555                                         TSI721_OBDMAC_DPTRL_MASK),
1556                         priv->regs + TSI721_OBDMAC_DPTRL(mbox));
1557
1558         /* Setup Outbound Message descriptor status FIFO */
1559         iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32),
1560                         priv->regs + TSI721_OBDMAC_DSBH(mbox));
1561         iowrite32(((u64)priv->omsg_ring[mbox].sts_phys &
1562                                         TSI721_OBDMAC_DSBL_MASK),
1563                         priv->regs + TSI721_OBDMAC_DSBL(mbox));
1564         iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size),
1565                 priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox));
1566
1567         /* Enable interrupts */
1568
1569 #ifdef CONFIG_PCI_MSI
1570         if (priv->flags & TSI721_USING_MSIX) {
1571                 /* Request interrupt service if we are in MSI-X mode */
1572                 rc = request_irq(
1573                         priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1574                         tsi721_omsg_msix, 0,
1575                         priv->msix[TSI721_VECT_OMB0_DONE + mbox].irq_name,
1576                         (void *)mport);
1577
1578                 if (rc) {
1579                         dev_dbg(&priv->pdev->dev,
1580                                 "Unable to allocate MSI-X interrupt for "
1581                                 "OBOX%d-DONE\n", mbox);
1582                         goto out_stat;
1583                 }
1584
1585                 rc = request_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
1586                         tsi721_omsg_msix, 0,
1587                         priv->msix[TSI721_VECT_OMB0_INT + mbox].irq_name,
1588                         (void *)mport);
1589
1590                 if (rc) {
1591                         dev_dbg(&priv->pdev->dev,
1592                                 "Unable to allocate MSI-X interrupt for "
1593                                 "MBOX%d-INT\n", mbox);
1594                         free_irq(
1595                                 priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1596                                 (void *)mport);
1597                         goto out_stat;
1598                 }
1599         }
1600 #endif /* CONFIG_PCI_MSI */
1601
1602         tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1603
1604         /* Initialize Outbound Message descriptors ring */
1605         bd_ptr = priv->omsg_ring[mbox].omd_base;
1606         bd_ptr[entries].type_id = cpu_to_le32(DTYPE5 << 29);
1607         bd_ptr[entries].msg_info = 0;
1608         bd_ptr[entries].next_lo =
1609                 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys &
1610                 TSI721_OBDMAC_DPTRL_MASK);
1611         bd_ptr[entries].next_hi =
1612                 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32);
1613         priv->omsg_ring[mbox].wr_count = 0;
1614         mb();
1615
1616         /* Initialize Outbound Message engine */
1617         iowrite32(TSI721_OBDMAC_CTL_INIT, priv->regs + TSI721_OBDMAC_CTL(mbox));
1618         ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1619         udelay(10);
1620
1621         priv->omsg_init[mbox] = 1;
1622
1623         return 0;
1624
1625 #ifdef CONFIG_PCI_MSI
1626 out_stat:
1627         dma_free_coherent(&priv->pdev->dev,
1628                 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
1629                 priv->omsg_ring[mbox].sts_base,
1630                 priv->omsg_ring[mbox].sts_phys);
1631
1632         priv->omsg_ring[mbox].sts_base = NULL;
1633 #endif /* CONFIG_PCI_MSI */
1634
1635 out_desc:
1636         dma_free_coherent(&priv->pdev->dev,
1637                 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1638                 priv->omsg_ring[mbox].omd_base,
1639                 priv->omsg_ring[mbox].omd_phys);
1640
1641         priv->omsg_ring[mbox].omd_base = NULL;
1642
1643 out_buf:
1644         for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
1645                 if (priv->omsg_ring[mbox].omq_base[i]) {
1646                         dma_free_coherent(&priv->pdev->dev,
1647                                 TSI721_MSG_BUFFER_SIZE,
1648                                 priv->omsg_ring[mbox].omq_base[i],
1649                                 priv->omsg_ring[mbox].omq_phys[i]);
1650
1651                         priv->omsg_ring[mbox].omq_base[i] = NULL;
1652                 }
1653         }
1654
1655 out:
1656         return rc;
1657 }
1658
1659 /**
1660  * tsi721_close_outb_mbox - Close Tsi721 outbound mailbox
1661  * @mport: Master port implementing the outbound message unit
1662  * @mbox: Mailbox to close
1663  */
1664 static void tsi721_close_outb_mbox(struct rio_mport *mport, int mbox)
1665 {
1666         struct tsi721_device *priv = mport->priv;
1667         u32 i;
1668
1669         if (!priv->omsg_init[mbox])
1670                 return;
1671         priv->omsg_init[mbox] = 0;
1672
1673         /* Disable Interrupts */
1674
1675         tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1676
1677 #ifdef CONFIG_PCI_MSI
1678         if (priv->flags & TSI721_USING_MSIX) {
1679                 free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
1680                          (void *)mport);
1681                 free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
1682                          (void *)mport);
1683         }
1684 #endif /* CONFIG_PCI_MSI */
1685
1686         /* Free OMSG Descriptor Status FIFO */
1687         dma_free_coherent(&priv->pdev->dev,
1688                 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
1689                 priv->omsg_ring[mbox].sts_base,
1690                 priv->omsg_ring[mbox].sts_phys);
1691
1692         priv->omsg_ring[mbox].sts_base = NULL;
1693
1694         /* Free OMSG descriptors */
1695         dma_free_coherent(&priv->pdev->dev,
1696                 (priv->omsg_ring[mbox].size + 1) *
1697                         sizeof(struct tsi721_omsg_desc),
1698                 priv->omsg_ring[mbox].omd_base,
1699                 priv->omsg_ring[mbox].omd_phys);
1700
1701         priv->omsg_ring[mbox].omd_base = NULL;
1702
1703         /* Free message buffers */
1704         for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
1705                 if (priv->omsg_ring[mbox].omq_base[i]) {
1706                         dma_free_coherent(&priv->pdev->dev,
1707                                 TSI721_MSG_BUFFER_SIZE,
1708                                 priv->omsg_ring[mbox].omq_base[i],
1709                                 priv->omsg_ring[mbox].omq_phys[i]);
1710
1711                         priv->omsg_ring[mbox].omq_base[i] = NULL;
1712                 }
1713         }
1714 }
1715
1716 /**
1717  * tsi721_imsg_handler - Inbound Message Interrupt Handler
1718  * @priv: pointer to tsi721 private data
1719  * @ch: inbound message channel number to service
1720  *
1721  * Services channel interrupts from inbound messaging engine.
1722  */
1723 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch)
1724 {
1725         u32 mbox = ch - 4;
1726         u32 imsg_int;
1727
1728         spin_lock(&priv->imsg_ring[mbox].lock);
1729
1730         imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch));
1731
1732         if (imsg_int & TSI721_IBDMAC_INT_SRTO)
1733                 dev_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout\n",
1734                         mbox);
1735
1736         if (imsg_int & TSI721_IBDMAC_INT_PC_ERROR)
1737                 dev_info(&priv->pdev->dev, "IB MBOX%d PCIe error\n",
1738                         mbox);
1739
1740         if (imsg_int & TSI721_IBDMAC_INT_FQ_LOW)
1741                 dev_info(&priv->pdev->dev,
1742                         "IB MBOX%d IB free queue low\n", mbox);
1743
1744         /* Clear IB channel interrupts */
1745         iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch));
1746
1747         /* If an IB Msg is received notify the upper layer */
1748         if (imsg_int & TSI721_IBDMAC_INT_DQ_RCV &&
1749                 priv->mport->inb_msg[mbox].mcback)
1750                 priv->mport->inb_msg[mbox].mcback(priv->mport,
1751                                 priv->imsg_ring[mbox].dev_id, mbox, -1);
1752
1753         if (!(priv->flags & TSI721_USING_MSIX)) {
1754                 u32 ch_inte;
1755
1756                 /* Re-enable channel interrupts */
1757                 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1758                 ch_inte |= TSI721_INT_IMSG_CHAN(ch);
1759                 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1760         }
1761
1762         spin_unlock(&priv->imsg_ring[mbox].lock);
1763 }
1764
1765 /**
1766  * tsi721_open_inb_mbox - Initialize Tsi721 inbound mailbox
1767  * @mport: Master port implementing the Inbound Messaging Engine
1768  * @dev_id: Device specific pointer to pass on event
1769  * @mbox: Mailbox to open
1770  * @entries: Number of entries in the inbound mailbox ring
1771  */
1772 static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id,
1773                                 int mbox, int entries)
1774 {
1775         struct tsi721_device *priv = mport->priv;
1776         int ch = mbox + 4;
1777         int i;
1778         u64 *free_ptr;
1779         int rc = 0;
1780
1781         if ((entries < TSI721_IMSGD_MIN_RING_SIZE) ||
1782             (entries > TSI721_IMSGD_RING_SIZE) ||
1783             (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1784                 rc = -EINVAL;
1785                 goto out;
1786         }
1787
1788         /* Initialize IB Messaging Ring */
1789         priv->imsg_ring[mbox].dev_id = dev_id;
1790         priv->imsg_ring[mbox].size = entries;
1791         priv->imsg_ring[mbox].rx_slot = 0;
1792         priv->imsg_ring[mbox].desc_rdptr = 0;
1793         priv->imsg_ring[mbox].fq_wrptr = 0;
1794         for (i = 0; i < priv->imsg_ring[mbox].size; i++)
1795                 priv->imsg_ring[mbox].imq_base[i] = NULL;
1796         spin_lock_init(&priv->imsg_ring[mbox].lock);
1797
1798         /* Allocate buffers for incoming messages */
1799         priv->imsg_ring[mbox].buf_base =
1800                 dma_alloc_coherent(&priv->pdev->dev,
1801                                    entries * TSI721_MSG_BUFFER_SIZE,
1802                                    &priv->imsg_ring[mbox].buf_phys,
1803                                    GFP_KERNEL);
1804
1805         if (priv->imsg_ring[mbox].buf_base == NULL) {
1806                 dev_err(&priv->pdev->dev,
1807                         "Failed to allocate buffers for IB MBOX%d\n", mbox);
1808                 rc = -ENOMEM;
1809                 goto out;
1810         }
1811
1812         /* Allocate memory for circular free list */
1813         priv->imsg_ring[mbox].imfq_base =
1814                 dma_alloc_coherent(&priv->pdev->dev,
1815                                    entries * 8,
1816                                    &priv->imsg_ring[mbox].imfq_phys,
1817                                    GFP_KERNEL);
1818
1819         if (priv->imsg_ring[mbox].imfq_base == NULL) {
1820                 dev_err(&priv->pdev->dev,
1821                         "Failed to allocate free queue for IB MBOX%d\n", mbox);
1822                 rc = -ENOMEM;
1823                 goto out_buf;
1824         }
1825
1826         /* Allocate memory for Inbound message descriptors */
1827         priv->imsg_ring[mbox].imd_base =
1828                 dma_alloc_coherent(&priv->pdev->dev,
1829                                    entries * sizeof(struct tsi721_imsg_desc),
1830                                    &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL);
1831
1832         if (priv->imsg_ring[mbox].imd_base == NULL) {
1833                 dev_err(&priv->pdev->dev,
1834                         "Failed to allocate descriptor memory for IB MBOX%d\n",
1835                         mbox);
1836                 rc = -ENOMEM;
1837                 goto out_dma;
1838         }
1839
1840         /* Fill free buffer pointer list */
1841         free_ptr = priv->imsg_ring[mbox].imfq_base;
1842         for (i = 0; i < entries; i++)
1843                 free_ptr[i] = cpu_to_le64(
1844                                 (u64)(priv->imsg_ring[mbox].buf_phys) +
1845                                 i * 0x1000);
1846
1847         mb();
1848
1849         /*
1850          * For mapping of inbound SRIO Messages into appropriate queues we need
1851          * to set Inbound Device ID register in the messaging engine. We do it
1852          * once when first inbound mailbox is requested.
1853          */
1854         if (!(priv->flags & TSI721_IMSGID_SET)) {
1855                 iowrite32((u32)priv->mport->host_deviceid,
1856                         priv->regs + TSI721_IB_DEVID);
1857                 priv->flags |= TSI721_IMSGID_SET;
1858         }
1859
1860         /*
1861          * Configure Inbound Messaging channel (ch = mbox + 4)
1862          */
1863
1864         /* Setup Inbound Message free queue */
1865         iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32),
1866                 priv->regs + TSI721_IBDMAC_FQBH(ch));
1867         iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys &
1868                         TSI721_IBDMAC_FQBL_MASK),
1869                 priv->regs+TSI721_IBDMAC_FQBL(ch));
1870         iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
1871                 priv->regs + TSI721_IBDMAC_FQSZ(ch));
1872
1873         /* Setup Inbound Message descriptor queue */
1874         iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32),
1875                 priv->regs + TSI721_IBDMAC_DQBH(ch));
1876         iowrite32(((u32)priv->imsg_ring[mbox].imd_phys &
1877                    (u32)TSI721_IBDMAC_DQBL_MASK),
1878                 priv->regs+TSI721_IBDMAC_DQBL(ch));
1879         iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
1880                 priv->regs + TSI721_IBDMAC_DQSZ(ch));
1881
1882         /* Enable interrupts */
1883
1884 #ifdef CONFIG_PCI_MSI
1885         if (priv->flags & TSI721_USING_MSIX) {
1886                 /* Request interrupt service if we are in MSI-X mode */
1887                 rc = request_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1888                         tsi721_imsg_msix, 0,
1889                         priv->msix[TSI721_VECT_IMB0_RCV + mbox].irq_name,
1890                         (void *)mport);
1891
1892                 if (rc) {
1893                         dev_dbg(&priv->pdev->dev,
1894                                 "Unable to allocate MSI-X interrupt for "
1895                                 "IBOX%d-DONE\n", mbox);
1896                         goto out_desc;
1897                 }
1898
1899                 rc = request_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
1900                         tsi721_imsg_msix, 0,
1901                         priv->msix[TSI721_VECT_IMB0_INT + mbox].irq_name,
1902                         (void *)mport);
1903
1904                 if (rc) {
1905                         dev_dbg(&priv->pdev->dev,
1906                                 "Unable to allocate MSI-X interrupt for "
1907                                 "IBOX%d-INT\n", mbox);
1908                         free_irq(
1909                                 priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1910                                 (void *)mport);
1911                         goto out_desc;
1912                 }
1913         }
1914 #endif /* CONFIG_PCI_MSI */
1915
1916         tsi721_imsg_interrupt_enable(priv, ch, TSI721_IBDMAC_INT_ALL);
1917
1918         /* Initialize Inbound Message Engine */
1919         iowrite32(TSI721_IBDMAC_CTL_INIT, priv->regs + TSI721_IBDMAC_CTL(ch));
1920         ioread32(priv->regs + TSI721_IBDMAC_CTL(ch));
1921         udelay(10);
1922         priv->imsg_ring[mbox].fq_wrptr = entries - 1;
1923         iowrite32(entries - 1, priv->regs + TSI721_IBDMAC_FQWP(ch));
1924
1925         priv->imsg_init[mbox] = 1;
1926         return 0;
1927
1928 #ifdef CONFIG_PCI_MSI
1929 out_desc:
1930         dma_free_coherent(&priv->pdev->dev,
1931                 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
1932                 priv->imsg_ring[mbox].imd_base,
1933                 priv->imsg_ring[mbox].imd_phys);
1934
1935         priv->imsg_ring[mbox].imd_base = NULL;
1936 #endif /* CONFIG_PCI_MSI */
1937
1938 out_dma:
1939         dma_free_coherent(&priv->pdev->dev,
1940                 priv->imsg_ring[mbox].size * 8,
1941                 priv->imsg_ring[mbox].imfq_base,
1942                 priv->imsg_ring[mbox].imfq_phys);
1943
1944         priv->imsg_ring[mbox].imfq_base = NULL;
1945
1946 out_buf:
1947         dma_free_coherent(&priv->pdev->dev,
1948                 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
1949                 priv->imsg_ring[mbox].buf_base,
1950                 priv->imsg_ring[mbox].buf_phys);
1951
1952         priv->imsg_ring[mbox].buf_base = NULL;
1953
1954 out:
1955         return rc;
1956 }
1957
1958 /**
1959  * tsi721_close_inb_mbox - Shut down Tsi721 inbound mailbox
1960  * @mport: Master port implementing the Inbound Messaging Engine
1961  * @mbox: Mailbox to close
1962  */
1963 static void tsi721_close_inb_mbox(struct rio_mport *mport, int mbox)
1964 {
1965         struct tsi721_device *priv = mport->priv;
1966         u32 rx_slot;
1967         int ch = mbox + 4;
1968
1969         if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */
1970                 return;
1971         priv->imsg_init[mbox] = 0;
1972
1973         /* Disable Inbound Messaging Engine */
1974
1975         /* Disable Interrupts */
1976         tsi721_imsg_interrupt_disable(priv, ch, TSI721_OBDMAC_INT_MASK);
1977
1978 #ifdef CONFIG_PCI_MSI
1979         if (priv->flags & TSI721_USING_MSIX) {
1980                 free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
1981                                 (void *)mport);
1982                 free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
1983                                 (void *)mport);
1984         }
1985 #endif /* CONFIG_PCI_MSI */
1986
1987         /* Clear Inbound Buffer Queue */
1988         for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++)
1989                 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
1990
1991         /* Free memory allocated for message buffers */
1992         dma_free_coherent(&priv->pdev->dev,
1993                 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
1994                 priv->imsg_ring[mbox].buf_base,
1995                 priv->imsg_ring[mbox].buf_phys);
1996
1997         priv->imsg_ring[mbox].buf_base = NULL;
1998
1999         /* Free memory allocated for free pointr list */
2000         dma_free_coherent(&priv->pdev->dev,
2001                 priv->imsg_ring[mbox].size * 8,
2002                 priv->imsg_ring[mbox].imfq_base,
2003                 priv->imsg_ring[mbox].imfq_phys);
2004
2005         priv->imsg_ring[mbox].imfq_base = NULL;
2006
2007         /* Free memory allocated for RX descriptors */
2008         dma_free_coherent(&priv->pdev->dev,
2009                 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
2010                 priv->imsg_ring[mbox].imd_base,
2011                 priv->imsg_ring[mbox].imd_phys);
2012
2013         priv->imsg_ring[mbox].imd_base = NULL;
2014 }
2015
2016 /**
2017  * tsi721_add_inb_buffer - Add buffer to the Tsi721 inbound message queue
2018  * @mport: Master port implementing the Inbound Messaging Engine
2019  * @mbox: Inbound mailbox number
2020  * @buf: Buffer to add to inbound queue
2021  */
2022 static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf)
2023 {
2024         struct tsi721_device *priv = mport->priv;
2025         u32 rx_slot;
2026         int rc = 0;
2027
2028         rx_slot = priv->imsg_ring[mbox].rx_slot;
2029         if (priv->imsg_ring[mbox].imq_base[rx_slot]) {
2030                 dev_err(&priv->pdev->dev,
2031                         "Error adding inbound buffer %d, buffer exists\n",
2032                         rx_slot);
2033                 rc = -EINVAL;
2034                 goto out;
2035         }
2036
2037         priv->imsg_ring[mbox].imq_base[rx_slot] = buf;
2038
2039         if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size)
2040                 priv->imsg_ring[mbox].rx_slot = 0;
2041
2042 out:
2043         return rc;
2044 }
2045
2046 /**
2047  * tsi721_get_inb_message - Fetch inbound message from the Tsi721 MSG Queue
2048  * @mport: Master port implementing the Inbound Messaging Engine
2049  * @mbox: Inbound mailbox number
2050  *
2051  * Returns pointer to the message on success or NULL on failure.
2052  */
2053 static void *tsi721_get_inb_message(struct rio_mport *mport, int mbox)
2054 {
2055         struct tsi721_device *priv = mport->priv;
2056         struct tsi721_imsg_desc *desc;
2057         u32 rx_slot;
2058         void *rx_virt = NULL;
2059         u64 rx_phys;
2060         void *buf = NULL;
2061         u64 *free_ptr;
2062         int ch = mbox + 4;
2063         int msg_size;
2064
2065         if (!priv->imsg_init[mbox])
2066                 return NULL;
2067
2068         desc = priv->imsg_ring[mbox].imd_base;
2069         desc += priv->imsg_ring[mbox].desc_rdptr;
2070
2071         if (!(le32_to_cpu(desc->msg_info) & TSI721_IMD_HO))
2072                 goto out;
2073
2074         rx_slot = priv->imsg_ring[mbox].rx_slot;
2075         while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) {
2076                 if (++rx_slot == priv->imsg_ring[mbox].size)
2077                         rx_slot = 0;
2078         }
2079
2080         rx_phys = ((u64)le32_to_cpu(desc->bufptr_hi) << 32) |
2081                         le32_to_cpu(desc->bufptr_lo);
2082
2083         rx_virt = priv->imsg_ring[mbox].buf_base +
2084                   (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys);
2085
2086         buf = priv->imsg_ring[mbox].imq_base[rx_slot];
2087         msg_size = le32_to_cpu(desc->msg_info) & TSI721_IMD_BCOUNT;
2088         if (msg_size == 0)
2089                 msg_size = RIO_MAX_MSG_SIZE;
2090
2091         memcpy(buf, rx_virt, msg_size);
2092         priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
2093
2094         desc->msg_info &= cpu_to_le32(~TSI721_IMD_HO);
2095         if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size)
2096                 priv->imsg_ring[mbox].desc_rdptr = 0;
2097
2098         iowrite32(priv->imsg_ring[mbox].desc_rdptr,
2099                 priv->regs + TSI721_IBDMAC_DQRP(ch));
2100
2101         /* Return free buffer into the pointer list */
2102         free_ptr = priv->imsg_ring[mbox].imfq_base;
2103         free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys);
2104
2105         if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size)
2106                 priv->imsg_ring[mbox].fq_wrptr = 0;
2107
2108         iowrite32(priv->imsg_ring[mbox].fq_wrptr,
2109                 priv->regs + TSI721_IBDMAC_FQWP(ch));
2110 out:
2111         return buf;
2112 }
2113
2114 /**
2115  * tsi721_messages_init - Initialization of Messaging Engine
2116  * @priv: pointer to tsi721 private data
2117  *
2118  * Configures Tsi721 messaging engine.
2119  */
2120 static int tsi721_messages_init(struct tsi721_device *priv)
2121 {
2122         int     ch;
2123
2124         iowrite32(0, priv->regs + TSI721_SMSG_ECC_LOG);
2125         iowrite32(0, priv->regs + TSI721_RETRY_GEN_CNT);
2126         iowrite32(0, priv->regs + TSI721_RETRY_RX_CNT);
2127
2128         /* Set SRIO Message Request/Response Timeout */
2129         iowrite32(TSI721_RQRPTO_VAL, priv->regs + TSI721_RQRPTO);
2130
2131         /* Initialize Inbound Messaging Engine Registers */
2132         for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++) {
2133                 /* Clear interrupt bits */
2134                 iowrite32(TSI721_IBDMAC_INT_MASK,
2135                         priv->regs + TSI721_IBDMAC_INT(ch));
2136                 /* Clear Status */
2137                 iowrite32(0, priv->regs + TSI721_IBDMAC_STS(ch));
2138
2139                 iowrite32(TSI721_SMSG_ECC_COR_LOG_MASK,
2140                                 priv->regs + TSI721_SMSG_ECC_COR_LOG(ch));
2141                 iowrite32(TSI721_SMSG_ECC_NCOR_MASK,
2142                                 priv->regs + TSI721_SMSG_ECC_NCOR(ch));
2143         }
2144
2145         return 0;
2146 }
2147
2148 /**
2149  * tsi721_disable_ints - disables all device interrupts
2150  * @priv: pointer to tsi721 private data
2151  */
2152 static void tsi721_disable_ints(struct tsi721_device *priv)
2153 {
2154         int ch;
2155
2156         /* Disable all device level interrupts */
2157         iowrite32(0, priv->regs + TSI721_DEV_INTE);
2158
2159         /* Disable all Device Channel interrupts */
2160         iowrite32(0, priv->regs + TSI721_DEV_CHAN_INTE);
2161
2162         /* Disable all Inbound Msg Channel interrupts */
2163         for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++)
2164                 iowrite32(0, priv->regs + TSI721_IBDMAC_INTE(ch));
2165
2166         /* Disable all Outbound Msg Channel interrupts */
2167         for (ch = 0; ch < TSI721_OMSG_CHNUM; ch++)
2168                 iowrite32(0, priv->regs + TSI721_OBDMAC_INTE(ch));
2169
2170         /* Disable all general messaging interrupts */
2171         iowrite32(0, priv->regs + TSI721_SMSG_INTE);
2172
2173         /* Disable all BDMA Channel interrupts */
2174         for (ch = 0; ch < TSI721_DMA_MAXCH; ch++)
2175                 iowrite32(0,
2176                         priv->regs + TSI721_DMAC_BASE(ch) + TSI721_DMAC_INTE);
2177
2178         /* Disable all general BDMA interrupts */
2179         iowrite32(0, priv->regs + TSI721_BDMA_INTE);
2180
2181         /* Disable all SRIO Channel interrupts */
2182         for (ch = 0; ch < TSI721_SRIO_MAXCH; ch++)
2183                 iowrite32(0, priv->regs + TSI721_SR_CHINTE(ch));
2184
2185         /* Disable all general SR2PC interrupts */
2186         iowrite32(0, priv->regs + TSI721_SR2PC_GEN_INTE);
2187
2188         /* Disable all PC2SR interrupts */
2189         iowrite32(0, priv->regs + TSI721_PC2SR_INTE);
2190
2191         /* Disable all I2C interrupts */
2192         iowrite32(0, priv->regs + TSI721_I2C_INT_ENABLE);
2193
2194         /* Disable SRIO MAC interrupts */
2195         iowrite32(0, priv->regs + TSI721_RIO_EM_INT_ENABLE);
2196         iowrite32(0, priv->regs + TSI721_RIO_EM_DEV_INT_EN);
2197 }
2198
2199 /**
2200  * tsi721_setup_mport - Setup Tsi721 as RapidIO subsystem master port
2201  * @priv: pointer to tsi721 private data
2202  *
2203  * Configures Tsi721 as RapidIO master port.
2204  */
2205 static int tsi721_setup_mport(struct tsi721_device *priv)
2206 {
2207         struct pci_dev *pdev = priv->pdev;
2208         int err = 0;
2209         struct rio_ops *ops;
2210
2211         struct rio_mport *mport;
2212
2213         ops = kzalloc(sizeof(struct rio_ops), GFP_KERNEL);
2214         if (!ops) {
2215                 dev_dbg(&pdev->dev, "Unable to allocate memory for rio_ops\n");
2216                 return -ENOMEM;
2217         }
2218
2219         ops->lcread = tsi721_lcread;
2220         ops->lcwrite = tsi721_lcwrite;
2221         ops->cread = tsi721_cread_dma;
2222         ops->cwrite = tsi721_cwrite_dma;
2223         ops->dsend = tsi721_dsend;
2224         ops->open_inb_mbox = tsi721_open_inb_mbox;
2225         ops->close_inb_mbox = tsi721_close_inb_mbox;
2226         ops->open_outb_mbox = tsi721_open_outb_mbox;
2227         ops->close_outb_mbox = tsi721_close_outb_mbox;
2228         ops->add_outb_message = tsi721_add_outb_message;
2229         ops->add_inb_buffer = tsi721_add_inb_buffer;
2230         ops->get_inb_message = tsi721_get_inb_message;
2231         ops->map_inb = tsi721_rio_map_inb_mem;
2232         ops->unmap_inb = tsi721_rio_unmap_inb_mem;
2233
2234         mport = kzalloc(sizeof(struct rio_mport), GFP_KERNEL);
2235         if (!mport) {
2236                 kfree(ops);
2237                 dev_dbg(&pdev->dev, "Unable to allocate memory for mport\n");
2238                 return -ENOMEM;
2239         }
2240
2241         mport->ops = ops;
2242         mport->index = 0;
2243         mport->sys_size = 0; /* small system */
2244         mport->phy_type = RIO_PHY_SERIAL;
2245         mport->priv = (void *)priv;
2246         mport->phys_efptr = 0x100;
2247         priv->mport = mport;
2248
2249         INIT_LIST_HEAD(&mport->dbells);
2250
2251         rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff);
2252         rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3);
2253         rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3);
2254         snprintf(mport->name, RIO_MAX_MPORT_NAME, "%s(%s)",
2255                  dev_driver_string(&pdev->dev), dev_name(&pdev->dev));
2256
2257         /* Hook up interrupt handler */
2258
2259 #ifdef CONFIG_PCI_MSI
2260         if (!tsi721_enable_msix(priv))
2261                 priv->flags |= TSI721_USING_MSIX;
2262         else if (!pci_enable_msi(pdev))
2263                 priv->flags |= TSI721_USING_MSI;
2264         else
2265                 dev_info(&pdev->dev,
2266                          "MSI/MSI-X is not available. Using legacy INTx.\n");
2267 #endif /* CONFIG_PCI_MSI */
2268
2269         err = tsi721_request_irq(mport);
2270
2271         if (!err) {
2272                 tsi721_interrupts_init(priv);
2273                 ops->pwenable = tsi721_pw_enable;
2274         } else {
2275                 dev_err(&pdev->dev, "Unable to get assigned PCI IRQ "
2276                         "vector %02X err=0x%x\n", pdev->irq, err);
2277                 goto err_exit;
2278         }
2279
2280 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
2281         tsi721_register_dma(priv);
2282 #endif
2283         /* Enable SRIO link */
2284         iowrite32(ioread32(priv->regs + TSI721_DEVCTL) |
2285                   TSI721_DEVCTL_SRBOOT_CMPL,
2286                   priv->regs + TSI721_DEVCTL);
2287
2288         rio_register_mport(mport);
2289
2290         if (mport->host_deviceid >= 0)
2291                 iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER |
2292                           RIO_PORT_GEN_DISCOVERED,
2293                           priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2294         else
2295                 iowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2296
2297         return 0;
2298
2299 err_exit:
2300         kfree(mport);
2301         kfree(ops);
2302         return err;
2303 }
2304
2305 static int tsi721_probe(struct pci_dev *pdev,
2306                                   const struct pci_device_id *id)
2307 {
2308         struct tsi721_device *priv;
2309         int err;
2310
2311         priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL);
2312         if (priv == NULL) {
2313                 dev_err(&pdev->dev, "Failed to allocate memory for device\n");
2314                 err = -ENOMEM;
2315                 goto err_exit;
2316         }
2317
2318         err = pci_enable_device(pdev);
2319         if (err) {
2320                 dev_err(&pdev->dev, "Failed to enable PCI device\n");
2321                 goto err_clean;
2322         }
2323
2324         priv->pdev = pdev;
2325
2326 #ifdef DEBUG
2327         {
2328         int i;
2329         for (i = 0; i <= PCI_STD_RESOURCE_END; i++) {
2330                 dev_dbg(&pdev->dev, "res[%d] @ 0x%llx (0x%lx, 0x%lx)\n",
2331                         i, (unsigned long long)pci_resource_start(pdev, i),
2332                         (unsigned long)pci_resource_len(pdev, i),
2333                         pci_resource_flags(pdev, i));
2334         }
2335         }
2336 #endif
2337         /*
2338          * Verify BAR configuration
2339          */
2340
2341         /* BAR_0 (registers) must be 512KB+ in 32-bit address space */
2342         if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) ||
2343             pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 ||
2344             pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) {
2345                 dev_err(&pdev->dev,
2346                         "Missing or misconfigured CSR BAR0, aborting.\n");
2347                 err = -ENODEV;
2348                 goto err_disable_pdev;
2349         }
2350
2351         /* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */
2352         if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) ||
2353             pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 ||
2354             pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) {
2355                 dev_err(&pdev->dev,
2356                         "Missing or misconfigured Doorbell BAR1, aborting.\n");
2357                 err = -ENODEV;
2358                 goto err_disable_pdev;
2359         }
2360
2361         /*
2362          * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address
2363          * space.
2364          * NOTE: BAR_2 and BAR_4 are not used by this version of driver.
2365          * It may be a good idea to keep them disabled using HW configuration
2366          * to save PCI memory space.
2367          */
2368         if ((pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM) &&
2369             (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64)) {
2370                 dev_info(&pdev->dev, "Outbound BAR2 is not used but enabled.\n");
2371         }
2372
2373         if ((pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM) &&
2374             (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64)) {
2375                 dev_info(&pdev->dev, "Outbound BAR4 is not used but enabled.\n");
2376         }
2377
2378         err = pci_request_regions(pdev, DRV_NAME);
2379         if (err) {
2380                 dev_err(&pdev->dev, "Cannot obtain PCI resources, "
2381                         "aborting.\n");
2382                 goto err_disable_pdev;
2383         }
2384
2385         pci_set_master(pdev);
2386
2387         priv->regs = pci_ioremap_bar(pdev, BAR_0);
2388         if (!priv->regs) {
2389                 dev_err(&pdev->dev,
2390                         "Unable to map device registers space, aborting\n");
2391                 err = -ENOMEM;
2392                 goto err_free_res;
2393         }
2394
2395         priv->odb_base = pci_ioremap_bar(pdev, BAR_1);
2396         if (!priv->odb_base) {
2397                 dev_err(&pdev->dev,
2398                         "Unable to map outbound doorbells space, aborting\n");
2399                 err = -ENOMEM;
2400                 goto err_unmap_bars;
2401         }
2402
2403         /* Configure DMA attributes. */
2404         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
2405                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2406                 if (err) {
2407                         dev_info(&pdev->dev, "Unable to set DMA mask\n");
2408                         goto err_unmap_bars;
2409                 }
2410
2411                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
2412                         dev_info(&pdev->dev, "Unable to set consistent DMA mask\n");
2413         } else {
2414                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2415                 if (err)
2416                         dev_info(&pdev->dev, "Unable to set consistent DMA mask\n");
2417         }
2418
2419         BUG_ON(!pci_is_pcie(pdev));
2420
2421         /* Clear "no snoop" and "relaxed ordering" bits, use default MRRS. */
2422         pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL,
2423                 PCI_EXP_DEVCTL_READRQ | PCI_EXP_DEVCTL_RELAX_EN |
2424                 PCI_EXP_DEVCTL_NOSNOOP_EN,
2425                 0x2 << MAX_READ_REQUEST_SZ_SHIFT);
2426
2427         /* Adjust PCIe completion timeout. */
2428         pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL2, 0xf, 0x2);
2429
2430         /*
2431          * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block
2432          */
2433         pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0x01);
2434         pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL,
2435                                                 TSI721_MSIXTBL_OFFSET);
2436         pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA,
2437                                                 TSI721_MSIXPBA_OFFSET);
2438         pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0);
2439         /* End of FIXUP */
2440
2441         tsi721_disable_ints(priv);
2442
2443         tsi721_init_pc2sr_mapping(priv);
2444         tsi721_init_sr2pc_mapping(priv);
2445
2446         if (tsi721_bdma_maint_init(priv)) {
2447                 dev_err(&pdev->dev, "BDMA initialization failed, aborting\n");
2448                 err = -ENOMEM;
2449                 goto err_unmap_bars;
2450         }
2451
2452         err = tsi721_doorbell_init(priv);
2453         if (err)
2454                 goto err_free_bdma;
2455
2456         tsi721_port_write_init(priv);
2457
2458         err = tsi721_messages_init(priv);
2459         if (err)
2460                 goto err_free_consistent;
2461
2462         err = tsi721_setup_mport(priv);
2463         if (err)
2464                 goto err_free_consistent;
2465
2466         return 0;
2467
2468 err_free_consistent:
2469         tsi721_doorbell_free(priv);
2470 err_free_bdma:
2471         tsi721_bdma_maint_free(priv);
2472 err_unmap_bars:
2473         if (priv->regs)
2474                 iounmap(priv->regs);
2475         if (priv->odb_base)
2476                 iounmap(priv->odb_base);
2477 err_free_res:
2478         pci_release_regions(pdev);
2479         pci_clear_master(pdev);
2480 err_disable_pdev:
2481         pci_disable_device(pdev);
2482 err_clean:
2483         kfree(priv);
2484 err_exit:
2485         return err;
2486 }
2487
2488 static DEFINE_PCI_DEVICE_TABLE(tsi721_pci_tbl) = {
2489         { PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721) },
2490         { 0, }  /* terminate list */
2491 };
2492
2493 MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl);
2494
2495 static struct pci_driver tsi721_driver = {
2496         .name           = "tsi721",
2497         .id_table       = tsi721_pci_tbl,
2498         .probe          = tsi721_probe,
2499 };
2500
2501 static int __init tsi721_init(void)
2502 {
2503         return pci_register_driver(&tsi721_driver);
2504 }
2505
2506 static void __exit tsi721_exit(void)
2507 {
2508         pci_unregister_driver(&tsi721_driver);
2509 }
2510
2511 device_initcall(tsi721_init);