drivers/infiniband/hw/ipath/ipath_sdma.c

   1 /*
   2  * Copyright (c) 2007, 2008 QLogic Corporation. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/spinlock.h>
  34 #include <linux/gfp.h>
  35
  36 #include "ipath_kernel.h"
  37 #include "ipath_verbs.h"
  38 #include "ipath_common.h"
  39
  40 #define SDMA_DESCQ_SZ PAGE_SIZE /* 256 entries per 4KB page */
  41
  42 static void vl15_watchdog_enq(struct ipath_devdata *dd)
  43 {
  44         /* ipath_sdma_lock must already be held */
  45         if (atomic_inc_return(&dd->ipath_sdma_vl15_count) == 1) {
  46                 unsigned long interval = (HZ + 19) / 20;
  47                 dd->ipath_sdma_vl15_timer.expires = jiffies + interval;
  48                 add_timer(&dd->ipath_sdma_vl15_timer);
  49         }
  50 }
  51
  52 static void vl15_watchdog_deq(struct ipath_devdata *dd)
  53 {
  54         /* ipath_sdma_lock must already be held */
  55         if (atomic_dec_return(&dd->ipath_sdma_vl15_count) != 0) {
  56                 unsigned long interval = (HZ + 19) / 20;
  57                 mod_timer(&dd->ipath_sdma_vl15_timer, jiffies + interval);
  58         } else {
  59                 del_timer(&dd->ipath_sdma_vl15_timer);
  60         }
  61 }
  62
  63 static void vl15_watchdog_timeout(unsigned long opaque)
  64 {
  65         struct ipath_devdata *dd = (struct ipath_devdata *)opaque;
  66
  67         if (atomic_read(&dd->ipath_sdma_vl15_count) != 0) {
  68                 ipath_dbg("vl15 watchdog timeout - clearing\n");
  69                 ipath_cancel_sends(dd, 1);
  70                 ipath_hol_down(dd);
  71         } else {
  72                 ipath_dbg("vl15 watchdog timeout - "
  73                           "condition already cleared\n");
  74         }
  75 }
  76
  77 static void unmap_desc(struct ipath_devdata *dd, unsigned head)
  78 {
  79         __le64 *descqp = &dd->ipath_sdma_descq[head].qw[0];
  80         u64 desc[2];
  81         dma_addr_t addr;
  82         size_t len;
  83
  84         desc[0] = le64_to_cpu(descqp[0]);
  85         desc[1] = le64_to_cpu(descqp[1]);
  86
  87         addr = (desc[1] << 32) | (desc[0] >> 32);
  88         len = (desc[0] >> 14) & (0x7ffULL << 2);
  89         dma_unmap_single(&dd->pcidev->dev, addr, len, DMA_TO_DEVICE);
  90 }
  91
  92 /*
  93  * ipath_sdma_lock should be locked before calling this.
  94  */
  95 int ipath_sdma_make_progress(struct ipath_devdata *dd)
  96 {
  97         struct list_head *lp = NULL;
  98         struct ipath_sdma_txreq *txp = NULL;
  99         u16 dmahead;
 100         u16 start_idx = 0;
 101         int progress = 0;
 102
 103         if (!list_empty(&dd->ipath_sdma_activelist)) {
 104                 lp = dd->ipath_sdma_activelist.next;
 105                 txp = list_entry(lp, struct ipath_sdma_txreq, list);
 106                 start_idx = txp->start_idx;
 107         }
 108
 109         /*
 110          * Read the SDMA head register in order to know that the
 111          * interrupt clear has been written to the chip.
 112          * Otherwise, we may not get an interrupt for the last
 113          * descriptor in the queue.
 114          */
 115         dmahead = (u16)ipath_read_kreg32(dd, dd->ipath_kregs->kr_senddmahead);
 116         /* sanity check return value for error handling (chip reset, etc.) */
 117         if (dmahead >= dd->ipath_sdma_descq_cnt)
 118                 goto done;
 119
 120         while (dd->ipath_sdma_descq_head != dmahead) {
 121                 if (txp && txp->flags & IPATH_SDMA_TXREQ_F_FREEDESC &&
 122                     dd->ipath_sdma_descq_head == start_idx) {
 123                         unmap_desc(dd, dd->ipath_sdma_descq_head);
 124                         start_idx++;
 125                         if (start_idx == dd->ipath_sdma_descq_cnt)
 126                                 start_idx = 0;
 127                 }
 128
 129                 /* increment free count and head */
 130                 dd->ipath_sdma_descq_removed++;
 131                 if (++dd->ipath_sdma_descq_head == dd->ipath_sdma_descq_cnt)
 132                         dd->ipath_sdma_descq_head = 0;
 133
 134                 if (txp && txp->next_descq_idx == dd->ipath_sdma_descq_head) {
 135                         /* move to notify list */
 136                         if (txp->flags & IPATH_SDMA_TXREQ_F_VL15)
 137                                 vl15_watchdog_deq(dd);
 138                         list_move_tail(lp, &dd->ipath_sdma_notifylist);
 139                         if (!list_empty(&dd->ipath_sdma_activelist)) {
 140                                 lp = dd->ipath_sdma_activelist.next;
 141                                 txp = list_entry(lp, struct ipath_sdma_txreq,
 142                                                  list);
 143                                 start_idx = txp->start_idx;
 144                         } else {
 145                                 lp = NULL;
 146                                 txp = NULL;
 147                         }
 148                 }
 149                 progress = 1;
 150         }
 151
 152         if (progress)
 153                 tasklet_hi_schedule(&dd->ipath_sdma_notify_task);
 154
 155 done:
 156         return progress;
 157 }
 158
 159 static void ipath_sdma_notify(struct ipath_devdata *dd, struct list_head *list)
 160 {
 161         struct ipath_sdma_txreq *txp, *txp_next;
 162
 163         list_for_each_entry_safe(txp, txp_next, list, list) {
 164                 list_del_init(&txp->list);
 165
 166                 if (txp->callback)
 167                         (*txp->callback)(txp->callback_cookie,
 168                                          txp->callback_status);
 169         }
 170 }
 171
 172 static void sdma_notify_taskbody(struct ipath_devdata *dd)
 173 {
 174         unsigned long flags;
 175         struct list_head list;
 176
 177         INIT_LIST_HEAD(&list);
 178
 179         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 180
 181         list_splice_init(&dd->ipath_sdma_notifylist, &list);
 182
 183         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 184
 185         ipath_sdma_notify(dd, &list);
 186
 187         /*
 188          * The IB verbs layer needs to see the callback before getting
 189          * the call to ipath_ib_piobufavail() because the callback
 190          * handles releasing resources the next send will need.
 191          * Otherwise, we could do these calls in
 192          * ipath_sdma_make_progress().
 193          */
 194         ipath_ib_piobufavail(dd->verbs_dev);
 195 }
 196
 197 static void sdma_notify_task(unsigned long opaque)
 198 {
 199         struct ipath_devdata *dd = (struct ipath_devdata *)opaque;
 200
 201         if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
 202                 sdma_notify_taskbody(dd);
 203 }
 204
 205 static void dump_sdma_state(struct ipath_devdata *dd)
 206 {
 207         unsigned long reg;
 208
 209         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmastatus);
 210         ipath_cdbg(VERBOSE, "kr_senddmastatus: 0x%016lx\n", reg);
 211
 212         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendctrl);
 213         ipath_cdbg(VERBOSE, "kr_sendctrl: 0x%016lx\n", reg);
 214
 215         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask0);
 216         ipath_cdbg(VERBOSE, "kr_senddmabufmask0: 0x%016lx\n", reg);
 217
 218         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask1);
 219         ipath_cdbg(VERBOSE, "kr_senddmabufmask1: 0x%016lx\n", reg);
 220
 221         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask2);
 222         ipath_cdbg(VERBOSE, "kr_senddmabufmask2: 0x%016lx\n", reg);
 223
 224         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmatail);
 225         ipath_cdbg(VERBOSE, "kr_senddmatail: 0x%016lx\n", reg);
 226
 227         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmahead);
 228         ipath_cdbg(VERBOSE, "kr_senddmahead: 0x%016lx\n", reg);
 229 }
 230
 231 static void sdma_abort_task(unsigned long opaque)
 232 {
 233         struct ipath_devdata *dd = (struct ipath_devdata *) opaque;
 234         u64 status;
 235         unsigned long flags;
 236
 237         if (test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
 238                 return;
 239
 240         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 241
 242         status = dd->ipath_sdma_status & IPATH_SDMA_ABORT_MASK;
 243
 244         /* nothing to do */
 245         if (status == IPATH_SDMA_ABORT_NONE)
 246                 goto unlock;
 247
 248         /* ipath_sdma_abort() is done, waiting for interrupt */
 249         if (status == IPATH_SDMA_ABORT_DISARMED) {
 250                 if (jiffies < dd->ipath_sdma_abort_intr_timeout)
 251                         goto resched_noprint;
 252                 /* give up, intr got lost somewhere */
 253                 ipath_dbg("give up waiting for SDMADISABLED intr\n");
 254                 __set_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status);
 255                 status = IPATH_SDMA_ABORT_ABORTED;
 256         }
 257
 258         /* everything is stopped, time to clean up and restart */
 259         if (status == IPATH_SDMA_ABORT_ABORTED) {
 260                 struct ipath_sdma_txreq *txp, *txpnext;
 261                 u64 hwstatus;
 262                 int notify = 0;
 263
 264                 hwstatus = ipath_read_kreg64(dd,
 265                                 dd->ipath_kregs->kr_senddmastatus);
 266
 267                 if ((hwstatus & (IPATH_SDMA_STATUS_SCORE_BOARD_DRAIN_IN_PROG |
 268                                  IPATH_SDMA_STATUS_ABORT_IN_PROG             |
 269                                  IPATH_SDMA_STATUS_INTERNAL_SDMA_ENABLE)) ||
 270                     !(hwstatus & IPATH_SDMA_STATUS_SCB_EMPTY)) {
 271                         if (dd->ipath_sdma_reset_wait > 0) {
 272                                 /* not done shutting down sdma */
 273                                 --dd->ipath_sdma_reset_wait;
 274                                 goto resched;
 275                         }
 276                         ipath_cdbg(VERBOSE, "gave up waiting for quiescent "
 277                                 "status after SDMA reset, continuing\n");
 278                         dump_sdma_state(dd);
 279                 }
 280
 281                 /* dequeue all "sent" requests */
 282                 list_for_each_entry_safe(txp, txpnext,
 283                                          &dd->ipath_sdma_activelist, list) {
 284                         txp->callback_status = IPATH_SDMA_TXREQ_S_ABORTED;
 285                         if (txp->flags & IPATH_SDMA_TXREQ_F_VL15)
 286                                 vl15_watchdog_deq(dd);
 287                         list_move_tail(&txp->list, &dd->ipath_sdma_notifylist);
 288                         notify = 1;
 289                 }
 290                 if (notify)
 291                         tasklet_hi_schedule(&dd->ipath_sdma_notify_task);
 292
 293                 /* reset our notion of head and tail */
 294                 dd->ipath_sdma_descq_tail = 0;
 295                 dd->ipath_sdma_descq_head = 0;
 296                 dd->ipath_sdma_head_dma[0] = 0;
 297                 dd->ipath_sdma_generation = 0;
 298                 dd->ipath_sdma_descq_removed = dd->ipath_sdma_descq_added;
 299
 300                 /* Reset SendDmaLenGen */
 301                 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen,
 302                         (u64) dd->ipath_sdma_descq_cnt | (1ULL << 18));
 303
 304                 /* done with sdma state for a bit */
 305                 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 306
 307                 /*
 308                  * Don't restart sdma here (with the exception
 309                  * below). Wait until link is up to ACTIVE.  VL15 MADs
 310                  * used to bring the link up use PIO, and multiple link
 311                  * transitions otherwise cause the sdma engine to be
 312                  * stopped and started multiple times.
 313                  * The disable is done here, including the shadow,
 314                  * so the state is kept consistent.
 315                  * See ipath_restart_sdma() for the actual starting
 316                  * of sdma.
 317                  */
 318                 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
 319                 dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
 320                 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
 321                                  dd->ipath_sendctrl);
 322                 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 323                 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
 324
 325                 /* make sure I see next message */
 326                 dd->ipath_sdma_abort_jiffies = 0;
 327
 328                 /*
 329                  * Not everything that takes SDMA offline is a link
 330                  * status change.  If the link was up, restart SDMA.
 331                  */
 332                 if (dd->ipath_flags & IPATH_LINKACTIVE)
 333                         ipath_restart_sdma(dd);
 334
 335                 goto done;
 336         }
 337
 338 resched:
 339         /*
 340          * for now, keep spinning
 341          * JAG - this is bad to just have default be a loop without
 342          * state change
 343          */
 344         if (jiffies > dd->ipath_sdma_abort_jiffies) {
 345                 ipath_dbg("looping with status 0x%08lx\n",
 346                           dd->ipath_sdma_status);
 347                 dd->ipath_sdma_abort_jiffies = jiffies + 5 * HZ;
 348         }
 349 resched_noprint:
 350         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 351         if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
 352                 tasklet_hi_schedule(&dd->ipath_sdma_abort_task);
 353         return;
 354
 355 unlock:
 356         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 357 done:
 358         return;
 359 }
 360
 361 /*
 362  * This is called from interrupt context.
 363  */
 364 void ipath_sdma_intr(struct ipath_devdata *dd)
 365 {
 366         unsigned long flags;
 367
 368         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 369
 370         (void) ipath_sdma_make_progress(dd);
 371
 372         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 373 }
 374
 375 static int alloc_sdma(struct ipath_devdata *dd)
 376 {
 377         int ret = 0;
 378
 379         /* Allocate memory for SendDMA descriptor FIFO */
 380         dd->ipath_sdma_descq = dma_alloc_coherent(&dd->pcidev->dev,
 381                 SDMA_DESCQ_SZ, &dd->ipath_sdma_descq_phys, GFP_KERNEL);
 382
 383         if (!dd->ipath_sdma_descq) {
 384                 ipath_dev_err(dd, "failed to allocate SendDMA descriptor "
 385                         "FIFO memory\n");
 386                 ret = -ENOMEM;
 387                 goto done;
 388         }
 389
 390         dd->ipath_sdma_descq_cnt =
 391                 SDMA_DESCQ_SZ / sizeof(struct ipath_sdma_desc);
 392
 393         /* Allocate memory for DMA of head register to memory */
 394         dd->ipath_sdma_head_dma = dma_alloc_coherent(&dd->pcidev->dev,
 395                 PAGE_SIZE, &dd->ipath_sdma_head_phys, GFP_KERNEL);
 396         if (!dd->ipath_sdma_head_dma) {
 397                 ipath_dev_err(dd, "failed to allocate SendDMA head memory\n");
 398                 ret = -ENOMEM;
 399                 goto cleanup_descq;
 400         }
 401         dd->ipath_sdma_head_dma[0] = 0;
 402
 403         init_timer(&dd->ipath_sdma_vl15_timer);
 404         dd->ipath_sdma_vl15_timer.function = vl15_watchdog_timeout;
 405         dd->ipath_sdma_vl15_timer.data = (unsigned long)dd;
 406         atomic_set(&dd->ipath_sdma_vl15_count, 0);
 407
 408         goto done;
 409
 410 cleanup_descq:
 411         dma_free_coherent(&dd->pcidev->dev, SDMA_DESCQ_SZ,
 412                 (void *)dd->ipath_sdma_descq, dd->ipath_sdma_descq_phys);
 413         dd->ipath_sdma_descq = NULL;
 414         dd->ipath_sdma_descq_phys = 0;
 415 done:
 416         return ret;
 417 }
 418
 419 int setup_sdma(struct ipath_devdata *dd)
 420 {
 421         int ret = 0;
 422         unsigned i, n;
 423         u64 tmp64;
 424         u64 senddmabufmask[3] = { 0 };
 425         unsigned long flags;
 426
 427         ret = alloc_sdma(dd);
 428         if (ret)
 429                 goto done;
 430
 431         if (!dd->ipath_sdma_descq) {
 432                 ipath_dev_err(dd, "SendDMA memory not allocated\n");
 433                 goto done;
 434         }
 435
 436         /*
 437          * Set initial status as if we had been up, then gone down.
 438          * This lets initial start on transition to ACTIVE be the
 439          * same as restart after link flap.
 440          */
 441         dd->ipath_sdma_status = IPATH_SDMA_ABORT_ABORTED;
 442         dd->ipath_sdma_abort_jiffies = 0;
 443         dd->ipath_sdma_generation = 0;
 444         dd->ipath_sdma_descq_tail = 0;
 445         dd->ipath_sdma_descq_head = 0;
 446         dd->ipath_sdma_descq_removed = 0;
 447         dd->ipath_sdma_descq_added = 0;
 448
 449         /* Set SendDmaBase */
 450         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabase,
 451                          dd->ipath_sdma_descq_phys);
 452         /* Set SendDmaLenGen */
 453         tmp64 = dd->ipath_sdma_descq_cnt;
 454         tmp64 |= 1<<18; /* enable generation checking */
 455         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen, tmp64);
 456         /* Set SendDmaTail */
 457         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail,
 458                          dd->ipath_sdma_descq_tail);
 459         /* Set SendDmaHeadAddr */
 460         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmaheadaddr,
 461                          dd->ipath_sdma_head_phys);
 462
 463         /*
 464          * Reserve all the former "kernel" piobufs, using high number range
 465          * so we get as many 4K buffers as possible
 466          */
 467         n = dd->ipath_piobcnt2k + dd->ipath_piobcnt4k;
 468         i = dd->ipath_lastport_piobuf + dd->ipath_pioreserved;
 469         ipath_chg_pioavailkernel(dd, i, n - i , 0);
 470         for (; i < n; ++i) {
 471                 unsigned word = i / 64;
 472                 unsigned bit = i & 63;
 473                 BUG_ON(word >= 3);
 474                 senddmabufmask[word] |= 1ULL << bit;
 475         }
 476         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask0,
 477                          senddmabufmask[0]);
 478         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask1,
 479                          senddmabufmask[1]);
 480         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask2,
 481                          senddmabufmask[2]);
 482
 483         INIT_LIST_HEAD(&dd->ipath_sdma_activelist);
 484         INIT_LIST_HEAD(&dd->ipath_sdma_notifylist);
 485
 486         tasklet_init(&dd->ipath_sdma_notify_task, sdma_notify_task,
 487                      (unsigned long) dd);
 488         tasklet_init(&dd->ipath_sdma_abort_task, sdma_abort_task,
 489                      (unsigned long) dd);
 490
 491         /*
 492          * No use to turn on SDMA here, as link is probably not ACTIVE
 493          * Just mark it RUNNING and enable the interrupt, and let the
 494          * ipath_restart_sdma() on link transition to ACTIVE actually
 495          * enable it.
 496          */
 497         spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
 498         dd->ipath_sendctrl |= INFINIPATH_S_SDMAINTENABLE;
 499         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
 500         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 501         __set_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status);
 502         spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
 503
 504 done:
 505         return ret;
 506 }
 507
 508 void teardown_sdma(struct ipath_devdata *dd)
 509 {
 510         struct ipath_sdma_txreq *txp, *txpnext;
 511         unsigned long flags;
 512         dma_addr_t sdma_head_phys = 0;
 513         dma_addr_t sdma_descq_phys = 0;
 514         void *sdma_descq = NULL;
 515         void *sdma_head_dma = NULL;
 516
 517         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 518         __clear_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status);
 519         __set_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
 520         __set_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status);
 521         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 522
 523         tasklet_kill(&dd->ipath_sdma_abort_task);
 524         tasklet_kill(&dd->ipath_sdma_notify_task);
 525
 526         /* turn off sdma */
 527         spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
 528         dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
 529         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
 530                 dd->ipath_sendctrl);
 531         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 532         spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
 533
 534         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 535         /* dequeue all "sent" requests */
 536         list_for_each_entry_safe(txp, txpnext, &dd->ipath_sdma_activelist,
 537                                  list) {
 538                 txp->callback_status = IPATH_SDMA_TXREQ_S_SHUTDOWN;
 539                 if (txp->flags & IPATH_SDMA_TXREQ_F_VL15)
 540                         vl15_watchdog_deq(dd);
 541                 list_move_tail(&txp->list, &dd->ipath_sdma_notifylist);
 542         }
 543         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 544
 545         sdma_notify_taskbody(dd);
 546
 547         del_timer_sync(&dd->ipath_sdma_vl15_timer);
 548
 549         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 550
 551         dd->ipath_sdma_abort_jiffies = 0;
 552
 553         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabase, 0);
 554         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen, 0);
 555         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail, 0);
 556         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmaheadaddr, 0);
 557         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask0, 0);
 558         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask1, 0);
 559         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask2, 0);
 560
 561         if (dd->ipath_sdma_head_dma) {
 562                 sdma_head_dma = (void *) dd->ipath_sdma_head_dma;
 563                 sdma_head_phys = dd->ipath_sdma_head_phys;
 564                 dd->ipath_sdma_head_dma = NULL;
 565                 dd->ipath_sdma_head_phys = 0;
 566         }
 567
 568         if (dd->ipath_sdma_descq) {
 569                 sdma_descq = dd->ipath_sdma_descq;
 570                 sdma_descq_phys = dd->ipath_sdma_descq_phys;
 571                 dd->ipath_sdma_descq = NULL;
 572                 dd->ipath_sdma_descq_phys = 0;
 573         }
 574
 575         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 576
 577         if (sdma_head_dma)
 578                 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
 579                                   sdma_head_dma, sdma_head_phys);
 580
 581         if (sdma_descq)
 582                 dma_free_coherent(&dd->pcidev->dev, SDMA_DESCQ_SZ,
 583                                   sdma_descq, sdma_descq_phys);
 584 }
 585
 586 /*
 587  * [Re]start SDMA, if we use it, and it's not already OK.
 588  * This is called on transition to link ACTIVE, either the first or
 589  * subsequent times.
 590  */
 591 void ipath_restart_sdma(struct ipath_devdata *dd)
 592 {
 593         unsigned long flags;
 594         int needed = 1;
 595
 596         if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA))
 597                 goto bail;
 598
 599         /*
 600          * First, make sure we should, which is to say,
 601          * check that we are "RUNNING" (not in teardown)
 602          * and not "SHUTDOWN"
 603          */
 604         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 605         if (!test_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status)
 606                 || test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
 607                         needed = 0;
 608         else {
 609                 __clear_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status);
 610                 __clear_bit(IPATH_SDMA_DISARMED, &dd->ipath_sdma_status);
 611                 __clear_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
 612         }
 613         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 614         if (!needed) {
 615                 ipath_dbg("invalid attempt to restart SDMA, status 0x%08lx\n",
 616                         dd->ipath_sdma_status);
 617                 goto bail;
 618         }
 619         spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
 620         /*
 621          * First clear, just to be safe. Enable is only done
 622          * in chip on 0->1 transition
 623          */
 624         dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
 625         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
 626         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 627         dd->ipath_sendctrl |= INFINIPATH_S_SDMAENABLE;
 628         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
 629         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 630         spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
 631
 632         /* notify upper layers */
 633         ipath_ib_piobufavail(dd->verbs_dev);
 634
 635 bail:
 636         return;
 637 }
 638
 639 static inline void make_sdma_desc(struct ipath_devdata *dd,
 640         u64 *sdmadesc, u64 addr, u64 dwlen, u64 dwoffset)
 641 {
 642         WARN_ON(addr & 3);
 643         /* SDmaPhyAddr[47:32] */
 644         sdmadesc[1] = addr >> 32;
 645         /* SDmaPhyAddr[31:0] */
 646         sdmadesc[0] = (addr & 0xfffffffcULL) << 32;
 647         /* SDmaGeneration[1:0] */
 648         sdmadesc[0] |= (dd->ipath_sdma_generation & 3ULL) << 30;
 649         /* SDmaDwordCount[10:0] */
 650         sdmadesc[0] |= (dwlen & 0x7ffULL) << 16;
 651         /* SDmaBufOffset[12:2] */
 652         sdmadesc[0] |= dwoffset & 0x7ffULL;
 653 }
 654
 655 /*
 656  * This function queues one IB packet onto the send DMA queue per call.
 657  * The caller is responsible for checking:
 658  * 1) The number of send DMA descriptor entries is less than the size of
 659  *    the descriptor queue.
 660  * 2) The IB SGE addresses and lengths are 32-bit aligned
 661  *    (except possibly the last SGE's length)
 662  * 3) The SGE addresses are suitable for passing to dma_map_single().
 663  */
 664 int ipath_sdma_verbs_send(struct ipath_devdata *dd,
 665         struct ipath_sge_state *ss, u32 dwords,
 666         struct ipath_verbs_txreq *tx)
 667 {
 668
 669         unsigned long flags;
 670         struct ipath_sge *sge;
 671         int ret = 0;
 672         u16 tail;
 673         __le64 *descqp;
 674         u64 sdmadesc[2];
 675         u32 dwoffset;
 676         dma_addr_t addr;
 677
 678         if ((tx->map_len + (dwords<<2)) > dd->ipath_ibmaxlen) {
 679                 ipath_dbg("packet size %X > ibmax %X, fail\n",
 680                         tx->map_len + (dwords<<2), dd->ipath_ibmaxlen);
 681                 ret = -EMSGSIZE;
 682                 goto fail;
 683         }
 684
 685         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 686
 687 retry:
 688         if (unlikely(test_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status))) {
 689                 ret = -EBUSY;
 690                 goto unlock;
 691         }
 692
 693         if (tx->txreq.sg_count > ipath_sdma_descq_freecnt(dd)) {
 694                 if (ipath_sdma_make_progress(dd))
 695                         goto retry;
 696                 ret = -ENOBUFS;
 697                 goto unlock;
 698         }
 699
 700         addr = dma_map_single(&dd->pcidev->dev, tx->txreq.map_addr,
 701                               tx->map_len, DMA_TO_DEVICE);
 702         if (dma_mapping_error(&dd->pcidev->dev, addr))
 703                 goto ioerr;
 704
 705         dwoffset = tx->map_len >> 2;
 706         make_sdma_desc(dd, sdmadesc, (u64) addr, dwoffset, 0);
 707
 708         /* SDmaFirstDesc */
 709         sdmadesc[0] |= 1ULL << 12;
 710         if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_USELARGEBUF)
 711                 sdmadesc[0] |= 1ULL << 14;      /* SDmaUseLargeBuf */
 712
 713         /* write to the descq */
 714         tail = dd->ipath_sdma_descq_tail;
 715         descqp = &dd->ipath_sdma_descq[tail].qw[0];
 716         *descqp++ = cpu_to_le64(sdmadesc[0]);
 717         *descqp++ = cpu_to_le64(sdmadesc[1]);
 718
 719         if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEDESC)
 720                 tx->txreq.start_idx = tail;
 721
 722         /* increment the tail */
 723         if (++tail == dd->ipath_sdma_descq_cnt) {
 724                 tail = 0;
 725                 descqp = &dd->ipath_sdma_descq[0].qw[0];
 726                 ++dd->ipath_sdma_generation;
 727         }
 728
 729         sge = &ss->sge;
 730         while (dwords) {
 731                 u32 dw;
 732                 u32 len;
 733
 734                 len = dwords << 2;
 735                 if (len > sge->length)
 736                         len = sge->length;
 737                 if (len > sge->sge_length)
 738                         len = sge->sge_length;
 739                 BUG_ON(len == 0);
 740                 dw = (len + 3) >> 2;
 741                 addr = dma_map_single(&dd->pcidev->dev, sge->vaddr, dw << 2,
 742                                       DMA_TO_DEVICE);
 743                 if (dma_mapping_error(&dd->pcidev->dev, addr))
 744                         goto unmap;
 745                 make_sdma_desc(dd, sdmadesc, (u64) addr, dw, dwoffset);
 746                 /* SDmaUseLargeBuf has to be set in every descriptor */
 747                 if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_USELARGEBUF)
 748                         sdmadesc[0] |= 1ULL << 14;
 749                 /* write to the descq */
 750                 *descqp++ = cpu_to_le64(sdmadesc[0]);
 751                 *descqp++ = cpu_to_le64(sdmadesc[1]);
 752
 753                 /* increment the tail */
 754                 if (++tail == dd->ipath_sdma_descq_cnt) {
 755                         tail = 0;
 756                         descqp = &dd->ipath_sdma_descq[0].qw[0];
 757                         ++dd->ipath_sdma_generation;
 758                 }
 759                 sge->vaddr += len;
 760                 sge->length -= len;
 761                 sge->sge_length -= len;
 762                 if (sge->sge_length == 0) {
 763                         if (--ss->num_sge)
 764                                 *sge = *ss->sg_list++;
 765                 } else if (sge->length == 0 && sge->mr != NULL) {
 766                         if (++sge->n >= IPATH_SEGSZ) {
 767                                 if (++sge->m >= sge->mr->mapsz)
 768                                         break;
 769                                 sge->n = 0;
 770                         }
 771                         sge->vaddr =
 772                                 sge->mr->map[sge->m]->segs[sge->n].vaddr;
 773                         sge->length =
 774                                 sge->mr->map[sge->m]->segs[sge->n].length;
 775                 }
 776
 777                 dwoffset += dw;
 778                 dwords -= dw;
 779         }
 780
 781         if (!tail)
 782                 descqp = &dd->ipath_sdma_descq[dd->ipath_sdma_descq_cnt].qw[0];
 783         descqp -= 2;
 784         /* SDmaLastDesc */
 785         descqp[0] |= cpu_to_le64(1ULL << 11);
 786         if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_INTREQ) {
 787                 /* SDmaIntReq */
 788                 descqp[0] |= cpu_to_le64(1ULL << 15);
 789         }
 790
 791         /* Commit writes to memory and advance the tail on the chip */
 792         wmb();
 793         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail, tail);
 794
 795         tx->txreq.next_descq_idx = tail;
 796         tx->txreq.callback_status = IPATH_SDMA_TXREQ_S_OK;
 797         dd->ipath_sdma_descq_tail = tail;
 798         dd->ipath_sdma_descq_added += tx->txreq.sg_count;
 799         list_add_tail(&tx->txreq.list, &dd->ipath_sdma_activelist);
 800         if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_VL15)
 801                 vl15_watchdog_enq(dd);
 802         goto unlock;
 803
 804 unmap:
 805         while (tail != dd->ipath_sdma_descq_tail) {
 806                 if (!tail)
 807                         tail = dd->ipath_sdma_descq_cnt - 1;
 808                 else
 809                         tail--;
 810                 unmap_desc(dd, tail);
 811         }
 812 ioerr:
 813         ret = -EIO;
 814 unlock:
 815         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 816 fail:
 817         return ret;
 818 }