drivers/infiniband/hw/qib/qib_sdma.c

   1 /*
   2  * Copyright (c) 2007, 2008, 2009, 2010 QLogic Corporation. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/spinlock.h>
  34 #include <linux/netdevice.h>
  35 #include <linux/moduleparam.h>
  36
  37 #include "qib.h"
  38 #include "qib_common.h"
  39
  40 /* default pio off, sdma on */
  41 static ushort sdma_descq_cnt = 256;
  42 module_param_named(sdma_descq_cnt, sdma_descq_cnt, ushort, S_IRUGO);
  43 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries");
  44
  45 /*
  46  * Bits defined in the send DMA descriptor.
  47  */
  48 #define SDMA_DESC_LAST          (1ULL << 11)
  49 #define SDMA_DESC_FIRST         (1ULL << 12)
  50 #define SDMA_DESC_DMA_HEAD      (1ULL << 13)
  51 #define SDMA_DESC_USE_LARGE_BUF (1ULL << 14)
  52 #define SDMA_DESC_INTR          (1ULL << 15)
  53 #define SDMA_DESC_COUNT_LSB     16
  54 #define SDMA_DESC_GEN_LSB       30
  55
  56 char *qib_sdma_state_names[] = {
  57         [qib_sdma_state_s00_hw_down]          = "s00_HwDown",
  58         [qib_sdma_state_s10_hw_start_up_wait] = "s10_HwStartUpWait",
  59         [qib_sdma_state_s20_idle]             = "s20_Idle",
  60         [qib_sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait",
  61         [qib_sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait",
  62         [qib_sdma_state_s50_hw_halt_wait]     = "s50_HwHaltWait",
  63         [qib_sdma_state_s99_running]          = "s99_Running",
  64 };
  65
  66 char *qib_sdma_event_names[] = {
  67         [qib_sdma_event_e00_go_hw_down]   = "e00_GoHwDown",
  68         [qib_sdma_event_e10_go_hw_start]  = "e10_GoHwStart",
  69         [qib_sdma_event_e20_hw_started]   = "e20_HwStarted",
  70         [qib_sdma_event_e30_go_running]   = "e30_GoRunning",
  71         [qib_sdma_event_e40_sw_cleaned]   = "e40_SwCleaned",
  72         [qib_sdma_event_e50_hw_cleaned]   = "e50_HwCleaned",
  73         [qib_sdma_event_e60_hw_halted]    = "e60_HwHalted",
  74         [qib_sdma_event_e70_go_idle]      = "e70_GoIdle",
  75         [qib_sdma_event_e7220_err_halted] = "e7220_ErrHalted",
  76         [qib_sdma_event_e7322_err_halted] = "e7322_ErrHalted",
  77         [qib_sdma_event_e90_timer_tick]   = "e90_TimerTick",
  78 };
  79
  80 /* declare all statics here rather than keep sorting */
  81 static int alloc_sdma(struct qib_pportdata *);
  82 static void sdma_complete(struct kref *);
  83 static void sdma_finalput(struct qib_sdma_state *);
  84 static void sdma_get(struct qib_sdma_state *);
  85 static void sdma_put(struct qib_sdma_state *);
  86 static void sdma_set_state(struct qib_pportdata *, enum qib_sdma_states);
  87 static void sdma_start_sw_clean_up(struct qib_pportdata *);
  88 static void sdma_sw_clean_up_task(unsigned long);
  89 static void unmap_desc(struct qib_pportdata *, unsigned);
  90
  91 static void sdma_get(struct qib_sdma_state *ss)
  92 {
  93         kref_get(&ss->kref);
  94 }
  95
  96 static void sdma_complete(struct kref *kref)
  97 {
  98         struct qib_sdma_state *ss =
  99                 container_of(kref, struct qib_sdma_state, kref);
 100
 101         complete(&ss->comp);
 102 }
 103
 104 static void sdma_put(struct qib_sdma_state *ss)
 105 {
 106         kref_put(&ss->kref, sdma_complete);
 107 }
 108
 109 static void sdma_finalput(struct qib_sdma_state *ss)
 110 {
 111         sdma_put(ss);
 112         wait_for_completion(&ss->comp);
 113 }
 114
 115 /*
 116  * Complete all the sdma requests on the active list, in the correct
 117  * order, and with appropriate processing.   Called when cleaning up
 118  * after sdma shutdown, and when new sdma requests are submitted for
 119  * a link that is down.   This matches what is done for requests
 120  * that complete normally, it's just the full list.
 121  *
 122  * Must be called with sdma_lock held
 123  */
 124 static void clear_sdma_activelist(struct qib_pportdata *ppd)
 125 {
 126         struct qib_sdma_txreq *txp, *txp_next;
 127
 128         list_for_each_entry_safe(txp, txp_next, &ppd->sdma_activelist, list) {
 129                 list_del_init(&txp->list);
 130                 if (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) {
 131                         unsigned idx;
 132
 133                         idx = txp->start_idx;
 134                         while (idx != txp->next_descq_idx) {
 135                                 unmap_desc(ppd, idx);
 136                                 if (++idx == ppd->sdma_descq_cnt)
 137                                         idx = 0;
 138                         }
 139                 }
 140                 if (txp->callback)
 141                         (*txp->callback)(txp, QIB_SDMA_TXREQ_S_ABORTED);
 142         }
 143 }
 144
 145 static void sdma_sw_clean_up_task(unsigned long opaque)
 146 {
 147         struct qib_pportdata *ppd = (struct qib_pportdata *) opaque;
 148         unsigned long flags;
 149
 150         spin_lock_irqsave(&ppd->sdma_lock, flags);
 151
 152         /*
 153          * At this point, the following should always be true:
 154          * - We are halted, so no more descriptors are getting retired.
 155          * - We are not running, so no one is submitting new work.
 156          * - Only we can send the e40_sw_cleaned, so we can't start
 157          *   running again until we say so.  So, the active list and
 158          *   descq are ours to play with.
 159          */
 160
 161         /* Process all retired requests. */
 162         qib_sdma_make_progress(ppd);
 163
 164         clear_sdma_activelist(ppd);
 165
 166         /*
 167          * Resync count of added and removed.  It is VERY important that
 168          * sdma_descq_removed NEVER decrement - user_sdma depends on it.
 169          */
 170         ppd->sdma_descq_removed = ppd->sdma_descq_added;
 171
 172         /*
 173          * Reset our notion of head and tail.
 174          * Note that the HW registers will be reset when switching states
 175          * due to calling __qib_sdma_process_event() below.
 176          */
 177         ppd->sdma_descq_tail = 0;
 178         ppd->sdma_descq_head = 0;
 179         ppd->sdma_head_dma[0] = 0;
 180         ppd->sdma_generation = 0;
 181
 182         __qib_sdma_process_event(ppd, qib_sdma_event_e40_sw_cleaned);
 183
 184         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 185 }
 186
 187 /*
 188  * This is called when changing to state qib_sdma_state_s10_hw_start_up_wait
 189  * as a result of send buffer errors or send DMA descriptor errors.
 190  * We want to disarm the buffers in these cases.
 191  */
 192 static void sdma_hw_start_up(struct qib_pportdata *ppd)
 193 {
 194         struct qib_sdma_state *ss = &ppd->sdma_state;
 195         unsigned bufno;
 196
 197         for (bufno = ss->first_sendbuf; bufno < ss->last_sendbuf; ++bufno)
 198                 ppd->dd->f_sendctrl(ppd, QIB_SENDCTRL_DISARM_BUF(bufno));
 199
 200         ppd->dd->f_sdma_hw_start_up(ppd);
 201 }
 202
 203 static void sdma_sw_tear_down(struct qib_pportdata *ppd)
 204 {
 205         struct qib_sdma_state *ss = &ppd->sdma_state;
 206
 207         /* Releasing this reference means the state machine has stopped. */
 208         sdma_put(ss);
 209 }
 210
 211 static void sdma_start_sw_clean_up(struct qib_pportdata *ppd)
 212 {
 213         tasklet_hi_schedule(&ppd->sdma_sw_clean_up_task);
 214 }
 215
 216 static void sdma_set_state(struct qib_pportdata *ppd,
 217         enum qib_sdma_states next_state)
 218 {
 219         struct qib_sdma_state *ss = &ppd->sdma_state;
 220         struct sdma_set_state_action *action = ss->set_state_action;
 221         unsigned op = 0;
 222
 223         /* debugging bookkeeping */
 224         ss->previous_state = ss->current_state;
 225         ss->previous_op = ss->current_op;
 226
 227         ss->current_state = next_state;
 228
 229         if (action[next_state].op_enable)
 230                 op |= QIB_SDMA_SENDCTRL_OP_ENABLE;
 231
 232         if (action[next_state].op_intenable)
 233                 op |= QIB_SDMA_SENDCTRL_OP_INTENABLE;
 234
 235         if (action[next_state].op_halt)
 236                 op |= QIB_SDMA_SENDCTRL_OP_HALT;
 237
 238         if (action[next_state].op_drain)
 239                 op |= QIB_SDMA_SENDCTRL_OP_DRAIN;
 240
 241         if (action[next_state].go_s99_running_tofalse)
 242                 ss->go_s99_running = 0;
 243
 244         if (action[next_state].go_s99_running_totrue)
 245                 ss->go_s99_running = 1;
 246
 247         ss->current_op = op;
 248
 249         ppd->dd->f_sdma_sendctrl(ppd, ss->current_op);
 250 }
 251
 252 static void unmap_desc(struct qib_pportdata *ppd, unsigned head)
 253 {
 254         __le64 *descqp = &ppd->sdma_descq[head].qw[0];
 255         u64 desc[2];
 256         dma_addr_t addr;
 257         size_t len;
 258
 259         desc[0] = le64_to_cpu(descqp[0]);
 260         desc[1] = le64_to_cpu(descqp[1]);
 261
 262         addr = (desc[1] << 32) | (desc[0] >> 32);
 263         len = (desc[0] >> 14) & (0x7ffULL << 2);
 264         dma_unmap_single(&ppd->dd->pcidev->dev, addr, len, DMA_TO_DEVICE);
 265 }
 266
 267 static int alloc_sdma(struct qib_pportdata *ppd)
 268 {
 269         ppd->sdma_descq_cnt = sdma_descq_cnt;
 270         if (!ppd->sdma_descq_cnt)
 271                 ppd->sdma_descq_cnt = 256;
 272
 273         /* Allocate memory for SendDMA descriptor FIFO */
 274         ppd->sdma_descq = dma_alloc_coherent(&ppd->dd->pcidev->dev,
 275                 ppd->sdma_descq_cnt * sizeof(u64[2]), &ppd->sdma_descq_phys,
 276                 GFP_KERNEL);
 277
 278         if (!ppd->sdma_descq) {
 279                 qib_dev_err(ppd->dd, "failed to allocate SendDMA descriptor "
 280                             "FIFO memory\n");
 281                 goto bail;
 282         }
 283
 284         /* Allocate memory for DMA of head register to memory */
 285         ppd->sdma_head_dma = dma_alloc_coherent(&ppd->dd->pcidev->dev,
 286                 PAGE_SIZE, &ppd->sdma_head_phys, GFP_KERNEL);
 287         if (!ppd->sdma_head_dma) {
 288                 qib_dev_err(ppd->dd, "failed to allocate SendDMA "
 289                             "head memory\n");
 290                 goto cleanup_descq;
 291         }
 292         ppd->sdma_head_dma[0] = 0;
 293         return 0;
 294
 295 cleanup_descq:
 296         dma_free_coherent(&ppd->dd->pcidev->dev,
 297                 ppd->sdma_descq_cnt * sizeof(u64[2]), (void *)ppd->sdma_descq,
 298                 ppd->sdma_descq_phys);
 299         ppd->sdma_descq = NULL;
 300         ppd->sdma_descq_phys = 0;
 301 bail:
 302         ppd->sdma_descq_cnt = 0;
 303         return -ENOMEM;
 304 }
 305
 306 static void free_sdma(struct qib_pportdata *ppd)
 307 {
 308         struct qib_devdata *dd = ppd->dd;
 309
 310         if (ppd->sdma_head_dma) {
 311                 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
 312                                   (void *)ppd->sdma_head_dma,
 313                                   ppd->sdma_head_phys);
 314                 ppd->sdma_head_dma = NULL;
 315                 ppd->sdma_head_phys = 0;
 316         }
 317
 318         if (ppd->sdma_descq) {
 319                 dma_free_coherent(&dd->pcidev->dev,
 320                                   ppd->sdma_descq_cnt * sizeof(u64[2]),
 321                                   ppd->sdma_descq, ppd->sdma_descq_phys);
 322                 ppd->sdma_descq = NULL;
 323                 ppd->sdma_descq_phys = 0;
 324         }
 325 }
 326
 327 static inline void make_sdma_desc(struct qib_pportdata *ppd,
 328                                   u64 *sdmadesc, u64 addr, u64 dwlen,
 329                                   u64 dwoffset)
 330 {
 331
 332         WARN_ON(addr & 3);
 333         /* SDmaPhyAddr[47:32] */
 334         sdmadesc[1] = addr >> 32;
 335         /* SDmaPhyAddr[31:0] */
 336         sdmadesc[0] = (addr & 0xfffffffcULL) << 32;
 337         /* SDmaGeneration[1:0] */
 338         sdmadesc[0] |= (ppd->sdma_generation & 3ULL) <<
 339                 SDMA_DESC_GEN_LSB;
 340         /* SDmaDwordCount[10:0] */
 341         sdmadesc[0] |= (dwlen & 0x7ffULL) << SDMA_DESC_COUNT_LSB;
 342         /* SDmaBufOffset[12:2] */
 343         sdmadesc[0] |= dwoffset & 0x7ffULL;
 344 }
 345
 346 /* sdma_lock must be held */
 347 int qib_sdma_make_progress(struct qib_pportdata *ppd)
 348 {
 349         struct list_head *lp = NULL;
 350         struct qib_sdma_txreq *txp = NULL;
 351         struct qib_devdata *dd = ppd->dd;
 352         int progress = 0;
 353         u16 hwhead;
 354         u16 idx = 0;
 355
 356         hwhead = dd->f_sdma_gethead(ppd);
 357
 358         /* The reason for some of the complexity of this code is that
 359          * not all descriptors have corresponding txps.  So, we have to
 360          * be able to skip over descs until we wander into the range of
 361          * the next txp on the list.
 362          */
 363
 364         if (!list_empty(&ppd->sdma_activelist)) {
 365                 lp = ppd->sdma_activelist.next;
 366                 txp = list_entry(lp, struct qib_sdma_txreq, list);
 367                 idx = txp->start_idx;
 368         }
 369
 370         while (ppd->sdma_descq_head != hwhead) {
 371                 /* if desc is part of this txp, unmap if needed */
 372                 if (txp && (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) &&
 373                     (idx == ppd->sdma_descq_head)) {
 374                         unmap_desc(ppd, ppd->sdma_descq_head);
 375                         if (++idx == ppd->sdma_descq_cnt)
 376                                 idx = 0;
 377                 }
 378
 379                 /* increment dequed desc count */
 380                 ppd->sdma_descq_removed++;
 381
 382                 /* advance head, wrap if needed */
 383                 if (++ppd->sdma_descq_head == ppd->sdma_descq_cnt)
 384                         ppd->sdma_descq_head = 0;
 385
 386                 /* if now past this txp's descs, do the callback */
 387                 if (txp && txp->next_descq_idx == ppd->sdma_descq_head) {
 388                         /* remove from active list */
 389                         list_del_init(&txp->list);
 390                         if (txp->callback)
 391                                 (*txp->callback)(txp, QIB_SDMA_TXREQ_S_OK);
 392                         /* see if there is another txp */
 393                         if (list_empty(&ppd->sdma_activelist))
 394                                 txp = NULL;
 395                         else {
 396                                 lp = ppd->sdma_activelist.next;
 397                                 txp = list_entry(lp, struct qib_sdma_txreq,
 398                                         list);
 399                                 idx = txp->start_idx;
 400                         }
 401                 }
 402                 progress = 1;
 403         }
 404         if (progress)
 405                 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
 406         return progress;
 407 }
 408
 409 /*
 410  * This is called from interrupt context.
 411  */
 412 void qib_sdma_intr(struct qib_pportdata *ppd)
 413 {
 414         unsigned long flags;
 415
 416         spin_lock_irqsave(&ppd->sdma_lock, flags);
 417
 418         __qib_sdma_intr(ppd);
 419
 420         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 421 }
 422
 423 void __qib_sdma_intr(struct qib_pportdata *ppd)
 424 {
 425         if (__qib_sdma_running(ppd))
 426                 qib_sdma_make_progress(ppd);
 427 }
 428
 429 int qib_setup_sdma(struct qib_pportdata *ppd)
 430 {
 431         struct qib_devdata *dd = ppd->dd;
 432         unsigned long flags;
 433         int ret = 0;
 434
 435         ret = alloc_sdma(ppd);
 436         if (ret)
 437                 goto bail;
 438
 439         /* set consistent sdma state */
 440         ppd->dd->f_sdma_init_early(ppd);
 441         spin_lock_irqsave(&ppd->sdma_lock, flags);
 442         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 443         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 444
 445         /* set up reference counting */
 446         kref_init(&ppd->sdma_state.kref);
 447         init_completion(&ppd->sdma_state.comp);
 448
 449         ppd->sdma_generation = 0;
 450         ppd->sdma_descq_head = 0;
 451         ppd->sdma_descq_removed = 0;
 452         ppd->sdma_descq_added = 0;
 453
 454         INIT_LIST_HEAD(&ppd->sdma_activelist);
 455
 456         tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task,
 457                 (unsigned long)ppd);
 458
 459         ret = dd->f_init_sdma_regs(ppd);
 460         if (ret)
 461                 goto bail_alloc;
 462
 463         qib_sdma_process_event(ppd, qib_sdma_event_e10_go_hw_start);
 464
 465         return 0;
 466
 467 bail_alloc:
 468         qib_teardown_sdma(ppd);
 469 bail:
 470         return ret;
 471 }
 472
 473 void qib_teardown_sdma(struct qib_pportdata *ppd)
 474 {
 475         qib_sdma_process_event(ppd, qib_sdma_event_e00_go_hw_down);
 476
 477         /*
 478          * This waits for the state machine to exit so it is not
 479          * necessary to kill the sdma_sw_clean_up_task to make sure
 480          * it is not running.
 481          */
 482         sdma_finalput(&ppd->sdma_state);
 483
 484         free_sdma(ppd);
 485 }
 486
 487 int qib_sdma_running(struct qib_pportdata *ppd)
 488 {
 489         unsigned long flags;
 490         int ret;
 491
 492         spin_lock_irqsave(&ppd->sdma_lock, flags);
 493         ret = __qib_sdma_running(ppd);
 494         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 495
 496         return ret;
 497 }
 498
 499 /*
 500  * Complete a request when sdma not running; likely only request
 501  * but to simplify the code, always queue it, then process the full
 502  * activelist.  We process the entire list to ensure that this particular
 503  * request does get it's callback, but in the correct order.
 504  * Must be called with sdma_lock held
 505  */
 506 static void complete_sdma_err_req(struct qib_pportdata *ppd,
 507                                   struct qib_verbs_txreq *tx)
 508 {
 509         atomic_inc(&tx->qp->s_dma_busy);
 510         /* no sdma descriptors, so no unmap_desc */
 511         tx->txreq.start_idx = 0;
 512         tx->txreq.next_descq_idx = 0;
 513         list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
 514         clear_sdma_activelist(ppd);
 515 }
 516
 517 /*
 518  * This function queues one IB packet onto the send DMA queue per call.
 519  * The caller is responsible for checking:
 520  * 1) The number of send DMA descriptor entries is less than the size of
 521  *    the descriptor queue.
 522  * 2) The IB SGE addresses and lengths are 32-bit aligned
 523  *    (except possibly the last SGE's length)
 524  * 3) The SGE addresses are suitable for passing to dma_map_single().
 525  */
 526 int qib_sdma_verbs_send(struct qib_pportdata *ppd,
 527                         struct qib_sge_state *ss, u32 dwords,
 528                         struct qib_verbs_txreq *tx)
 529 {
 530         unsigned long flags;
 531         struct qib_sge *sge;
 532         struct qib_qp *qp;
 533         int ret = 0;
 534         u16 tail;
 535         __le64 *descqp;
 536         u64 sdmadesc[2];
 537         u32 dwoffset;
 538         dma_addr_t addr;
 539
 540         spin_lock_irqsave(&ppd->sdma_lock, flags);
 541
 542 retry:
 543         if (unlikely(!__qib_sdma_running(ppd))) {
 544                 complete_sdma_err_req(ppd, tx);
 545                 goto unlock;
 546         }
 547
 548         if (tx->txreq.sg_count > qib_sdma_descq_freecnt(ppd)) {
 549                 if (qib_sdma_make_progress(ppd))
 550                         goto retry;
 551                 if (ppd->dd->flags & QIB_HAS_SDMA_TIMEOUT)
 552                         ppd->dd->f_sdma_set_desc_cnt(ppd,
 553                                         ppd->sdma_descq_cnt / 2);
 554                 goto busy;
 555         }
 556
 557         dwoffset = tx->hdr_dwords;
 558         make_sdma_desc(ppd, sdmadesc, (u64) tx->txreq.addr, dwoffset, 0);
 559
 560         sdmadesc[0] |= SDMA_DESC_FIRST;
 561         if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
 562                 sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
 563
 564         /* write to the descq */
 565         tail = ppd->sdma_descq_tail;
 566         descqp = &ppd->sdma_descq[tail].qw[0];
 567         *descqp++ = cpu_to_le64(sdmadesc[0]);
 568         *descqp++ = cpu_to_le64(sdmadesc[1]);
 569
 570         /* increment the tail */
 571         if (++tail == ppd->sdma_descq_cnt) {
 572                 tail = 0;
 573                 descqp = &ppd->sdma_descq[0].qw[0];
 574                 ++ppd->sdma_generation;
 575         }
 576
 577         tx->txreq.start_idx = tail;
 578
 579         sge = &ss->sge;
 580         while (dwords) {
 581                 u32 dw;
 582                 u32 len;
 583
 584                 len = dwords << 2;
 585                 if (len > sge->length)
 586                         len = sge->length;
 587                 if (len > sge->sge_length)
 588                         len = sge->sge_length;
 589                 BUG_ON(len == 0);
 590                 dw = (len + 3) >> 2;
 591                 addr = dma_map_single(&ppd->dd->pcidev->dev, sge->vaddr,
 592                                       dw << 2, DMA_TO_DEVICE);
 593                 if (dma_mapping_error(&ppd->dd->pcidev->dev, addr))
 594                         goto unmap;
 595                 sdmadesc[0] = 0;
 596                 make_sdma_desc(ppd, sdmadesc, (u64) addr, dw, dwoffset);
 597                 /* SDmaUseLargeBuf has to be set in every descriptor */
 598                 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
 599                         sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
 600                 /* write to the descq */
 601                 *descqp++ = cpu_to_le64(sdmadesc[0]);
 602                 *descqp++ = cpu_to_le64(sdmadesc[1]);
 603
 604                 /* increment the tail */
 605                 if (++tail == ppd->sdma_descq_cnt) {
 606                         tail = 0;
 607                         descqp = &ppd->sdma_descq[0].qw[0];
 608                         ++ppd->sdma_generation;
 609                 }
 610                 sge->vaddr += len;
 611                 sge->length -= len;
 612                 sge->sge_length -= len;
 613                 if (sge->sge_length == 0) {
 614                         if (--ss->num_sge)
 615                                 *sge = *ss->sg_list++;
 616                 } else if (sge->length == 0 && sge->mr->lkey) {
 617                         if (++sge->n >= QIB_SEGSZ) {
 618                                 if (++sge->m >= sge->mr->mapsz)
 619                                         break;
 620                                 sge->n = 0;
 621                         }
 622                         sge->vaddr =
 623                                 sge->mr->map[sge->m]->segs[sge->n].vaddr;
 624                         sge->length =
 625                                 sge->mr->map[sge->m]->segs[sge->n].length;
 626                 }
 627
 628                 dwoffset += dw;
 629                 dwords -= dw;
 630         }
 631
 632         if (!tail)
 633                 descqp = &ppd->sdma_descq[ppd->sdma_descq_cnt].qw[0];
 634         descqp -= 2;
 635         descqp[0] |= cpu_to_le64(SDMA_DESC_LAST);
 636         if (tx->txreq.flags & QIB_SDMA_TXREQ_F_HEADTOHOST)
 637                 descqp[0] |= cpu_to_le64(SDMA_DESC_DMA_HEAD);
 638         if (tx->txreq.flags & QIB_SDMA_TXREQ_F_INTREQ)
 639                 descqp[0] |= cpu_to_le64(SDMA_DESC_INTR);
 640
 641         atomic_inc(&tx->qp->s_dma_busy);
 642         tx->txreq.next_descq_idx = tail;
 643         ppd->dd->f_sdma_update_tail(ppd, tail);
 644         ppd->sdma_descq_added += tx->txreq.sg_count;
 645         list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
 646         goto unlock;
 647
 648 unmap:
 649         for (;;) {
 650                 if (!tail)
 651                         tail = ppd->sdma_descq_cnt - 1;
 652                 else
 653                         tail--;
 654                 if (tail == ppd->sdma_descq_tail)
 655                         break;
 656                 unmap_desc(ppd, tail);
 657         }
 658         qp = tx->qp;
 659         qib_put_txreq(tx);
 660         spin_lock(&qp->r_lock);
 661         spin_lock(&qp->s_lock);
 662         if (qp->ibqp.qp_type == IB_QPT_RC) {
 663                 /* XXX what about error sending RDMA read responses? */
 664                 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)
 665                         qib_error_qp(qp, IB_WC_GENERAL_ERR);
 666         } else if (qp->s_wqe)
 667                 qib_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR);
 668         spin_unlock(&qp->s_lock);
 669         spin_unlock(&qp->r_lock);
 670         /* return zero to process the next send work request */
 671         goto unlock;
 672
 673 busy:
 674         qp = tx->qp;
 675         spin_lock(&qp->s_lock);
 676         if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) {
 677                 struct qib_ibdev *dev;
 678
 679                 /*
 680                  * If we couldn't queue the DMA request, save the info
 681                  * and try again later rather than destroying the
 682                  * buffer and undoing the side effects of the copy.
 683                  */
 684                 tx->ss = ss;
 685                 tx->dwords = dwords;
 686                 qp->s_tx = tx;
 687                 dev = &ppd->dd->verbs_dev;
 688                 spin_lock(&dev->pending_lock);
 689                 if (list_empty(&qp->iowait)) {
 690                         struct qib_ibport *ibp;
 691
 692                         ibp = &ppd->ibport_data;
 693                         ibp->n_dmawait++;
 694                         qp->s_flags |= QIB_S_WAIT_DMA_DESC;
 695                         list_add_tail(&qp->iowait, &dev->dmawait);
 696                 }
 697                 spin_unlock(&dev->pending_lock);
 698                 qp->s_flags &= ~QIB_S_BUSY;
 699                 spin_unlock(&qp->s_lock);
 700                 ret = -EBUSY;
 701         } else {
 702                 spin_unlock(&qp->s_lock);
 703                 qib_put_txreq(tx);
 704         }
 705 unlock:
 706         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 707         return ret;
 708 }
 709
 710 void qib_sdma_process_event(struct qib_pportdata *ppd,
 711         enum qib_sdma_events event)
 712 {
 713         unsigned long flags;
 714
 715         spin_lock_irqsave(&ppd->sdma_lock, flags);
 716
 717         __qib_sdma_process_event(ppd, event);
 718
 719         if (ppd->sdma_state.current_state == qib_sdma_state_s99_running)
 720                 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
 721
 722         spin_unlock_irqrestore(&ppd->sdma_lock, flags);
 723 }
 724
 725 void __qib_sdma_process_event(struct qib_pportdata *ppd,
 726         enum qib_sdma_events event)
 727 {
 728         struct qib_sdma_state *ss = &ppd->sdma_state;
 729
 730         switch (ss->current_state) {
 731         case qib_sdma_state_s00_hw_down:
 732                 switch (event) {
 733                 case qib_sdma_event_e00_go_hw_down:
 734                         break;
 735                 case qib_sdma_event_e30_go_running:
 736                         /*
 737                          * If down, but running requested (usually result
 738                          * of link up, then we need to start up.
 739                          * This can happen when hw down is requested while
 740                          * bringing the link up with traffic active on
 741                          * 7220, e.g. */
 742                         ss->go_s99_running = 1;
 743                         /* fall through and start dma engine */
 744                 case qib_sdma_event_e10_go_hw_start:
 745                         /* This reference means the state machine is started */
 746                         sdma_get(&ppd->sdma_state);
 747                         sdma_set_state(ppd,
 748                                        qib_sdma_state_s10_hw_start_up_wait);
 749                         break;
 750                 case qib_sdma_event_e20_hw_started:
 751                         break;
 752                 case qib_sdma_event_e40_sw_cleaned:
 753                         sdma_sw_tear_down(ppd);
 754                         break;
 755                 case qib_sdma_event_e50_hw_cleaned:
 756                         break;
 757                 case qib_sdma_event_e60_hw_halted:
 758                         break;
 759                 case qib_sdma_event_e70_go_idle:
 760                         break;
 761                 case qib_sdma_event_e7220_err_halted:
 762                         break;
 763                 case qib_sdma_event_e7322_err_halted:
 764                         break;
 765                 case qib_sdma_event_e90_timer_tick:
 766                         break;
 767                 }
 768                 break;
 769
 770         case qib_sdma_state_s10_hw_start_up_wait:
 771                 switch (event) {
 772                 case qib_sdma_event_e00_go_hw_down:
 773                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 774                         sdma_sw_tear_down(ppd);
 775                         break;
 776                 case qib_sdma_event_e10_go_hw_start:
 777                         break;
 778                 case qib_sdma_event_e20_hw_started:
 779                         sdma_set_state(ppd, ss->go_s99_running ?
 780                                        qib_sdma_state_s99_running :
 781                                        qib_sdma_state_s20_idle);
 782                         break;
 783                 case qib_sdma_event_e30_go_running:
 784                         ss->go_s99_running = 1;
 785                         break;
 786                 case qib_sdma_event_e40_sw_cleaned:
 787                         break;
 788                 case qib_sdma_event_e50_hw_cleaned:
 789                         break;
 790                 case qib_sdma_event_e60_hw_halted:
 791                         break;
 792                 case qib_sdma_event_e70_go_idle:
 793                         ss->go_s99_running = 0;
 794                         break;
 795                 case qib_sdma_event_e7220_err_halted:
 796                         break;
 797                 case qib_sdma_event_e7322_err_halted:
 798                         break;
 799                 case qib_sdma_event_e90_timer_tick:
 800                         break;
 801                 }
 802                 break;
 803
 804         case qib_sdma_state_s20_idle:
 805                 switch (event) {
 806                 case qib_sdma_event_e00_go_hw_down:
 807                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 808                         sdma_sw_tear_down(ppd);
 809                         break;
 810                 case qib_sdma_event_e10_go_hw_start:
 811                         break;
 812                 case qib_sdma_event_e20_hw_started:
 813                         break;
 814                 case qib_sdma_event_e30_go_running:
 815                         sdma_set_state(ppd, qib_sdma_state_s99_running);
 816                         ss->go_s99_running = 1;
 817                         break;
 818                 case qib_sdma_event_e40_sw_cleaned:
 819                         break;
 820                 case qib_sdma_event_e50_hw_cleaned:
 821                         break;
 822                 case qib_sdma_event_e60_hw_halted:
 823                         break;
 824                 case qib_sdma_event_e70_go_idle:
 825                         break;
 826                 case qib_sdma_event_e7220_err_halted:
 827                         break;
 828                 case qib_sdma_event_e7322_err_halted:
 829                         break;
 830                 case qib_sdma_event_e90_timer_tick:
 831                         break;
 832                 }
 833                 break;
 834
 835         case qib_sdma_state_s30_sw_clean_up_wait:
 836                 switch (event) {
 837                 case qib_sdma_event_e00_go_hw_down:
 838                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 839                         break;
 840                 case qib_sdma_event_e10_go_hw_start:
 841                         break;
 842                 case qib_sdma_event_e20_hw_started:
 843                         break;
 844                 case qib_sdma_event_e30_go_running:
 845                         ss->go_s99_running = 1;
 846                         break;
 847                 case qib_sdma_event_e40_sw_cleaned:
 848                         sdma_set_state(ppd,
 849                                        qib_sdma_state_s10_hw_start_up_wait);
 850                         sdma_hw_start_up(ppd);
 851                         break;
 852                 case qib_sdma_event_e50_hw_cleaned:
 853                         break;
 854                 case qib_sdma_event_e60_hw_halted:
 855                         break;
 856                 case qib_sdma_event_e70_go_idle:
 857                         ss->go_s99_running = 0;
 858                         break;
 859                 case qib_sdma_event_e7220_err_halted:
 860                         break;
 861                 case qib_sdma_event_e7322_err_halted:
 862                         break;
 863                 case qib_sdma_event_e90_timer_tick:
 864                         break;
 865                 }
 866                 break;
 867
 868         case qib_sdma_state_s40_hw_clean_up_wait:
 869                 switch (event) {
 870                 case qib_sdma_event_e00_go_hw_down:
 871                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 872                         sdma_start_sw_clean_up(ppd);
 873                         break;
 874                 case qib_sdma_event_e10_go_hw_start:
 875                         break;
 876                 case qib_sdma_event_e20_hw_started:
 877                         break;
 878                 case qib_sdma_event_e30_go_running:
 879                         ss->go_s99_running = 1;
 880                         break;
 881                 case qib_sdma_event_e40_sw_cleaned:
 882                         break;
 883                 case qib_sdma_event_e50_hw_cleaned:
 884                         sdma_set_state(ppd,
 885                                        qib_sdma_state_s30_sw_clean_up_wait);
 886                         sdma_start_sw_clean_up(ppd);
 887                         break;
 888                 case qib_sdma_event_e60_hw_halted:
 889                         break;
 890                 case qib_sdma_event_e70_go_idle:
 891                         ss->go_s99_running = 0;
 892                         break;
 893                 case qib_sdma_event_e7220_err_halted:
 894                         break;
 895                 case qib_sdma_event_e7322_err_halted:
 896                         break;
 897                 case qib_sdma_event_e90_timer_tick:
 898                         break;
 899                 }
 900                 break;
 901
 902         case qib_sdma_state_s50_hw_halt_wait:
 903                 switch (event) {
 904                 case qib_sdma_event_e00_go_hw_down:
 905                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 906                         sdma_start_sw_clean_up(ppd);
 907                         break;
 908                 case qib_sdma_event_e10_go_hw_start:
 909                         break;
 910                 case qib_sdma_event_e20_hw_started:
 911                         break;
 912                 case qib_sdma_event_e30_go_running:
 913                         ss->go_s99_running = 1;
 914                         break;
 915                 case qib_sdma_event_e40_sw_cleaned:
 916                         break;
 917                 case qib_sdma_event_e50_hw_cleaned:
 918                         break;
 919                 case qib_sdma_event_e60_hw_halted:
 920                         sdma_set_state(ppd,
 921                                        qib_sdma_state_s40_hw_clean_up_wait);
 922                         ppd->dd->f_sdma_hw_clean_up(ppd);
 923                         break;
 924                 case qib_sdma_event_e70_go_idle:
 925                         ss->go_s99_running = 0;
 926                         break;
 927                 case qib_sdma_event_e7220_err_halted:
 928                         break;
 929                 case qib_sdma_event_e7322_err_halted:
 930                         break;
 931                 case qib_sdma_event_e90_timer_tick:
 932                         break;
 933                 }
 934                 break;
 935
 936         case qib_sdma_state_s99_running:
 937                 switch (event) {
 938                 case qib_sdma_event_e00_go_hw_down:
 939                         sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
 940                         sdma_start_sw_clean_up(ppd);
 941                         break;
 942                 case qib_sdma_event_e10_go_hw_start:
 943                         break;
 944                 case qib_sdma_event_e20_hw_started:
 945                         break;
 946                 case qib_sdma_event_e30_go_running:
 947                         break;
 948                 case qib_sdma_event_e40_sw_cleaned:
 949                         break;
 950                 case qib_sdma_event_e50_hw_cleaned:
 951                         break;
 952                 case qib_sdma_event_e60_hw_halted:
 953                         sdma_set_state(ppd,
 954                                        qib_sdma_state_s30_sw_clean_up_wait);
 955                         sdma_start_sw_clean_up(ppd);
 956                         break;
 957                 case qib_sdma_event_e70_go_idle:
 958                         sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
 959                         ss->go_s99_running = 0;
 960                         break;
 961                 case qib_sdma_event_e7220_err_halted:
 962                         sdma_set_state(ppd,
 963                                        qib_sdma_state_s30_sw_clean_up_wait);
 964                         sdma_start_sw_clean_up(ppd);
 965                         break;
 966                 case qib_sdma_event_e7322_err_halted:
 967                         sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
 968                         break;
 969                 case qib_sdma_event_e90_timer_tick:
 970                         break;
 971                 }
 972                 break;
 973         }
 974
 975         ss->last_event = event;
 976 }