drivers/usb/host/ehci-sched.c

   1 /*
   2  * Copyright (c) 2001-2004 by David Brownell
   3  * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License as published by the
   7  * Free Software Foundation; either version 2 of the License, or (at your
   8  * option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software Foundation,
  17  * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18  */
  19
  20 /* this file is part of ehci-hcd.c */
  21
  22 /*-------------------------------------------------------------------------*/
  23
  24 /*
  25  * EHCI scheduled transaction support:  interrupt, iso, split iso
  26  * These are called "periodic" transactions in the EHCI spec.
  27  *
  28  * Note that for interrupt transfers, the QH/QTD manipulation is shared
  29  * with the "asynchronous" transaction support (control/bulk transfers).
  30  * The only real difference is in how interrupt transfers are scheduled.
  31  *
  32  * For ISO, we make an "iso_stream" head to serve the same role as a QH.
  33  * It keeps track of every ITD (or SITD) that's linked, and holds enough
  34  * pre-calculated schedule data to make appending to the queue be quick.
  35  */
  36
  37 static int ehci_get_frame (struct usb_hcd *hcd);
  38
  39 /*-------------------------------------------------------------------------*/
  40
  41 /*
  42  * periodic_next_shadow - return "next" pointer on shadow list
  43  * @periodic: host pointer to qh/itd/sitd
  44  * @tag: hardware tag for type of this record
  45  */
  46 static union ehci_shadow *
  47 periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
  48                 __hc32 tag)
  49 {
  50         switch (hc32_to_cpu(ehci, tag)) {
  51         case Q_TYPE_QH:
  52                 return &periodic->qh->qh_next;
  53         case Q_TYPE_FSTN:
  54                 return &periodic->fstn->fstn_next;
  55         case Q_TYPE_ITD:
  56                 return &periodic->itd->itd_next;
  57         // case Q_TYPE_SITD:
  58         default:
  59                 return &periodic->sitd->sitd_next;
  60         }
  61 }
  62
  63 static __hc32 *
  64 shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic,
  65                 __hc32 tag)
  66 {
  67         switch (hc32_to_cpu(ehci, tag)) {
  68         /* our ehci_shadow.qh is actually software part */
  69         case Q_TYPE_QH:
  70                 return &periodic->qh->hw->hw_next;
  71         /* others are hw parts */
  72         default:
  73                 return periodic->hw_next;
  74         }
  75 }
  76
  77 /* caller must hold ehci->lock */
  78 static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr)
  79 {
  80         union ehci_shadow       *prev_p = &ehci->pshadow[frame];
  81         __hc32                  *hw_p = &ehci->periodic[frame];
  82         union ehci_shadow       here = *prev_p;
  83
  84         /* find predecessor of "ptr"; hw and shadow lists are in sync */
  85         while (here.ptr && here.ptr != ptr) {
  86                 prev_p = periodic_next_shadow(ehci, prev_p,
  87                                 Q_NEXT_TYPE(ehci, *hw_p));
  88                 hw_p = shadow_next_periodic(ehci, &here,
  89                                 Q_NEXT_TYPE(ehci, *hw_p));
  90                 here = *prev_p;
  91         }
  92         /* an interrupt entry (at list end) could have been shared */
  93         if (!here.ptr)
  94                 return;
  95
  96         /* update shadow and hardware lists ... the old "next" pointers
  97          * from ptr may still be in use, the caller updates them.
  98          */
  99         *prev_p = *periodic_next_shadow(ehci, &here,
 100                         Q_NEXT_TYPE(ehci, *hw_p));
 101         *hw_p = *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p));
 102 }
 103
 104 /* how many of the uframe's 125 usecs are allocated? */
 105 static unsigned short
 106 periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
 107 {
 108         __hc32                  *hw_p = &ehci->periodic [frame];
 109         union ehci_shadow       *q = &ehci->pshadow [frame];
 110         unsigned                usecs = 0;
 111         struct ehci_qh_hw       *hw;
 112
 113         while (q->ptr) {
 114                 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
 115                 case Q_TYPE_QH:
 116                         hw = q->qh->hw;
 117                         /* is it in the S-mask? */
 118                         if (hw->hw_info2 & cpu_to_hc32(ehci, 1 << uframe))
 119                                 usecs += q->qh->usecs;
 120                         /* ... or C-mask? */
 121                         if (hw->hw_info2 & cpu_to_hc32(ehci,
 122                                         1 << (8 + uframe)))
 123                                 usecs += q->qh->c_usecs;
 124                         hw_p = &hw->hw_next;
 125                         q = &q->qh->qh_next;
 126                         break;
 127                 // case Q_TYPE_FSTN:
 128                 default:
 129                         /* for "save place" FSTNs, count the relevant INTR
 130                          * bandwidth from the previous frame
 131                          */
 132                         if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) {
 133                                 ehci_dbg (ehci, "ignoring FSTN cost ...\n");
 134                         }
 135                         hw_p = &q->fstn->hw_next;
 136                         q = &q->fstn->fstn_next;
 137                         break;
 138                 case Q_TYPE_ITD:
 139                         if (q->itd->hw_transaction[uframe])
 140                                 usecs += q->itd->stream->usecs;
 141                         hw_p = &q->itd->hw_next;
 142                         q = &q->itd->itd_next;
 143                         break;
 144                 case Q_TYPE_SITD:
 145                         /* is it in the S-mask?  (count SPLIT, DATA) */
 146                         if (q->sitd->hw_uframe & cpu_to_hc32(ehci,
 147                                         1 << uframe)) {
 148                                 if (q->sitd->hw_fullspeed_ep &
 149                                                 cpu_to_hc32(ehci, 1<<31))
 150                                         usecs += q->sitd->stream->usecs;
 151                                 else    /* worst case for OUT start-split */
 152                                         usecs += HS_USECS_ISO (188);
 153                         }
 154
 155                         /* ... C-mask?  (count CSPLIT, DATA) */
 156                         if (q->sitd->hw_uframe &
 157                                         cpu_to_hc32(ehci, 1 << (8 + uframe))) {
 158                                 /* worst case for IN complete-split */
 159                                 usecs += q->sitd->stream->c_usecs;
 160                         }
 161
 162                         hw_p = &q->sitd->hw_next;
 163                         q = &q->sitd->sitd_next;
 164                         break;
 165                 }
 166         }
 167 #ifdef  DEBUG
 168         if (usecs > 100)
 169                 ehci_err (ehci, "uframe %d sched overrun: %d usecs\n",
 170                         frame * 8 + uframe, usecs);
 171 #endif
 172         return usecs;
 173 }
 174
 175 /*-------------------------------------------------------------------------*/
 176
 177 static int same_tt (struct usb_device *dev1, struct usb_device *dev2)
 178 {
 179         if (!dev1->tt || !dev2->tt)
 180                 return 0;
 181         if (dev1->tt != dev2->tt)
 182                 return 0;
 183         if (dev1->tt->multi)
 184                 return dev1->ttport == dev2->ttport;
 185         else
 186                 return 1;
 187 }
 188
 189 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
 190
 191 /* Which uframe does the low/fullspeed transfer start in?
 192  *
 193  * The parameter is the mask of ssplits in "H-frame" terms
 194  * and this returns the transfer start uframe in "B-frame" terms,
 195  * which allows both to match, e.g. a ssplit in "H-frame" uframe 0
 196  * will cause a transfer in "B-frame" uframe 0.  "B-frames" lag
 197  * "H-frames" by 1 uframe.  See the EHCI spec sec 4.5 and figure 4.7.
 198  */
 199 static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
 200 {
 201         unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask);
 202         if (!smask) {
 203                 ehci_err(ehci, "invalid empty smask!\n");
 204                 /* uframe 7 can't have bw so this will indicate failure */
 205                 return 7;
 206         }
 207         return ffs(smask) - 1;
 208 }
 209
 210 static const unsigned char
 211 max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
 212
 213 /* carryover low/fullspeed bandwidth that crosses uframe boundries */
 214 static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
 215 {
 216         int i;
 217         for (i=0; i<7; i++) {
 218                 if (max_tt_usecs[i] < tt_usecs[i]) {
 219                         tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
 220                         tt_usecs[i] = max_tt_usecs[i];
 221                 }
 222         }
 223 }
 224
 225 /* How many of the tt's periodic downstream 1000 usecs are allocated?
 226  *
 227  * While this measures the bandwidth in terms of usecs/uframe,
 228  * the low/fullspeed bus has no notion of uframes, so any particular
 229  * low/fullspeed transfer can "carry over" from one uframe to the next,
 230  * since the TT just performs downstream transfers in sequence.
 231  *
 232  * For example two separate 100 usec transfers can start in the same uframe,
 233  * and the second one would "carry over" 75 usecs into the next uframe.
 234  */
 235 static void
 236 periodic_tt_usecs (
 237         struct ehci_hcd *ehci,
 238         struct usb_device *dev,
 239         unsigned frame,
 240         unsigned short tt_usecs[8]
 241 )
 242 {
 243         __hc32                  *hw_p = &ehci->periodic [frame];
 244         union ehci_shadow       *q = &ehci->pshadow [frame];
 245         unsigned char           uf;
 246
 247         memset(tt_usecs, 0, 16);
 248
 249         while (q->ptr) {
 250                 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
 251                 case Q_TYPE_ITD:
 252                         hw_p = &q->itd->hw_next;
 253                         q = &q->itd->itd_next;
 254                         continue;
 255                 case Q_TYPE_QH:
 256                         if (same_tt(dev, q->qh->dev)) {
 257                                 uf = tt_start_uframe(ehci, q->qh->hw->hw_info2);
 258                                 tt_usecs[uf] += q->qh->tt_usecs;
 259                         }
 260                         hw_p = &q->qh->hw->hw_next;
 261                         q = &q->qh->qh_next;
 262                         continue;
 263                 case Q_TYPE_SITD:
 264                         if (same_tt(dev, q->sitd->urb->dev)) {
 265                                 uf = tt_start_uframe(ehci, q->sitd->hw_uframe);
 266                                 tt_usecs[uf] += q->sitd->stream->tt_usecs;
 267                         }
 268                         hw_p = &q->sitd->hw_next;
 269                         q = &q->sitd->sitd_next;
 270                         continue;
 271                 // case Q_TYPE_FSTN:
 272                 default:
 273                         ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n",
 274                                         frame);
 275                         hw_p = &q->fstn->hw_next;
 276                         q = &q->fstn->fstn_next;
 277                 }
 278         }
 279
 280         carryover_tt_bandwidth(tt_usecs);
 281
 282         if (max_tt_usecs[7] < tt_usecs[7])
 283                 ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n",
 284                         frame, tt_usecs[7] - max_tt_usecs[7]);
 285 }
 286
 287 /*
 288  * Return true if the device's tt's downstream bus is available for a
 289  * periodic transfer of the specified length (usecs), starting at the
 290  * specified frame/uframe.  Note that (as summarized in section 11.19
 291  * of the usb 2.0 spec) TTs can buffer multiple transactions for each
 292  * uframe.
 293  *
 294  * The uframe parameter is when the fullspeed/lowspeed transfer
 295  * should be executed in "B-frame" terms, which is the same as the
 296  * highspeed ssplit's uframe (which is in "H-frame" terms).  For example
 297  * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0.
 298  * See the EHCI spec sec 4.5 and fig 4.7.
 299  *
 300  * This checks if the full/lowspeed bus, at the specified starting uframe,
 301  * has the specified bandwidth available, according to rules listed
 302  * in USB 2.0 spec section 11.18.1 fig 11-60.
 303  *
 304  * This does not check if the transfer would exceed the max ssplit
 305  * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4,
 306  * since proper scheduling limits ssplits to less than 16 per uframe.
 307  */
 308 static int tt_available (
 309         struct ehci_hcd         *ehci,
 310         unsigned                period,
 311         struct usb_device       *dev,
 312         unsigned                frame,
 313         unsigned                uframe,
 314         u16                     usecs
 315 )
 316 {
 317         if ((period == 0) || (uframe >= 7))     /* error */
 318                 return 0;
 319
 320         for (; frame < ehci->periodic_size; frame += period) {
 321                 unsigned short tt_usecs[8];
 322
 323                 periodic_tt_usecs (ehci, dev, frame, tt_usecs);
 324
 325                 ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in"
 326                         " schedule %d/%d/%d/%d/%d/%d/%d/%d\n",
 327                         frame, usecs, uframe,
 328                         tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3],
 329                         tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]);
 330
 331                 if (max_tt_usecs[uframe] <= tt_usecs[uframe]) {
 332                         ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n",
 333                                 frame, uframe);
 334                         return 0;
 335                 }
 336
 337                 /* special case for isoc transfers larger than 125us:
 338                  * the first and each subsequent fully used uframe
 339                  * must be empty, so as to not illegally delay
 340                  * already scheduled transactions
 341                  */
 342                 if (125 < usecs) {
 343                         int ufs = (usecs / 125);
 344                         int i;
 345                         for (i = uframe; i < (uframe + ufs) && i < 8; i++)
 346                                 if (0 < tt_usecs[i]) {
 347                                         ehci_vdbg(ehci,
 348                                                 "multi-uframe xfer can't fit "
 349                                                 "in frame %d uframe %d\n",
 350                                                 frame, i);
 351                                         return 0;
 352                                 }
 353                 }
 354
 355                 tt_usecs[uframe] += usecs;
 356
 357                 carryover_tt_bandwidth(tt_usecs);
 358
 359                 /* fail if the carryover pushed bw past the last uframe's limit */
 360                 if (max_tt_usecs[7] < tt_usecs[7]) {
 361                         ehci_vdbg(ehci,
 362                                 "tt unavailable usecs %d frame %d uframe %d\n",
 363                                 usecs, frame, uframe);
 364                         return 0;
 365                 }
 366         }
 367
 368         return 1;
 369 }
 370
 371 #else
 372
 373 /* return true iff the device's transaction translator is available
 374  * for a periodic transfer starting at the specified frame, using
 375  * all the uframes in the mask.
 376  */
 377 static int tt_no_collision (
 378         struct ehci_hcd         *ehci,
 379         unsigned                period,
 380         struct usb_device       *dev,
 381         unsigned                frame,
 382         u32                     uf_mask
 383 )
 384 {
 385         if (period == 0)        /* error */
 386                 return 0;
 387
 388         /* note bandwidth wastage:  split never follows csplit
 389          * (different dev or endpoint) until the next uframe.
 390          * calling convention doesn't make that distinction.
 391          */
 392         for (; frame < ehci->periodic_size; frame += period) {
 393                 union ehci_shadow       here;
 394                 __hc32                  type;
 395                 struct ehci_qh_hw       *hw;
 396
 397                 here = ehci->pshadow [frame];
 398                 type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]);
 399                 while (here.ptr) {
 400                         switch (hc32_to_cpu(ehci, type)) {
 401                         case Q_TYPE_ITD:
 402                                 type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
 403                                 here = here.itd->itd_next;
 404                                 continue;
 405                         case Q_TYPE_QH:
 406                                 hw = here.qh->hw;
 407                                 if (same_tt (dev, here.qh->dev)) {
 408                                         u32             mask;
 409
 410                                         mask = hc32_to_cpu(ehci,
 411                                                         hw->hw_info2);
 412                                         /* "knows" no gap is needed */
 413                                         mask |= mask >> 8;
 414                                         if (mask & uf_mask)
 415                                                 break;
 416                                 }
 417                                 type = Q_NEXT_TYPE(ehci, hw->hw_next);
 418                                 here = here.qh->qh_next;
 419                                 continue;
 420                         case Q_TYPE_SITD:
 421                                 if (same_tt (dev, here.sitd->urb->dev)) {
 422                                         u16             mask;
 423
 424                                         mask = hc32_to_cpu(ehci, here.sitd
 425                                                                 ->hw_uframe);
 426                                         /* FIXME assumes no gap for IN! */
 427                                         mask |= mask >> 8;
 428                                         if (mask & uf_mask)
 429                                                 break;
 430                                 }
 431                                 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
 432                                 here = here.sitd->sitd_next;
 433                                 continue;
 434                         // case Q_TYPE_FSTN:
 435                         default:
 436                                 ehci_dbg (ehci,
 437                                         "periodic frame %d bogus type %d\n",
 438                                         frame, type);
 439                         }
 440
 441                         /* collision or error */
 442                         return 0;
 443                 }
 444         }
 445
 446         /* no collision */
 447         return 1;
 448 }
 449
 450 #endif /* CONFIG_USB_EHCI_TT_NEWSCHED */
 451
 452 /*-------------------------------------------------------------------------*/
 453
 454 static int enable_periodic (struct ehci_hcd *ehci)
 455 {
 456         u32     cmd;
 457         int     status;
 458
 459         if (ehci->periodic_sched++)
 460                 return 0;
 461
 462         /* did clearing PSE did take effect yet?
 463          * takes effect only at frame boundaries...
 464          */
 465         status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
 466                                              STS_PSS, 0, 9 * 125);
 467         if (status)
 468                 return status;
 469
 470         cmd = ehci_readl(ehci, &ehci->regs->command) | CMD_PSE;
 471         ehci_writel(ehci, cmd, &ehci->regs->command);
 472         /* posted write ... PSS happens later */
 473         ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
 474
 475         /* make sure ehci_work scans these */
 476         ehci->next_uframe = ehci_readl(ehci, &ehci->regs->frame_index)
 477                 % (ehci->periodic_size << 3);
 478         return 0;
 479 }
 480
 481 static int disable_periodic (struct ehci_hcd *ehci)
 482 {
 483         u32     cmd;
 484         int     status;
 485
 486         if (--ehci->periodic_sched)
 487                 return 0;
 488
 489         /* did setting PSE not take effect yet?
 490          * takes effect only at frame boundaries...
 491          */
 492         status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
 493                                              STS_PSS, STS_PSS, 9 * 125);
 494         if (status)
 495                 return status;
 496
 497         cmd = ehci_readl(ehci, &ehci->regs->command) & ~CMD_PSE;
 498         ehci_writel(ehci, cmd, &ehci->regs->command);
 499         /* posted write ... */
 500
 501         ehci->next_uframe = -1;
 502         return 0;
 503 }
 504
 505 /*-------------------------------------------------------------------------*/
 506
 507 /* periodic schedule slots have iso tds (normal or split) first, then a
 508  * sparse tree for active interrupt transfers.
 509  *
 510  * this just links in a qh; caller guarantees uframe masks are set right.
 511  * no FSTN support (yet; ehci 0.96+)
 512  */
 513 static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
 514 {
 515         unsigned        i;
 516         unsigned        period = qh->period;
 517
 518         dev_dbg (&qh->dev->dev,
 519                 "link qh%d-%04x/%p start %d [%d/%d us]\n",
 520                 period, hc32_to_cpup(ehci, &qh->hw->hw_info2)
 521                         & (QH_CMASK | QH_SMASK),
 522                 qh, qh->start, qh->usecs, qh->c_usecs);
 523
 524         /* high bandwidth, or otherwise every microframe */
 525         if (period == 0)
 526                 period = 1;
 527
 528         for (i = qh->start; i < ehci->periodic_size; i += period) {
 529                 union ehci_shadow       *prev = &ehci->pshadow[i];
 530                 __hc32                  *hw_p = &ehci->periodic[i];
 531                 union ehci_shadow       here = *prev;
 532                 __hc32                  type = 0;
 533
 534                 /* skip the iso nodes at list head */
 535                 while (here.ptr) {
 536                         type = Q_NEXT_TYPE(ehci, *hw_p);
 537                         if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
 538                                 break;
 539                         prev = periodic_next_shadow(ehci, prev, type);
 540                         hw_p = shadow_next_periodic(ehci, &here, type);
 541                         here = *prev;
 542                 }
 543
 544                 /* sorting each branch by period (slow-->fast)
 545                  * enables sharing interior tree nodes
 546                  */
 547                 while (here.ptr && qh != here.qh) {
 548                         if (qh->period > here.qh->period)
 549                                 break;
 550                         prev = &here.qh->qh_next;
 551                         hw_p = &here.qh->hw->hw_next;
 552                         here = *prev;
 553                 }
 554                 /* link in this qh, unless some earlier pass did that */
 555                 if (qh != here.qh) {
 556                         qh->qh_next = here;
 557                         if (here.qh)
 558                                 qh->hw->hw_next = *hw_p;
 559                         wmb ();
 560                         prev->qh = qh;
 561                         *hw_p = QH_NEXT (ehci, qh->qh_dma);
 562                 }
 563         }
 564         qh->qh_state = QH_STATE_LINKED;
 565         qh->xacterrs = 0;
 566         qh_get (qh);
 567
 568         /* update per-qh bandwidth for usbfs */
 569         ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period
 570                 ? ((qh->usecs + qh->c_usecs) / qh->period)
 571                 : (qh->usecs * 8);
 572
 573         /* maybe enable periodic schedule processing */
 574         return enable_periodic(ehci);
 575 }
 576
 577 static int qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
 578 {
 579         unsigned        i;
 580         unsigned        period;
 581
 582         // FIXME:
 583         // IF this isn't high speed
 584         //   and this qh is active in the current uframe
 585         //   (and overlay token SplitXstate is false?)
 586         // THEN
 587         //   qh->hw_info1 |= cpu_to_hc32(1 << 7 /* "ignore" */);
 588
 589         /* high bandwidth, or otherwise part of every microframe */
 590         if ((period = qh->period) == 0)
 591                 period = 1;
 592
 593         for (i = qh->start; i < ehci->periodic_size; i += period)
 594                 periodic_unlink (ehci, i, qh);
 595
 596         /* update per-qh bandwidth for usbfs */
 597         ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period
 598                 ? ((qh->usecs + qh->c_usecs) / qh->period)
 599                 : (qh->usecs * 8);
 600
 601         dev_dbg (&qh->dev->dev,
 602                 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
 603                 qh->period,
 604                 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK),
 605                 qh, qh->start, qh->usecs, qh->c_usecs);
 606
 607         /* qh->qh_next still "live" to HC */
 608         qh->qh_state = QH_STATE_UNLINK;
 609         qh->qh_next.ptr = NULL;
 610         qh_put (qh);
 611
 612         /* maybe turn off periodic schedule */
 613         return disable_periodic(ehci);
 614 }
 615
 616 static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
 617 {
 618         unsigned                wait;
 619         struct ehci_qh_hw       *hw = qh->hw;
 620         int                     rc;
 621
 622         /* If the QH isn't linked then there's nothing we can do
 623          * unless we were called during a giveback, in which case
 624          * qh_completions() has to deal with it.
 625          */
 626         if (qh->qh_state != QH_STATE_LINKED) {
 627                 if (qh->qh_state == QH_STATE_COMPLETING)
 628                         qh->needs_rescan = 1;
 629                 return;
 630         }
 631
 632         qh_unlink_periodic (ehci, qh);
 633
 634         /* simple/paranoid:  always delay, expecting the HC needs to read
 635          * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
 636          * expect khubd to clean up after any CSPLITs we won't issue.
 637          * active high speed queues may need bigger delays...
 638          */
 639         if (list_empty (&qh->qtd_list)
 640                         || (cpu_to_hc32(ehci, QH_CMASK)
 641                                         & hw->hw_info2) != 0)
 642                 wait = 2;
 643         else
 644                 wait = 55;      /* worst case: 3 * 1024 */
 645
 646         udelay (wait);
 647         qh->qh_state = QH_STATE_IDLE;
 648         hw->hw_next = EHCI_LIST_END(ehci);
 649         wmb ();
 650
 651         qh_completions(ehci, qh);
 652
 653         /* reschedule QH iff another request is queued */
 654         if (!list_empty(&qh->qtd_list) &&
 655                         HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
 656                 rc = qh_schedule(ehci, qh);
 657
 658                 /* An error here likely indicates handshake failure
 659                  * or no space left in the schedule.  Neither fault
 660                  * should happen often ...
 661                  *
 662                  * FIXME kill the now-dysfunctional queued urbs
 663                  */
 664                 if (rc != 0)
 665                         ehci_err(ehci, "can't reschedule qh %p, err %d\n",
 666                                         qh, rc);
 667         }
 668 }
 669
 670 /*-------------------------------------------------------------------------*/
 671
 672 static int check_period (
 673         struct ehci_hcd *ehci,
 674         unsigned        frame,
 675         unsigned        uframe,
 676         unsigned        period,
 677         unsigned        usecs
 678 ) {
 679         int             claimed;
 680
 681         /* complete split running into next frame?
 682          * given FSTN support, we could sometimes check...
 683          */
 684         if (uframe >= 8)
 685                 return 0;
 686
 687         /*
 688          * 80% periodic == 100 usec/uframe available
 689          * convert "usecs we need" to "max already claimed"
 690          */
 691         usecs = 100 - usecs;
 692
 693         /* we "know" 2 and 4 uframe intervals were rejected; so
 694          * for period 0, check _every_ microframe in the schedule.
 695          */
 696         if (unlikely (period == 0)) {
 697                 do {
 698                         for (uframe = 0; uframe < 7; uframe++) {
 699                                 claimed = periodic_usecs (ehci, frame, uframe);
 700                                 if (claimed > usecs)
 701                                         return 0;
 702                         }
 703                 } while ((frame += 1) < ehci->periodic_size);
 704
 705         /* just check the specified uframe, at that period */
 706         } else {
 707                 do {
 708                         claimed = periodic_usecs (ehci, frame, uframe);
 709                         if (claimed > usecs)
 710                                 return 0;
 711                 } while ((frame += period) < ehci->periodic_size);
 712         }
 713
 714         // success!
 715         return 1;
 716 }
 717
 718 static int check_intr_schedule (
 719         struct ehci_hcd         *ehci,
 720         unsigned                frame,
 721         unsigned                uframe,
 722         const struct ehci_qh    *qh,
 723         __hc32                  *c_maskp
 724 )
 725 {
 726         int             retval = -ENOSPC;
 727         u8              mask = 0;
 728
 729         if (qh->c_usecs && uframe >= 6)         /* FSTN territory? */
 730                 goto done;
 731
 732         if (!check_period (ehci, frame, uframe, qh->period, qh->usecs))
 733                 goto done;
 734         if (!qh->c_usecs) {
 735                 retval = 0;
 736                 *c_maskp = 0;
 737                 goto done;
 738         }
 739
 740 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
 741         if (tt_available (ehci, qh->period, qh->dev, frame, uframe,
 742                                 qh->tt_usecs)) {
 743                 unsigned i;
 744
 745                 /* TODO : this may need FSTN for SSPLIT in uframe 5. */
 746                 for (i=uframe+1; i<8 && i<uframe+4; i++)
 747                         if (!check_period (ehci, frame, i,
 748                                                 qh->period, qh->c_usecs))
 749                                 goto done;
 750                         else
 751                                 mask |= 1 << i;
 752
 753                 retval = 0;
 754
 755                 *c_maskp = cpu_to_hc32(ehci, mask << 8);
 756         }
 757 #else
 758         /* Make sure this tt's buffer is also available for CSPLITs.
 759          * We pessimize a bit; probably the typical full speed case
 760          * doesn't need the second CSPLIT.
 761          *
 762          * NOTE:  both SPLIT and CSPLIT could be checked in just
 763          * one smart pass...
 764          */
 765         mask = 0x03 << (uframe + qh->gap_uf);
 766         *c_maskp = cpu_to_hc32(ehci, mask << 8);
 767
 768         mask |= 1 << uframe;
 769         if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) {
 770                 if (!check_period (ehci, frame, uframe + qh->gap_uf + 1,
 771                                         qh->period, qh->c_usecs))
 772                         goto done;
 773                 if (!check_period (ehci, frame, uframe + qh->gap_uf,
 774                                         qh->period, qh->c_usecs))
 775                         goto done;
 776                 retval = 0;
 777         }
 778 #endif
 779 done:
 780         return retval;
 781 }
 782
 783 /* "first fit" scheduling policy used the first time through,
 784  * or when the previous schedule slot can't be re-used.
 785  */
 786 static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
 787 {
 788         int             status;
 789         unsigned        uframe;
 790         __hc32          c_mask;
 791         unsigned        frame;          /* 0..(qh->period - 1), or NO_FRAME */
 792         struct ehci_qh_hw       *hw = qh->hw;
 793
 794         qh_refresh(ehci, qh);
 795         hw->hw_next = EHCI_LIST_END(ehci);
 796         frame = qh->start;
 797
 798         /* reuse the previous schedule slots, if we can */
 799         if (frame < qh->period) {
 800                 uframe = ffs(hc32_to_cpup(ehci, &hw->hw_info2) & QH_SMASK);
 801                 status = check_intr_schedule (ehci, frame, --uframe,
 802                                 qh, &c_mask);
 803         } else {
 804                 uframe = 0;
 805                 c_mask = 0;
 806                 status = -ENOSPC;
 807         }
 808
 809         /* else scan the schedule to find a group of slots such that all
 810          * uframes have enough periodic bandwidth available.
 811          */
 812         if (status) {
 813                 /* "normal" case, uframing flexible except with splits */
 814                 if (qh->period) {
 815                         int             i;
 816
 817                         for (i = qh->period; status && i > 0; --i) {
 818                                 frame = ++ehci->random_frame % qh->period;
 819                                 for (uframe = 0; uframe < 8; uframe++) {
 820                                         status = check_intr_schedule (ehci,
 821                                                         frame, uframe, qh,
 822                                                         &c_mask);
 823                                         if (status == 0)
 824                                                 break;
 825                                 }
 826                         }
 827
 828                 /* qh->period == 0 means every uframe */
 829                 } else {
 830                         frame = 0;
 831                         status = check_intr_schedule (ehci, 0, 0, qh, &c_mask);
 832                 }
 833                 if (status)
 834                         goto done;
 835                 qh->start = frame;
 836
 837                 /* reset S-frame and (maybe) C-frame masks */
 838                 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
 839                 hw->hw_info2 |= qh->period
 840                         ? cpu_to_hc32(ehci, 1 << uframe)
 841                         : cpu_to_hc32(ehci, QH_SMASK);
 842                 hw->hw_info2 |= c_mask;
 843         } else
 844                 ehci_dbg (ehci, "reused qh %p schedule\n", qh);
 845
 846         /* stuff into the periodic schedule */
 847         status = qh_link_periodic (ehci, qh);
 848 done:
 849         return status;
 850 }
 851
 852 static int intr_submit (
 853         struct ehci_hcd         *ehci,
 854         struct urb              *urb,
 855         struct list_head        *qtd_list,
 856         gfp_t                   mem_flags
 857 ) {
 858         unsigned                epnum;
 859         unsigned long           flags;
 860         struct ehci_qh          *qh;
 861         int                     status;
 862         struct list_head        empty;
 863
 864         /* get endpoint and transfer/schedule data */
 865         epnum = urb->ep->desc.bEndpointAddress;
 866
 867         spin_lock_irqsave (&ehci->lock, flags);
 868
 869         if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
 870                         &ehci_to_hcd(ehci)->flags))) {
 871                 status = -ESHUTDOWN;
 872                 goto done_not_linked;
 873         }
 874         status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
 875         if (unlikely(status))
 876                 goto done_not_linked;
 877
 878         /* get qh and force any scheduling errors */
 879         INIT_LIST_HEAD (&empty);
 880         qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
 881         if (qh == NULL) {
 882                 status = -ENOMEM;
 883                 goto done;
 884         }
 885         if (qh->qh_state == QH_STATE_IDLE) {
 886                 if ((status = qh_schedule (ehci, qh)) != 0)
 887                         goto done;
 888         }
 889
 890         /* then queue the urb's tds to the qh */
 891         qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
 892         BUG_ON (qh == NULL);
 893
 894         /* ... update usbfs periodic stats */
 895         ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
 896
 897 done:
 898         if (unlikely(status))
 899                 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
 900 done_not_linked:
 901         spin_unlock_irqrestore (&ehci->lock, flags);
 902         if (status)
 903                 qtd_list_free (ehci, urb, qtd_list);
 904
 905         return status;
 906 }
 907
 908 /*-------------------------------------------------------------------------*/
 909
 910 /* ehci_iso_stream ops work with both ITD and SITD */
 911
 912 static struct ehci_iso_stream *
 913 iso_stream_alloc (gfp_t mem_flags)
 914 {
 915         struct ehci_iso_stream *stream;
 916
 917         stream = kzalloc(sizeof *stream, mem_flags);
 918         if (likely (stream != NULL)) {
 919                 INIT_LIST_HEAD(&stream->td_list);
 920                 INIT_LIST_HEAD(&stream->free_list);
 921                 stream->next_uframe = -1;
 922                 stream->refcount = 1;
 923         }
 924         return stream;
 925 }
 926
 927 static void
 928 iso_stream_init (
 929         struct ehci_hcd         *ehci,
 930         struct ehci_iso_stream  *stream,
 931         struct usb_device       *dev,
 932         int                     pipe,
 933         unsigned                interval
 934 )
 935 {
 936         static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
 937
 938         u32                     buf1;
 939         unsigned                epnum, maxp;
 940         int                     is_input;
 941         long                    bandwidth;
 942
 943         /*
 944          * this might be a "high bandwidth" highspeed endpoint,
 945          * as encoded in the ep descriptor's wMaxPacket field
 946          */
 947         epnum = usb_pipeendpoint (pipe);
 948         is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
 949         maxp = usb_maxpacket(dev, pipe, !is_input);
 950         if (is_input) {
 951                 buf1 = (1 << 11);
 952         } else {
 953                 buf1 = 0;
 954         }
 955
 956         /* knows about ITD vs SITD */
 957         if (dev->speed == USB_SPEED_HIGH) {
 958                 unsigned multi = hb_mult(maxp);
 959
 960                 stream->highspeed = 1;
 961
 962                 maxp = max_packet(maxp);
 963                 buf1 |= maxp;
 964                 maxp *= multi;
 965
 966                 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
 967                 stream->buf1 = cpu_to_hc32(ehci, buf1);
 968                 stream->buf2 = cpu_to_hc32(ehci, multi);
 969
 970                 /* usbfs wants to report the average usecs per frame tied up
 971                  * when transfers on this endpoint are scheduled ...
 972                  */
 973                 stream->usecs = HS_USECS_ISO (maxp);
 974                 bandwidth = stream->usecs * 8;
 975                 bandwidth /= interval;
 976
 977         } else {
 978                 u32             addr;
 979                 int             think_time;
 980                 int             hs_transfers;
 981
 982                 addr = dev->ttport << 24;
 983                 if (!ehci_is_TDI(ehci)
 984                                 || (dev->tt->hub !=
 985                                         ehci_to_hcd(ehci)->self.root_hub))
 986                         addr |= dev->tt->hub->devnum << 16;
 987                 addr |= epnum << 8;
 988                 addr |= dev->devnum;
 989                 stream->usecs = HS_USECS_ISO (maxp);
 990                 think_time = dev->tt ? dev->tt->think_time : 0;
 991                 stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time (
 992                                 dev->speed, is_input, 1, maxp));
 993                 hs_transfers = max (1u, (maxp + 187) / 188);
 994                 if (is_input) {
 995                         u32     tmp;
 996
 997                         addr |= 1 << 31;
 998                         stream->c_usecs = stream->usecs;
 999                         stream->usecs = HS_USECS_ISO (1);
1000                         stream->raw_mask = 1;
1001
1002                         /* c-mask as specified in USB 2.0 11.18.4 3.c */
1003                         tmp = (1 << (hs_transfers + 2)) - 1;
1004                         stream->raw_mask |= tmp << (8 + 2);
1005                 } else
1006                         stream->raw_mask = smask_out [hs_transfers - 1];
1007                 bandwidth = stream->usecs + stream->c_usecs;
1008                 bandwidth /= interval << 3;
1009
1010                 /* stream->splits gets created from raw_mask later */
1011                 stream->address = cpu_to_hc32(ehci, addr);
1012         }
1013         stream->bandwidth = bandwidth;
1014
1015         stream->udev = dev;
1016
1017         stream->bEndpointAddress = is_input | epnum;
1018         stream->interval = interval;
1019         stream->maxp = maxp;
1020 }
1021
1022 static void
1023 iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
1024 {
1025         stream->refcount--;
1026
1027         /* free whenever just a dev->ep reference remains.
1028          * not like a QH -- no persistent state (toggle, halt)
1029          */
1030         if (stream->refcount == 1) {
1031                 int             is_in;
1032
1033                 // BUG_ON (!list_empty(&stream->td_list));
1034
1035                 while (!list_empty (&stream->free_list)) {
1036                         struct list_head        *entry;
1037
1038                         entry = stream->free_list.next;
1039                         list_del (entry);
1040
1041                         /* knows about ITD vs SITD */
1042                         if (stream->highspeed) {
1043                                 struct ehci_itd         *itd;
1044
1045                                 itd = list_entry (entry, struct ehci_itd,
1046                                                 itd_list);
1047                                 dma_pool_free (ehci->itd_pool, itd,
1048                                                 itd->itd_dma);
1049                         } else {
1050                                 struct ehci_sitd        *sitd;
1051
1052                                 sitd = list_entry (entry, struct ehci_sitd,
1053                                                 sitd_list);
1054                                 dma_pool_free (ehci->sitd_pool, sitd,
1055                                                 sitd->sitd_dma);
1056                         }
1057                 }
1058
1059                 is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0;
1060                 stream->bEndpointAddress &= 0x0f;
1061                 if (stream->ep)
1062                         stream->ep->hcpriv = NULL;
1063
1064                 if (stream->rescheduled) {
1065                         ehci_info (ehci, "ep%d%s-iso rescheduled "
1066                                 "%lu times in %lu seconds\n",
1067                                 stream->bEndpointAddress, is_in ? "in" : "out",
1068                                 stream->rescheduled,
1069                                 ((jiffies - stream->start)/HZ)
1070                                 );
1071                 }
1072
1073                 kfree(stream);
1074         }
1075 }
1076
1077 static inline struct ehci_iso_stream *
1078 iso_stream_get (struct ehci_iso_stream *stream)
1079 {
1080         if (likely (stream != NULL))
1081                 stream->refcount++;
1082         return stream;
1083 }
1084
1085 static struct ehci_iso_stream *
1086 iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
1087 {
1088         unsigned                epnum;
1089         struct ehci_iso_stream  *stream;
1090         struct usb_host_endpoint *ep;
1091         unsigned long           flags;
1092
1093         epnum = usb_pipeendpoint (urb->pipe);
1094         if (usb_pipein(urb->pipe))
1095                 ep = urb->dev->ep_in[epnum];
1096         else
1097                 ep = urb->dev->ep_out[epnum];
1098
1099         spin_lock_irqsave (&ehci->lock, flags);
1100         stream = ep->hcpriv;
1101
1102         if (unlikely (stream == NULL)) {
1103                 stream = iso_stream_alloc(GFP_ATOMIC);
1104                 if (likely (stream != NULL)) {
1105                         /* dev->ep owns the initial refcount */
1106                         ep->hcpriv = stream;
1107                         stream->ep = ep;
1108                         iso_stream_init(ehci, stream, urb->dev, urb->pipe,
1109                                         urb->interval);
1110                 }
1111
1112         /* if dev->ep [epnum] is a QH, info1.maxpacket is nonzero */
1113         } else if (unlikely (stream->hw_info1 != 0)) {
1114                 ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
1115                         urb->dev->devpath, epnum,
1116                         usb_pipein(urb->pipe) ? "in" : "out");
1117                 stream = NULL;
1118         }
1119
1120         /* caller guarantees an eventual matching iso_stream_put */
1121         stream = iso_stream_get (stream);
1122
1123         spin_unlock_irqrestore (&ehci->lock, flags);
1124         return stream;
1125 }
1126
1127 /*-------------------------------------------------------------------------*/
1128
1129 /* ehci_iso_sched ops can be ITD-only or SITD-only */
1130
1131 static struct ehci_iso_sched *
1132 iso_sched_alloc (unsigned packets, gfp_t mem_flags)
1133 {
1134         struct ehci_iso_sched   *iso_sched;
1135         int                     size = sizeof *iso_sched;
1136
1137         size += packets * sizeof (struct ehci_iso_packet);
1138         iso_sched = kzalloc(size, mem_flags);
1139         if (likely (iso_sched != NULL)) {
1140                 INIT_LIST_HEAD (&iso_sched->td_list);
1141         }
1142         return iso_sched;
1143 }
1144
1145 static inline void
1146 itd_sched_init(
1147         struct ehci_hcd         *ehci,
1148         struct ehci_iso_sched   *iso_sched,
1149         struct ehci_iso_stream  *stream,
1150         struct urb              *urb
1151 )
1152 {
1153         unsigned        i;
1154         dma_addr_t      dma = urb->transfer_dma;
1155
1156         /* how many uframes are needed for these transfers */
1157         iso_sched->span = urb->number_of_packets * stream->interval;
1158
1159         /* figure out per-uframe itd fields that we'll need later
1160          * when we fit new itds into the schedule.
1161          */
1162         for (i = 0; i < urb->number_of_packets; i++) {
1163                 struct ehci_iso_packet  *uframe = &iso_sched->packet [i];
1164                 unsigned                length;
1165                 dma_addr_t              buf;
1166                 u32                     trans;
1167
1168                 length = urb->iso_frame_desc [i].length;
1169                 buf = dma + urb->iso_frame_desc [i].offset;
1170
1171                 trans = EHCI_ISOC_ACTIVE;
1172                 trans |= buf & 0x0fff;
1173                 if (unlikely (((i + 1) == urb->number_of_packets))
1174                                 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1175                         trans |= EHCI_ITD_IOC;
1176                 trans |= length << 16;
1177                 uframe->transaction = cpu_to_hc32(ehci, trans);
1178
1179                 /* might need to cross a buffer page within a uframe */
1180                 uframe->bufp = (buf & ~(u64)0x0fff);
1181                 buf += length;
1182                 if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
1183                         uframe->cross = 1;
1184         }
1185 }
1186
1187 static void
1188 iso_sched_free (
1189         struct ehci_iso_stream  *stream,
1190         struct ehci_iso_sched   *iso_sched
1191 )
1192 {
1193         if (!iso_sched)
1194                 return;
1195         // caller must hold ehci->lock!
1196         list_splice (&iso_sched->td_list, &stream->free_list);
1197         kfree (iso_sched);
1198 }
1199
1200 static int
1201 itd_urb_transaction (
1202         struct ehci_iso_stream  *stream,
1203         struct ehci_hcd         *ehci,
1204         struct urb              *urb,
1205         gfp_t                   mem_flags
1206 )
1207 {
1208         struct ehci_itd         *itd;
1209         dma_addr_t              itd_dma;
1210         int                     i;
1211         unsigned                num_itds;
1212         struct ehci_iso_sched   *sched;
1213         unsigned long           flags;
1214
1215         sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1216         if (unlikely (sched == NULL))
1217                 return -ENOMEM;
1218
1219         itd_sched_init(ehci, sched, stream, urb);
1220
1221         if (urb->interval < 8)
1222                 num_itds = 1 + (sched->span + 7) / 8;
1223         else
1224                 num_itds = urb->number_of_packets;
1225
1226         /* allocate/init ITDs */
1227         spin_lock_irqsave (&ehci->lock, flags);
1228         for (i = 0; i < num_itds; i++) {
1229
1230                 /* free_list.next might be cache-hot ... but maybe
1231                  * the HC caches it too. avoid that issue for now.
1232                  */
1233
1234                 /* prefer previously-allocated itds */
1235                 if (likely (!list_empty(&stream->free_list))) {
1236                         itd = list_entry (stream->free_list.prev,
1237                                         struct ehci_itd, itd_list);
1238                         list_del (&itd->itd_list);
1239                         itd_dma = itd->itd_dma;
1240                 } else {
1241                         spin_unlock_irqrestore (&ehci->lock, flags);
1242                         itd = dma_pool_alloc (ehci->itd_pool, mem_flags,
1243                                         &itd_dma);
1244                         spin_lock_irqsave (&ehci->lock, flags);
1245                         if (!itd) {
1246                                 iso_sched_free(stream, sched);
1247                                 spin_unlock_irqrestore(&ehci->lock, flags);
1248                                 return -ENOMEM;
1249                         }
1250                 }
1251
1252                 memset (itd, 0, sizeof *itd);
1253                 itd->itd_dma = itd_dma;
1254                 list_add (&itd->itd_list, &sched->td_list);
1255         }
1256         spin_unlock_irqrestore (&ehci->lock, flags);
1257
1258         /* temporarily store schedule info in hcpriv */
1259         urb->hcpriv = sched;
1260         urb->error_count = 0;
1261         return 0;
1262 }
1263
1264 /*-------------------------------------------------------------------------*/
1265
1266 static inline int
1267 itd_slot_ok (
1268         struct ehci_hcd         *ehci,
1269         u32                     mod,
1270         u32                     uframe,
1271         u8                      usecs,
1272         u32                     period
1273 )
1274 {
1275         uframe %= period;
1276         do {
1277                 /* can't commit more than 80% periodic == 100 usec */
1278                 if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
1279                                 > (100 - usecs))
1280                         return 0;
1281
1282                 /* we know urb->interval is 2^N uframes */
1283                 uframe += period;
1284         } while (uframe < mod);
1285         return 1;
1286 }
1287
1288 static inline int
1289 sitd_slot_ok (
1290         struct ehci_hcd         *ehci,
1291         u32                     mod,
1292         struct ehci_iso_stream  *stream,
1293         u32                     uframe,
1294         struct ehci_iso_sched   *sched,
1295         u32                     period_uframes
1296 )
1297 {
1298         u32                     mask, tmp;
1299         u32                     frame, uf;
1300
1301         mask = stream->raw_mask << (uframe & 7);
1302
1303         /* for IN, don't wrap CSPLIT into the next frame */
1304         if (mask & ~0xffff)
1305                 return 0;
1306
1307         /* this multi-pass logic is simple, but performance may
1308          * suffer when the schedule data isn't cached.
1309          */
1310
1311         /* check bandwidth */
1312         uframe %= period_uframes;
1313         do {
1314                 u32             max_used;
1315
1316                 frame = uframe >> 3;
1317                 uf = uframe & 7;
1318
1319 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1320                 /* The tt's fullspeed bus bandwidth must be available.
1321                  * tt_available scheduling guarantees 10+% for control/bulk.
1322                  */
1323                 if (!tt_available (ehci, period_uframes << 3,
1324                                 stream->udev, frame, uf, stream->tt_usecs))
1325                         return 0;
1326 #else
1327                 /* tt must be idle for start(s), any gap, and csplit.
1328                  * assume scheduling slop leaves 10+% for control/bulk.
1329                  */
1330                 if (!tt_no_collision (ehci, period_uframes << 3,
1331                                 stream->udev, frame, mask))
1332                         return 0;
1333 #endif
1334
1335                 /* check starts (OUT uses more than one) */
1336                 max_used = 100 - stream->usecs;
1337                 for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) {
1338                         if (periodic_usecs (ehci, frame, uf) > max_used)
1339                                 return 0;
1340                 }
1341
1342                 /* for IN, check CSPLIT */
1343                 if (stream->c_usecs) {
1344                         uf = uframe & 7;
1345                         max_used = 100 - stream->c_usecs;
1346                         do {
1347                                 tmp = 1 << uf;
1348                                 tmp <<= 8;
1349                                 if ((stream->raw_mask & tmp) == 0)
1350                                         continue;
1351                                 if (periodic_usecs (ehci, frame, uf)
1352                                                 > max_used)
1353                                         return 0;
1354                         } while (++uf < 8);
1355                 }
1356
1357                 /* we know urb->interval is 2^N uframes */
1358                 uframe += period_uframes;
1359         } while (uframe < mod);
1360
1361         stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7));
1362         return 1;
1363 }
1364
1365 /*
1366  * This scheduler plans almost as far into the future as it has actual
1367  * periodic schedule slots.  (Affected by TUNE_FLS, which defaults to
1368  * "as small as possible" to be cache-friendlier.)  That limits the size
1369  * transfers you can stream reliably; avoid more than 64 msec per urb.
1370  * Also avoid queue depths of less than ehci's worst irq latency (affected
1371  * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1372  * and other factors); or more than about 230 msec total (for portability,
1373  * given EHCI_TUNE_FLS and the slop).  Or, write a smarter scheduler!
1374  */
1375
1376 #define SCHEDULE_SLOP   10      /* frames */
1377
1378 static int
1379 iso_stream_schedule (
1380         struct ehci_hcd         *ehci,
1381         struct urb              *urb,
1382         struct ehci_iso_stream  *stream
1383 )
1384 {
1385         u32                     now, start, max, period;
1386         int                     status;
1387         unsigned                mod = ehci->periodic_size << 3;
1388         struct ehci_iso_sched   *sched = urb->hcpriv;
1389
1390         if (sched->span > (mod - 8 * SCHEDULE_SLOP)) {
1391                 ehci_dbg (ehci, "iso request %p too long\n", urb);
1392                 status = -EFBIG;
1393                 goto fail;
1394         }
1395
1396         if ((stream->depth + sched->span) > mod) {
1397                 ehci_dbg (ehci, "request %p would overflow (%d+%d>%d)\n",
1398                         urb, stream->depth, sched->span, mod);
1399                 status = -EFBIG;
1400                 goto fail;
1401         }
1402
1403         now = ehci_readl(ehci, &ehci->regs->frame_index) % mod;
1404
1405         /* when's the last uframe this urb could start? */
1406         max = now + mod;
1407
1408         /* Typical case: reuse current schedule, stream is still active.
1409          * Hopefully there are no gaps from the host falling behind
1410          * (irq delays etc), but if there are we'll take the next
1411          * slot in the schedule, implicitly assuming URB_ISO_ASAP.
1412          */
1413         if (likely (!list_empty (&stream->td_list))) {
1414                 start = stream->next_uframe;
1415                 if (start < now)
1416                         start += mod;
1417
1418                 /* Fell behind (by up to twice the slop amount)? */
1419                 if (start >= max - 2 * 8 * SCHEDULE_SLOP)
1420                         start += stream->interval * DIV_ROUND_UP(
1421                                         max - start, stream->interval) - mod;
1422
1423                 /* Tried to schedule too far into the future? */
1424                 if (unlikely((start + sched->span) >= max)) {
1425                         status = -EFBIG;
1426                         goto fail;
1427                 }
1428                 goto ready;
1429         }
1430
1431         /* need to schedule; when's the next (u)frame we could start?
1432          * this is bigger than ehci->i_thresh allows; scheduling itself
1433          * isn't free, the slop should handle reasonably slow cpus.  it
1434          * can also help high bandwidth if the dma and irq loads don't
1435          * jump until after the queue is primed.
1436          */
1437         start = SCHEDULE_SLOP * 8 + (now & ~0x07);
1438         start %= mod;
1439         stream->next_uframe = start;
1440
1441         /* NOTE:  assumes URB_ISO_ASAP, to limit complexity/bugs */
1442
1443         period = urb->interval;
1444         if (!stream->highspeed)
1445                 period <<= 3;
1446
1447         /* find a uframe slot with enough bandwidth */
1448         for (; start < (stream->next_uframe + period); start++) {
1449                 int             enough_space;
1450
1451                 /* check schedule: enough space? */
1452                 if (stream->highspeed)
1453                         enough_space = itd_slot_ok (ehci, mod, start,
1454                                         stream->usecs, period);
1455                 else {
1456                         if ((start % 8) >= 6)
1457                                 continue;
1458                         enough_space = sitd_slot_ok (ehci, mod, stream,
1459                                         start, sched, period);
1460                 }
1461
1462                 /* schedule it here if there's enough bandwidth */
1463                 if (enough_space) {
1464                         stream->next_uframe = start % mod;
1465                         goto ready;
1466                 }
1467         }
1468
1469         /* no room in the schedule */
1470         ehci_dbg (ehci, "iso %ssched full %p (now %d max %d)\n",
1471                 list_empty (&stream->td_list) ? "" : "re",
1472                 urb, now, max);
1473         status = -ENOSPC;
1474
1475 fail:
1476         iso_sched_free (stream, sched);
1477         urb->hcpriv = NULL;
1478         return status;
1479
1480 ready:
1481         /* report high speed start in uframes; full speed, in frames */
1482         urb->start_frame = stream->next_uframe;
1483         if (!stream->highspeed)
1484                 urb->start_frame >>= 3;
1485         return 0;
1486 }
1487
1488 /*-------------------------------------------------------------------------*/
1489
1490 static inline void
1491 itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1492                 struct ehci_itd *itd)
1493 {
1494         int i;
1495
1496         /* it's been recently zeroed */
1497         itd->hw_next = EHCI_LIST_END(ehci);
1498         itd->hw_bufp [0] = stream->buf0;
1499         itd->hw_bufp [1] = stream->buf1;
1500         itd->hw_bufp [2] = stream->buf2;
1501
1502         for (i = 0; i < 8; i++)
1503                 itd->index[i] = -1;
1504
1505         /* All other fields are filled when scheduling */
1506 }
1507
1508 static inline void
1509 itd_patch(
1510         struct ehci_hcd         *ehci,
1511         struct ehci_itd         *itd,
1512         struct ehci_iso_sched   *iso_sched,
1513         unsigned                index,
1514         u16                     uframe
1515 )
1516 {
1517         struct ehci_iso_packet  *uf = &iso_sched->packet [index];
1518         unsigned                pg = itd->pg;
1519
1520         // BUG_ON (pg == 6 && uf->cross);
1521
1522         uframe &= 0x07;
1523         itd->index [uframe] = index;
1524
1525         itd->hw_transaction[uframe] = uf->transaction;
1526         itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1527         itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1528         itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
1529
1530         /* iso_frame_desc[].offset must be strictly increasing */
1531         if (unlikely (uf->cross)) {
1532                 u64     bufp = uf->bufp + 4096;
1533
1534                 itd->pg = ++pg;
1535                 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1536                 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
1537         }
1538 }
1539
1540 static inline void
1541 itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1542 {
1543         /* always prepend ITD/SITD ... only QH tree is order-sensitive */
1544         itd->itd_next = ehci->pshadow [frame];
1545         itd->hw_next = ehci->periodic [frame];
1546         ehci->pshadow [frame].itd = itd;
1547         itd->frame = frame;
1548         wmb ();
1549         ehci->periodic[frame] = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
1550 }
1551
1552 /* fit urb's itds into the selected schedule slot; activate as needed */
1553 static int
1554 itd_link_urb (
1555         struct ehci_hcd         *ehci,
1556         struct urb              *urb,
1557         unsigned                mod,
1558         struct ehci_iso_stream  *stream
1559 )
1560 {
1561         int                     packet;
1562         unsigned                next_uframe, uframe, frame;
1563         struct ehci_iso_sched   *iso_sched = urb->hcpriv;
1564         struct ehci_itd         *itd;
1565
1566         next_uframe = stream->next_uframe % mod;
1567
1568         if (unlikely (list_empty(&stream->td_list))) {
1569                 ehci_to_hcd(ehci)->self.bandwidth_allocated
1570                                 += stream->bandwidth;
1571                 ehci_vdbg (ehci,
1572                         "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1573                         urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1574                         (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1575                         urb->interval,
1576                         next_uframe >> 3, next_uframe & 0x7);
1577                 stream->start = jiffies;
1578         }
1579         ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1580
1581         /* fill iTDs uframe by uframe */
1582         for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) {
1583                 if (itd == NULL) {
1584                         /* ASSERT:  we have all necessary itds */
1585                         // BUG_ON (list_empty (&iso_sched->td_list));
1586
1587                         /* ASSERT:  no itds for this endpoint in this uframe */
1588
1589                         itd = list_entry (iso_sched->td_list.next,
1590                                         struct ehci_itd, itd_list);
1591                         list_move_tail (&itd->itd_list, &stream->td_list);
1592                         itd->stream = iso_stream_get (stream);
1593                         itd->urb = urb;
1594                         itd_init (ehci, stream, itd);
1595                 }
1596
1597                 uframe = next_uframe & 0x07;
1598                 frame = next_uframe >> 3;
1599
1600                 itd_patch(ehci, itd, iso_sched, packet, uframe);
1601
1602                 next_uframe += stream->interval;
1603                 stream->depth += stream->interval;
1604                 next_uframe %= mod;
1605                 packet++;
1606
1607                 /* link completed itds into the schedule */
1608                 if (((next_uframe >> 3) != frame)
1609                                 || packet == urb->number_of_packets) {
1610                         itd_link (ehci, frame % ehci->periodic_size, itd);
1611                         itd = NULL;
1612                 }
1613         }
1614         stream->next_uframe = next_uframe;
1615
1616         /* don't need that schedule data any more */
1617         iso_sched_free (stream, iso_sched);
1618         urb->hcpriv = NULL;
1619
1620         timer_action (ehci, TIMER_IO_WATCHDOG);
1621         return enable_periodic(ehci);
1622 }
1623
1624 #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1625
1626 /* Process and recycle a completed ITD.  Return true iff its urb completed,
1627  * and hence its completion callback probably added things to the hardware
1628  * schedule.
1629  *
1630  * Note that we carefully avoid recycling this descriptor until after any
1631  * completion callback runs, so that it won't be reused quickly.  That is,
1632  * assuming (a) no more than two urbs per frame on this endpoint, and also
1633  * (b) only this endpoint's completions submit URBs.  It seems some silicon
1634  * corrupts things if you reuse completed descriptors very quickly...
1635  */
1636 static unsigned
1637 itd_complete (
1638         struct ehci_hcd *ehci,
1639         struct ehci_itd *itd
1640 ) {
1641         struct urb                              *urb = itd->urb;
1642         struct usb_iso_packet_descriptor        *desc;
1643         u32                                     t;
1644         unsigned                                uframe;
1645         int                                     urb_index = -1;
1646         struct ehci_iso_stream                  *stream = itd->stream;
1647         struct usb_device                       *dev;
1648         unsigned                                retval = false;
1649
1650         /* for each uframe with a packet */
1651         for (uframe = 0; uframe < 8; uframe++) {
1652                 if (likely (itd->index[uframe] == -1))
1653                         continue;
1654                 urb_index = itd->index[uframe];
1655                 desc = &urb->iso_frame_desc [urb_index];
1656
1657                 t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]);
1658                 itd->hw_transaction [uframe] = 0;
1659                 stream->depth -= stream->interval;
1660
1661                 /* report transfer status */
1662                 if (unlikely (t & ISO_ERRS)) {
1663                         urb->error_count++;
1664                         if (t & EHCI_ISOC_BUF_ERR)
1665                                 desc->status = usb_pipein (urb->pipe)
1666                                         ? -ENOSR  /* hc couldn't read */
1667                                         : -ECOMM; /* hc couldn't write */
1668                         else if (t & EHCI_ISOC_BABBLE)
1669                                 desc->status = -EOVERFLOW;
1670                         else /* (t & EHCI_ISOC_XACTERR) */
1671                                 desc->status = -EPROTO;
1672
1673                         /* HC need not update length with this error */
1674                         if (!(t & EHCI_ISOC_BABBLE)) {
1675                                 desc->actual_length = EHCI_ITD_LENGTH(t);
1676                                 urb->actual_length += desc->actual_length;
1677                         }
1678                 } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) {
1679                         desc->status = 0;
1680                         desc->actual_length = EHCI_ITD_LENGTH(t);
1681                         urb->actual_length += desc->actual_length;
1682                 } else {
1683                         /* URB was too late */
1684                         desc->status = -EXDEV;
1685                 }
1686         }
1687
1688         /* handle completion now? */
1689         if (likely ((urb_index + 1) != urb->number_of_packets))
1690                 goto done;
1691
1692         /* ASSERT: it's really the last itd for this urb
1693         list_for_each_entry (itd, &stream->td_list, itd_list)
1694                 BUG_ON (itd->urb == urb);
1695          */
1696
1697         /* give urb back to the driver; completion often (re)submits */
1698         dev = urb->dev;
1699         ehci_urb_done(ehci, urb, 0);
1700         retval = true;
1701         urb = NULL;
1702         (void) disable_periodic(ehci);
1703         ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1704
1705         if (unlikely(list_is_singular(&stream->td_list))) {
1706                 ehci_to_hcd(ehci)->self.bandwidth_allocated
1707                                 -= stream->bandwidth;
1708                 ehci_vdbg (ehci,
1709                         "deschedule devp %s ep%d%s-iso\n",
1710                         dev->devpath, stream->bEndpointAddress & 0x0f,
1711                         (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1712         }
1713         iso_stream_put (ehci, stream);
1714
1715 done:
1716         itd->urb = NULL;
1717         if (ehci->clock_frame != itd->frame || itd->index[7] != -1) {
1718                 /* OK to recycle this ITD now. */
1719                 itd->stream = NULL;
1720                 list_move(&itd->itd_list, &stream->free_list);
1721                 iso_stream_put(ehci, stream);
1722         } else {
1723                 /* HW might remember this ITD, so we can't recycle it yet.
1724                  * Move it to a safe place until a new frame starts.
1725                  */
1726                 list_move(&itd->itd_list, &ehci->cached_itd_list);
1727                 if (stream->refcount == 2) {
1728                         /* If iso_stream_put() were called here, stream
1729                          * would be freed.  Instead, just prevent reuse.
1730                          */
1731                         stream->ep->hcpriv = NULL;
1732                         stream->ep = NULL;
1733                 }
1734         }
1735         return retval;
1736 }
1737
1738 /*-------------------------------------------------------------------------*/
1739
1740 static int itd_submit (struct ehci_hcd *ehci, struct urb *urb,
1741         gfp_t mem_flags)
1742 {
1743         int                     status = -EINVAL;
1744         unsigned long           flags;
1745         struct ehci_iso_stream  *stream;
1746
1747         /* Get iso_stream head */
1748         stream = iso_stream_find (ehci, urb);
1749         if (unlikely (stream == NULL)) {
1750                 ehci_dbg (ehci, "can't get iso stream\n");
1751                 return -ENOMEM;
1752         }
1753         if (unlikely (urb->interval != stream->interval)) {
1754                 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1755                         stream->interval, urb->interval);
1756                 goto done;
1757         }
1758
1759 #ifdef EHCI_URB_TRACE
1760         ehci_dbg (ehci,
1761                 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1762                 __func__, urb->dev->devpath, urb,
1763                 usb_pipeendpoint (urb->pipe),
1764                 usb_pipein (urb->pipe) ? "in" : "out",
1765                 urb->transfer_buffer_length,
1766                 urb->number_of_packets, urb->interval,
1767                 stream);
1768 #endif
1769
1770         /* allocate ITDs w/o locking anything */
1771         status = itd_urb_transaction (stream, ehci, urb, mem_flags);
1772         if (unlikely (status < 0)) {
1773                 ehci_dbg (ehci, "can't init itds\n");
1774                 goto done;
1775         }
1776
1777         /* schedule ... need to lock */
1778         spin_lock_irqsave (&ehci->lock, flags);
1779         if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
1780                                &ehci_to_hcd(ehci)->flags))) {
1781                 status = -ESHUTDOWN;
1782                 goto done_not_linked;
1783         }
1784         status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1785         if (unlikely(status))
1786                 goto done_not_linked;
1787         status = iso_stream_schedule(ehci, urb, stream);
1788         if (likely (status == 0))
1789                 itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1790         else
1791                 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1792 done_not_linked:
1793         spin_unlock_irqrestore (&ehci->lock, flags);
1794
1795 done:
1796         if (unlikely (status < 0))
1797                 iso_stream_put (ehci, stream);
1798         return status;
1799 }
1800
1801 /*-------------------------------------------------------------------------*/
1802
1803 /*
1804  * "Split ISO TDs" ... used for USB 1.1 devices going through the
1805  * TTs in USB 2.0 hubs.  These need microframe scheduling.
1806  */
1807
1808 static inline void
1809 sitd_sched_init(
1810         struct ehci_hcd         *ehci,
1811         struct ehci_iso_sched   *iso_sched,
1812         struct ehci_iso_stream  *stream,
1813         struct urb              *urb
1814 )
1815 {
1816         unsigned        i;
1817         dma_addr_t      dma = urb->transfer_dma;
1818
1819         /* how many frames are needed for these transfers */
1820         iso_sched->span = urb->number_of_packets * stream->interval;
1821
1822         /* figure out per-frame sitd fields that we'll need later
1823          * when we fit new sitds into the schedule.
1824          */
1825         for (i = 0; i < urb->number_of_packets; i++) {
1826                 struct ehci_iso_packet  *packet = &iso_sched->packet [i];
1827                 unsigned                length;
1828                 dma_addr_t              buf;
1829                 u32                     trans;
1830
1831                 length = urb->iso_frame_desc [i].length & 0x03ff;
1832                 buf = dma + urb->iso_frame_desc [i].offset;
1833
1834                 trans = SITD_STS_ACTIVE;
1835                 if (((i + 1) == urb->number_of_packets)
1836                                 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1837                         trans |= SITD_IOC;
1838                 trans |= length << 16;
1839                 packet->transaction = cpu_to_hc32(ehci, trans);
1840
1841                 /* might need to cross a buffer page within a td */
1842                 packet->bufp = buf;
1843                 packet->buf1 = (buf + length) & ~0x0fff;
1844                 if (packet->buf1 != (buf & ~(u64)0x0fff))
1845                         packet->cross = 1;
1846
1847                 /* OUT uses multiple start-splits */
1848                 if (stream->bEndpointAddress & USB_DIR_IN)
1849                         continue;
1850                 length = (length + 187) / 188;
1851                 if (length > 1) /* BEGIN vs ALL */
1852                         length |= 1 << 3;
1853                 packet->buf1 |= length;
1854         }
1855 }
1856
1857 static int
1858 sitd_urb_transaction (
1859         struct ehci_iso_stream  *stream,
1860         struct ehci_hcd         *ehci,
1861         struct urb              *urb,
1862         gfp_t                   mem_flags
1863 )
1864 {
1865         struct ehci_sitd        *sitd;
1866         dma_addr_t              sitd_dma;
1867         int                     i;
1868         struct ehci_iso_sched   *iso_sched;
1869         unsigned long           flags;
1870
1871         iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1872         if (iso_sched == NULL)
1873                 return -ENOMEM;
1874
1875         sitd_sched_init(ehci, iso_sched, stream, urb);
1876
1877         /* allocate/init sITDs */
1878         spin_lock_irqsave (&ehci->lock, flags);
1879         for (i = 0; i < urb->number_of_packets; i++) {
1880
1881                 /* NOTE:  for now, we don't try to handle wraparound cases
1882                  * for IN (using sitd->hw_backpointer, like a FSTN), which
1883                  * means we never need two sitds for full speed packets.
1884                  */
1885
1886                 /* free_list.next might be cache-hot ... but maybe
1887                  * the HC caches it too. avoid that issue for now.
1888                  */
1889
1890                 /* prefer previously-allocated sitds */
1891                 if (!list_empty(&stream->free_list)) {
1892                         sitd = list_entry (stream->free_list.prev,
1893                                          struct ehci_sitd, sitd_list);
1894                         list_del (&sitd->sitd_list);
1895                         sitd_dma = sitd->sitd_dma;
1896                 } else {
1897                         spin_unlock_irqrestore (&ehci->lock, flags);
1898                         sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags,
1899                                         &sitd_dma);
1900                         spin_lock_irqsave (&ehci->lock, flags);
1901                         if (!sitd) {
1902                                 iso_sched_free(stream, iso_sched);
1903                                 spin_unlock_irqrestore(&ehci->lock, flags);
1904                                 return -ENOMEM;
1905                         }
1906                 }
1907
1908                 memset (sitd, 0, sizeof *sitd);
1909                 sitd->sitd_dma = sitd_dma;
1910                 list_add (&sitd->sitd_list, &iso_sched->td_list);
1911         }
1912
1913         /* temporarily store schedule info in hcpriv */
1914         urb->hcpriv = iso_sched;
1915         urb->error_count = 0;
1916
1917         spin_unlock_irqrestore (&ehci->lock, flags);
1918         return 0;
1919 }
1920
1921 /*-------------------------------------------------------------------------*/
1922
1923 static inline void
1924 sitd_patch(
1925         struct ehci_hcd         *ehci,
1926         struct ehci_iso_stream  *stream,
1927         struct ehci_sitd        *sitd,
1928         struct ehci_iso_sched   *iso_sched,
1929         unsigned                index
1930 )
1931 {
1932         struct ehci_iso_packet  *uf = &iso_sched->packet [index];
1933         u64                     bufp = uf->bufp;
1934
1935         sitd->hw_next = EHCI_LIST_END(ehci);
1936         sitd->hw_fullspeed_ep = stream->address;
1937         sitd->hw_uframe = stream->splits;
1938         sitd->hw_results = uf->transaction;
1939         sitd->hw_backpointer = EHCI_LIST_END(ehci);
1940
1941         bufp = uf->bufp;
1942         sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
1943         sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
1944
1945         sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
1946         if (uf->cross)
1947                 bufp += 4096;
1948         sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
1949         sitd->index = index;
1950 }
1951
1952 static inline void
1953 sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
1954 {
1955         /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
1956         sitd->sitd_next = ehci->pshadow [frame];
1957         sitd->hw_next = ehci->periodic [frame];
1958         ehci->pshadow [frame].sitd = sitd;
1959         sitd->frame = frame;
1960         wmb ();
1961         ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
1962 }
1963
1964 /* fit urb's sitds into the selected schedule slot; activate as needed */
1965 static int
1966 sitd_link_urb (
1967         struct ehci_hcd         *ehci,
1968         struct urb              *urb,
1969         unsigned                mod,
1970         struct ehci_iso_stream  *stream
1971 )
1972 {
1973         int                     packet;
1974         unsigned                next_uframe;
1975         struct ehci_iso_sched   *sched = urb->hcpriv;
1976         struct ehci_sitd        *sitd;
1977
1978         next_uframe = stream->next_uframe;
1979
1980         if (list_empty(&stream->td_list)) {
1981                 /* usbfs ignores TT bandwidth */
1982                 ehci_to_hcd(ehci)->self.bandwidth_allocated
1983                                 += stream->bandwidth;
1984                 ehci_vdbg (ehci,
1985                         "sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
1986                         urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1987                         (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1988                         (next_uframe >> 3) % ehci->periodic_size,
1989                         stream->interval, hc32_to_cpu(ehci, stream->splits));
1990                 stream->start = jiffies;
1991         }
1992         ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1993
1994         /* fill sITDs frame by frame */
1995         for (packet = 0, sitd = NULL;
1996                         packet < urb->number_of_packets;
1997                         packet++) {
1998
1999                 /* ASSERT:  we have all necessary sitds */
2000                 BUG_ON (list_empty (&sched->td_list));
2001
2002                 /* ASSERT:  no itds for this endpoint in this frame */
2003
2004                 sitd = list_entry (sched->td_list.next,
2005                                 struct ehci_sitd, sitd_list);
2006                 list_move_tail (&sitd->sitd_list, &stream->td_list);
2007                 sitd->stream = iso_stream_get (stream);
2008                 sitd->urb = urb;
2009
2010                 sitd_patch(ehci, stream, sitd, sched, packet);
2011                 sitd_link (ehci, (next_uframe >> 3) % ehci->periodic_size,
2012                                 sitd);
2013
2014                 next_uframe += stream->interval << 3;
2015                 stream->depth += stream->interval << 3;
2016         }
2017         stream->next_uframe = next_uframe % mod;
2018
2019         /* don't need that schedule data any more */
2020         iso_sched_free (stream, sched);
2021         urb->hcpriv = NULL;
2022
2023         timer_action (ehci, TIMER_IO_WATCHDOG);
2024         return enable_periodic(ehci);
2025 }
2026
2027 /*-------------------------------------------------------------------------*/
2028
2029 #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
2030                                 | SITD_STS_XACT | SITD_STS_MMF)
2031
2032 /* Process and recycle a completed SITD.  Return true iff its urb completed,
2033  * and hence its completion callback probably added things to the hardware
2034  * schedule.
2035  *
2036  * Note that we carefully avoid recycling this descriptor until after any
2037  * completion callback runs, so that it won't be reused quickly.  That is,
2038  * assuming (a) no more than two urbs per frame on this endpoint, and also
2039  * (b) only this endpoint's completions submit URBs.  It seems some silicon
2040  * corrupts things if you reuse completed descriptors very quickly...
2041  */
2042 static unsigned
2043 sitd_complete (
2044         struct ehci_hcd         *ehci,
2045         struct ehci_sitd        *sitd
2046 ) {
2047         struct urb                              *urb = sitd->urb;
2048         struct usb_iso_packet_descriptor        *desc;
2049         u32                                     t;
2050         int                                     urb_index = -1;
2051         struct ehci_iso_stream                  *stream = sitd->stream;
2052         struct usb_device                       *dev;
2053         unsigned                                retval = false;
2054
2055         urb_index = sitd->index;
2056         desc = &urb->iso_frame_desc [urb_index];
2057         t = hc32_to_cpup(ehci, &sitd->hw_results);
2058
2059         /* report transfer status */
2060         if (t & SITD_ERRS) {
2061                 urb->error_count++;
2062                 if (t & SITD_STS_DBE)
2063                         desc->status = usb_pipein (urb->pipe)
2064                                 ? -ENOSR  /* hc couldn't read */
2065                                 : -ECOMM; /* hc couldn't write */
2066                 else if (t & SITD_STS_BABBLE)
2067                         desc->status = -EOVERFLOW;
2068                 else /* XACT, MMF, etc */
2069                         desc->status = -EPROTO;
2070         } else {
2071                 desc->status = 0;
2072                 desc->actual_length = desc->length - SITD_LENGTH(t);
2073                 urb->actual_length += desc->actual_length;
2074         }
2075         stream->depth -= stream->interval << 3;
2076
2077         /* handle completion now? */
2078         if ((urb_index + 1) != urb->number_of_packets)
2079                 goto done;
2080
2081         /* ASSERT: it's really the last sitd for this urb
2082         list_for_each_entry (sitd, &stream->td_list, sitd_list)
2083                 BUG_ON (sitd->urb == urb);
2084          */
2085
2086         /* give urb back to the driver; completion often (re)submits */
2087         dev = urb->dev;
2088         ehci_urb_done(ehci, urb, 0);
2089         retval = true;
2090         urb = NULL;
2091         (void) disable_periodic(ehci);
2092         ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
2093
2094         if (list_is_singular(&stream->td_list)) {
2095                 ehci_to_hcd(ehci)->self.bandwidth_allocated
2096                                 -= stream->bandwidth;
2097                 ehci_vdbg (ehci,
2098                         "deschedule devp %s ep%d%s-iso\n",
2099                         dev->devpath, stream->bEndpointAddress & 0x0f,
2100                         (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
2101         }
2102         iso_stream_put (ehci, stream);
2103         /* OK to recycle this SITD now that its completion callback ran. */
2104 done:
2105         sitd->urb = NULL;
2106         sitd->stream = NULL;
2107         list_move(&sitd->sitd_list, &stream->free_list);
2108         iso_stream_put(ehci, stream);
2109
2110         return retval;
2111 }
2112
2113
2114 static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
2115         gfp_t mem_flags)
2116 {
2117         int                     status = -EINVAL;
2118         unsigned long           flags;
2119         struct ehci_iso_stream  *stream;
2120
2121         /* Get iso_stream head */
2122         stream = iso_stream_find (ehci, urb);
2123         if (stream == NULL) {
2124                 ehci_dbg (ehci, "can't get iso stream\n");
2125                 return -ENOMEM;
2126         }
2127         if (urb->interval != stream->interval) {
2128                 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
2129                         stream->interval, urb->interval);
2130                 goto done;
2131         }
2132
2133 #ifdef EHCI_URB_TRACE
2134         ehci_dbg (ehci,
2135                 "submit %p dev%s ep%d%s-iso len %d\n",
2136                 urb, urb->dev->devpath,
2137                 usb_pipeendpoint (urb->pipe),
2138                 usb_pipein (urb->pipe) ? "in" : "out",
2139                 urb->transfer_buffer_length);
2140 #endif
2141
2142         /* allocate SITDs */
2143         status = sitd_urb_transaction (stream, ehci, urb, mem_flags);
2144         if (status < 0) {
2145                 ehci_dbg (ehci, "can't init sitds\n");
2146                 goto done;
2147         }
2148
2149         /* schedule ... need to lock */
2150         spin_lock_irqsave (&ehci->lock, flags);
2151         if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE,
2152                                &ehci_to_hcd(ehci)->flags))) {
2153                 status = -ESHUTDOWN;
2154                 goto done_not_linked;
2155         }
2156         status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2157         if (unlikely(status))
2158                 goto done_not_linked;
2159         status = iso_stream_schedule(ehci, urb, stream);
2160         if (status == 0)
2161                 sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
2162         else
2163                 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
2164 done_not_linked:
2165         spin_unlock_irqrestore (&ehci->lock, flags);
2166
2167 done:
2168         if (status < 0)
2169                 iso_stream_put (ehci, stream);
2170         return status;
2171 }
2172
2173 /*-------------------------------------------------------------------------*/
2174
2175 static void free_cached_itd_list(struct ehci_hcd *ehci)
2176 {
2177         struct ehci_itd *itd, *n;
2178
2179         list_for_each_entry_safe(itd, n, &ehci->cached_itd_list, itd_list) {
2180                 struct ehci_iso_stream  *stream = itd->stream;
2181                 itd->stream = NULL;
2182                 list_move(&itd->itd_list, &stream->free_list);
2183                 iso_stream_put(ehci, stream);
2184         }
2185 }
2186
2187 /*-------------------------------------------------------------------------*/
2188
2189 static void
2190 scan_periodic (struct ehci_hcd *ehci)
2191 {
2192         unsigned        now_uframe, frame, clock, clock_frame, mod;
2193         unsigned        modified;
2194
2195         mod = ehci->periodic_size << 3;
2196
2197         /*
2198          * When running, scan from last scan point up to "now"
2199          * else clean up by scanning everything that's left.
2200          * Touches as few pages as possible:  cache-friendly.
2201          */
2202         now_uframe = ehci->next_uframe;
2203         if (HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2204                 clock = ehci_readl(ehci, &ehci->regs->frame_index);
2205                 clock_frame = (clock >> 3) % ehci->periodic_size;
2206         } else  {
2207                 clock = now_uframe + mod - 1;
2208                 clock_frame = -1;
2209         }
2210         if (ehci->clock_frame != clock_frame) {
2211                 free_cached_itd_list(ehci);
2212                 ehci->clock_frame = clock_frame;
2213         }
2214         clock %= mod;
2215         clock_frame = clock >> 3;
2216
2217         for (;;) {
2218                 union ehci_shadow       q, *q_p;
2219                 __hc32                  type, *hw_p;
2220                 unsigned                incomplete = false;
2221
2222                 frame = now_uframe >> 3;
2223
2224 restart:
2225                 /* scan each element in frame's queue for completions */
2226                 q_p = &ehci->pshadow [frame];
2227                 hw_p = &ehci->periodic [frame];
2228                 q.ptr = q_p->ptr;
2229                 type = Q_NEXT_TYPE(ehci, *hw_p);
2230                 modified = 0;
2231
2232                 while (q.ptr != NULL) {
2233                         unsigned                uf;
2234                         union ehci_shadow       temp;
2235                         int                     live;
2236
2237                         live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state);
2238                         switch (hc32_to_cpu(ehci, type)) {
2239                         case Q_TYPE_QH:
2240                                 /* handle any completions */
2241                                 temp.qh = qh_get (q.qh);
2242                                 type = Q_NEXT_TYPE(ehci, q.qh->hw->hw_next);
2243                                 q = q.qh->qh_next;
2244                                 modified = qh_completions (ehci, temp.qh);
2245                                 if (unlikely(list_empty(&temp.qh->qtd_list) ||
2246                                                 temp.qh->needs_rescan))
2247                                         intr_deschedule (ehci, temp.qh);
2248                                 qh_put (temp.qh);
2249                                 break;
2250                         case Q_TYPE_FSTN:
2251                                 /* for "save place" FSTNs, look at QH entries
2252                                  * in the previous frame for completions.
2253                                  */
2254                                 if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) {
2255                                         dbg ("ignoring completions from FSTNs");
2256                                 }
2257                                 type = Q_NEXT_TYPE(ehci, q.fstn->hw_next);
2258                                 q = q.fstn->fstn_next;
2259                                 break;
2260                         case Q_TYPE_ITD:
2261                                 /* If this ITD is still active, leave it for
2262                                  * later processing ... check the next entry.
2263                                  * No need to check for activity unless the
2264                                  * frame is current.
2265                                  */
2266                                 if (frame == clock_frame && live) {
2267                                         rmb();
2268                                         for (uf = 0; uf < 8; uf++) {
2269                                                 if (q.itd->hw_transaction[uf] &
2270                                                             ITD_ACTIVE(ehci))
2271                                                         break;
2272                                         }
2273                                         if (uf < 8) {
2274                                                 incomplete = true;
2275                                                 q_p = &q.itd->itd_next;
2276                                                 hw_p = &q.itd->hw_next;
2277                                                 type = Q_NEXT_TYPE(ehci,
2278                                                         q.itd->hw_next);
2279                                                 q = *q_p;
2280                                                 break;
2281                                         }
2282                                 }
2283
2284                                 /* Take finished ITDs out of the schedule
2285                                  * and process them:  recycle, maybe report
2286                                  * URB completion.  HC won't cache the
2287                                  * pointer for much longer, if at all.
2288                                  */
2289                                 *q_p = q.itd->itd_next;
2290                                 *hw_p = q.itd->hw_next;
2291                                 type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2292                                 wmb();
2293                                 modified = itd_complete (ehci, q.itd);
2294                                 q = *q_p;
2295                                 break;
2296                         case Q_TYPE_SITD:
2297                                 /* If this SITD is still active, leave it for
2298                                  * later processing ... check the next entry.
2299                                  * No need to check for activity unless the
2300                                  * frame is current.
2301                                  */
2302                                 if (frame == clock_frame && live &&
2303                                                 (q.sitd->hw_results &
2304                                                         SITD_ACTIVE(ehci))) {
2305                                         incomplete = true;
2306                                         q_p = &q.sitd->sitd_next;
2307                                         hw_p = &q.sitd->hw_next;
2308                                         type = Q_NEXT_TYPE(ehci,
2309                                                         q.sitd->hw_next);
2310                                         q = *q_p;
2311                                         break;
2312                                 }
2313
2314                                 /* Take finished SITDs out of the schedule
2315                                  * and process them:  recycle, maybe report
2316                                  * URB completion.
2317                                  */
2318                                 *q_p = q.sitd->sitd_next;
2319                                 *hw_p = q.sitd->hw_next;
2320                                 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2321                                 wmb();
2322                                 modified = sitd_complete (ehci, q.sitd);
2323                                 q = *q_p;
2324                                 break;
2325                         default:
2326                                 dbg ("corrupt type %d frame %d shadow %p",
2327                                         type, frame, q.ptr);
2328                                 // BUG ();
2329                                 q.ptr = NULL;
2330                         }
2331
2332                         /* assume completion callbacks modify the queue */
2333                         if (unlikely (modified)) {
2334                                 if (likely(ehci->periodic_sched > 0))
2335                                         goto restart;
2336                                 /* short-circuit this scan */
2337                                 now_uframe = clock;
2338                                 break;
2339                         }
2340                 }
2341
2342                 /* If we can tell we caught up to the hardware, stop now.
2343                  * We can't advance our scan without collecting the ISO
2344                  * transfers that are still pending in this frame.
2345                  */
2346                 if (incomplete && HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2347                         ehci->next_uframe = now_uframe;
2348                         break;
2349                 }
2350
2351                 // FIXME:  this assumes we won't get lapped when
2352                 // latencies climb; that should be rare, but...
2353                 // detect it, and just go all the way around.
2354                 // FLR might help detect this case, so long as latencies
2355                 // don't exceed periodic_size msec (default 1.024 sec).
2356
2357                 // FIXME:  likewise assumes HC doesn't halt mid-scan
2358
2359                 if (now_uframe == clock) {
2360                         unsigned        now;
2361
2362                         if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state)
2363                                         || ehci->periodic_sched == 0)
2364                                 break;
2365                         ehci->next_uframe = now_uframe;
2366                         now = ehci_readl(ehci, &ehci->regs->frame_index) % mod;
2367                         if (now_uframe == now)
2368                                 break;
2369
2370                         /* rescan the rest of this frame, then ... */
2371                         clock = now;
2372                         clock_frame = clock >> 3;
2373                         if (ehci->clock_frame != clock_frame) {
2374                                 free_cached_itd_list(ehci);
2375                                 ehci->clock_frame = clock_frame;
2376                         }
2377                 } else {
2378                         now_uframe++;
2379                         now_uframe %= mod;
2380                 }
2381         }
2382 }