usb: xhci: exit early in xhci_setup_device() if we're halted or dying
[pandora-kernel.git] / drivers / usb / host / xhci-ring.c
1 /*
2  * xHCI host controller driver
3  *
4  * Copyright (C) 2008 Intel Corp.
5  *
6  * Author: Sarah Sharp
7  * Some code borrowed from the Linux EHCI driver.
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License version 2 as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
16  * for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software Foundation,
20  * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21  */
22
23 /*
24  * Ring initialization rules:
25  * 1. Each segment is initialized to zero, except for link TRBs.
26  * 2. Ring cycle state = 0.  This represents Producer Cycle State (PCS) or
27  *    Consumer Cycle State (CCS), depending on ring function.
28  * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment.
29  *
30  * Ring behavior rules:
31  * 1. A ring is empty if enqueue == dequeue.  This means there will always be at
32  *    least one free TRB in the ring.  This is useful if you want to turn that
33  *    into a link TRB and expand the ring.
34  * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a
35  *    link TRB, then load the pointer with the address in the link TRB.  If the
36  *    link TRB had its toggle bit set, you may need to update the ring cycle
37  *    state (see cycle bit rules).  You may have to do this multiple times
38  *    until you reach a non-link TRB.
39  * 3. A ring is full if enqueue++ (for the definition of increment above)
40  *    equals the dequeue pointer.
41  *
42  * Cycle bit rules:
43  * 1. When a consumer increments a dequeue pointer and encounters a toggle bit
44  *    in a link TRB, it must toggle the ring cycle state.
45  * 2. When a producer increments an enqueue pointer and encounters a toggle bit
46  *    in a link TRB, it must toggle the ring cycle state.
47  *
48  * Producer rules:
49  * 1. Check if ring is full before you enqueue.
50  * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing.
51  *    Update enqueue pointer between each write (which may update the ring
52  *    cycle state).
53  * 3. Notify consumer.  If SW is producer, it rings the doorbell for command
54  *    and endpoint rings.  If HC is the producer for the event ring,
55  *    and it generates an interrupt according to interrupt modulation rules.
56  *
57  * Consumer rules:
58  * 1. Check if TRB belongs to you.  If the cycle bit == your ring cycle state,
59  *    the TRB is owned by the consumer.
60  * 2. Update dequeue pointer (which may update the ring cycle state) and
61  *    continue processing TRBs until you reach a TRB which is not owned by you.
62  * 3. Notify the producer.  SW is the consumer for the event ring, and it
63  *   updates event ring dequeue pointer.  HC is the consumer for the command and
64  *   endpoint rings; it generates events on the event ring for these.
65  */
66
67 #include <linux/scatterlist.h>
68 #include <linux/slab.h>
69 #include "xhci.h"
70 #include "xhci-trace.h"
71
72 /*
73  * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
74  * address of the TRB.
75  */
76 dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg,
77                 union xhci_trb *trb)
78 {
79         unsigned long segment_offset;
80
81         if (!seg || !trb || trb < seg->trbs)
82                 return 0;
83         /* offset in TRBs */
84         segment_offset = trb - seg->trbs;
85         if (segment_offset >= TRBS_PER_SEGMENT)
86                 return 0;
87         return seg->dma + (segment_offset * sizeof(*trb));
88 }
89
90 /* Does this link TRB point to the first segment in a ring,
91  * or was the previous TRB the last TRB on the last segment in the ERST?
92  */
93 static bool last_trb_on_last_seg(struct xhci_hcd *xhci, struct xhci_ring *ring,
94                 struct xhci_segment *seg, union xhci_trb *trb)
95 {
96         if (ring == xhci->event_ring)
97                 return (trb == &seg->trbs[TRBS_PER_SEGMENT]) &&
98                         (seg->next == xhci->event_ring->first_seg);
99         else
100                 return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
101 }
102
103 /* Is this TRB a link TRB or was the last TRB the last TRB in this event ring
104  * segment?  I.e. would the updated event TRB pointer step off the end of the
105  * event seg?
106  */
107 static int last_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
108                 struct xhci_segment *seg, union xhci_trb *trb)
109 {
110         if (ring == xhci->event_ring)
111                 return trb == &seg->trbs[TRBS_PER_SEGMENT];
112         else
113                 return TRB_TYPE_LINK_LE32(trb->link.control);
114 }
115
116 static int enqueue_is_link_trb(struct xhci_ring *ring)
117 {
118         struct xhci_link_trb *link = &ring->enqueue->link;
119         return TRB_TYPE_LINK_LE32(link->control);
120 }
121
122 /* Updates trb to point to the next TRB in the ring, and updates seg if the next
123  * TRB is in a new segment.  This does not skip over link TRBs, and it does not
124  * effect the ring dequeue or enqueue pointers.
125  */
126 static void next_trb(struct xhci_hcd *xhci,
127                 struct xhci_ring *ring,
128                 struct xhci_segment **seg,
129                 union xhci_trb **trb)
130 {
131         if (last_trb(xhci, ring, *seg, *trb)) {
132                 *seg = (*seg)->next;
133                 *trb = ((*seg)->trbs);
134         } else {
135                 (*trb)++;
136         }
137 }
138
139 /*
140  * See Cycle bit rules. SW is the consumer for the event ring only.
141  * Don't make a ring full of link TRBs.  That would be dumb and this would loop.
142  */
143 static void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring)
144 {
145         ring->deq_updates++;
146
147         /*
148          * If this is not event ring, and the dequeue pointer
149          * is not on a link TRB, there is one more usable TRB
150          */
151         if (ring->type != TYPE_EVENT &&
152                         !last_trb(xhci, ring, ring->deq_seg, ring->dequeue))
153                 ring->num_trbs_free++;
154
155         do {
156                 /*
157                  * Update the dequeue pointer further if that was a link TRB or
158                  * we're at the end of an event ring segment (which doesn't have
159                  * link TRBS)
160                  */
161                 if (last_trb(xhci, ring, ring->deq_seg, ring->dequeue)) {
162                         if (ring->type == TYPE_EVENT &&
163                                         last_trb_on_last_seg(xhci, ring,
164                                                 ring->deq_seg, ring->dequeue)) {
165                                 ring->cycle_state ^= 1;
166                         }
167                         ring->deq_seg = ring->deq_seg->next;
168                         ring->dequeue = ring->deq_seg->trbs;
169                 } else {
170                         ring->dequeue++;
171                 }
172         } while (last_trb(xhci, ring, ring->deq_seg, ring->dequeue));
173 }
174
175 /*
176  * See Cycle bit rules. SW is the consumer for the event ring only.
177  * Don't make a ring full of link TRBs.  That would be dumb and this would loop.
178  *
179  * If we've just enqueued a TRB that is in the middle of a TD (meaning the
180  * chain bit is set), then set the chain bit in all the following link TRBs.
181  * If we've enqueued the last TRB in a TD, make sure the following link TRBs
182  * have their chain bit cleared (so that each Link TRB is a separate TD).
183  *
184  * Section 6.4.4.1 of the 0.95 spec says link TRBs cannot have the chain bit
185  * set, but other sections talk about dealing with the chain bit set.  This was
186  * fixed in the 0.96 specification errata, but we have to assume that all 0.95
187  * xHCI hardware can't handle the chain bit being cleared on a link TRB.
188  *
189  * @more_trbs_coming:   Will you enqueue more TRBs before calling
190  *                      prepare_transfer()?
191  */
192 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
193                         bool more_trbs_coming)
194 {
195         u32 chain;
196         union xhci_trb *next;
197
198         chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
199         /* If this is not event ring, there is one less usable TRB */
200         if (ring->type != TYPE_EVENT &&
201                         !last_trb(xhci, ring, ring->enq_seg, ring->enqueue))
202                 ring->num_trbs_free--;
203         next = ++(ring->enqueue);
204
205         ring->enq_updates++;
206         /* Update the dequeue pointer further if that was a link TRB or we're at
207          * the end of an event ring segment (which doesn't have link TRBS)
208          */
209         while (last_trb(xhci, ring, ring->enq_seg, next)) {
210                 if (ring->type != TYPE_EVENT) {
211                         /*
212                          * If the caller doesn't plan on enqueueing more
213                          * TDs before ringing the doorbell, then we
214                          * don't want to give the link TRB to the
215                          * hardware just yet.  We'll give the link TRB
216                          * back in prepare_ring() just before we enqueue
217                          * the TD at the top of the ring.
218                          */
219                         if (!chain && !more_trbs_coming)
220                                 break;
221
222                         /* If we're not dealing with 0.95 hardware or
223                          * isoc rings on AMD 0.96 host,
224                          * carry over the chain bit of the previous TRB
225                          * (which may mean the chain bit is cleared).
226                          */
227                         if (!(ring->type == TYPE_ISOC &&
228                                         (xhci->quirks & XHCI_AMD_0x96_HOST))
229                                                 && !xhci_link_trb_quirk(xhci)) {
230                                 next->link.control &=
231                                         cpu_to_le32(~TRB_CHAIN);
232                                 next->link.control |=
233                                         cpu_to_le32(chain);
234                         }
235                         /* Give this link TRB to the hardware */
236                         wmb();
237                         next->link.control ^= cpu_to_le32(TRB_CYCLE);
238
239                         /* Toggle the cycle bit after the last ring segment. */
240                         if (last_trb_on_last_seg(xhci, ring, ring->enq_seg, next)) {
241                                 ring->cycle_state ^= 1;
242                         }
243                 }
244                 ring->enq_seg = ring->enq_seg->next;
245                 ring->enqueue = ring->enq_seg->trbs;
246                 next = ring->enqueue;
247         }
248 }
249
250 /*
251  * Check to see if there's room to enqueue num_trbs on the ring and make sure
252  * enqueue pointer will not advance into dequeue segment. See rules above.
253  */
254 static inline int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring,
255                 unsigned int num_trbs)
256 {
257         int num_trbs_in_deq_seg;
258
259         if (ring->num_trbs_free < num_trbs)
260                 return 0;
261
262         if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) {
263                 num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs;
264                 if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg)
265                         return 0;
266         }
267
268         return 1;
269 }
270
271 /* Ring the host controller doorbell after placing a command on the ring */
272 void xhci_ring_cmd_db(struct xhci_hcd *xhci)
273 {
274         if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING))
275                 return;
276
277         xhci_dbg(xhci, "// Ding dong!\n");
278         writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]);
279         /* Flush PCI posted writes */
280         readl(&xhci->dba->doorbell[0]);
281 }
282
283 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci)
284 {
285         u64 temp_64;
286         int ret;
287
288         xhci_dbg(xhci, "Abort command ring\n");
289
290         temp_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
291         xhci->cmd_ring_state = CMD_RING_STATE_ABORTED;
292         xhci_write_64(xhci, temp_64 | CMD_RING_ABORT,
293                         &xhci->op_regs->cmd_ring);
294
295         /* Section 4.6.1.2 of xHCI 1.0 spec says software should
296          * time the completion od all xHCI commands, including
297          * the Command Abort operation. If software doesn't see
298          * CRR negated in a timely manner (e.g. longer than 5
299          * seconds), then it should assume that the there are
300          * larger problems with the xHC and assert HCRST.
301          */
302         ret = xhci_handshake(&xhci->op_regs->cmd_ring,
303                         CMD_RING_RUNNING, 0, 5 * 1000 * 1000);
304         if (ret < 0) {
305                 /* we are about to kill xhci, give it one more chance */
306                 xhci_write_64(xhci, temp_64 | CMD_RING_ABORT,
307                               &xhci->op_regs->cmd_ring);
308                 udelay(1000);
309                 ret = xhci_handshake(&xhci->op_regs->cmd_ring,
310                                      CMD_RING_RUNNING, 0, 3 * 1000 * 1000);
311                 if (ret == 0)
312                         return 0;
313
314                 xhci_err(xhci, "Stopped the command ring failed, "
315                                 "maybe the host is dead\n");
316                 xhci->xhc_state |= XHCI_STATE_DYING;
317                 xhci_quiesce(xhci);
318                 xhci_halt(xhci);
319                 return -ESHUTDOWN;
320         }
321
322         return 0;
323 }
324
325 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci,
326                 unsigned int slot_id,
327                 unsigned int ep_index,
328                 unsigned int stream_id)
329 {
330         __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id];
331         struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
332         unsigned int ep_state = ep->ep_state;
333
334         /* Don't ring the doorbell for this endpoint if there are pending
335          * cancellations because we don't want to interrupt processing.
336          * We don't want to restart any stream rings if there's a set dequeue
337          * pointer command pending because the device can choose to start any
338          * stream once the endpoint is on the HW schedule.
339          */
340         if ((ep_state & EP_HALT_PENDING) || (ep_state & SET_DEQ_PENDING) ||
341             (ep_state & EP_HALTED))
342                 return;
343         writel(DB_VALUE(ep_index, stream_id), db_addr);
344         /* The CPU has better things to do at this point than wait for a
345          * write-posting flush.  It'll get there soon enough.
346          */
347 }
348
349 /* Ring the doorbell for any rings with pending URBs */
350 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
351                 unsigned int slot_id,
352                 unsigned int ep_index)
353 {
354         unsigned int stream_id;
355         struct xhci_virt_ep *ep;
356
357         ep = &xhci->devs[slot_id]->eps[ep_index];
358
359         /* A ring has pending URBs if its TD list is not empty */
360         if (!(ep->ep_state & EP_HAS_STREAMS)) {
361                 if (ep->ring && !(list_empty(&ep->ring->td_list)))
362                         xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0);
363                 return;
364         }
365
366         for (stream_id = 1; stream_id < ep->stream_info->num_streams;
367                         stream_id++) {
368                 struct xhci_stream_info *stream_info = ep->stream_info;
369                 if (!list_empty(&stream_info->stream_rings[stream_id]->td_list))
370                         xhci_ring_ep_doorbell(xhci, slot_id, ep_index,
371                                                 stream_id);
372         }
373 }
374
375 static struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci,
376                 unsigned int slot_id, unsigned int ep_index,
377                 unsigned int stream_id)
378 {
379         struct xhci_virt_ep *ep;
380
381         ep = &xhci->devs[slot_id]->eps[ep_index];
382         /* Common case: no streams */
383         if (!(ep->ep_state & EP_HAS_STREAMS))
384                 return ep->ring;
385
386         if (stream_id == 0) {
387                 xhci_warn(xhci,
388                                 "WARN: Slot ID %u, ep index %u has streams, "
389                                 "but URB has no stream ID.\n",
390                                 slot_id, ep_index);
391                 return NULL;
392         }
393
394         if (stream_id < ep->stream_info->num_streams)
395                 return ep->stream_info->stream_rings[stream_id];
396
397         xhci_warn(xhci,
398                         "WARN: Slot ID %u, ep index %u has "
399                         "stream IDs 1 to %u allocated, "
400                         "but stream ID %u is requested.\n",
401                         slot_id, ep_index,
402                         ep->stream_info->num_streams - 1,
403                         stream_id);
404         return NULL;
405 }
406
407 /* Get the right ring for the given URB.
408  * If the endpoint supports streams, boundary check the URB's stream ID.
409  * If the endpoint doesn't support streams, return the singular endpoint ring.
410  */
411 static struct xhci_ring *xhci_urb_to_transfer_ring(struct xhci_hcd *xhci,
412                 struct urb *urb)
413 {
414         return xhci_triad_to_transfer_ring(xhci, urb->dev->slot_id,
415                 xhci_get_endpoint_index(&urb->ep->desc), urb->stream_id);
416 }
417
418 /*
419  * Move the xHC's endpoint ring dequeue pointer past cur_td.
420  * Record the new state of the xHC's endpoint ring dequeue segment,
421  * dequeue pointer, and new consumer cycle state in state.
422  * Update our internal representation of the ring's dequeue pointer.
423  *
424  * We do this in three jumps:
425  *  - First we update our new ring state to be the same as when the xHC stopped.
426  *  - Then we traverse the ring to find the segment that contains
427  *    the last TRB in the TD.  We toggle the xHC's new cycle state when we pass
428  *    any link TRBs with the toggle cycle bit set.
429  *  - Finally we move the dequeue state one TRB further, toggling the cycle bit
430  *    if we've moved it past a link TRB with the toggle cycle bit set.
431  *
432  * Some of the uses of xhci_generic_trb are grotty, but if they're done
433  * with correct __le32 accesses they should work fine.  Only users of this are
434  * in here.
435  */
436 void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
437                 unsigned int slot_id, unsigned int ep_index,
438                 unsigned int stream_id, struct xhci_td *cur_td,
439                 struct xhci_dequeue_state *state)
440 {
441         struct xhci_virt_device *dev = xhci->devs[slot_id];
442         struct xhci_virt_ep *ep = &dev->eps[ep_index];
443         struct xhci_ring *ep_ring;
444         struct xhci_segment *new_seg;
445         union xhci_trb *new_deq;
446         dma_addr_t addr;
447         u64 hw_dequeue;
448         bool cycle_found = false;
449         bool td_last_trb_found = false;
450
451         ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,
452                         ep_index, stream_id);
453         if (!ep_ring) {
454                 xhci_warn(xhci, "WARN can't find new dequeue state "
455                                 "for invalid stream ID %u.\n",
456                                 stream_id);
457                 return;
458         }
459
460         /* Dig out the cycle state saved by the xHC during the stop ep cmd */
461         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
462                         "Finding endpoint context");
463         /* 4.6.9 the css flag is written to the stream context for streams */
464         if (ep->ep_state & EP_HAS_STREAMS) {
465                 struct xhci_stream_ctx *ctx =
466                         &ep->stream_info->stream_ctx_array[stream_id];
467                 hw_dequeue = le64_to_cpu(ctx->stream_ring);
468         } else {
469                 struct xhci_ep_ctx *ep_ctx
470                         = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
471                 hw_dequeue = le64_to_cpu(ep_ctx->deq);
472         }
473
474         new_seg = ep_ring->deq_seg;
475         new_deq = ep_ring->dequeue;
476         state->new_cycle_state = hw_dequeue & 0x1;
477
478         /*
479          * We want to find the pointer, segment and cycle state of the new trb
480          * (the one after current TD's last_trb). We know the cycle state at
481          * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are
482          * found.
483          */
484         do {
485                 if (!cycle_found && xhci_trb_virt_to_dma(new_seg, new_deq)
486                     == (dma_addr_t)(hw_dequeue & ~0xf)) {
487                         cycle_found = true;
488                         if (td_last_trb_found)
489                                 break;
490                 }
491                 if (new_deq == cur_td->last_trb)
492                         td_last_trb_found = true;
493
494                 if (cycle_found &&
495                     TRB_TYPE_LINK_LE32(new_deq->generic.field[3]) &&
496                     new_deq->generic.field[3] & cpu_to_le32(LINK_TOGGLE))
497                         state->new_cycle_state ^= 0x1;
498
499                 next_trb(xhci, ep_ring, &new_seg, &new_deq);
500
501                 /* Search wrapped around, bail out */
502                 if (new_deq == ep->ring->dequeue) {
503                         xhci_err(xhci, "Error: Failed finding new dequeue state\n");
504                         state->new_deq_seg = NULL;
505                         state->new_deq_ptr = NULL;
506                         return;
507                 }
508
509         } while (!cycle_found || !td_last_trb_found);
510
511         state->new_deq_seg = new_seg;
512         state->new_deq_ptr = new_deq;
513
514         /* Don't update the ring cycle state for the producer (us). */
515         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
516                         "Cycle state = 0x%x", state->new_cycle_state);
517
518         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
519                         "New dequeue segment = %p (virtual)",
520                         state->new_deq_seg);
521         addr = xhci_trb_virt_to_dma(state->new_deq_seg, state->new_deq_ptr);
522         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
523                         "New dequeue pointer = 0x%llx (DMA)",
524                         (unsigned long long) addr);
525 }
526
527 /* flip_cycle means flip the cycle bit of all but the first and last TRB.
528  * (The last TRB actually points to the ring enqueue pointer, which is not part
529  * of this TD.)  This is used to remove partially enqueued isoc TDs from a ring.
530  */
531 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
532                 struct xhci_td *cur_td, bool flip_cycle)
533 {
534         struct xhci_segment *cur_seg;
535         union xhci_trb *cur_trb;
536
537         for (cur_seg = cur_td->start_seg, cur_trb = cur_td->first_trb;
538                         true;
539                         next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) {
540                 if (TRB_TYPE_LINK_LE32(cur_trb->generic.field[3])) {
541                         /* Unchain any chained Link TRBs, but
542                          * leave the pointers intact.
543                          */
544                         cur_trb->generic.field[3] &= cpu_to_le32(~TRB_CHAIN);
545                         /* Flip the cycle bit (link TRBs can't be the first
546                          * or last TRB).
547                          */
548                         if (flip_cycle)
549                                 cur_trb->generic.field[3] ^=
550                                         cpu_to_le32(TRB_CYCLE);
551                         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
552                                         "Cancel (unchain) link TRB");
553                         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
554                                         "Address = %p (0x%llx dma); "
555                                         "in seg %p (0x%llx dma)",
556                                         cur_trb,
557                                         (unsigned long long)xhci_trb_virt_to_dma(cur_seg, cur_trb),
558                                         cur_seg,
559                                         (unsigned long long)cur_seg->dma);
560                 } else {
561                         cur_trb->generic.field[0] = 0;
562                         cur_trb->generic.field[1] = 0;
563                         cur_trb->generic.field[2] = 0;
564                         /* Preserve only the cycle bit of this TRB */
565                         cur_trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
566                         /* Flip the cycle bit except on the first or last TRB */
567                         if (flip_cycle && cur_trb != cur_td->first_trb &&
568                                         cur_trb != cur_td->last_trb)
569                                 cur_trb->generic.field[3] ^=
570                                         cpu_to_le32(TRB_CYCLE);
571                         cur_trb->generic.field[3] |= cpu_to_le32(
572                                 TRB_TYPE(TRB_TR_NOOP));
573                         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
574                                         "TRB to noop at offset 0x%llx",
575                                         (unsigned long long)
576                                         xhci_trb_virt_to_dma(cur_seg, cur_trb));
577                 }
578                 if (cur_trb == cur_td->last_trb)
579                         break;
580         }
581 }
582
583 static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci,
584                 struct xhci_virt_ep *ep)
585 {
586         ep->ep_state &= ~EP_HALT_PENDING;
587         /* Can't del_timer_sync in interrupt, so we attempt to cancel.  If the
588          * timer is running on another CPU, we don't decrement stop_cmds_pending
589          * (since we didn't successfully stop the watchdog timer).
590          */
591         if (del_timer(&ep->stop_cmd_timer))
592                 ep->stop_cmds_pending--;
593 }
594
595 /* Must be called with xhci->lock held in interrupt context */
596 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
597                 struct xhci_td *cur_td, int status)
598 {
599         struct usb_hcd *hcd;
600         struct urb      *urb;
601         struct urb_priv *urb_priv;
602
603         urb = cur_td->urb;
604         urb_priv = urb->hcpriv;
605         urb_priv->td_cnt++;
606         hcd = bus_to_hcd(urb->dev->bus);
607
608         /* Only giveback urb when this is the last td in urb */
609         if (urb_priv->td_cnt == urb_priv->length) {
610                 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
611                         xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
612                         if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
613                                 if (xhci->quirks & XHCI_AMD_PLL_FIX)
614                                         usb_amd_quirk_pll_enable();
615                         }
616                 }
617                 usb_hcd_unlink_urb_from_ep(hcd, urb);
618
619                 spin_unlock(&xhci->lock);
620                 usb_hcd_giveback_urb(hcd, urb, status);
621                 xhci_urb_free_priv(urb_priv);
622                 spin_lock(&xhci->lock);
623         }
624 }
625
626 /*
627  * When we get a command completion for a Stop Endpoint Command, we need to
628  * unlink any cancelled TDs from the ring.  There are two ways to do that:
629  *
630  *  1. If the HW was in the middle of processing the TD that needs to be
631  *     cancelled, then we must move the ring's dequeue pointer past the last TRB
632  *     in the TD with a Set Dequeue Pointer Command.
633  *  2. Otherwise, we turn all the TRBs in the TD into No-op TRBs (with the chain
634  *     bit cleared) so that the HW will skip over them.
635  */
636 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
637                 union xhci_trb *trb, struct xhci_event_cmd *event)
638 {
639         unsigned int ep_index;
640         struct xhci_ring *ep_ring;
641         struct xhci_virt_ep *ep;
642         struct list_head *entry;
643         struct xhci_td *cur_td = NULL;
644         struct xhci_td *last_unlinked_td;
645
646         struct xhci_dequeue_state deq_state;
647
648         if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) {
649                 if (!xhci->devs[slot_id])
650                         xhci_warn(xhci, "Stop endpoint command "
651                                 "completion for disabled slot %u\n",
652                                 slot_id);
653                 return;
654         }
655
656         memset(&deq_state, 0, sizeof(deq_state));
657         ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
658         ep = &xhci->devs[slot_id]->eps[ep_index];
659
660         if (list_empty(&ep->cancelled_td_list)) {
661                 xhci_stop_watchdog_timer_in_irq(xhci, ep);
662                 ep->stopped_td = NULL;
663                 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
664                 return;
665         }
666
667         /* Fix up the ep ring first, so HW stops executing cancelled TDs.
668          * We have the xHCI lock, so nothing can modify this list until we drop
669          * it.  We're also in the event handler, so we can't get re-interrupted
670          * if another Stop Endpoint command completes
671          */
672         list_for_each(entry, &ep->cancelled_td_list) {
673                 cur_td = list_entry(entry, struct xhci_td, cancelled_td_list);
674                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
675                                 "Removing canceled TD starting at 0x%llx (dma).",
676                                 (unsigned long long)xhci_trb_virt_to_dma(
677                                         cur_td->start_seg, cur_td->first_trb));
678                 ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
679                 if (!ep_ring) {
680                         /* This shouldn't happen unless a driver is mucking
681                          * with the stream ID after submission.  This will
682                          * leave the TD on the hardware ring, and the hardware
683                          * will try to execute it, and may access a buffer
684                          * that has already been freed.  In the best case, the
685                          * hardware will execute it, and the event handler will
686                          * ignore the completion event for that TD, since it was
687                          * removed from the td_list for that endpoint.  In
688                          * short, don't muck with the stream ID after
689                          * submission.
690                          */
691                         xhci_warn(xhci, "WARN Cancelled URB %p "
692                                         "has invalid stream ID %u.\n",
693                                         cur_td->urb,
694                                         cur_td->urb->stream_id);
695                         goto remove_finished_td;
696                 }
697                 /*
698                  * If we stopped on the TD we need to cancel, then we have to
699                  * move the xHC endpoint ring dequeue pointer past this TD.
700                  */
701                 if (cur_td == ep->stopped_td)
702                         xhci_find_new_dequeue_state(xhci, slot_id, ep_index,
703                                         cur_td->urb->stream_id,
704                                         cur_td, &deq_state);
705                 else
706                         td_to_noop(xhci, ep_ring, cur_td, false);
707 remove_finished_td:
708                 /*
709                  * The event handler won't see a completion for this TD anymore,
710                  * so remove it from the endpoint ring's TD list.  Keep it in
711                  * the cancelled TD list for URB completion later.
712                  */
713                 list_del_init(&cur_td->td_list);
714         }
715         last_unlinked_td = cur_td;
716         xhci_stop_watchdog_timer_in_irq(xhci, ep);
717
718         /* If necessary, queue a Set Transfer Ring Dequeue Pointer command */
719         if (deq_state.new_deq_ptr && deq_state.new_deq_seg) {
720                 xhci_queue_new_dequeue_state(xhci, slot_id, ep_index,
721                                 ep->stopped_td->urb->stream_id, &deq_state);
722                 xhci_ring_cmd_db(xhci);
723         } else {
724                 /* Otherwise ring the doorbell(s) to restart queued transfers */
725                 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
726         }
727
728         ep->stopped_td = NULL;
729
730         /*
731          * Drop the lock and complete the URBs in the cancelled TD list.
732          * New TDs to be cancelled might be added to the end of the list before
733          * we can complete all the URBs for the TDs we already unlinked.
734          * So stop when we've completed the URB for the last TD we unlinked.
735          */
736         do {
737                 cur_td = list_entry(ep->cancelled_td_list.next,
738                                 struct xhci_td, cancelled_td_list);
739                 list_del_init(&cur_td->cancelled_td_list);
740
741                 /* Clean up the cancelled URB */
742                 /* Doesn't matter what we pass for status, since the core will
743                  * just overwrite it (because the URB has been unlinked).
744                  */
745                 xhci_giveback_urb_in_irq(xhci, cur_td, 0);
746
747                 /* Stop processing the cancelled list if the watchdog timer is
748                  * running.
749                  */
750                 if (xhci->xhc_state & XHCI_STATE_DYING)
751                         return;
752         } while (cur_td != last_unlinked_td);
753
754         /* Return to the event handler with xhci->lock re-acquired */
755 }
756
757 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring)
758 {
759         struct xhci_td *cur_td;
760
761         while (!list_empty(&ring->td_list)) {
762                 cur_td = list_first_entry(&ring->td_list,
763                                 struct xhci_td, td_list);
764                 list_del_init(&cur_td->td_list);
765                 if (!list_empty(&cur_td->cancelled_td_list))
766                         list_del_init(&cur_td->cancelled_td_list);
767                 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
768         }
769 }
770
771 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci,
772                 int slot_id, int ep_index)
773 {
774         struct xhci_td *cur_td;
775         struct xhci_virt_ep *ep;
776         struct xhci_ring *ring;
777
778         ep = &xhci->devs[slot_id]->eps[ep_index];
779         if ((ep->ep_state & EP_HAS_STREAMS) ||
780                         (ep->ep_state & EP_GETTING_NO_STREAMS)) {
781                 int stream_id;
782
783                 for (stream_id = 0; stream_id < ep->stream_info->num_streams;
784                                 stream_id++) {
785                         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
786                                         "Killing URBs for slot ID %u, ep index %u, stream %u",
787                                         slot_id, ep_index, stream_id + 1);
788                         xhci_kill_ring_urbs(xhci,
789                                         ep->stream_info->stream_rings[stream_id]);
790                 }
791         } else {
792                 ring = ep->ring;
793                 if (!ring)
794                         return;
795                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
796                                 "Killing URBs for slot ID %u, ep index %u",
797                                 slot_id, ep_index);
798                 xhci_kill_ring_urbs(xhci, ring);
799         }
800         while (!list_empty(&ep->cancelled_td_list)) {
801                 cur_td = list_first_entry(&ep->cancelled_td_list,
802                                 struct xhci_td, cancelled_td_list);
803                 list_del_init(&cur_td->cancelled_td_list);
804                 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
805         }
806 }
807
808 /* Watchdog timer function for when a stop endpoint command fails to complete.
809  * In this case, we assume the host controller is broken or dying or dead.  The
810  * host may still be completing some other events, so we have to be careful to
811  * let the event ring handler and the URB dequeueing/enqueueing functions know
812  * through xhci->state.
813  *
814  * The timer may also fire if the host takes a very long time to respond to the
815  * command, and the stop endpoint command completion handler cannot delete the
816  * timer before the timer function is called.  Another endpoint cancellation may
817  * sneak in before the timer function can grab the lock, and that may queue
818  * another stop endpoint command and add the timer back.  So we cannot use a
819  * simple flag to say whether there is a pending stop endpoint command for a
820  * particular endpoint.
821  *
822  * Instead we use a combination of that flag and a counter for the number of
823  * pending stop endpoint commands.  If the timer is the tail end of the last
824  * stop endpoint command, and the endpoint's command is still pending, we assume
825  * the host is dying.
826  */
827 void xhci_stop_endpoint_command_watchdog(unsigned long arg)
828 {
829         struct xhci_hcd *xhci;
830         struct xhci_virt_ep *ep;
831         int ret, i, j;
832         unsigned long flags;
833
834         ep = (struct xhci_virt_ep *) arg;
835         xhci = ep->xhci;
836
837         spin_lock_irqsave(&xhci->lock, flags);
838
839         ep->stop_cmds_pending--;
840         if (xhci->xhc_state & XHCI_STATE_DYING) {
841                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
842                                 "Stop EP timer ran, but another timer marked "
843                                 "xHCI as DYING, exiting.");
844                 spin_unlock_irqrestore(&xhci->lock, flags);
845                 return;
846         }
847         if (!(ep->stop_cmds_pending == 0 && (ep->ep_state & EP_HALT_PENDING))) {
848                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
849                                 "Stop EP timer ran, but no command pending, "
850                                 "exiting.");
851                 spin_unlock_irqrestore(&xhci->lock, flags);
852                 return;
853         }
854
855         xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n");
856         xhci_warn(xhci, "Assuming host is dying, halting host.\n");
857         /* Oops, HC is dead or dying or at least not responding to the stop
858          * endpoint command.
859          */
860         xhci->xhc_state |= XHCI_STATE_DYING;
861         /* Disable interrupts from the host controller and start halting it */
862         xhci_quiesce(xhci);
863         spin_unlock_irqrestore(&xhci->lock, flags);
864
865         ret = xhci_halt(xhci);
866
867         spin_lock_irqsave(&xhci->lock, flags);
868         if (ret < 0) {
869                 /* This is bad; the host is not responding to commands and it's
870                  * not allowing itself to be halted.  At least interrupts are
871                  * disabled. If we call usb_hc_died(), it will attempt to
872                  * disconnect all device drivers under this host.  Those
873                  * disconnect() methods will wait for all URBs to be unlinked,
874                  * so we must complete them.
875                  */
876                 xhci_warn(xhci, "Non-responsive xHCI host is not halting.\n");
877                 xhci_warn(xhci, "Completing active URBs anyway.\n");
878                 /* We could turn all TDs on the rings to no-ops.  This won't
879                  * help if the host has cached part of the ring, and is slow if
880                  * we want to preserve the cycle bit.  Skip it and hope the host
881                  * doesn't touch the memory.
882                  */
883         }
884         for (i = 0; i < MAX_HC_SLOTS; i++) {
885                 if (!xhci->devs[i])
886                         continue;
887                 for (j = 0; j < 31; j++)
888                         xhci_kill_endpoint_urbs(xhci, i, j);
889         }
890         spin_unlock_irqrestore(&xhci->lock, flags);
891         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
892                         "Calling usb_hc_died()");
893         usb_hc_died(xhci_to_hcd(xhci)->primary_hcd);
894         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
895                         "xHCI host controller is dead.");
896 }
897
898
899 static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci,
900                 struct xhci_virt_device *dev,
901                 struct xhci_ring *ep_ring,
902                 unsigned int ep_index)
903 {
904         union xhci_trb *dequeue_temp;
905         int num_trbs_free_temp;
906         bool revert = false;
907
908         num_trbs_free_temp = ep_ring->num_trbs_free;
909         dequeue_temp = ep_ring->dequeue;
910
911         /* If we get two back-to-back stalls, and the first stalled transfer
912          * ends just before a link TRB, the dequeue pointer will be left on
913          * the link TRB by the code in the while loop.  So we have to update
914          * the dequeue pointer one segment further, or we'll jump off
915          * the segment into la-la-land.
916          */
917         if (last_trb(xhci, ep_ring, ep_ring->deq_seg, ep_ring->dequeue)) {
918                 ep_ring->deq_seg = ep_ring->deq_seg->next;
919                 ep_ring->dequeue = ep_ring->deq_seg->trbs;
920         }
921
922         while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) {
923                 /* We have more usable TRBs */
924                 ep_ring->num_trbs_free++;
925                 ep_ring->dequeue++;
926                 if (last_trb(xhci, ep_ring, ep_ring->deq_seg,
927                                 ep_ring->dequeue)) {
928                         if (ep_ring->dequeue ==
929                                         dev->eps[ep_index].queued_deq_ptr)
930                                 break;
931                         ep_ring->deq_seg = ep_ring->deq_seg->next;
932                         ep_ring->dequeue = ep_ring->deq_seg->trbs;
933                 }
934                 if (ep_ring->dequeue == dequeue_temp) {
935                         revert = true;
936                         break;
937                 }
938         }
939
940         if (revert) {
941                 xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
942                 ep_ring->num_trbs_free = num_trbs_free_temp;
943         }
944 }
945
946 /*
947  * When we get a completion for a Set Transfer Ring Dequeue Pointer command,
948  * we need to clear the set deq pending flag in the endpoint ring state, so that
949  * the TD queueing code can ring the doorbell again.  We also need to ring the
950  * endpoint doorbell to restart the ring, but only if there aren't more
951  * cancellations pending.
952  */
953 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
954                 union xhci_trb *trb, u32 cmd_comp_code)
955 {
956         unsigned int ep_index;
957         unsigned int stream_id;
958         struct xhci_ring *ep_ring;
959         struct xhci_virt_device *dev;
960         struct xhci_virt_ep *ep;
961         struct xhci_ep_ctx *ep_ctx;
962         struct xhci_slot_ctx *slot_ctx;
963
964         ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
965         stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2]));
966         dev = xhci->devs[slot_id];
967         ep = &dev->eps[ep_index];
968
969         ep_ring = xhci_stream_id_to_ring(dev, ep_index, stream_id);
970         if (!ep_ring) {
971                 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n",
972                                 stream_id);
973                 /* XXX: Harmless??? */
974                 goto cleanup;
975         }
976
977         ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
978         slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx);
979
980         if (cmd_comp_code != COMP_SUCCESS) {
981                 unsigned int ep_state;
982                 unsigned int slot_state;
983
984                 switch (cmd_comp_code) {
985                 case COMP_TRB_ERR:
986                         xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n");
987                         break;
988                 case COMP_CTX_STATE:
989                         xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n");
990                         ep_state = le32_to_cpu(ep_ctx->ep_info);
991                         ep_state &= EP_STATE_MASK;
992                         slot_state = le32_to_cpu(slot_ctx->dev_state);
993                         slot_state = GET_SLOT_STATE(slot_state);
994                         xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
995                                         "Slot state = %u, EP state = %u",
996                                         slot_state, ep_state);
997                         break;
998                 case COMP_EBADSLT:
999                         xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n",
1000                                         slot_id);
1001                         break;
1002                 default:
1003                         xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n",
1004                                         cmd_comp_code);
1005                         break;
1006                 }
1007                 /* OK what do we do now?  The endpoint state is hosed, and we
1008                  * should never get to this point if the synchronization between
1009                  * queueing, and endpoint state are correct.  This might happen
1010                  * if the device gets disconnected after we've finished
1011                  * cancelling URBs, which might not be an error...
1012                  */
1013         } else {
1014                 u64 deq;
1015                 /* 4.6.10 deq ptr is written to the stream ctx for streams */
1016                 if (ep->ep_state & EP_HAS_STREAMS) {
1017                         struct xhci_stream_ctx *ctx =
1018                                 &ep->stream_info->stream_ctx_array[stream_id];
1019                         deq = le64_to_cpu(ctx->stream_ring) & SCTX_DEQ_MASK;
1020                 } else {
1021                         deq = le64_to_cpu(ep_ctx->deq) & ~EP_CTX_CYCLE_MASK;
1022                 }
1023                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1024                         "Successful Set TR Deq Ptr cmd, deq = @%08llx", deq);
1025                 if (xhci_trb_virt_to_dma(ep->queued_deq_seg,
1026                                          ep->queued_deq_ptr) == deq) {
1027                         /* Update the ring's dequeue segment and dequeue pointer
1028                          * to reflect the new position.
1029                          */
1030                         update_ring_for_set_deq_completion(xhci, dev,
1031                                 ep_ring, ep_index);
1032                 } else {
1033                         xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n");
1034                         xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n",
1035                                   ep->queued_deq_seg, ep->queued_deq_ptr);
1036                 }
1037         }
1038
1039 cleanup:
1040         dev->eps[ep_index].ep_state &= ~SET_DEQ_PENDING;
1041         dev->eps[ep_index].queued_deq_seg = NULL;
1042         dev->eps[ep_index].queued_deq_ptr = NULL;
1043         /* Restart any rings with pending URBs */
1044         ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1045 }
1046
1047 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
1048                 union xhci_trb *trb, u32 cmd_comp_code)
1049 {
1050         unsigned int ep_index;
1051
1052         ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1053         /* This command will only fail if the endpoint wasn't halted,
1054          * but we don't care.
1055          */
1056         xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
1057                 "Ignoring reset ep completion code of %u", cmd_comp_code);
1058
1059         /* HW with the reset endpoint quirk needs to have a configure endpoint
1060          * command complete before the endpoint can be used.  Queue that here
1061          * because the HW can't handle two commands being queued in a row.
1062          */
1063         if (xhci->quirks & XHCI_RESET_EP_QUIRK) {
1064                 struct xhci_command *command;
1065                 command = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
1066                 if (!command) {
1067                         xhci_warn(xhci, "WARN Cannot submit cfg ep: ENOMEM\n");
1068                         return;
1069                 }
1070                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1071                                 "Queueing configure endpoint command");
1072                 xhci_queue_configure_endpoint(xhci, command,
1073                                 xhci->devs[slot_id]->in_ctx->dma, slot_id,
1074                                 false);
1075                 xhci_ring_cmd_db(xhci);
1076         } else {
1077                 /* Clear our internal halted state */
1078                 xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_HALTED;
1079         }
1080 }
1081
1082 static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
1083                 u32 cmd_comp_code)
1084 {
1085         if (cmd_comp_code == COMP_SUCCESS)
1086                 xhci->slot_id = slot_id;
1087         else
1088                 xhci->slot_id = 0;
1089 }
1090
1091 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id)
1092 {
1093         struct xhci_virt_device *virt_dev;
1094
1095         virt_dev = xhci->devs[slot_id];
1096         if (!virt_dev)
1097                 return;
1098         if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
1099                 /* Delete default control endpoint resources */
1100                 xhci_free_device_endpoint_resources(xhci, virt_dev, true);
1101         xhci_free_virt_device(xhci, slot_id);
1102 }
1103
1104 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
1105                 struct xhci_event_cmd *event, u32 cmd_comp_code)
1106 {
1107         struct xhci_virt_device *virt_dev;
1108         struct xhci_input_control_ctx *ctrl_ctx;
1109         unsigned int ep_index;
1110         unsigned int ep_state;
1111         u32 add_flags, drop_flags;
1112
1113         /*
1114          * Configure endpoint commands can come from the USB core
1115          * configuration or alt setting changes, or because the HW
1116          * needed an extra configure endpoint command after a reset
1117          * endpoint command or streams were being configured.
1118          * If the command was for a halted endpoint, the xHCI driver
1119          * is not waiting on the configure endpoint command.
1120          */
1121         virt_dev = xhci->devs[slot_id];
1122         ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
1123         if (!ctrl_ctx) {
1124                 xhci_warn(xhci, "Could not get input context, bad type.\n");
1125                 return;
1126         }
1127
1128         add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1129         drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1130         /* Input ctx add_flags are the endpoint index plus one */
1131         ep_index = xhci_last_valid_endpoint(add_flags) - 1;
1132
1133         /* A usb_set_interface() call directly after clearing a halted
1134          * condition may race on this quirky hardware.  Not worth
1135          * worrying about, since this is prototype hardware.  Not sure
1136          * if this will work for streams, but streams support was
1137          * untested on this prototype.
1138          */
1139         if (xhci->quirks & XHCI_RESET_EP_QUIRK &&
1140                         ep_index != (unsigned int) -1 &&
1141                         add_flags - SLOT_FLAG == drop_flags) {
1142                 ep_state = virt_dev->eps[ep_index].ep_state;
1143                 if (!(ep_state & EP_HALTED))
1144                         return;
1145                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1146                                 "Completed config ep cmd - "
1147                                 "last ep index = %d, state = %d",
1148                                 ep_index, ep_state);
1149                 /* Clear internal halted state and restart ring(s) */
1150                 virt_dev->eps[ep_index].ep_state &= ~EP_HALTED;
1151                 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1152                 return;
1153         }
1154         return;
1155 }
1156
1157 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id,
1158                 struct xhci_event_cmd *event)
1159 {
1160         xhci_dbg(xhci, "Completed reset device command.\n");
1161         if (!xhci->devs[slot_id])
1162                 xhci_warn(xhci, "Reset device command completion "
1163                                 "for disabled slot %u\n", slot_id);
1164 }
1165
1166 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
1167                 struct xhci_event_cmd *event)
1168 {
1169         if (!(xhci->quirks & XHCI_NEC_HOST)) {
1170                 xhci->error_bitmask |= 1 << 6;
1171                 return;
1172         }
1173         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1174                         "NEC firmware version %2x.%02x",
1175                         NEC_FW_MAJOR(le32_to_cpu(event->status)),
1176                         NEC_FW_MINOR(le32_to_cpu(event->status)));
1177 }
1178
1179 static void xhci_complete_del_and_free_cmd(struct xhci_command *cmd, u32 status)
1180 {
1181         list_del(&cmd->cmd_list);
1182
1183         if (cmd->completion) {
1184                 cmd->status = status;
1185                 complete(cmd->completion);
1186         } else {
1187                 kfree(cmd);
1188         }
1189 }
1190
1191 void xhci_cleanup_command_queue(struct xhci_hcd *xhci)
1192 {
1193         struct xhci_command *cur_cmd, *tmp_cmd;
1194         list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list)
1195                 xhci_complete_del_and_free_cmd(cur_cmd, COMP_CMD_ABORT);
1196 }
1197
1198 /*
1199  * Turn all commands on command ring with status set to "aborted" to no-op trbs.
1200  * If there are other commands waiting then restart the ring and kick the timer.
1201  * This must be called with command ring stopped and xhci->lock held.
1202  */
1203 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci,
1204                                          struct xhci_command *cur_cmd)
1205 {
1206         struct xhci_command *i_cmd, *tmp_cmd;
1207         u32 cycle_state;
1208
1209         /* Turn all aborted commands in list to no-ops, then restart */
1210         list_for_each_entry_safe(i_cmd, tmp_cmd, &xhci->cmd_list,
1211                                  cmd_list) {
1212
1213                 if (i_cmd->status != COMP_CMD_ABORT)
1214                         continue;
1215
1216                 i_cmd->status = COMP_CMD_STOP;
1217
1218                 xhci_dbg(xhci, "Turn aborted command %p to no-op\n",
1219                          i_cmd->command_trb);
1220                 /* get cycle state from the original cmd trb */
1221                 cycle_state = le32_to_cpu(
1222                         i_cmd->command_trb->generic.field[3]) & TRB_CYCLE;
1223                 /* modify the command trb to no-op command */
1224                 i_cmd->command_trb->generic.field[0] = 0;
1225                 i_cmd->command_trb->generic.field[1] = 0;
1226                 i_cmd->command_trb->generic.field[2] = 0;
1227                 i_cmd->command_trb->generic.field[3] = cpu_to_le32(
1228                         TRB_TYPE(TRB_CMD_NOOP) | cycle_state);
1229
1230                 /*
1231                  * caller waiting for completion is called when command
1232                  *  completion event is received for these no-op commands
1233                  */
1234         }
1235
1236         xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
1237
1238         /* ring command ring doorbell to restart the command ring */
1239         if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) &&
1240             !(xhci->xhc_state & XHCI_STATE_DYING)) {
1241                 xhci->current_cmd = cur_cmd;
1242                 mod_timer(&xhci->cmd_timer, jiffies + XHCI_CMD_DEFAULT_TIMEOUT);
1243                 xhci_ring_cmd_db(xhci);
1244         }
1245         return;
1246 }
1247
1248
1249 void xhci_handle_command_timeout(unsigned long data)
1250 {
1251         struct xhci_hcd *xhci;
1252         int ret;
1253         unsigned long flags;
1254         u64 hw_ring_state;
1255         struct xhci_command *cur_cmd = NULL;
1256         xhci = (struct xhci_hcd *) data;
1257
1258         /* mark this command to be cancelled */
1259         spin_lock_irqsave(&xhci->lock, flags);
1260         if (xhci->current_cmd) {
1261                 cur_cmd = xhci->current_cmd;
1262                 cur_cmd->status = COMP_CMD_ABORT;
1263         }
1264
1265
1266         /* Make sure command ring is running before aborting it */
1267         hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
1268         if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) &&
1269             (hw_ring_state & CMD_RING_RUNNING))  {
1270
1271                 spin_unlock_irqrestore(&xhci->lock, flags);
1272                 xhci_dbg(xhci, "Command timeout\n");
1273                 ret = xhci_abort_cmd_ring(xhci);
1274                 if (unlikely(ret == -ESHUTDOWN)) {
1275                         xhci_err(xhci, "Abort command ring failed\n");
1276                         xhci_cleanup_command_queue(xhci);
1277                         usb_hc_died(xhci_to_hcd(xhci)->primary_hcd);
1278                         xhci_dbg(xhci, "xHCI host controller is dead.\n");
1279                 }
1280                 return;
1281         }
1282         /* command timeout on stopped ring, ring can't be aborted */
1283         xhci_dbg(xhci, "Command timeout on stopped ring\n");
1284         xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd);
1285         spin_unlock_irqrestore(&xhci->lock, flags);
1286         return;
1287 }
1288
1289 static void handle_cmd_completion(struct xhci_hcd *xhci,
1290                 struct xhci_event_cmd *event)
1291 {
1292         int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1293         u64 cmd_dma;
1294         dma_addr_t cmd_dequeue_dma;
1295         u32 cmd_comp_code;
1296         union xhci_trb *cmd_trb;
1297         struct xhci_command *cmd;
1298         u32 cmd_type;
1299
1300         cmd_dma = le64_to_cpu(event->cmd_trb);
1301         cmd_trb = xhci->cmd_ring->dequeue;
1302         cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
1303                         cmd_trb);
1304         /* Is the command ring deq ptr out of sync with the deq seg ptr? */
1305         if (cmd_dequeue_dma == 0) {
1306                 xhci->error_bitmask |= 1 << 4;
1307                 return;
1308         }
1309         /* Does the DMA address match our internal dequeue pointer address? */
1310         if (cmd_dma != (u64) cmd_dequeue_dma) {
1311                 xhci->error_bitmask |= 1 << 5;
1312                 return;
1313         }
1314
1315         cmd = list_entry(xhci->cmd_list.next, struct xhci_command, cmd_list);
1316
1317         if (cmd->command_trb != xhci->cmd_ring->dequeue) {
1318                 xhci_err(xhci,
1319                          "Command completion event does not match command\n");
1320                 return;
1321         }
1322
1323         del_timer(&xhci->cmd_timer);
1324
1325         trace_xhci_cmd_completion(cmd_trb, (struct xhci_generic_trb *) event);
1326
1327         cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status));
1328
1329         /* If CMD ring stopped we own the trbs between enqueue and dequeue */
1330         if (cmd_comp_code == COMP_CMD_STOP) {
1331                 xhci_handle_stopped_cmd_ring(xhci, cmd);
1332                 return;
1333         }
1334         /*
1335          * Host aborted the command ring, check if the current command was
1336          * supposed to be aborted, otherwise continue normally.
1337          * The command ring is stopped now, but the xHC will issue a Command
1338          * Ring Stopped event which will cause us to restart it.
1339          */
1340         if (cmd_comp_code == COMP_CMD_ABORT) {
1341                 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
1342                 if (cmd->status == COMP_CMD_ABORT)
1343                         goto event_handled;
1344         }
1345
1346         cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3]));
1347         switch (cmd_type) {
1348         case TRB_ENABLE_SLOT:
1349                 xhci_handle_cmd_enable_slot(xhci, slot_id, cmd_comp_code);
1350                 break;
1351         case TRB_DISABLE_SLOT:
1352                 xhci_handle_cmd_disable_slot(xhci, slot_id);
1353                 break;
1354         case TRB_CONFIG_EP:
1355                 if (!cmd->completion)
1356                         xhci_handle_cmd_config_ep(xhci, slot_id, event,
1357                                                   cmd_comp_code);
1358                 break;
1359         case TRB_EVAL_CONTEXT:
1360                 break;
1361         case TRB_ADDR_DEV:
1362                 break;
1363         case TRB_STOP_RING:
1364                 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1365                                 le32_to_cpu(cmd_trb->generic.field[3])));
1366                 xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, event);
1367                 break;
1368         case TRB_SET_DEQ:
1369                 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1370                                 le32_to_cpu(cmd_trb->generic.field[3])));
1371                 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code);
1372                 break;
1373         case TRB_CMD_NOOP:
1374                 /* Is this an aborted command turned to NO-OP? */
1375                 if (cmd->status == COMP_CMD_STOP)
1376                         cmd_comp_code = COMP_CMD_STOP;
1377                 break;
1378         case TRB_RESET_EP:
1379                 WARN_ON(slot_id != TRB_TO_SLOT_ID(
1380                                 le32_to_cpu(cmd_trb->generic.field[3])));
1381                 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code);
1382                 break;
1383         case TRB_RESET_DEV:
1384                 /* SLOT_ID field in reset device cmd completion event TRB is 0.
1385                  * Use the SLOT_ID from the command TRB instead (xhci 4.6.11)
1386                  */
1387                 slot_id = TRB_TO_SLOT_ID(
1388                                 le32_to_cpu(cmd_trb->generic.field[3]));
1389                 xhci_handle_cmd_reset_dev(xhci, slot_id, event);
1390                 break;
1391         case TRB_NEC_GET_FW:
1392                 xhci_handle_cmd_nec_get_fw(xhci, event);
1393                 break;
1394         default:
1395                 /* Skip over unknown commands on the event ring */
1396                 xhci->error_bitmask |= 1 << 6;
1397                 break;
1398         }
1399
1400         /* restart timer if this wasn't the last command */
1401         if (cmd->cmd_list.next != &xhci->cmd_list) {
1402                 xhci->current_cmd = list_entry(cmd->cmd_list.next,
1403                                                struct xhci_command, cmd_list);
1404                 mod_timer(&xhci->cmd_timer, jiffies + XHCI_CMD_DEFAULT_TIMEOUT);
1405         }
1406
1407 event_handled:
1408         xhci_complete_del_and_free_cmd(cmd, cmd_comp_code);
1409
1410         inc_deq(xhci, xhci->cmd_ring);
1411 }
1412
1413 static void handle_vendor_event(struct xhci_hcd *xhci,
1414                 union xhci_trb *event)
1415 {
1416         u32 trb_type;
1417
1418         trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->generic.field[3]));
1419         xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
1420         if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
1421                 handle_cmd_completion(xhci, &event->event_cmd);
1422 }
1423
1424 /* @port_id: the one-based port ID from the hardware (indexed from array of all
1425  * port registers -- USB 3.0 and USB 2.0).
1426  *
1427  * Returns a zero-based port number, which is suitable for indexing into each of
1428  * the split roothubs' port arrays and bus state arrays.
1429  * Add one to it in order to call xhci_find_slot_id_by_port.
1430  */
1431 static unsigned int find_faked_portnum_from_hw_portnum(struct usb_hcd *hcd,
1432                 struct xhci_hcd *xhci, u32 port_id)
1433 {
1434         unsigned int i;
1435         unsigned int num_similar_speed_ports = 0;
1436
1437         /* port_id from the hardware is 1-based, but port_array[], usb3_ports[],
1438          * and usb2_ports are 0-based indexes.  Count the number of similar
1439          * speed ports, up to 1 port before this port.
1440          */
1441         for (i = 0; i < (port_id - 1); i++) {
1442                 u8 port_speed = xhci->port_array[i];
1443
1444                 /*
1445                  * Skip ports that don't have known speeds, or have duplicate
1446                  * Extended Capabilities port speed entries.
1447                  */
1448                 if (port_speed == 0 || port_speed == DUPLICATE_ENTRY)
1449                         continue;
1450
1451                 /*
1452                  * USB 3.0 ports are always under a USB 3.0 hub.  USB 2.0 and
1453                  * 1.1 ports are under the USB 2.0 hub.  If the port speed
1454                  * matches the device speed, it's a similar speed port.
1455                  */
1456                 if ((port_speed == 0x03) == (hcd->speed == HCD_USB3))
1457                         num_similar_speed_ports++;
1458         }
1459         return num_similar_speed_ports;
1460 }
1461
1462 static void handle_device_notification(struct xhci_hcd *xhci,
1463                 union xhci_trb *event)
1464 {
1465         u32 slot_id;
1466         struct usb_device *udev;
1467
1468         slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3]));
1469         if (!xhci->devs[slot_id]) {
1470                 xhci_warn(xhci, "Device Notification event for "
1471                                 "unused slot %u\n", slot_id);
1472                 return;
1473         }
1474
1475         xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n",
1476                         slot_id);
1477         udev = xhci->devs[slot_id]->udev;
1478         if (udev && udev->parent)
1479                 usb_wakeup_notification(udev->parent, udev->portnum);
1480 }
1481
1482 static void handle_port_status(struct xhci_hcd *xhci,
1483                 union xhci_trb *event)
1484 {
1485         struct usb_hcd *hcd;
1486         u32 port_id;
1487         u32 temp, temp1;
1488         int max_ports;
1489         int slot_id;
1490         unsigned int faked_port_index;
1491         u8 major_revision;
1492         struct xhci_bus_state *bus_state;
1493         __le32 __iomem **port_array;
1494         bool bogus_port_status = false;
1495
1496         /* Port status change events always have a successful completion code */
1497         if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) {
1498                 xhci_warn(xhci, "WARN: xHC returned failed port status event\n");
1499                 xhci->error_bitmask |= 1 << 8;
1500         }
1501         port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
1502         xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id);
1503
1504         max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1505         if ((port_id <= 0) || (port_id > max_ports)) {
1506                 xhci_warn(xhci, "Invalid port id %d\n", port_id);
1507                 inc_deq(xhci, xhci->event_ring);
1508                 return;
1509         }
1510
1511         /* Figure out which usb_hcd this port is attached to:
1512          * is it a USB 3.0 port or a USB 2.0/1.1 port?
1513          */
1514         major_revision = xhci->port_array[port_id - 1];
1515
1516         /* Find the right roothub. */
1517         hcd = xhci_to_hcd(xhci);
1518         if ((major_revision == 0x03) != (hcd->speed == HCD_USB3))
1519                 hcd = xhci->shared_hcd;
1520
1521         if (major_revision == 0) {
1522                 xhci_warn(xhci, "Event for port %u not in "
1523                                 "Extended Capabilities, ignoring.\n",
1524                                 port_id);
1525                 bogus_port_status = true;
1526                 goto cleanup;
1527         }
1528         if (major_revision == DUPLICATE_ENTRY) {
1529                 xhci_warn(xhci, "Event for port %u duplicated in"
1530                                 "Extended Capabilities, ignoring.\n",
1531                                 port_id);
1532                 bogus_port_status = true;
1533                 goto cleanup;
1534         }
1535
1536         /*
1537          * Hardware port IDs reported by a Port Status Change Event include USB
1538          * 3.0 and USB 2.0 ports.  We want to check if the port has reported a
1539          * resume event, but we first need to translate the hardware port ID
1540          * into the index into the ports on the correct split roothub, and the
1541          * correct bus_state structure.
1542          */
1543         bus_state = &xhci->bus_state[hcd_index(hcd)];
1544         if (hcd->speed == HCD_USB3)
1545                 port_array = xhci->usb3_ports;
1546         else
1547                 port_array = xhci->usb2_ports;
1548         /* Find the faked port hub number */
1549         faked_port_index = find_faked_portnum_from_hw_portnum(hcd, xhci,
1550                         port_id);
1551
1552         temp = readl(port_array[faked_port_index]);
1553         if (hcd->state == HC_STATE_SUSPENDED) {
1554                 xhci_dbg(xhci, "resume root hub\n");
1555                 usb_hcd_resume_root_hub(hcd);
1556         }
1557
1558         if (hcd->speed == HCD_USB3 && (temp & PORT_PLS_MASK) == XDEV_INACTIVE)
1559                 bus_state->port_remote_wakeup &= ~(1 << faked_port_index);
1560
1561         if ((temp & PORT_PLC) && (temp & PORT_PLS_MASK) == XDEV_RESUME) {
1562                 xhci_dbg(xhci, "port resume event for port %d\n", port_id);
1563
1564                 temp1 = readl(&xhci->op_regs->command);
1565                 if (!(temp1 & CMD_RUN)) {
1566                         xhci_warn(xhci, "xHC is not running.\n");
1567                         goto cleanup;
1568                 }
1569
1570                 if (DEV_SUPERSPEED(temp)) {
1571                         xhci_dbg(xhci, "remote wake SS port %d\n", port_id);
1572                         /* Set a flag to say the port signaled remote wakeup,
1573                          * so we can tell the difference between the end of
1574                          * device and host initiated resume.
1575                          */
1576                         bus_state->port_remote_wakeup |= 1 << faked_port_index;
1577                         xhci_test_and_clear_bit(xhci, port_array,
1578                                         faked_port_index, PORT_PLC);
1579                         xhci_set_link_state(xhci, port_array, faked_port_index,
1580                                                 XDEV_U0);
1581                         /* Need to wait until the next link state change
1582                          * indicates the device is actually in U0.
1583                          */
1584                         bogus_port_status = true;
1585                         goto cleanup;
1586                 } else {
1587                         xhci_dbg(xhci, "resume HS port %d\n", port_id);
1588                         bus_state->resume_done[faked_port_index] = jiffies +
1589                                 msecs_to_jiffies(USB_RESUME_TIMEOUT);
1590                         set_bit(faked_port_index, &bus_state->resuming_ports);
1591                         mod_timer(&hcd->rh_timer,
1592                                   bus_state->resume_done[faked_port_index]);
1593                         /* Do the rest in GetPortStatus */
1594                 }
1595         }
1596
1597         if ((temp & PORT_PLC) && (temp & PORT_PLS_MASK) == XDEV_U0 &&
1598                         DEV_SUPERSPEED(temp)) {
1599                 xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
1600                 /* We've just brought the device into U0 through either the
1601                  * Resume state after a device remote wakeup, or through the
1602                  * U3Exit state after a host-initiated resume.  If it's a device
1603                  * initiated remote wake, don't pass up the link state change,
1604                  * so the roothub behavior is consistent with external
1605                  * USB 3.0 hub behavior.
1606                  */
1607                 slot_id = xhci_find_slot_id_by_port(hcd, xhci,
1608                                 faked_port_index + 1);
1609                 if (slot_id && xhci->devs[slot_id])
1610                         xhci_ring_device(xhci, slot_id);
1611                 if (bus_state->port_remote_wakeup & (1 << faked_port_index)) {
1612                         bus_state->port_remote_wakeup &=
1613                                 ~(1 << faked_port_index);
1614                         xhci_test_and_clear_bit(xhci, port_array,
1615                                         faked_port_index, PORT_PLC);
1616                         usb_wakeup_notification(hcd->self.root_hub,
1617                                         faked_port_index + 1);
1618                         bogus_port_status = true;
1619                         goto cleanup;
1620                 }
1621         }
1622
1623         /*
1624          * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or
1625          * RExit to a disconnect state).  If so, let the the driver know it's
1626          * out of the RExit state.
1627          */
1628         if (!DEV_SUPERSPEED(temp) &&
1629                         test_and_clear_bit(faked_port_index,
1630                                 &bus_state->rexit_ports)) {
1631                 complete(&bus_state->rexit_done[faked_port_index]);
1632                 bogus_port_status = true;
1633                 goto cleanup;
1634         }
1635
1636         if (hcd->speed != HCD_USB3)
1637                 xhci_test_and_clear_bit(xhci, port_array, faked_port_index,
1638                                         PORT_PLC);
1639
1640 cleanup:
1641         /* Update event ring dequeue pointer before dropping the lock */
1642         inc_deq(xhci, xhci->event_ring);
1643
1644         /* Don't make the USB core poll the roothub if we got a bad port status
1645          * change event.  Besides, at that point we can't tell which roothub
1646          * (USB 2.0 or USB 3.0) to kick.
1647          */
1648         if (bogus_port_status)
1649                 return;
1650
1651         /*
1652          * xHCI port-status-change events occur when the "or" of all the
1653          * status-change bits in the portsc register changes from 0 to 1.
1654          * New status changes won't cause an event if any other change
1655          * bits are still set.  When an event occurs, switch over to
1656          * polling to avoid losing status changes.
1657          */
1658         xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
1659         set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1660         spin_unlock(&xhci->lock);
1661         /* Pass this up to the core */
1662         usb_hcd_poll_rh_status(hcd);
1663         spin_lock(&xhci->lock);
1664 }
1665
1666 /*
1667  * This TD is defined by the TRBs starting at start_trb in start_seg and ending
1668  * at end_trb, which may be in another segment.  If the suspect DMA address is a
1669  * TRB in this TD, this function returns that TRB's segment.  Otherwise it
1670  * returns 0.
1671  */
1672 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci,
1673                 struct xhci_segment *start_seg,
1674                 union xhci_trb  *start_trb,
1675                 union xhci_trb  *end_trb,
1676                 dma_addr_t      suspect_dma,
1677                 bool            debug)
1678 {
1679         dma_addr_t start_dma;
1680         dma_addr_t end_seg_dma;
1681         dma_addr_t end_trb_dma;
1682         struct xhci_segment *cur_seg;
1683
1684         start_dma = xhci_trb_virt_to_dma(start_seg, start_trb);
1685         cur_seg = start_seg;
1686
1687         do {
1688                 if (start_dma == 0)
1689                         return NULL;
1690                 /* We may get an event for a Link TRB in the middle of a TD */
1691                 end_seg_dma = xhci_trb_virt_to_dma(cur_seg,
1692                                 &cur_seg->trbs[TRBS_PER_SEGMENT - 1]);
1693                 /* If the end TRB isn't in this segment, this is set to 0 */
1694                 end_trb_dma = xhci_trb_virt_to_dma(cur_seg, end_trb);
1695
1696                 if (debug)
1697                         xhci_warn(xhci,
1698                                 "Looking for event-dma %016llx trb-start %016llx trb-end %016llx seg-start %016llx seg-end %016llx\n",
1699                                 (unsigned long long)suspect_dma,
1700                                 (unsigned long long)start_dma,
1701                                 (unsigned long long)end_trb_dma,
1702                                 (unsigned long long)cur_seg->dma,
1703                                 (unsigned long long)end_seg_dma);
1704
1705                 if (end_trb_dma > 0) {
1706                         /* The end TRB is in this segment, so suspect should be here */
1707                         if (start_dma <= end_trb_dma) {
1708                                 if (suspect_dma >= start_dma && suspect_dma <= end_trb_dma)
1709                                         return cur_seg;
1710                         } else {
1711                                 /* Case for one segment with
1712                                  * a TD wrapped around to the top
1713                                  */
1714                                 if ((suspect_dma >= start_dma &&
1715                                                         suspect_dma <= end_seg_dma) ||
1716                                                 (suspect_dma >= cur_seg->dma &&
1717                                                  suspect_dma <= end_trb_dma))
1718                                         return cur_seg;
1719                         }
1720                         return NULL;
1721                 } else {
1722                         /* Might still be somewhere in this segment */
1723                         if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma)
1724                                 return cur_seg;
1725                 }
1726                 cur_seg = cur_seg->next;
1727                 start_dma = xhci_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]);
1728         } while (cur_seg != start_seg);
1729
1730         return NULL;
1731 }
1732
1733 static void xhci_cleanup_halted_endpoint(struct xhci_hcd *xhci,
1734                 unsigned int slot_id, unsigned int ep_index,
1735                 unsigned int stream_id,
1736                 struct xhci_td *td, union xhci_trb *event_trb)
1737 {
1738         struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
1739         struct xhci_command *command;
1740         command = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
1741         if (!command)
1742                 return;
1743
1744         ep->ep_state |= EP_HALTED;
1745         ep->stopped_stream = stream_id;
1746
1747         xhci_queue_reset_ep(xhci, command, slot_id, ep_index);
1748         xhci_cleanup_stalled_ring(xhci, ep_index, td);
1749
1750         ep->stopped_stream = 0;
1751
1752         xhci_ring_cmd_db(xhci);
1753 }
1754
1755 /* Check if an error has halted the endpoint ring.  The class driver will
1756  * cleanup the halt for a non-default control endpoint if we indicate a stall.
1757  * However, a babble and other errors also halt the endpoint ring, and the class
1758  * driver won't clear the halt in that case, so we need to issue a Set Transfer
1759  * Ring Dequeue Pointer command manually.
1760  */
1761 static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci,
1762                 struct xhci_ep_ctx *ep_ctx,
1763                 unsigned int trb_comp_code)
1764 {
1765         /* TRB completion codes that may require a manual halt cleanup */
1766         if (trb_comp_code == COMP_TX_ERR ||
1767                         trb_comp_code == COMP_BABBLE ||
1768                         trb_comp_code == COMP_SPLIT_ERR)
1769                 /* The 0.96 spec says a babbling control endpoint
1770                  * is not halted. The 0.96 spec says it is.  Some HW
1771                  * claims to be 0.95 compliant, but it halts the control
1772                  * endpoint anyway.  Check if a babble halted the
1773                  * endpoint.
1774                  */
1775                 if ((ep_ctx->ep_info & cpu_to_le32(EP_STATE_MASK)) ==
1776                     cpu_to_le32(EP_STATE_HALTED))
1777                         return 1;
1778
1779         return 0;
1780 }
1781
1782 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code)
1783 {
1784         if (trb_comp_code >= 224 && trb_comp_code <= 255) {
1785                 /* Vendor defined "informational" completion code,
1786                  * treat as not-an-error.
1787                  */
1788                 xhci_dbg(xhci, "Vendor defined info completion code %u\n",
1789                                 trb_comp_code);
1790                 xhci_dbg(xhci, "Treating code as success.\n");
1791                 return 1;
1792         }
1793         return 0;
1794 }
1795
1796 /*
1797  * Finish the td processing, remove the td from td list;
1798  * Return 1 if the urb can be given back.
1799  */
1800 static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
1801         union xhci_trb *event_trb, struct xhci_transfer_event *event,
1802         struct xhci_virt_ep *ep, int *status, bool skip)
1803 {
1804         struct xhci_virt_device *xdev;
1805         struct xhci_ring *ep_ring;
1806         unsigned int slot_id;
1807         int ep_index;
1808         struct urb *urb = NULL;
1809         struct xhci_ep_ctx *ep_ctx;
1810         int ret = 0;
1811         struct urb_priv *urb_priv;
1812         u32 trb_comp_code;
1813
1814         slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1815         xdev = xhci->devs[slot_id];
1816         ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1817         ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1818         ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1819         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1820
1821         if (skip)
1822                 goto td_cleanup;
1823
1824         if (trb_comp_code == COMP_STOP_INVAL ||
1825                         trb_comp_code == COMP_STOP ||
1826                         trb_comp_code == COMP_STOP_SHORT) {
1827                 /* The Endpoint Stop Command completion will take care of any
1828                  * stopped TDs.  A stopped TD may be restarted, so don't update
1829                  * the ring dequeue pointer or take this TD off any lists yet.
1830                  */
1831                 ep->stopped_td = td;
1832                 return 0;
1833         }
1834         if (trb_comp_code == COMP_STALL ||
1835                 xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
1836                                                 trb_comp_code)) {
1837                 /* Issue a reset endpoint command to clear the host side
1838                  * halt, followed by a set dequeue command to move the
1839                  * dequeue pointer past the TD.
1840                  * The class driver clears the device side halt later.
1841                  */
1842                 xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
1843                                         ep_ring->stream_id, td, event_trb);
1844         } else {
1845                 /* Update ring dequeue pointer */
1846                 while (ep_ring->dequeue != td->last_trb)
1847                         inc_deq(xhci, ep_ring);
1848                 inc_deq(xhci, ep_ring);
1849         }
1850
1851 td_cleanup:
1852         /* Clean up the endpoint's TD list */
1853         urb = td->urb;
1854         urb_priv = urb->hcpriv;
1855
1856         /* Do one last check of the actual transfer length.
1857          * If the host controller said we transferred more data than the buffer
1858          * length, urb->actual_length will be a very big number (since it's
1859          * unsigned).  Play it safe and say we didn't transfer anything.
1860          */
1861         if (urb->actual_length > urb->transfer_buffer_length) {
1862                 xhci_warn(xhci, "URB transfer length is wrong, xHC issue? req. len = %u, act. len = %u\n",
1863                         urb->transfer_buffer_length,
1864                         urb->actual_length);
1865                 urb->actual_length = 0;
1866                 if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
1867                         *status = -EREMOTEIO;
1868                 else
1869                         *status = 0;
1870         }
1871         list_del_init(&td->td_list);
1872         /* Was this TD slated to be cancelled but completed anyway? */
1873         if (!list_empty(&td->cancelled_td_list))
1874                 list_del_init(&td->cancelled_td_list);
1875
1876         urb_priv->td_cnt++;
1877         /* Giveback the urb when all the tds are completed */
1878         if (urb_priv->td_cnt == urb_priv->length) {
1879                 ret = 1;
1880                 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
1881                         xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
1882                         if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
1883                                 if (xhci->quirks & XHCI_AMD_PLL_FIX)
1884                                         usb_amd_quirk_pll_enable();
1885                         }
1886                 }
1887         }
1888
1889         return ret;
1890 }
1891
1892 /*
1893  * Process control tds, update urb status and actual_length.
1894  */
1895 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
1896         union xhci_trb *event_trb, struct xhci_transfer_event *event,
1897         struct xhci_virt_ep *ep, int *status)
1898 {
1899         struct xhci_virt_device *xdev;
1900         struct xhci_ring *ep_ring;
1901         unsigned int slot_id;
1902         int ep_index;
1903         struct xhci_ep_ctx *ep_ctx;
1904         u32 trb_comp_code;
1905
1906         slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1907         xdev = xhci->devs[slot_id];
1908         ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1909         ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1910         ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1911         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1912
1913         switch (trb_comp_code) {
1914         case COMP_SUCCESS:
1915                 if (event_trb == ep_ring->dequeue) {
1916                         xhci_warn(xhci, "WARN: Success on ctrl setup TRB "
1917                                         "without IOC set??\n");
1918                         *status = -ESHUTDOWN;
1919                 } else if (event_trb != td->last_trb) {
1920                         xhci_warn(xhci, "WARN: Success on ctrl data TRB "
1921                                         "without IOC set??\n");
1922                         *status = -ESHUTDOWN;
1923                 } else {
1924                         *status = 0;
1925                 }
1926                 break;
1927         case COMP_SHORT_TX:
1928                 if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
1929                         *status = -EREMOTEIO;
1930                 else
1931                         *status = 0;
1932                 break;
1933         case COMP_STOP_SHORT:
1934                 if (event_trb == ep_ring->dequeue || event_trb == td->last_trb)
1935                         xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n");
1936                 else
1937                         td->urb->actual_length =
1938                                 EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1939
1940                 return finish_td(xhci, td, event_trb, event, ep, status, false);
1941         case COMP_STOP:
1942                 /* Did we stop at data stage? */
1943                 if (event_trb != ep_ring->dequeue && event_trb != td->last_trb)
1944                         td->urb->actual_length =
1945                                 td->urb->transfer_buffer_length -
1946                                 EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1947                 /* fall through */
1948         case COMP_STOP_INVAL:
1949                 return finish_td(xhci, td, event_trb, event, ep, status, false);
1950         default:
1951                 if (!xhci_requires_manual_halt_cleanup(xhci,
1952                                         ep_ctx, trb_comp_code))
1953                         break;
1954                 xhci_dbg(xhci, "TRB error code %u, "
1955                                 "halted endpoint index = %u\n",
1956                                 trb_comp_code, ep_index);
1957                 /* else fall through */
1958         case COMP_STALL:
1959                 /* Did we transfer part of the data (middle) phase? */
1960                 if (event_trb != ep_ring->dequeue &&
1961                                 event_trb != td->last_trb)
1962                         td->urb->actual_length =
1963                                 td->urb->transfer_buffer_length -
1964                                 EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1965                 else if (!td->urb_length_set)
1966                         td->urb->actual_length = 0;
1967
1968                 return finish_td(xhci, td, event_trb, event, ep, status, false);
1969         }
1970         /*
1971          * Did we transfer any data, despite the errors that might have
1972          * happened?  I.e. did we get past the setup stage?
1973          */
1974         if (event_trb != ep_ring->dequeue) {
1975                 /* The event was for the status stage */
1976                 if (event_trb == td->last_trb) {
1977                         if (td->urb_length_set) {
1978                                 /* Don't overwrite a previously set error code
1979                                  */
1980                                 if ((*status == -EINPROGRESS || *status == 0) &&
1981                                                 (td->urb->transfer_flags
1982                                                  & URB_SHORT_NOT_OK))
1983                                         /* Did we already see a short data
1984                                          * stage? */
1985                                         *status = -EREMOTEIO;
1986                         } else {
1987                                 td->urb->actual_length =
1988                                         td->urb->transfer_buffer_length;
1989                         }
1990                 } else {
1991                         /*
1992                          * Maybe the event was for the data stage? If so, update
1993                          * already the actual_length of the URB and flag it as
1994                          * set, so that it is not overwritten in the event for
1995                          * the last TRB.
1996                          */
1997                         td->urb_length_set = true;
1998                         td->urb->actual_length =
1999                                 td->urb->transfer_buffer_length -
2000                                 EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2001                         xhci_dbg(xhci, "Waiting for status "
2002                                         "stage event\n");
2003                         return 0;
2004                 }
2005         }
2006
2007         return finish_td(xhci, td, event_trb, event, ep, status, false);
2008 }
2009
2010 /*
2011  * Process isochronous tds, update urb packet status and actual_length.
2012  */
2013 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2014         union xhci_trb *event_trb, struct xhci_transfer_event *event,
2015         struct xhci_virt_ep *ep, int *status)
2016 {
2017         struct xhci_ring *ep_ring;
2018         struct urb_priv *urb_priv;
2019         int idx;
2020         int len = 0;
2021         union xhci_trb *cur_trb;
2022         struct xhci_segment *cur_seg;
2023         struct usb_iso_packet_descriptor *frame;
2024         u32 trb_comp_code;
2025         bool skip_td = false;
2026
2027         ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2028         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2029         urb_priv = td->urb->hcpriv;
2030         idx = urb_priv->td_cnt;
2031         frame = &td->urb->iso_frame_desc[idx];
2032
2033         /* handle completion code */
2034         switch (trb_comp_code) {
2035         case COMP_SUCCESS:
2036                 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0) {
2037                         frame->status = 0;
2038                         break;
2039                 }
2040                 if ((xhci->quirks & XHCI_TRUST_TX_LENGTH))
2041                         trb_comp_code = COMP_SHORT_TX;
2042         /* fallthrough */
2043         case COMP_STOP_SHORT:
2044         case COMP_SHORT_TX:
2045                 frame->status = td->urb->transfer_flags & URB_SHORT_NOT_OK ?
2046                                 -EREMOTEIO : 0;
2047                 break;
2048         case COMP_BW_OVER:
2049                 frame->status = -ECOMM;
2050                 skip_td = true;
2051                 break;
2052         case COMP_BUFF_OVER:
2053         case COMP_BABBLE:
2054                 frame->status = -EOVERFLOW;
2055                 skip_td = true;
2056                 break;
2057         case COMP_DEV_ERR:
2058         case COMP_STALL:
2059                 frame->status = -EPROTO;
2060                 skip_td = true;
2061                 break;
2062         case COMP_TX_ERR:
2063                 frame->status = -EPROTO;
2064                 if (event_trb != td->last_trb)
2065                         return 0;
2066                 skip_td = true;
2067                 break;
2068         case COMP_STOP:
2069         case COMP_STOP_INVAL:
2070                 break;
2071         default:
2072                 frame->status = -1;
2073                 break;
2074         }
2075
2076         if (trb_comp_code == COMP_SUCCESS || skip_td) {
2077                 frame->actual_length = frame->length;
2078                 td->urb->actual_length += frame->length;
2079         } else if (trb_comp_code == COMP_STOP_SHORT) {
2080                 frame->actual_length =
2081                         EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2082                 td->urb->actual_length += frame->actual_length;
2083         } else {
2084                 for (cur_trb = ep_ring->dequeue,
2085                      cur_seg = ep_ring->deq_seg; cur_trb != event_trb;
2086                      next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) {
2087                         if (!TRB_TYPE_NOOP_LE32(cur_trb->generic.field[3]) &&
2088                             !TRB_TYPE_LINK_LE32(cur_trb->generic.field[3]))
2089                                 len += TRB_LEN(le32_to_cpu(cur_trb->generic.field[2]));
2090                 }
2091                 len += TRB_LEN(le32_to_cpu(cur_trb->generic.field[2])) -
2092                         EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2093
2094                 if (trb_comp_code != COMP_STOP_INVAL) {
2095                         frame->actual_length = len;
2096                         td->urb->actual_length += len;
2097                 }
2098         }
2099
2100         return finish_td(xhci, td, event_trb, event, ep, status, false);
2101 }
2102
2103 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2104                         struct xhci_transfer_event *event,
2105                         struct xhci_virt_ep *ep, int *status)
2106 {
2107         struct xhci_ring *ep_ring;
2108         struct urb_priv *urb_priv;
2109         struct usb_iso_packet_descriptor *frame;
2110         int idx;
2111
2112         ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2113         urb_priv = td->urb->hcpriv;
2114         idx = urb_priv->td_cnt;
2115         frame = &td->urb->iso_frame_desc[idx];
2116
2117         /* The transfer is partly done. */
2118         frame->status = -EXDEV;
2119
2120         /* calc actual length */
2121         frame->actual_length = 0;
2122
2123         /* Update ring dequeue pointer */
2124         while (ep_ring->dequeue != td->last_trb)
2125                 inc_deq(xhci, ep_ring);
2126         inc_deq(xhci, ep_ring);
2127
2128         return finish_td(xhci, td, NULL, event, ep, status, true);
2129 }
2130
2131 /*
2132  * Process bulk and interrupt tds, update urb status and actual_length.
2133  */
2134 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
2135         union xhci_trb *event_trb, struct xhci_transfer_event *event,
2136         struct xhci_virt_ep *ep, int *status)
2137 {
2138         struct xhci_ring *ep_ring;
2139         union xhci_trb *cur_trb;
2140         struct xhci_segment *cur_seg;
2141         u32 trb_comp_code;
2142
2143         ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2144         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2145
2146         switch (trb_comp_code) {
2147         case COMP_SUCCESS:
2148                 /* Double check that the HW transferred everything. */
2149                 if (event_trb != td->last_trb ||
2150                     EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) != 0) {
2151                         xhci_warn(xhci, "WARN Successful completion "
2152                                         "on short TX\n");
2153                         if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
2154                                 *status = -EREMOTEIO;
2155                         else
2156                                 *status = 0;
2157                         if ((xhci->quirks & XHCI_TRUST_TX_LENGTH))
2158                                 trb_comp_code = COMP_SHORT_TX;
2159                 } else {
2160                         *status = 0;
2161                 }
2162                 break;
2163         case COMP_STOP_SHORT:
2164         case COMP_SHORT_TX:
2165                 if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
2166                         *status = -EREMOTEIO;
2167                 else
2168                         *status = 0;
2169                 break;
2170         default:
2171                 /* Others already handled above */
2172                 break;
2173         }
2174         if (trb_comp_code == COMP_SHORT_TX)
2175                 xhci_dbg(xhci, "ep %#x - asked for %d bytes, "
2176                                 "%d bytes untransferred\n",
2177                                 td->urb->ep->desc.bEndpointAddress,
2178                                 td->urb->transfer_buffer_length,
2179                                 EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)));
2180         /* Stopped - short packet completion */
2181         if (trb_comp_code == COMP_STOP_SHORT) {
2182                 td->urb->actual_length =
2183                         EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2184
2185                 if (td->urb->transfer_buffer_length <
2186                                 td->urb->actual_length) {
2187                         xhci_warn(xhci, "HC gave bad length of %d bytes txed\n",
2188                                 EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)));
2189                         td->urb->actual_length = 0;
2190                          /* status will be set by usb core for canceled urbs */
2191                 }
2192         /* Fast path - was this the last TRB in the TD for this URB? */
2193         } else if (event_trb == td->last_trb) {
2194                 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) != 0) {
2195                         td->urb->actual_length =
2196                                 td->urb->transfer_buffer_length -
2197                                 EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2198                         if (td->urb->transfer_buffer_length <
2199                                         td->urb->actual_length) {
2200                                 xhci_warn(xhci, "HC gave bad length "
2201                                                 "of %d bytes left\n",
2202                                           EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)));
2203                                 td->urb->actual_length = 0;
2204                                 if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
2205                                         *status = -EREMOTEIO;
2206                                 else
2207                                         *status = 0;
2208                         }
2209                         /* Don't overwrite a previously set error code */
2210                         if (*status == -EINPROGRESS) {
2211                                 if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
2212                                         *status = -EREMOTEIO;
2213                                 else
2214                                         *status = 0;
2215                         }
2216                 } else {
2217                         td->urb->actual_length =
2218                                 td->urb->transfer_buffer_length;
2219                         /* Ignore a short packet completion if the
2220                          * untransferred length was zero.
2221                          */
2222                         if (*status == -EREMOTEIO)
2223                                 *status = 0;
2224                 }
2225         } else {
2226                 /* Slow path - walk the list, starting from the dequeue
2227                  * pointer, to get the actual length transferred.
2228                  */
2229                 td->urb->actual_length = 0;
2230                 for (cur_trb = ep_ring->dequeue, cur_seg = ep_ring->deq_seg;
2231                                 cur_trb != event_trb;
2232                                 next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) {
2233                         if (!TRB_TYPE_NOOP_LE32(cur_trb->generic.field[3]) &&
2234                             !TRB_TYPE_LINK_LE32(cur_trb->generic.field[3]))
2235                                 td->urb->actual_length +=
2236                                         TRB_LEN(le32_to_cpu(cur_trb->generic.field[2]));
2237                 }
2238                 /* If the ring didn't stop on a Link or No-op TRB, add
2239                  * in the actual bytes transferred from the Normal TRB
2240                  */
2241                 if (trb_comp_code != COMP_STOP_INVAL)
2242                         td->urb->actual_length +=
2243                                 TRB_LEN(le32_to_cpu(cur_trb->generic.field[2])) -
2244                                 EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2245         }
2246
2247         return finish_td(xhci, td, event_trb, event, ep, status, false);
2248 }
2249
2250 /*
2251  * If this function returns an error condition, it means it got a Transfer
2252  * event with a corrupted Slot ID, Endpoint ID, or TRB DMA address.
2253  * At this point, the host controller is probably hosed and should be reset.
2254  */
2255 static int handle_tx_event(struct xhci_hcd *xhci,
2256                 struct xhci_transfer_event *event)
2257         __releases(&xhci->lock)
2258         __acquires(&xhci->lock)
2259 {
2260         struct xhci_virt_device *xdev;
2261         struct xhci_virt_ep *ep;
2262         struct xhci_ring *ep_ring;
2263         unsigned int slot_id;
2264         int ep_index;
2265         struct xhci_td *td = NULL;
2266         dma_addr_t event_dma;
2267         struct xhci_segment *event_seg;
2268         union xhci_trb *event_trb;
2269         struct urb *urb = NULL;
2270         int status = -EINPROGRESS;
2271         struct urb_priv *urb_priv;
2272         struct xhci_ep_ctx *ep_ctx;
2273         struct list_head *tmp;
2274         u32 trb_comp_code;
2275         int ret = 0;
2276         int td_num = 0;
2277
2278         slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
2279         xdev = xhci->devs[slot_id];
2280         if (!xdev) {
2281                 xhci_err(xhci, "ERROR Transfer event pointed to bad slot\n");
2282                 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
2283                          (unsigned long long) xhci_trb_virt_to_dma(
2284                                  xhci->event_ring->deq_seg,
2285                                  xhci->event_ring->dequeue),
2286                          lower_32_bits(le64_to_cpu(event->buffer)),
2287                          upper_32_bits(le64_to_cpu(event->buffer)),
2288                          le32_to_cpu(event->transfer_len),
2289                          le32_to_cpu(event->flags));
2290                 xhci_dbg(xhci, "Event ring:\n");
2291                 xhci_debug_segment(xhci, xhci->event_ring->deq_seg);
2292                 return -ENODEV;
2293         }
2294
2295         /* Endpoint ID is 1 based, our index is zero based */
2296         ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
2297         ep = &xdev->eps[ep_index];
2298         ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2299         ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2300         if (!ep_ring ||
2301             (le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK) ==
2302             EP_STATE_DISABLED) {
2303                 xhci_err(xhci, "ERROR Transfer event for disabled endpoint "
2304                                 "or incorrect stream ring\n");
2305                 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
2306                          (unsigned long long) xhci_trb_virt_to_dma(
2307                                  xhci->event_ring->deq_seg,
2308                                  xhci->event_ring->dequeue),
2309                          lower_32_bits(le64_to_cpu(event->buffer)),
2310                          upper_32_bits(le64_to_cpu(event->buffer)),
2311                          le32_to_cpu(event->transfer_len),
2312                          le32_to_cpu(event->flags));
2313                 xhci_dbg(xhci, "Event ring:\n");
2314                 xhci_debug_segment(xhci, xhci->event_ring->deq_seg);
2315                 return -ENODEV;
2316         }
2317
2318         /* Count current td numbers if ep->skip is set */
2319         if (ep->skip) {
2320                 list_for_each(tmp, &ep_ring->td_list)
2321                         td_num++;
2322         }
2323
2324         event_dma = le64_to_cpu(event->buffer);
2325         trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2326         /* Look for common error cases */
2327         switch (trb_comp_code) {
2328         /* Skip codes that require special handling depending on
2329          * transfer type
2330          */
2331         case COMP_SUCCESS:
2332                 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0)
2333                         break;
2334                 if (xhci->quirks & XHCI_TRUST_TX_LENGTH)
2335                         trb_comp_code = COMP_SHORT_TX;
2336                 else
2337                         xhci_warn_ratelimited(xhci,
2338                                         "WARN Successful completion on short TX: needs XHCI_TRUST_TX_LENGTH quirk?\n");
2339         case COMP_SHORT_TX:
2340                 break;
2341         case COMP_STOP:
2342                 xhci_dbg(xhci, "Stopped on Transfer TRB\n");
2343                 break;
2344         case COMP_STOP_INVAL:
2345                 xhci_dbg(xhci, "Stopped on No-op or Link TRB\n");
2346                 break;
2347         case COMP_STOP_SHORT:
2348                 xhci_dbg(xhci, "Stopped with short packet transfer detected\n");
2349                 break;
2350         case COMP_STALL:
2351                 xhci_dbg(xhci, "Stalled endpoint\n");
2352                 ep->ep_state |= EP_HALTED;
2353                 status = -EPIPE;
2354                 break;
2355         case COMP_TRB_ERR:
2356                 xhci_warn(xhci, "WARN: TRB error on endpoint\n");
2357                 status = -EILSEQ;
2358                 break;
2359         case COMP_SPLIT_ERR:
2360         case COMP_TX_ERR:
2361                 xhci_dbg(xhci, "Transfer error on endpoint\n");
2362                 status = -EPROTO;
2363                 break;
2364         case COMP_BABBLE:
2365                 xhci_dbg(xhci, "Babble error on endpoint\n");
2366                 status = -EOVERFLOW;
2367                 break;
2368         case COMP_DB_ERR:
2369                 xhci_warn(xhci, "WARN: HC couldn't access mem fast enough\n");
2370                 status = -ENOSR;
2371                 break;
2372         case COMP_BW_OVER:
2373                 xhci_warn(xhci, "WARN: bandwidth overrun event on endpoint\n");
2374                 break;
2375         case COMP_BUFF_OVER:
2376                 xhci_warn(xhci, "WARN: buffer overrun event on endpoint\n");
2377                 break;
2378         case COMP_UNDERRUN:
2379                 /*
2380                  * When the Isoch ring is empty, the xHC will generate
2381                  * a Ring Overrun Event for IN Isoch endpoint or Ring
2382                  * Underrun Event for OUT Isoch endpoint.
2383                  */
2384                 xhci_dbg(xhci, "underrun event on endpoint\n");
2385                 if (!list_empty(&ep_ring->td_list))
2386                         xhci_dbg(xhci, "Underrun Event for slot %d ep %d "
2387                                         "still with TDs queued?\n",
2388                                  TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2389                                  ep_index);
2390                 goto cleanup;
2391         case COMP_OVERRUN:
2392                 xhci_dbg(xhci, "overrun event on endpoint\n");
2393                 if (!list_empty(&ep_ring->td_list))
2394                         xhci_dbg(xhci, "Overrun Event for slot %d ep %d "
2395                                         "still with TDs queued?\n",
2396                                  TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2397                                  ep_index);
2398                 goto cleanup;
2399         case COMP_DEV_ERR:
2400                 xhci_warn(xhci, "WARN: detect an incompatible device");
2401                 status = -EPROTO;
2402                 break;
2403         case COMP_MISSED_INT:
2404                 /*
2405                  * When encounter missed service error, one or more isoc tds
2406                  * may be missed by xHC.
2407                  * Set skip flag of the ep_ring; Complete the missed tds as
2408                  * short transfer when process the ep_ring next time.
2409                  */
2410                 ep->skip = true;
2411                 xhci_dbg(xhci, "Miss service interval error, set skip flag\n");
2412                 goto cleanup;
2413         default:
2414                 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) {
2415                         status = 0;
2416                         break;
2417                 }
2418                 xhci_warn(xhci, "ERROR Unknown event condition %u, HC probably busted\n",
2419                           trb_comp_code);
2420                 goto cleanup;
2421         }
2422
2423         do {
2424                 /* This TRB should be in the TD at the head of this ring's
2425                  * TD list.
2426                  */
2427                 if (list_empty(&ep_ring->td_list)) {
2428                         /*
2429                          * A stopped endpoint may generate an extra completion
2430                          * event if the device was suspended.  Don't print
2431                          * warnings.
2432                          */
2433                         if (!(trb_comp_code == COMP_STOP ||
2434                                                 trb_comp_code == COMP_STOP_INVAL)) {
2435                                 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n",
2436                                                 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2437                                                 ep_index);
2438                                 xhci_dbg(xhci, "Event TRB with TRB type ID %u\n",
2439                                                 (le32_to_cpu(event->flags) &
2440                                                  TRB_TYPE_BITMASK)>>10);
2441                                 xhci_print_trb_offsets(xhci, (union xhci_trb *) event);
2442                         }
2443                         if (ep->skip) {
2444                                 ep->skip = false;
2445                                 xhci_dbg(xhci, "td_list is empty while skip "
2446                                                 "flag set. Clear skip flag.\n");
2447                         }
2448                         ret = 0;
2449                         goto cleanup;
2450                 }
2451
2452                 /* We've skipped all the TDs on the ep ring when ep->skip set */
2453                 if (ep->skip && td_num == 0) {
2454                         ep->skip = false;
2455                         xhci_dbg(xhci, "All tds on the ep_ring skipped. "
2456                                                 "Clear skip flag.\n");
2457                         ret = 0;
2458                         goto cleanup;
2459                 }
2460
2461                 td = list_entry(ep_ring->td_list.next, struct xhci_td, td_list);
2462                 if (ep->skip)
2463                         td_num--;
2464
2465                 /* Is this a TRB in the currently executing TD? */
2466                 event_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,
2467                                 td->last_trb, event_dma, false);
2468
2469                 /*
2470                  * Skip the Force Stopped Event. The event_trb(event_dma) of FSE
2471                  * is not in the current TD pointed by ep_ring->dequeue because
2472                  * that the hardware dequeue pointer still at the previous TRB
2473                  * of the current TD. The previous TRB maybe a Link TD or the
2474                  * last TRB of the previous TD. The command completion handle
2475                  * will take care the rest.
2476                  */
2477                 if (!event_seg && (trb_comp_code == COMP_STOP ||
2478                                    trb_comp_code == COMP_STOP_INVAL)) {
2479                         ret = 0;
2480                         goto cleanup;
2481                 }
2482
2483                 if (!event_seg) {
2484                         if (!ep->skip ||
2485                             !usb_endpoint_xfer_isoc(&td->urb->ep->desc)) {
2486                                 /* Some host controllers give a spurious
2487                                  * successful event after a short transfer.
2488                                  * Ignore it.
2489                                  */
2490                                 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) &&
2491                                                 ep_ring->last_td_was_short) {
2492                                         ep_ring->last_td_was_short = false;
2493                                         ret = 0;
2494                                         goto cleanup;
2495                                 }
2496                                 /* HC is busted, give up! */
2497                                 xhci_err(xhci,
2498                                         "ERROR Transfer event TRB DMA ptr not "
2499                                         "part of current TD ep_index %d "
2500                                         "comp_code %u\n", ep_index,
2501                                         trb_comp_code);
2502                                 trb_in_td(xhci, ep_ring->deq_seg,
2503                                           ep_ring->dequeue, td->last_trb,
2504                                           event_dma, true);
2505                                 return -ESHUTDOWN;
2506                         }
2507
2508                         ret = skip_isoc_td(xhci, td, event, ep, &status);
2509                         goto cleanup;
2510                 }
2511                 if (trb_comp_code == COMP_SHORT_TX)
2512                         ep_ring->last_td_was_short = true;
2513                 else
2514                         ep_ring->last_td_was_short = false;
2515
2516                 if (ep->skip) {
2517                         xhci_dbg(xhci, "Found td. Clear skip flag.\n");
2518                         ep->skip = false;
2519                 }
2520
2521                 event_trb = &event_seg->trbs[(event_dma - event_seg->dma) /
2522                                                 sizeof(*event_trb)];
2523                 /*
2524                  * No-op TRB should not trigger interrupts.
2525                  * If event_trb is a no-op TRB, it means the
2526                  * corresponding TD has been cancelled. Just ignore
2527                  * the TD.
2528                  */
2529                 if (TRB_TYPE_NOOP_LE32(event_trb->generic.field[3])) {
2530                         xhci_dbg(xhci,
2531                                  "event_trb is a no-op TRB. Skip it\n");
2532                         goto cleanup;
2533                 }
2534
2535                 /* Now update the urb's actual_length and give back to
2536                  * the core
2537                  */
2538                 if (usb_endpoint_xfer_control(&td->urb->ep->desc))
2539                         ret = process_ctrl_td(xhci, td, event_trb, event, ep,
2540                                                  &status);
2541                 else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc))
2542                         ret = process_isoc_td(xhci, td, event_trb, event, ep,
2543                                                  &status);
2544                 else
2545                         ret = process_bulk_intr_td(xhci, td, event_trb, event,
2546                                                  ep, &status);
2547
2548 cleanup:
2549                 /*
2550                  * Do not update event ring dequeue pointer if ep->skip is set.
2551                  * Will roll back to continue process missed tds.
2552                  */
2553                 if (trb_comp_code == COMP_MISSED_INT || !ep->skip) {
2554                         inc_deq(xhci, xhci->event_ring);
2555                 }
2556
2557                 if (ret) {
2558                         urb = td->urb;
2559                         urb_priv = urb->hcpriv;
2560
2561                         xhci_urb_free_priv(urb_priv);
2562
2563                         usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb);
2564                         if ((urb->actual_length != urb->transfer_buffer_length &&
2565                                                 (urb->transfer_flags &
2566                                                  URB_SHORT_NOT_OK)) ||
2567                                         (status != 0 &&
2568                                          !usb_endpoint_xfer_isoc(&urb->ep->desc)))
2569                                 xhci_dbg(xhci, "Giveback URB %p, len = %d, "
2570                                                 "expected = %d, status = %d\n",
2571                                                 urb, urb->actual_length,
2572                                                 urb->transfer_buffer_length,
2573                                                 status);
2574                         spin_unlock(&xhci->lock);
2575                         /* EHCI, UHCI, and OHCI always unconditionally set the
2576                          * urb->status of an isochronous endpoint to 0.
2577                          */
2578                         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
2579                                 status = 0;
2580                         usb_hcd_giveback_urb(bus_to_hcd(urb->dev->bus), urb, status);
2581                         spin_lock(&xhci->lock);
2582                 }
2583
2584         /*
2585          * If ep->skip is set, it means there are missed tds on the
2586          * endpoint ring need to take care of.
2587          * Process them as short transfer until reach the td pointed by
2588          * the event.
2589          */
2590         } while (ep->skip && trb_comp_code != COMP_MISSED_INT);
2591
2592         return 0;
2593 }
2594
2595 /*
2596  * This function handles all OS-owned events on the event ring.  It may drop
2597  * xhci->lock between event processing (e.g. to pass up port status changes).
2598  * Returns >0 for "possibly more events to process" (caller should call again),
2599  * otherwise 0 if done.  In future, <0 returns should indicate error code.
2600  */
2601 static int xhci_handle_event(struct xhci_hcd *xhci)
2602 {
2603         union xhci_trb *event;
2604         int update_ptrs = 1;
2605         int ret;
2606
2607         if (!xhci->event_ring || !xhci->event_ring->dequeue) {
2608                 xhci->error_bitmask |= 1 << 1;
2609                 return 0;
2610         }
2611
2612         event = xhci->event_ring->dequeue;
2613         /* Does the HC or OS own the TRB? */
2614         if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
2615             xhci->event_ring->cycle_state) {
2616                 xhci->error_bitmask |= 1 << 2;
2617                 return 0;
2618         }
2619
2620         /*
2621          * Barrier between reading the TRB_CYCLE (valid) flag above and any
2622          * speculative reads of the event's flags/data below.
2623          */
2624         rmb();
2625         /* FIXME: Handle more event types. */
2626         switch ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK)) {
2627         case TRB_TYPE(TRB_COMPLETION):
2628                 handle_cmd_completion(xhci, &event->event_cmd);
2629                 break;
2630         case TRB_TYPE(TRB_PORT_STATUS):
2631                 handle_port_status(xhci, event);
2632                 update_ptrs = 0;
2633                 break;
2634         case TRB_TYPE(TRB_TRANSFER):
2635                 ret = handle_tx_event(xhci, &event->trans_event);
2636                 if (ret < 0)
2637                         xhci->error_bitmask |= 1 << 9;
2638                 else
2639                         update_ptrs = 0;
2640                 break;
2641         case TRB_TYPE(TRB_DEV_NOTE):
2642                 handle_device_notification(xhci, event);
2643                 break;
2644         default:
2645                 if ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) >=
2646                     TRB_TYPE(48))
2647                         handle_vendor_event(xhci, event);
2648                 else
2649                         xhci->error_bitmask |= 1 << 3;
2650         }
2651         /* Any of the above functions may drop and re-acquire the lock, so check
2652          * to make sure a watchdog timer didn't mark the host as non-responsive.
2653          */
2654         if (xhci->xhc_state & XHCI_STATE_DYING) {
2655                 xhci_dbg(xhci, "xHCI host dying, returning from "
2656                                 "event handler.\n");
2657                 return 0;
2658         }
2659
2660         if (update_ptrs)
2661                 /* Update SW event ring dequeue pointer */
2662                 inc_deq(xhci, xhci->event_ring);
2663
2664         /* Are there more items on the event ring?  Caller will call us again to
2665          * check.
2666          */
2667         return 1;
2668 }
2669
2670 /*
2671  * xHCI spec says we can get an interrupt, and if the HC has an error condition,
2672  * we might get bad data out of the event ring.  Section 4.10.2.7 has a list of
2673  * indicators of an event TRB error, but we check the status *first* to be safe.
2674  */
2675 irqreturn_t xhci_irq(struct usb_hcd *hcd)
2676 {
2677         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
2678         u32 status;
2679         u64 temp_64;
2680         union xhci_trb *event_ring_deq;
2681         dma_addr_t deq;
2682
2683         spin_lock(&xhci->lock);
2684         /* Check if the xHC generated the interrupt, or the irq is shared */
2685         status = readl(&xhci->op_regs->status);
2686         if (status == 0xffffffff)
2687                 goto hw_died;
2688
2689         if (!(status & STS_EINT)) {
2690                 spin_unlock(&xhci->lock);
2691                 return IRQ_NONE;
2692         }
2693         if (status & STS_FATAL) {
2694                 xhci_warn(xhci, "WARNING: Host System Error\n");
2695                 xhci_halt(xhci);
2696 hw_died:
2697                 spin_unlock(&xhci->lock);
2698                 return IRQ_HANDLED;
2699         }
2700
2701         /*
2702          * Clear the op reg interrupt status first,
2703          * so we can receive interrupts from other MSI-X interrupters.
2704          * Write 1 to clear the interrupt status.
2705          */
2706         status |= STS_EINT;
2707         writel(status, &xhci->op_regs->status);
2708         /* FIXME when MSI-X is supported and there are multiple vectors */
2709         /* Clear the MSI-X event interrupt status */
2710
2711         if (hcd->irq) {
2712                 u32 irq_pending;
2713                 /* Acknowledge the PCI interrupt */
2714                 irq_pending = readl(&xhci->ir_set->irq_pending);
2715                 irq_pending |= IMAN_IP;
2716                 writel(irq_pending, &xhci->ir_set->irq_pending);
2717         }
2718
2719         if (xhci->xhc_state & XHCI_STATE_DYING) {
2720                 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. "
2721                                 "Shouldn't IRQs be disabled?\n");
2722                 /* Clear the event handler busy flag (RW1C);
2723                  * the event ring should be empty.
2724                  */
2725                 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2726                 xhci_write_64(xhci, temp_64 | ERST_EHB,
2727                                 &xhci->ir_set->erst_dequeue);
2728                 spin_unlock(&xhci->lock);
2729
2730                 return IRQ_HANDLED;
2731         }
2732
2733         event_ring_deq = xhci->event_ring->dequeue;
2734         /* FIXME this should be a delayed service routine
2735          * that clears the EHB.
2736          */
2737         while (xhci_handle_event(xhci) > 0) {}
2738
2739         temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2740         /* If necessary, update the HW's version of the event ring deq ptr. */
2741         if (event_ring_deq != xhci->event_ring->dequeue) {
2742                 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
2743                                 xhci->event_ring->dequeue);
2744                 if (deq == 0)
2745                         xhci_warn(xhci, "WARN something wrong with SW event "
2746                                         "ring dequeue ptr.\n");
2747                 /* Update HC event ring dequeue pointer */
2748                 temp_64 &= ERST_PTR_MASK;
2749                 temp_64 |= ((u64) deq & (u64) ~ERST_PTR_MASK);
2750         }
2751
2752         /* Clear the event handler busy flag (RW1C); event ring is empty. */
2753         temp_64 |= ERST_EHB;
2754         xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue);
2755
2756         spin_unlock(&xhci->lock);
2757
2758         return IRQ_HANDLED;
2759 }
2760
2761 irqreturn_t xhci_msi_irq(int irq, void *hcd)
2762 {
2763         return xhci_irq(hcd);
2764 }
2765
2766 /****           Endpoint Ring Operations        ****/
2767
2768 /*
2769  * Generic function for queueing a TRB on a ring.
2770  * The caller must have checked to make sure there's room on the ring.
2771  *
2772  * @more_trbs_coming:   Will you enqueue more TRBs before calling
2773  *                      prepare_transfer()?
2774  */
2775 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
2776                 bool more_trbs_coming,
2777                 u32 field1, u32 field2, u32 field3, u32 field4)
2778 {
2779         struct xhci_generic_trb *trb;
2780
2781         trb = &ring->enqueue->generic;
2782         trb->field[0] = cpu_to_le32(field1);
2783         trb->field[1] = cpu_to_le32(field2);
2784         trb->field[2] = cpu_to_le32(field3);
2785         trb->field[3] = cpu_to_le32(field4);
2786         inc_enq(xhci, ring, more_trbs_coming);
2787 }
2788
2789 /*
2790  * Does various checks on the endpoint ring, and makes it ready to queue num_trbs.
2791  * FIXME allocate segments if the ring is full.
2792  */
2793 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
2794                 u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
2795 {
2796         unsigned int num_trbs_needed;
2797
2798         /* Make sure the endpoint has been added to xHC schedule */
2799         switch (ep_state) {
2800         case EP_STATE_DISABLED:
2801                 /*
2802                  * USB core changed config/interfaces without notifying us,
2803                  * or hardware is reporting the wrong state.
2804                  */
2805                 xhci_warn(xhci, "WARN urb submitted to disabled ep\n");
2806                 return -ENOENT;
2807         case EP_STATE_ERROR:
2808                 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n");
2809                 /* FIXME event handling code for error needs to clear it */
2810                 /* XXX not sure if this should be -ENOENT or not */
2811                 return -EINVAL;
2812         case EP_STATE_HALTED:
2813                 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n");
2814         case EP_STATE_STOPPED:
2815         case EP_STATE_RUNNING:
2816                 break;
2817         default:
2818                 xhci_err(xhci, "ERROR unknown endpoint state for ep\n");
2819                 /*
2820                  * FIXME issue Configure Endpoint command to try to get the HC
2821                  * back into a known state.
2822                  */
2823                 return -EINVAL;
2824         }
2825
2826         while (1) {
2827                 if (room_on_ring(xhci, ep_ring, num_trbs))
2828                         break;
2829
2830                 if (ep_ring == xhci->cmd_ring) {
2831                         xhci_err(xhci, "Do not support expand command ring\n");
2832                         return -ENOMEM;
2833                 }
2834
2835                 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion,
2836                                 "ERROR no room on ep ring, try ring expansion");
2837                 num_trbs_needed = num_trbs - ep_ring->num_trbs_free;
2838                 if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed,
2839                                         mem_flags)) {
2840                         xhci_err(xhci, "Ring expansion failed\n");
2841                         return -ENOMEM;
2842                 }
2843         }
2844
2845         if (enqueue_is_link_trb(ep_ring)) {
2846                 struct xhci_ring *ring = ep_ring;
2847                 union xhci_trb *next;
2848
2849                 next = ring->enqueue;
2850
2851                 while (last_trb(xhci, ring, ring->enq_seg, next)) {
2852                         /* If we're not dealing with 0.95 hardware or isoc rings
2853                          * on AMD 0.96 host, clear the chain bit.
2854                          */
2855                         if (!xhci_link_trb_quirk(xhci) &&
2856                                         !(ring->type == TYPE_ISOC &&
2857                                          (xhci->quirks & XHCI_AMD_0x96_HOST)))
2858                                 next->link.control &= cpu_to_le32(~TRB_CHAIN);
2859                         else
2860                                 next->link.control |= cpu_to_le32(TRB_CHAIN);
2861
2862                         wmb();
2863                         next->link.control ^= cpu_to_le32(TRB_CYCLE);