96e6bdadcbd46eb3adbf961d28a9f586866a0f68
[pandora-kernel.git] / drivers / usb / host / xhci.c
1 /*
2  * xHCI host controller driver
3  *
4  * Copyright (C) 2008 Intel Corp.
5  *
6  * Author: Sarah Sharp
7  * Some code borrowed from the Linux EHCI driver.
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License version 2 as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
16  * for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software Foundation,
20  * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21  */
22
23 #include <linux/pci.h>
24 #include <linux/irq.h>
25 #include <linux/log2.h>
26 #include <linux/module.h>
27 #include <linux/moduleparam.h>
28 #include <linux/slab.h>
29 #include <linux/dmi.h>
30
31 #include "xhci.h"
32
33 #define DRIVER_AUTHOR "Sarah Sharp"
34 #define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver"
35
36 /* Some 0.95 hardware can't handle the chain bit on a Link TRB being cleared */
37 static int link_quirk;
38 module_param(link_quirk, int, S_IRUGO | S_IWUSR);
39 MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB");
40
41 /* TODO: copied from ehci-hcd.c - can this be refactored? */
42 /*
43  * handshake - spin reading hc until handshake completes or fails
44  * @ptr: address of hc register to be read
45  * @mask: bits to look at in result of read
46  * @done: value of those bits when handshake succeeds
47  * @usec: timeout in microseconds
48  *
49  * Returns negative errno, or zero on success
50  *
51  * Success happens when the "mask" bits have the specified value (hardware
52  * handshake done).  There are two failure modes:  "usec" have passed (major
53  * hardware flakeout), or the register reads as all-ones (hardware removed).
54  */
55 int handshake(struct xhci_hcd *xhci, void __iomem *ptr,
56                       u32 mask, u32 done, int usec)
57 {
58         u32     result;
59
60         do {
61                 result = xhci_readl(xhci, ptr);
62                 if (result == ~(u32)0)          /* card removed */
63                         return -ENODEV;
64                 result &= mask;
65                 if (result == done)
66                         return 0;
67                 udelay(1);
68                 usec--;
69         } while (usec > 0);
70         return -ETIMEDOUT;
71 }
72
73 /*
74  * Disable interrupts and begin the xHCI halting process.
75  */
76 void xhci_quiesce(struct xhci_hcd *xhci)
77 {
78         u32 halted;
79         u32 cmd;
80         u32 mask;
81
82         mask = ~(XHCI_IRQS);
83         halted = xhci_readl(xhci, &xhci->op_regs->status) & STS_HALT;
84         if (!halted)
85                 mask &= ~CMD_RUN;
86
87         cmd = xhci_readl(xhci, &xhci->op_regs->command);
88         cmd &= mask;
89         xhci_writel(xhci, cmd, &xhci->op_regs->command);
90 }
91
92 /*
93  * Force HC into halt state.
94  *
95  * Disable any IRQs and clear the run/stop bit.
96  * HC will complete any current and actively pipelined transactions, and
97  * should halt within 16 ms of the run/stop bit being cleared.
98  * Read HC Halted bit in the status register to see when the HC is finished.
99  */
100 int xhci_halt(struct xhci_hcd *xhci)
101 {
102         int ret;
103         xhci_dbg(xhci, "// Halt the HC\n");
104         xhci_quiesce(xhci);
105
106         ret = handshake(xhci, &xhci->op_regs->status,
107                         STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
108         if (!ret) {
109                 xhci->xhc_state |= XHCI_STATE_HALTED;
110                 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
111         } else
112                 xhci_warn(xhci, "Host not halted after %u microseconds.\n",
113                                 XHCI_MAX_HALT_USEC);
114         return ret;
115 }
116
117 /*
118  * Set the run bit and wait for the host to be running.
119  */
120 static int xhci_start(struct xhci_hcd *xhci)
121 {
122         u32 temp;
123         int ret;
124
125         temp = xhci_readl(xhci, &xhci->op_regs->command);
126         temp |= (CMD_RUN);
127         xhci_dbg(xhci, "// Turn on HC, cmd = 0x%x.\n",
128                         temp);
129         xhci_writel(xhci, temp, &xhci->op_regs->command);
130
131         /*
132          * Wait for the HCHalted Status bit to be 0 to indicate the host is
133          * running.
134          */
135         ret = handshake(xhci, &xhci->op_regs->status,
136                         STS_HALT, 0, XHCI_MAX_HALT_USEC);
137         if (ret == -ETIMEDOUT)
138                 xhci_err(xhci, "Host took too long to start, "
139                                 "waited %u microseconds.\n",
140                                 XHCI_MAX_HALT_USEC);
141         if (!ret)
142                 xhci->xhc_state &= ~XHCI_STATE_HALTED;
143         return ret;
144 }
145
146 /*
147  * Reset a halted HC.
148  *
149  * This resets pipelines, timers, counters, state machines, etc.
150  * Transactions will be terminated immediately, and operational registers
151  * will be set to their defaults.
152  */
153 int xhci_reset(struct xhci_hcd *xhci)
154 {
155         u32 command;
156         u32 state;
157         int ret;
158
159         state = xhci_readl(xhci, &xhci->op_regs->status);
160         if ((state & STS_HALT) == 0) {
161                 xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
162                 return 0;
163         }
164
165         xhci_dbg(xhci, "// Reset the HC\n");
166         command = xhci_readl(xhci, &xhci->op_regs->command);
167         command |= CMD_RESET;
168         xhci_writel(xhci, command, &xhci->op_regs->command);
169
170         ret = handshake(xhci, &xhci->op_regs->command,
171                         CMD_RESET, 0, 10 * 1000 * 1000);
172         if (ret)
173                 return ret;
174
175         xhci_dbg(xhci, "Wait for controller to be ready for doorbell rings\n");
176         /*
177          * xHCI cannot write to any doorbells or operational registers other
178          * than status until the "Controller Not Ready" flag is cleared.
179          */
180         return handshake(xhci, &xhci->op_regs->status,
181                          STS_CNR, 0, 10 * 1000 * 1000);
182 }
183
184 #ifdef CONFIG_PCI
185 static int xhci_free_msi(struct xhci_hcd *xhci)
186 {
187         int i;
188
189         if (!xhci->msix_entries)
190                 return -EINVAL;
191
192         for (i = 0; i < xhci->msix_count; i++)
193                 if (xhci->msix_entries[i].vector)
194                         free_irq(xhci->msix_entries[i].vector,
195                                         xhci_to_hcd(xhci));
196         return 0;
197 }
198
199 /*
200  * Set up MSI
201  */
202 static int xhci_setup_msi(struct xhci_hcd *xhci)
203 {
204         int ret;
205         struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
206
207         ret = pci_enable_msi(pdev);
208         if (ret) {
209                 xhci_dbg(xhci, "failed to allocate MSI entry\n");
210                 return ret;
211         }
212
213         ret = request_irq(pdev->irq, (irq_handler_t)xhci_msi_irq,
214                                 0, "xhci_hcd", xhci_to_hcd(xhci));
215         if (ret) {
216                 xhci_dbg(xhci, "disable MSI interrupt\n");
217                 pci_disable_msi(pdev);
218         }
219
220         return ret;
221 }
222
223 /*
224  * Free IRQs
225  * free all IRQs request
226  */
227 static void xhci_free_irq(struct xhci_hcd *xhci)
228 {
229         struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
230         int ret;
231
232         /* return if using legacy interrupt */
233         if (xhci_to_hcd(xhci)->irq >= 0)
234                 return;
235
236         ret = xhci_free_msi(xhci);
237         if (!ret)
238                 return;
239         if (pdev->irq >= 0)
240                 free_irq(pdev->irq, xhci_to_hcd(xhci));
241
242         return;
243 }
244
245 /*
246  * Set up MSI-X
247  */
248 static int xhci_setup_msix(struct xhci_hcd *xhci)
249 {
250         int i, ret = 0;
251         struct usb_hcd *hcd = xhci_to_hcd(xhci);
252         struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
253
254         /*
255          * calculate number of msi-x vectors supported.
256          * - HCS_MAX_INTRS: the max number of interrupts the host can handle,
257          *   with max number of interrupters based on the xhci HCSPARAMS1.
258          * - num_online_cpus: maximum msi-x vectors per CPUs core.
259          *   Add additional 1 vector to ensure always available interrupt.
260          */
261         xhci->msix_count = min(num_online_cpus() + 1,
262                                 HCS_MAX_INTRS(xhci->hcs_params1));
263
264         xhci->msix_entries =
265                 kmalloc((sizeof(struct msix_entry))*xhci->msix_count,
266                                 GFP_KERNEL);
267         if (!xhci->msix_entries) {
268                 xhci_err(xhci, "Failed to allocate MSI-X entries\n");
269                 return -ENOMEM;
270         }
271
272         for (i = 0; i < xhci->msix_count; i++) {
273                 xhci->msix_entries[i].entry = i;
274                 xhci->msix_entries[i].vector = 0;
275         }
276
277         ret = pci_enable_msix(pdev, xhci->msix_entries, xhci->msix_count);
278         if (ret) {
279                 xhci_dbg(xhci, "Failed to enable MSI-X\n");
280                 goto free_entries;
281         }
282
283         for (i = 0; i < xhci->msix_count; i++) {
284                 ret = request_irq(xhci->msix_entries[i].vector,
285                                 (irq_handler_t)xhci_msi_irq,
286                                 0, "xhci_hcd", xhci_to_hcd(xhci));
287                 if (ret)
288                         goto disable_msix;
289         }
290
291         hcd->msix_enabled = 1;
292         return ret;
293
294 disable_msix:
295         xhci_dbg(xhci, "disable MSI-X interrupt\n");
296         xhci_free_irq(xhci);
297         pci_disable_msix(pdev);
298 free_entries:
299         kfree(xhci->msix_entries);
300         xhci->msix_entries = NULL;
301         return ret;
302 }
303
304 /* Free any IRQs and disable MSI-X */
305 static void xhci_cleanup_msix(struct xhci_hcd *xhci)
306 {
307         struct usb_hcd *hcd = xhci_to_hcd(xhci);
308         struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
309
310         xhci_free_irq(xhci);
311
312         if (xhci->msix_entries) {
313                 pci_disable_msix(pdev);
314                 kfree(xhci->msix_entries);
315                 xhci->msix_entries = NULL;
316         } else {
317                 pci_disable_msi(pdev);
318         }
319
320         hcd->msix_enabled = 0;
321         return;
322 }
323
324 static void xhci_msix_sync_irqs(struct xhci_hcd *xhci)
325 {
326         int i;
327
328         if (xhci->msix_entries) {
329                 for (i = 0; i < xhci->msix_count; i++)
330                         synchronize_irq(xhci->msix_entries[i].vector);
331         }
332 }
333
334 static int xhci_try_enable_msi(struct usb_hcd *hcd)
335 {
336         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
337         struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
338         int ret;
339
340         /*
341          * Some Fresco Logic host controllers advertise MSI, but fail to
342          * generate interrupts.  Don't even try to enable MSI.
343          */
344         if (xhci->quirks & XHCI_BROKEN_MSI)
345                 goto legacy_irq;
346
347         /* unregister the legacy interrupt */
348         if (hcd->irq)
349                 free_irq(hcd->irq, hcd);
350         hcd->irq = -1;
351
352         ret = xhci_setup_msix(xhci);
353         if (ret)
354                 /* fall back to msi*/
355                 ret = xhci_setup_msi(xhci);
356
357         if (!ret)
358                 /* hcd->irq is -1, we have MSI */
359                 return 0;
360
361         if (!pdev->irq) {
362                 xhci_err(xhci, "No msi-x/msi found and no IRQ in BIOS\n");
363                 return -EINVAL;
364         }
365
366  legacy_irq:
367         /* fall back to legacy interrupt*/
368         ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,
369                         hcd->irq_descr, hcd);
370         if (ret) {
371                 xhci_err(xhci, "request interrupt %d failed\n",
372                                 pdev->irq);
373                 return ret;
374         }
375         hcd->irq = pdev->irq;
376         return 0;
377 }
378
379 #else
380
381 static int xhci_try_enable_msi(struct usb_hcd *hcd)
382 {
383         return 0;
384 }
385
386 static void xhci_cleanup_msix(struct xhci_hcd *xhci)
387 {
388 }
389
390 static void xhci_msix_sync_irqs(struct xhci_hcd *xhci)
391 {
392 }
393
394 #endif
395
396 static void compliance_mode_recovery(unsigned long arg)
397 {
398         struct xhci_hcd *xhci;
399         struct usb_hcd *hcd;
400         u32 temp;
401         int i;
402
403         xhci = (struct xhci_hcd *)arg;
404
405         for (i = 0; i < xhci->num_usb3_ports; i++) {
406                 temp = xhci_readl(xhci, xhci->usb3_ports[i]);
407                 if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) {
408                         /*
409                          * Compliance Mode Detected. Letting USB Core
410                          * handle the Warm Reset
411                          */
412                         xhci_dbg(xhci, "Compliance Mode Detected->Port %d!\n",
413                                         i + 1);
414                         xhci_dbg(xhci, "Attempting Recovery routine!\n");
415                         hcd = xhci->shared_hcd;
416
417                         if (hcd->state == HC_STATE_SUSPENDED)
418                                 usb_hcd_resume_root_hub(hcd);
419
420                         usb_hcd_poll_rh_status(hcd);
421                 }
422         }
423
424         if (xhci->port_status_u0 != ((1 << xhci->num_usb3_ports)-1))
425                 mod_timer(&xhci->comp_mode_recovery_timer,
426                         jiffies + msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
427 }
428
429 /*
430  * Quirk to work around issue generated by the SN65LVPE502CP USB3.0 re-driver
431  * that causes ports behind that hardware to enter compliance mode sometimes.
432  * The quirk creates a timer that polls every 2 seconds the link state of
433  * each host controller's port and recovers it by issuing a Warm reset
434  * if Compliance mode is detected, otherwise the port will become "dead" (no
435  * device connections or disconnections will be detected anymore). Becasue no
436  * status event is generated when entering compliance mode (per xhci spec),
437  * this quirk is needed on systems that have the failing hardware installed.
438  */
439 static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci)
440 {
441         xhci->port_status_u0 = 0;
442         init_timer(&xhci->comp_mode_recovery_timer);
443
444         xhci->comp_mode_recovery_timer.data = (unsigned long) xhci;
445         xhci->comp_mode_recovery_timer.function = compliance_mode_recovery;
446         xhci->comp_mode_recovery_timer.expires = jiffies +
447                         msecs_to_jiffies(COMP_MODE_RCVRY_MSECS);
448
449         set_timer_slack(&xhci->comp_mode_recovery_timer,
450                         msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
451         add_timer(&xhci->comp_mode_recovery_timer);
452         xhci_dbg(xhci, "Compliance Mode Recovery Timer Initialized.\n");
453 }
454
455 /*
456  * This function identifies the systems that have installed the SN65LVPE502CP
457  * USB3.0 re-driver and that need the Compliance Mode Quirk.
458  * Systems:
459  * Vendor: Hewlett-Packard -> System Models: Z420, Z620 and Z820
460  */
461 static bool compliance_mode_recovery_timer_quirk_check(void)
462 {
463         const char *dmi_product_name, *dmi_sys_vendor;
464
465         dmi_product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
466         dmi_sys_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
467         if (!dmi_product_name || !dmi_sys_vendor)
468                 return false;
469
470         if (!(strstr(dmi_sys_vendor, "Hewlett-Packard")))
471                 return false;
472
473         if (strstr(dmi_product_name, "Z420") ||
474                         strstr(dmi_product_name, "Z620") ||
475                         strstr(dmi_product_name, "Z820") ||
476                         strstr(dmi_product_name, "Z1 Workstation"))
477                 return true;
478
479         return false;
480 }
481
482 static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci)
483 {
484         return (xhci->port_status_u0 == ((1 << xhci->num_usb3_ports)-1));
485 }
486
487
488 /*
489  * Initialize memory for HCD and xHC (one-time init).
490  *
491  * Program the PAGESIZE register, initialize the device context array, create
492  * device contexts (?), set up a command ring segment (or two?), create event
493  * ring (one for now).
494  */
495 int xhci_init(struct usb_hcd *hcd)
496 {
497         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
498         int retval = 0;
499
500         xhci_dbg(xhci, "xhci_init\n");
501         spin_lock_init(&xhci->lock);
502         if (xhci->hci_version == 0x95 && link_quirk) {
503                 xhci_dbg(xhci, "QUIRK: Not clearing Link TRB chain bits.\n");
504                 xhci->quirks |= XHCI_LINK_TRB_QUIRK;
505         } else {
506                 xhci_dbg(xhci, "xHCI doesn't need link TRB QUIRK\n");
507         }
508         retval = xhci_mem_init(xhci, GFP_KERNEL);
509         xhci_dbg(xhci, "Finished xhci_init\n");
510
511         /* Initializing Compliance Mode Recovery Data If Needed */
512         if (compliance_mode_recovery_timer_quirk_check()) {
513                 xhci->quirks |= XHCI_COMP_MODE_QUIRK;
514                 compliance_mode_recovery_timer_init(xhci);
515         }
516
517         return retval;
518 }
519
520 /*-------------------------------------------------------------------------*/
521
522
523 #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
524 static void xhci_event_ring_work(unsigned long arg)
525 {
526         unsigned long flags;
527         int temp;
528         u64 temp_64;
529         struct xhci_hcd *xhci = (struct xhci_hcd *) arg;
530         int i, j;
531
532         xhci_dbg(xhci, "Poll event ring: %lu\n", jiffies);
533
534         spin_lock_irqsave(&xhci->lock, flags);
535         temp = xhci_readl(xhci, &xhci->op_regs->status);
536         xhci_dbg(xhci, "op reg status = 0x%x\n", temp);
537         if (temp == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||
538                         (xhci->xhc_state & XHCI_STATE_HALTED)) {
539                 xhci_dbg(xhci, "HW died, polling stopped.\n");
540                 spin_unlock_irqrestore(&xhci->lock, flags);
541                 return;
542         }
543
544         temp = xhci_readl(xhci, &xhci->ir_set->irq_pending);
545         xhci_dbg(xhci, "ir_set 0 pending = 0x%x\n", temp);
546         xhci_dbg(xhci, "HC error bitmask = 0x%x\n", xhci->error_bitmask);
547         xhci->error_bitmask = 0;
548         xhci_dbg(xhci, "Event ring:\n");
549         xhci_debug_segment(xhci, xhci->event_ring->deq_seg);
550         xhci_dbg_ring_ptrs(xhci, xhci->event_ring);
551         temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
552         temp_64 &= ~ERST_PTR_MASK;
553         xhci_dbg(xhci, "ERST deq = 64'h%0lx\n", (long unsigned int) temp_64);
554         xhci_dbg(xhci, "Command ring:\n");
555         xhci_debug_segment(xhci, xhci->cmd_ring->deq_seg);
556         xhci_dbg_ring_ptrs(xhci, xhci->cmd_ring);
557         xhci_dbg_cmd_ptrs(xhci);
558         for (i = 0; i < MAX_HC_SLOTS; ++i) {
559                 if (!xhci->devs[i])
560                         continue;
561                 for (j = 0; j < 31; ++j) {
562                         xhci_dbg_ep_rings(xhci, i, j, &xhci->devs[i]->eps[j]);
563                 }
564         }
565         spin_unlock_irqrestore(&xhci->lock, flags);
566
567         if (!xhci->zombie)
568                 mod_timer(&xhci->event_ring_timer, jiffies + POLL_TIMEOUT * HZ);
569         else
570                 xhci_dbg(xhci, "Quit polling the event ring.\n");
571 }
572 #endif
573
574 static int xhci_run_finished(struct xhci_hcd *xhci)
575 {
576         if (xhci_start(xhci)) {
577                 xhci_halt(xhci);
578                 return -ENODEV;
579         }
580         xhci->shared_hcd->state = HC_STATE_RUNNING;
581         xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
582
583         if (xhci->quirks & XHCI_NEC_HOST)
584                 xhci_ring_cmd_db(xhci);
585
586         xhci_dbg(xhci, "Finished xhci_run for USB3 roothub\n");
587         return 0;
588 }
589
590 /*
591  * Start the HC after it was halted.
592  *
593  * This function is called by the USB core when the HC driver is added.
594  * Its opposite is xhci_stop().
595  *
596  * xhci_init() must be called once before this function can be called.
597  * Reset the HC, enable device slot contexts, program DCBAAP, and
598  * set command ring pointer and event ring pointer.
599  *
600  * Setup MSI-X vectors and enable interrupts.
601  */
602 int xhci_run(struct usb_hcd *hcd)
603 {
604         u32 temp;
605         u64 temp_64;
606         int ret;
607         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
608
609         /* Start the xHCI host controller running only after the USB 2.0 roothub
610          * is setup.
611          */
612
613         hcd->uses_new_polling = 1;
614         if (!usb_hcd_is_primary_hcd(hcd))
615                 return xhci_run_finished(xhci);
616
617         xhci_dbg(xhci, "xhci_run\n");
618
619         ret = xhci_try_enable_msi(hcd);
620         if (ret)
621                 return ret;
622
623 #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
624         init_timer(&xhci->event_ring_timer);
625         xhci->event_ring_timer.data = (unsigned long) xhci;
626         xhci->event_ring_timer.function = xhci_event_ring_work;
627         /* Poll the event ring */
628         xhci->event_ring_timer.expires = jiffies + POLL_TIMEOUT * HZ;
629         xhci->zombie = 0;
630         xhci_dbg(xhci, "Setting event ring polling timer\n");
631         add_timer(&xhci->event_ring_timer);
632 #endif
633
634         xhci_dbg(xhci, "Command ring memory map follows:\n");
635         xhci_debug_ring(xhci, xhci->cmd_ring);
636         xhci_dbg_ring_ptrs(xhci, xhci->cmd_ring);
637         xhci_dbg_cmd_ptrs(xhci);
638
639         xhci_dbg(xhci, "ERST memory map follows:\n");
640         xhci_dbg_erst(xhci, &xhci->erst);
641         xhci_dbg(xhci, "Event ring:\n");
642         xhci_debug_ring(xhci, xhci->event_ring);
643         xhci_dbg_ring_ptrs(xhci, xhci->event_ring);
644         temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
645         temp_64 &= ~ERST_PTR_MASK;
646         xhci_dbg(xhci, "ERST deq = 64'h%0lx\n", (long unsigned int) temp_64);
647
648         xhci_dbg(xhci, "// Set the interrupt modulation register\n");
649         temp = xhci_readl(xhci, &xhci->ir_set->irq_control);
650         temp &= ~ER_IRQ_INTERVAL_MASK;
651         temp |= (u32) 160;
652         xhci_writel(xhci, temp, &xhci->ir_set->irq_control);
653
654         /* Set the HCD state before we enable the irqs */
655         temp = xhci_readl(xhci, &xhci->op_regs->command);
656         temp |= (CMD_EIE);
657         xhci_dbg(xhci, "// Enable interrupts, cmd = 0x%x.\n",
658                         temp);
659         xhci_writel(xhci, temp, &xhci->op_regs->command);
660
661         temp = xhci_readl(xhci, &xhci->ir_set->irq_pending);
662         xhci_dbg(xhci, "// Enabling event ring interrupter %p by writing 0x%x to irq_pending\n",
663                         xhci->ir_set, (unsigned int) ER_IRQ_ENABLE(temp));
664         xhci_writel(xhci, ER_IRQ_ENABLE(temp),
665                         &xhci->ir_set->irq_pending);
666         xhci_print_ir_set(xhci, 0);
667
668         if (xhci->quirks & XHCI_NEC_HOST)
669                 xhci_queue_vendor_command(xhci, 0, 0, 0,
670                                 TRB_TYPE(TRB_NEC_GET_FW));
671
672         xhci_dbg(xhci, "Finished xhci_run for USB2 roothub\n");
673         return 0;
674 }
675
676 static void xhci_only_stop_hcd(struct usb_hcd *hcd)
677 {
678         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
679
680         spin_lock_irq(&xhci->lock);
681         xhci_halt(xhci);
682
683         /* The shared_hcd is going to be deallocated shortly (the USB core only
684          * calls this function when allocation fails in usb_add_hcd(), or
685          * usb_remove_hcd() is called).  So we need to unset xHCI's pointer.
686          */
687         xhci->shared_hcd = NULL;
688         spin_unlock_irq(&xhci->lock);
689 }
690
691 /*
692  * Stop xHCI driver.
693  *
694  * This function is called by the USB core when the HC driver is removed.
695  * Its opposite is xhci_run().
696  *
697  * Disable device contexts, disable IRQs, and quiesce the HC.
698  * Reset the HC, finish any completed transactions, and cleanup memory.
699  */
700 void xhci_stop(struct usb_hcd *hcd)
701 {
702         u32 temp;
703         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
704
705         if (!usb_hcd_is_primary_hcd(hcd)) {
706                 xhci_only_stop_hcd(xhci->shared_hcd);
707                 return;
708         }
709
710         spin_lock_irq(&xhci->lock);
711         /* Make sure the xHC is halted for a USB3 roothub
712          * (xhci_stop() could be called as part of failed init).
713          */
714         xhci_halt(xhci);
715         xhci_reset(xhci);
716         spin_unlock_irq(&xhci->lock);
717
718         xhci_cleanup_msix(xhci);
719
720 #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
721         /* Tell the event ring poll function not to reschedule */
722         xhci->zombie = 1;
723         del_timer_sync(&xhci->event_ring_timer);
724 #endif
725
726         /* Deleting Compliance Mode Recovery Timer */
727         if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
728                         (!(xhci_all_ports_seen_u0(xhci))))
729                 del_timer_sync(&xhci->comp_mode_recovery_timer);
730
731         if (xhci->quirks & XHCI_AMD_PLL_FIX)
732                 usb_amd_dev_put();
733
734         xhci_dbg(xhci, "// Disabling event ring interrupts\n");
735         temp = xhci_readl(xhci, &xhci->op_regs->status);
736         xhci_writel(xhci, temp & ~STS_EINT, &xhci->op_regs->status);
737         temp = xhci_readl(xhci, &xhci->ir_set->irq_pending);
738         xhci_writel(xhci, ER_IRQ_DISABLE(temp),
739                         &xhci->ir_set->irq_pending);
740         xhci_print_ir_set(xhci, 0);
741
742         xhci_dbg(xhci, "cleaning up memory\n");
743         xhci_mem_cleanup(xhci);
744         xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
745                     xhci_readl(xhci, &xhci->op_regs->status));
746 }
747
748 /*
749  * Shutdown HC (not bus-specific)
750  *
751  * This is called when the machine is rebooting or halting.  We assume that the
752  * machine will be powered off, and the HC's internal state will be reset.
753  * Don't bother to free memory.
754  *
755  * This will only ever be called with the main usb_hcd (the USB3 roothub).
756  */
757 void xhci_shutdown(struct usb_hcd *hcd)
758 {
759         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
760
761         if (xhci->quirks & XHCI_SPURIOUS_REBOOT)
762                 usb_disable_xhci_ports(to_pci_dev(hcd->self.controller));
763
764         spin_lock_irq(&xhci->lock);
765         xhci_halt(xhci);
766         spin_unlock_irq(&xhci->lock);
767
768         xhci_cleanup_msix(xhci);
769
770         xhci_dbg(xhci, "xhci_shutdown completed - status = %x\n",
771                     xhci_readl(xhci, &xhci->op_regs->status));
772 }
773
774 #ifdef CONFIG_PM
775 static void xhci_save_registers(struct xhci_hcd *xhci)
776 {
777         xhci->s3.command = xhci_readl(xhci, &xhci->op_regs->command);
778         xhci->s3.dev_nt = xhci_readl(xhci, &xhci->op_regs->dev_notification);
779         xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
780         xhci->s3.config_reg = xhci_readl(xhci, &xhci->op_regs->config_reg);
781         xhci->s3.erst_size = xhci_readl(xhci, &xhci->ir_set->erst_size);
782         xhci->s3.erst_base = xhci_read_64(xhci, &xhci->ir_set->erst_base);
783         xhci->s3.erst_dequeue = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
784         xhci->s3.irq_pending = xhci_readl(xhci, &xhci->ir_set->irq_pending);
785         xhci->s3.irq_control = xhci_readl(xhci, &xhci->ir_set->irq_control);
786 }
787
788 static void xhci_restore_registers(struct xhci_hcd *xhci)
789 {
790         xhci_writel(xhci, xhci->s3.command, &xhci->op_regs->command);
791         xhci_writel(xhci, xhci->s3.dev_nt, &xhci->op_regs->dev_notification);
792         xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);
793         xhci_writel(xhci, xhci->s3.config_reg, &xhci->op_regs->config_reg);
794         xhci_writel(xhci, xhci->s3.erst_size, &xhci->ir_set->erst_size);
795         xhci_write_64(xhci, xhci->s3.erst_base, &xhci->ir_set->erst_base);
796         xhci_write_64(xhci, xhci->s3.erst_dequeue, &xhci->ir_set->erst_dequeue);
797         xhci_writel(xhci, xhci->s3.irq_pending, &xhci->ir_set->irq_pending);
798         xhci_writel(xhci, xhci->s3.irq_control, &xhci->ir_set->irq_control);
799 }
800
801 static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)
802 {
803         u64     val_64;
804
805         /* step 2: initialize command ring buffer */
806         val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
807         val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
808                 (xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
809                                       xhci->cmd_ring->dequeue) &
810                  (u64) ~CMD_RING_RSVD_BITS) |
811                 xhci->cmd_ring->cycle_state;
812         xhci_dbg(xhci, "// Setting command ring address to 0x%llx\n",
813                         (long unsigned long) val_64);
814         xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
815 }
816
817 /*
818  * The whole command ring must be cleared to zero when we suspend the host.
819  *
820  * The host doesn't save the command ring pointer in the suspend well, so we
821  * need to re-program it on resume.  Unfortunately, the pointer must be 64-byte
822  * aligned, because of the reserved bits in the command ring dequeue pointer
823  * register.  Therefore, we can't just set the dequeue pointer back in the
824  * middle of the ring (TRBs are 16-byte aligned).
825  */
826 static void xhci_clear_command_ring(struct xhci_hcd *xhci)
827 {
828         struct xhci_ring *ring;
829         struct xhci_segment *seg;
830
831         ring = xhci->cmd_ring;
832         seg = ring->deq_seg;
833         do {
834                 memset(seg->trbs, 0,
835                         sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));
836                 seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
837                         cpu_to_le32(~TRB_CYCLE);
838                 seg = seg->next;
839         } while (seg != ring->deq_seg);
840
841         /* Reset the software enqueue and dequeue pointers */
842         ring->deq_seg = ring->first_seg;
843         ring->dequeue = ring->first_seg->trbs;
844         ring->enq_seg = ring->deq_seg;
845         ring->enqueue = ring->dequeue;
846
847         /*
848          * Ring is now zeroed, so the HW should look for change of ownership
849          * when the cycle bit is set to 1.
850          */
851         ring->cycle_state = 1;
852
853         /*
854          * Reset the hardware dequeue pointer.
855          * Yes, this will need to be re-written after resume, but we're paranoid
856          * and want to make sure the hardware doesn't access bogus memory
857          * because, say, the BIOS or an SMI started the host without changing
858          * the command ring pointers.
859          */
860         xhci_set_cmd_ring_deq(xhci);
861 }
862
863 /*
864  * Stop HC (not bus-specific)
865  *
866  * This is called when the machine transition into S3/S4 mode.
867  *
868  */
869 int xhci_suspend(struct xhci_hcd *xhci)
870 {
871         int                     rc = 0;
872         unsigned int            delay = XHCI_MAX_HALT_USEC;
873         struct usb_hcd          *hcd = xhci_to_hcd(xhci);
874         u32                     command;
875
876         /* Don't poll the roothubs on bus suspend. */
877         xhci_dbg(xhci, "%s: stopping port polling.\n", __func__);
878         clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
879         del_timer_sync(&hcd->rh_timer);
880
881         spin_lock_irq(&xhci->lock);
882         clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
883         clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
884         /* step 1: stop endpoint */
885         /* skipped assuming that port suspend has done */
886
887         /* step 2: clear Run/Stop bit */
888         command = xhci_readl(xhci, &xhci->op_regs->command);
889         command &= ~CMD_RUN;
890         xhci_writel(xhci, command, &xhci->op_regs->command);
891
892         /* Some chips from Fresco Logic need an extraordinary delay */
893         delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1;
894
895         if (handshake(xhci, &xhci->op_regs->status,
896                       STS_HALT, STS_HALT, delay)) {
897                 xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");
898                 spin_unlock_irq(&xhci->lock);
899                 return -ETIMEDOUT;
900         }
901         xhci_clear_command_ring(xhci);
902
903         /* step 3: save registers */
904         xhci_save_registers(xhci);
905
906         /* step 4: set CSS flag */
907         command = xhci_readl(xhci, &xhci->op_regs->command);
908         command |= CMD_CSS;
909         xhci_writel(xhci, command, &xhci->op_regs->command);
910         if (handshake(xhci, &xhci->op_regs->status, STS_SAVE, 0, 10 * 1000)) {
911                 xhci_warn(xhci, "WARN: xHC save state timeout\n");
912                 spin_unlock_irq(&xhci->lock);
913                 return -ETIMEDOUT;
914         }
915         spin_unlock_irq(&xhci->lock);
916
917         /*
918          * Deleting Compliance Mode Recovery Timer because the xHCI Host
919          * is about to be suspended.
920          */
921         if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
922                         (!(xhci_all_ports_seen_u0(xhci)))) {
923                 del_timer_sync(&xhci->comp_mode_recovery_timer);
924                 xhci_dbg(xhci, "Compliance Mode Recovery Timer Deleted!\n");
925         }
926
927         /* step 5: remove core well power */
928         /* synchronize irq when using MSI-X */
929         xhci_msix_sync_irqs(xhci);
930
931         return rc;
932 }
933
934 /*
935  * start xHC (not bus-specific)
936  *
937  * This is called when the machine transition from S3/S4 mode.
938  *
939  */
940 int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
941 {
942         u32                     command, temp = 0;
943         struct usb_hcd          *hcd = xhci_to_hcd(xhci);
944         struct usb_hcd          *secondary_hcd;
945         int                     retval = 0;
946         bool                    comp_timer_running = false;
947
948         /* Wait a bit if either of the roothubs need to settle from the
949          * transition into bus suspend.
950          */
951         if (time_before(jiffies, xhci->bus_state[0].next_statechange) ||
952                         time_before(jiffies,
953                                 xhci->bus_state[1].next_statechange))
954                 msleep(100);
955
956         set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
957         set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
958
959         spin_lock_irq(&xhci->lock);
960         if (xhci->quirks & XHCI_RESET_ON_RESUME)
961                 hibernated = true;
962
963         if (!hibernated) {
964                 /* step 1: restore register */
965                 xhci_restore_registers(xhci);
966                 /* step 2: initialize command ring buffer */
967                 xhci_set_cmd_ring_deq(xhci);
968                 /* step 3: restore state and start state*/
969                 /* step 3: set CRS flag */
970                 command = xhci_readl(xhci, &xhci->op_regs->command);
971                 command |= CMD_CRS;
972                 xhci_writel(xhci, command, &xhci->op_regs->command);
973                 if (handshake(xhci, &xhci->op_regs->status,
974                               STS_RESTORE, 0, 10 * 1000)) {
975                         xhci_warn(xhci, "WARN: xHC restore state timeout\n");
976                         spin_unlock_irq(&xhci->lock);
977                         return -ETIMEDOUT;
978                 }
979                 temp = xhci_readl(xhci, &xhci->op_regs->status);
980         }
981
982         /* If restore operation fails, re-initialize the HC during resume */
983         if ((temp & STS_SRE) || hibernated) {
984
985                 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
986                                 !(xhci_all_ports_seen_u0(xhci))) {
987                         del_timer_sync(&xhci->comp_mode_recovery_timer);
988                         xhci_dbg(xhci, "Compliance Mode Recovery Timer deleted!\n");
989                 }
990
991                 /* Let the USB core know _both_ roothubs lost power. */
992                 usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);
993                 usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);
994
995                 xhci_dbg(xhci, "Stop HCD\n");
996                 xhci_halt(xhci);
997                 xhci_reset(xhci);
998                 spin_unlock_irq(&xhci->lock);
999                 xhci_cleanup_msix(xhci);
1000
1001 #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
1002                 /* Tell the event ring poll function not to reschedule */
1003                 xhci->zombie = 1;
1004                 del_timer_sync(&xhci->event_ring_timer);
1005 #endif
1006
1007                 xhci_dbg(xhci, "// Disabling event ring interrupts\n");
1008                 temp = xhci_readl(xhci, &xhci->op_regs->status);
1009                 xhci_writel(xhci, temp & ~STS_EINT, &xhci->op_regs->status);
1010                 temp = xhci_readl(xhci, &xhci->ir_set->irq_pending);
1011                 xhci_writel(xhci, ER_IRQ_DISABLE(temp),
1012                                 &xhci->ir_set->irq_pending);
1013                 xhci_print_ir_set(xhci, 0);
1014
1015                 xhci_dbg(xhci, "cleaning up memory\n");
1016                 xhci_mem_cleanup(xhci);
1017                 xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
1018                             xhci_readl(xhci, &xhci->op_regs->status));
1019
1020                 /* USB core calls the PCI reinit and start functions twice:
1021                  * first with the primary HCD, and then with the secondary HCD.
1022                  * If we don't do the same, the host will never be started.
1023                  */
1024                 if (!usb_hcd_is_primary_hcd(hcd))
1025                         secondary_hcd = hcd;
1026                 else
1027                         secondary_hcd = xhci->shared_hcd;
1028
1029                 xhci_dbg(xhci, "Initialize the xhci_hcd\n");
1030                 retval = xhci_init(hcd->primary_hcd);
1031                 if (retval)
1032                         return retval;
1033                 comp_timer_running = true;
1034
1035                 xhci_dbg(xhci, "Start the primary HCD\n");
1036                 retval = xhci_run(hcd->primary_hcd);
1037                 if (!retval) {
1038                         xhci_dbg(xhci, "Start the secondary HCD\n");
1039                         retval = xhci_run(secondary_hcd);
1040                 }
1041                 hcd->state = HC_STATE_SUSPENDED;
1042                 xhci->shared_hcd->state = HC_STATE_SUSPENDED;
1043                 goto done;
1044         }
1045
1046         /* step 4: set Run/Stop bit */
1047         command = xhci_readl(xhci, &xhci->op_regs->command);
1048         command |= CMD_RUN;
1049         xhci_writel(xhci, command, &xhci->op_regs->command);
1050         handshake(xhci, &xhci->op_regs->status, STS_HALT,
1051                   0, 250 * 1000);
1052
1053         /* step 5: walk topology and initialize portsc,
1054          * portpmsc and portli
1055          */
1056         /* this is done in bus_resume */
1057
1058         /* step 6: restart each of the previously
1059          * Running endpoints by ringing their doorbells
1060          */
1061
1062         spin_unlock_irq(&xhci->lock);
1063
1064  done:
1065         if (retval == 0) {
1066                 usb_hcd_resume_root_hub(hcd);
1067                 usb_hcd_resume_root_hub(xhci->shared_hcd);
1068         }
1069
1070         /*
1071          * If system is subject to the Quirk, Compliance Mode Timer needs to
1072          * be re-initialized Always after a system resume. Ports are subject
1073          * to suffer the Compliance Mode issue again. It doesn't matter if
1074          * ports have entered previously to U0 before system's suspension.
1075          */
1076         if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running)
1077                 compliance_mode_recovery_timer_init(xhci);
1078
1079         /* Re-enable port polling. */
1080         xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
1081         set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1082         usb_hcd_poll_rh_status(hcd);
1083
1084         return retval;
1085 }
1086 #endif  /* CONFIG_PM */
1087
1088 /*-------------------------------------------------------------------------*/
1089
1090 /**
1091  * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
1092  * HCDs.  Find the index for an endpoint given its descriptor.  Use the return
1093  * value to right shift 1 for the bitmask.
1094  *
1095  * Index  = (epnum * 2) + direction - 1,
1096  * where direction = 0 for OUT, 1 for IN.
1097  * For control endpoints, the IN index is used (OUT index is unused), so
1098  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
1099  */
1100 unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc)
1101 {
1102         unsigned int index;
1103         if (usb_endpoint_xfer_control(desc))
1104                 index = (unsigned int) (usb_endpoint_num(desc)*2);
1105         else
1106                 index = (unsigned int) (usb_endpoint_num(desc)*2) +
1107                         (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
1108         return index;
1109 }
1110
1111 /* Find the flag for this endpoint (for use in the control context).  Use the
1112  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
1113  * bit 1, etc.
1114  */
1115 unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc)
1116 {
1117         return 1 << (xhci_get_endpoint_index(desc) + 1);
1118 }
1119
1120 /* Find the flag for this endpoint (for use in the control context).  Use the
1121  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
1122  * bit 1, etc.
1123  */
1124 unsigned int xhci_get_endpoint_flag_from_index(unsigned int ep_index)
1125 {
1126         return 1 << (ep_index + 1);
1127 }
1128
1129 /* Compute the last valid endpoint context index.  Basically, this is the
1130  * endpoint index plus one.  For slot contexts with more than valid endpoint,
1131  * we find the most significant bit set in the added contexts flags.
1132  * e.g. ep 1 IN (with epnum 0x81) => added_ctxs = 0b1000
1133  * fls(0b1000) = 4, but the endpoint context index is 3, so subtract one.
1134  */
1135 unsigned int xhci_last_valid_endpoint(u32 added_ctxs)
1136 {
1137         return fls(added_ctxs) - 1;
1138 }
1139
1140 /* Returns 1 if the arguments are OK;
1141  * returns 0 this is a root hub; returns -EINVAL for NULL pointers.
1142  */
1143 static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev,
1144                 struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev,
1145                 const char *func) {
1146         struct xhci_hcd *xhci;
1147         struct xhci_virt_device *virt_dev;
1148
1149         if (!hcd || (check_ep && !ep) || !udev) {
1150                 printk(KERN_DEBUG "xHCI %s called with invalid args\n",
1151                                 func);
1152                 return -EINVAL;
1153         }
1154         if (!udev->parent) {
1155                 printk(KERN_DEBUG "xHCI %s called for root hub\n",
1156                                 func);
1157                 return 0;
1158         }
1159
1160         xhci = hcd_to_xhci(hcd);
1161         if (check_virt_dev) {
1162                 if (!udev->slot_id || !xhci->devs[udev->slot_id]) {
1163                         printk(KERN_DEBUG "xHCI %s called with unaddressed "
1164                                                 "device\n", func);
1165                         return -EINVAL;
1166                 }
1167
1168                 virt_dev = xhci->devs[udev->slot_id];
1169                 if (virt_dev->udev != udev) {
1170                         printk(KERN_DEBUG "xHCI %s called with udev and "
1171                                           "virt_dev does not match\n", func);
1172                         return -EINVAL;
1173                 }
1174         }
1175
1176         if (xhci->xhc_state & XHCI_STATE_HALTED)
1177                 return -ENODEV;
1178
1179         return 1;
1180 }
1181
1182 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
1183                 struct usb_device *udev, struct xhci_command *command,
1184                 bool ctx_change, bool must_succeed);
1185
1186 /*
1187  * Full speed devices may have a max packet size greater than 8 bytes, but the
1188  * USB core doesn't know that until it reads the first 8 bytes of the
1189  * descriptor.  If the usb_device's max packet size changes after that point,
1190  * we need to issue an evaluate context command and wait on it.
1191  */
1192 static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
1193                 unsigned int ep_index, struct urb *urb)
1194 {
1195         struct xhci_container_ctx *in_ctx;
1196         struct xhci_container_ctx *out_ctx;
1197         struct xhci_input_control_ctx *ctrl_ctx;
1198         struct xhci_ep_ctx *ep_ctx;
1199         int max_packet_size;
1200         int hw_max_packet_size;
1201         int ret = 0;
1202
1203         out_ctx = xhci->devs[slot_id]->out_ctx;
1204         ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1205         hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));
1206         max_packet_size = usb_endpoint_maxp(&urb->dev->ep0.desc);
1207         if (hw_max_packet_size != max_packet_size) {
1208                 xhci_dbg(xhci, "Max Packet Size for ep 0 changed.\n");
1209                 xhci_dbg(xhci, "Max packet size in usb_device = %d\n",
1210                                 max_packet_size);
1211                 xhci_dbg(xhci, "Max packet size in xHCI HW = %d\n",
1212                                 hw_max_packet_size);
1213                 xhci_dbg(xhci, "Issuing evaluate context command.\n");
1214
1215                 /* Set up the modified control endpoint 0 */
1216                 xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
1217                                 xhci->devs[slot_id]->out_ctx, ep_index);
1218                 in_ctx = xhci->devs[slot_id]->in_ctx;
1219                 ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
1220                 ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);
1221                 ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));
1222
1223                 /* Set up the input context flags for the command */
1224                 /* FIXME: This won't work if a non-default control endpoint
1225                  * changes max packet sizes.
1226                  */
1227                 ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
1228                 ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);
1229                 ctrl_ctx->drop_flags = 0;
1230
1231                 xhci_dbg(xhci, "Slot %d input context\n", slot_id);
1232                 xhci_dbg_ctx(xhci, in_ctx, ep_index);
1233                 xhci_dbg(xhci, "Slot %d output context\n", slot_id);
1234                 xhci_dbg_ctx(xhci, out_ctx, ep_index);
1235
1236                 ret = xhci_configure_endpoint(xhci, urb->dev, NULL,
1237                                 true, false);
1238
1239                 /* Clean up the input context for later use by bandwidth
1240                  * functions.
1241                  */
1242                 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);
1243         }
1244         return ret;
1245 }
1246
1247 /*
1248  * non-error returns are a promise to giveback() the urb later
1249  * we drop ownership so next owner (or urb unlink) can get it
1250  */
1251 int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
1252 {
1253         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
1254         struct xhci_td *buffer;
1255         unsigned long flags;
1256         int ret = 0;
1257         unsigned int slot_id, ep_index;
1258         struct urb_priv *urb_priv;
1259         int size, i;
1260
1261         if (!urb || xhci_check_args(hcd, urb->dev, urb->ep,
1262                                         true, true, __func__) <= 0)
1263                 return -EINVAL;
1264
1265         slot_id = urb->dev->slot_id;
1266         ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1267
1268         if (!HCD_HW_ACCESSIBLE(hcd)) {
1269                 if (!in_interrupt())
1270                         xhci_dbg(xhci, "urb submitted during PCI suspend\n");
1271                 ret = -ESHUTDOWN;
1272                 goto exit;
1273         }
1274
1275         if (usb_endpoint_xfer_isoc(&urb->ep->desc))
1276                 size = urb->number_of_packets;
1277         else
1278                 size = 1;
1279
1280         urb_priv = kzalloc(sizeof(struct urb_priv) +
1281                                   size * sizeof(struct xhci_td *), mem_flags);
1282         if (!urb_priv)
1283                 return -ENOMEM;
1284
1285         buffer = kzalloc(size * sizeof(struct xhci_td), mem_flags);
1286         if (!buffer) {
1287                 kfree(urb_priv);
1288                 return -ENOMEM;
1289         }
1290
1291         for (i = 0; i < size; i++) {
1292                 urb_priv->td[i] = buffer;
1293                 buffer++;
1294         }
1295
1296         urb_priv->length = size;
1297         urb_priv->td_cnt = 0;
1298         urb->hcpriv = urb_priv;
1299
1300         if (usb_endpoint_xfer_control(&urb->ep->desc)) {
1301                 /* Check to see if the max packet size for the default control
1302                  * endpoint changed during FS device enumeration
1303                  */
1304                 if (urb->dev->speed == USB_SPEED_FULL) {
1305                         ret = xhci_check_maxpacket(xhci, slot_id,
1306                                         ep_index, urb);
1307                         if (ret < 0) {
1308                                 xhci_urb_free_priv(xhci, urb_priv);
1309                                 urb->hcpriv = NULL;
1310                                 return ret;
1311                         }
1312                 }
1313
1314                 /* We have a spinlock and interrupts disabled, so we must pass
1315                  * atomic context to this function, which may allocate memory.
1316                  */
1317                 spin_lock_irqsave(&xhci->lock, flags);
1318                 if (xhci->xhc_state & XHCI_STATE_DYING)
1319                         goto dying;
1320                 ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
1321                                 slot_id, ep_index);
1322                 if (ret)
1323                         goto free_priv;
1324                 spin_unlock_irqrestore(&xhci->lock, flags);
1325         } else if (usb_endpoint_xfer_bulk(&urb->ep->desc)) {
1326                 spin_lock_irqsave(&xhci->lock, flags);
1327                 if (xhci->xhc_state & XHCI_STATE_DYING)
1328                         goto dying;
1329                 if (xhci->devs[slot_id]->eps[ep_index].ep_state &
1330                                 EP_GETTING_STREAMS) {
1331                         xhci_warn(xhci, "WARN: Can't enqueue URB while bulk ep "
1332                                         "is transitioning to using streams.\n");
1333                         ret = -EINVAL;
1334                 } else if (xhci->devs[slot_id]->eps[ep_index].ep_state &
1335                                 EP_GETTING_NO_STREAMS) {
1336                         xhci_warn(xhci, "WARN: Can't enqueue URB while bulk ep "
1337                                         "is transitioning to "
1338                                         "not having streams.\n");
1339                         ret = -EINVAL;
1340                 } else {
1341                         ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
1342                                         slot_id, ep_index);
1343                 }
1344                 if (ret)
1345                         goto free_priv;
1346                 spin_unlock_irqrestore(&xhci->lock, flags);
1347         } else if (usb_endpoint_xfer_int(&urb->ep->desc)) {
1348                 spin_lock_irqsave(&xhci->lock, flags);
1349                 if (xhci->xhc_state & XHCI_STATE_DYING)
1350                         goto dying;
1351                 ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
1352                                 slot_id, ep_index);
1353                 if (ret)
1354                         goto free_priv;
1355                 spin_unlock_irqrestore(&xhci->lock, flags);
1356         } else {
1357                 spin_lock_irqsave(&xhci->lock, flags);
1358                 if (xhci->xhc_state & XHCI_STATE_DYING)
1359                         goto dying;
1360                 ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
1361                                 slot_id, ep_index);
1362                 if (ret)
1363                         goto free_priv;
1364                 spin_unlock_irqrestore(&xhci->lock, flags);
1365         }
1366 exit:
1367         return ret;
1368 dying:
1369         xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for "
1370                         "non-responsive xHCI host.\n",
1371                         urb->ep->desc.bEndpointAddress, urb);
1372         ret = -ESHUTDOWN;
1373 free_priv:
1374         xhci_urb_free_priv(xhci, urb_priv);
1375         urb->hcpriv = NULL;
1376         spin_unlock_irqrestore(&xhci->lock, flags);
1377         return ret;
1378 }
1379
1380 /* Get the right ring for the given URB.
1381  * If the endpoint supports streams, boundary check the URB's stream ID.
1382  * If the endpoint doesn't support streams, return the singular endpoint ring.
1383  */
1384 static struct xhci_ring *xhci_urb_to_transfer_ring(struct xhci_hcd *xhci,
1385                 struct urb *urb)
1386 {
1387         unsigned int slot_id;
1388         unsigned int ep_index;
1389         unsigned int stream_id;
1390         struct xhci_virt_ep *ep;
1391
1392         slot_id = urb->dev->slot_id;
1393         ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1394         stream_id = urb->stream_id;
1395         ep = &xhci->devs[slot_id]->eps[ep_index];
1396         /* Common case: no streams */
1397         if (!(ep->ep_state & EP_HAS_STREAMS))
1398                 return ep->ring;
1399
1400         if (stream_id == 0) {
1401                 xhci_warn(xhci,
1402                                 "WARN: Slot ID %u, ep index %u has streams, "
1403                                 "but URB has no stream ID.\n",
1404                                 slot_id, ep_index);
1405                 return NULL;
1406         }
1407
1408         if (stream_id < ep->stream_info->num_streams)
1409                 return ep->stream_info->stream_rings[stream_id];
1410
1411         xhci_warn(xhci,
1412                         "WARN: Slot ID %u, ep index %u has "
1413                         "stream IDs 1 to %u allocated, "
1414                         "but stream ID %u is requested.\n",
1415                         slot_id, ep_index,
1416                         ep->stream_info->num_streams - 1,
1417                         stream_id);
1418         return NULL;
1419 }
1420
1421 /*
1422  * Remove the URB's TD from the endpoint ring.  This may cause the HC to stop
1423  * USB transfers, potentially stopping in the middle of a TRB buffer.  The HC
1424  * should pick up where it left off in the TD, unless a Set Transfer Ring
1425  * Dequeue Pointer is issued.
1426  *
1427  * The TRBs that make up the buffers for the canceled URB will be "removed" from
1428  * the ring.  Since the ring is a contiguous structure, they can't be physically
1429  * removed.  Instead, there are two options:
1430  *
1431  *  1) If the HC is in the middle of processing the URB to be canceled, we
1432  *     simply move the ring's dequeue pointer past those TRBs using the Set
1433  *     Transfer Ring Dequeue Pointer command.  This will be the common case,
1434  *     when drivers timeout on the last submitted URB and attempt to cancel.
1435  *
1436  *  2) If the HC is in the middle of a different TD, we turn the TRBs into a
1437  *     series of 1-TRB transfer no-op TDs.  (No-ops shouldn't be chained.)  The
1438  *     HC will need to invalidate the any TRBs it has cached after the stop
1439  *     endpoint command, as noted in the xHCI 0.95 errata.
1440  *
1441  *  3) The TD may have completed by the time the Stop Endpoint Command
1442  *     completes, so software needs to handle that case too.
1443  *
1444  * This function should protect against the TD enqueueing code ringing the
1445  * doorbell while this code is waiting for a Stop Endpoint command to complete.
1446  * It also needs to account for multiple cancellations on happening at the same
1447  * time for the same endpoint.
1448  *
1449  * Note that this function can be called in any context, or so says
1450  * usb_hcd_unlink_urb()
1451  */
1452 int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
1453 {
1454         unsigned long flags;
1455         int ret, i;
1456         u32 temp;
1457         struct xhci_hcd *xhci;
1458         struct urb_priv *urb_priv;
1459         struct xhci_td *td;
1460         unsigned int ep_index;
1461         struct xhci_ring *ep_ring;
1462         struct xhci_virt_ep *ep;
1463
1464         xhci = hcd_to_xhci(hcd);
1465         spin_lock_irqsave(&xhci->lock, flags);
1466         /* Make sure the URB hasn't completed or been unlinked already */
1467         ret = usb_hcd_check_unlink_urb(hcd, urb, status);
1468         if (ret || !urb->hcpriv)
1469                 goto done;
1470         temp = xhci_readl(xhci, &xhci->op_regs->status);
1471         if (temp == 0xffffffff || (xhci->xhc_state & XHCI_STATE_HALTED)) {
1472                 xhci_dbg(xhci, "HW died, freeing TD.\n");
1473                 urb_priv = urb->hcpriv;
1474                 for (i = urb_priv->td_cnt; i < urb_priv->length; i++) {
1475                         td = urb_priv->td[i];
1476                         if (!list_empty(&td->td_list))
1477                                 list_del_init(&td->td_list);
1478                         if (!list_empty(&td->cancelled_td_list))
1479                                 list_del_init(&td->cancelled_td_list);
1480                 }
1481
1482                 usb_hcd_unlink_urb_from_ep(hcd, urb);
1483                 spin_unlock_irqrestore(&xhci->lock, flags);
1484                 usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);
1485                 xhci_urb_free_priv(xhci, urb_priv);
1486                 return ret;
1487         }
1488         if ((xhci->xhc_state & XHCI_STATE_DYING) ||
1489                         (xhci->xhc_state & XHCI_STATE_HALTED)) {
1490                 xhci_dbg(xhci, "Ep 0x%x: URB %p to be canceled on "
1491                                 "non-responsive xHCI host.\n",
1492                                 urb->ep->desc.bEndpointAddress, urb);
1493                 /* Let the stop endpoint command watchdog timer (which set this
1494                  * state) finish cleaning up the endpoint TD lists.  We must
1495                  * have caught it in the middle of dropping a lock and giving
1496                  * back an URB.
1497                  */
1498                 goto done;
1499         }
1500
1501         xhci_dbg(xhci, "Cancel URB %p\n", urb);
1502         xhci_dbg(xhci, "Event ring:\n");
1503         xhci_debug_ring(xhci, xhci->event_ring);
1504         ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1505         ep = &xhci->devs[urb->dev->slot_id]->eps[ep_index];
1506         ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
1507         if (!ep_ring) {
1508                 ret = -EINVAL;
1509                 goto done;
1510         }
1511
1512         xhci_dbg(xhci, "Endpoint ring:\n");
1513         xhci_debug_ring(xhci, ep_ring);
1514
1515         urb_priv = urb->hcpriv;
1516
1517         for (i = urb_priv->td_cnt; i < urb_priv->length; i++) {
1518                 td = urb_priv->td[i];
1519                 list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list);
1520         }
1521
1522         /* Queue a stop endpoint command, but only if this is
1523          * the first cancellation to be handled.
1524          */
1525         if (!(ep->ep_state & EP_HALT_PENDING)) {
1526                 ep->ep_state |= EP_HALT_PENDING;
1527                 ep->stop_cmds_pending++;
1528                 ep->stop_cmd_timer.expires = jiffies +
1529                         XHCI_STOP_EP_CMD_TIMEOUT * HZ;
1530                 add_timer(&ep->stop_cmd_timer);
1531                 xhci_queue_stop_endpoint(xhci, urb->dev->slot_id, ep_index, 0);
1532                 xhci_ring_cmd_db(xhci);
1533         }
1534 done:
1535         spin_unlock_irqrestore(&xhci->lock, flags);
1536         return ret;
1537 }
1538
1539 /* Drop an endpoint from a new bandwidth configuration for this device.
1540  * Only one call to this function is allowed per endpoint before
1541  * check_bandwidth() or reset_bandwidth() must be called.
1542  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1543  * add the endpoint to the schedule with possibly new parameters denoted by a
1544  * different endpoint descriptor in usb_host_endpoint.
1545  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1546  * not allowed.
1547  *
1548  * The USB core will not allow URBs to be queued to an endpoint that is being
1549  * disabled, so there's no need for mutual exclusion to protect
1550  * the xhci->devs[slot_id] structure.
1551  */
1552 int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1553                 struct usb_host_endpoint *ep)
1554 {
1555         struct xhci_hcd *xhci;
1556         struct xhci_container_ctx *in_ctx, *out_ctx;
1557         struct xhci_input_control_ctx *ctrl_ctx;
1558         struct xhci_slot_ctx *slot_ctx;
1559         unsigned int last_ctx;
1560         unsigned int ep_index;
1561         struct xhci_ep_ctx *ep_ctx;
1562         u32 drop_flag;
1563         u32 new_add_flags, new_drop_flags, new_slot_info;
1564         int ret;
1565
1566         ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1567         if (ret <= 0)
1568                 return ret;
1569         xhci = hcd_to_xhci(hcd);
1570         if (xhci->xhc_state & XHCI_STATE_DYING)
1571                 return -ENODEV;
1572
1573         xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1574         drop_flag = xhci_get_endpoint_flag(&ep->desc);
1575         if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) {
1576                 xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",
1577                                 __func__, drop_flag);
1578                 return 0;
1579         }
1580
1581         in_ctx = xhci->devs[udev->slot_id]->in_ctx;
1582         out_ctx = xhci->devs[udev->slot_id]->out_ctx;
1583         ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
1584         ep_index = xhci_get_endpoint_index(&ep->desc);
1585         ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1586         /* If the HC already knows the endpoint is disabled,
1587          * or the HCD has noted it is disabled, ignore this request
1588          */
1589         if (((ep_ctx->ep_info & cpu_to_le32(EP_STATE_MASK)) ==
1590              cpu_to_le32(EP_STATE_DISABLED)) ||
1591             le32_to_cpu(ctrl_ctx->drop_flags) &
1592             xhci_get_endpoint_flag(&ep->desc)) {
1593                 xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
1594                                 __func__, ep);
1595                 return 0;
1596         }
1597
1598         ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);
1599         new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1600
1601         ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);
1602         new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1603
1604         last_ctx = xhci_last_valid_endpoint(le32_to_cpu(ctrl_ctx->add_flags));
1605         slot_ctx = xhci_get_slot_ctx(xhci, in_ctx);
1606         /* Update the last valid endpoint context, if we deleted the last one */
1607         if ((le32_to_cpu(slot_ctx->dev_info) & LAST_CTX_MASK) >
1608             LAST_CTX(last_ctx)) {
1609                 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
1610                 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(last_ctx));
1611         }
1612         new_slot_info = le32_to_cpu(slot_ctx->dev_info);
1613
1614         xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);
1615
1616         xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x, new slot info = %#x\n",
1617                         (unsigned int) ep->desc.bEndpointAddress,
1618                         udev->slot_id,
1619                         (unsigned int) new_drop_flags,
1620                         (unsigned int) new_add_flags,
1621                         (unsigned int) new_slot_info);
1622         return 0;
1623 }
1624
1625 /* Add an endpoint to a new possible bandwidth configuration for this device.
1626  * Only one call to this function is allowed per endpoint before
1627  * check_bandwidth() or reset_bandwidth() must be called.
1628  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1629  * add the endpoint to the schedule with possibly new parameters denoted by a
1630  * different endpoint descriptor in usb_host_endpoint.
1631  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1632  * not allowed.
1633  *
1634  * The USB core will not allow URBs to be queued to an endpoint until the
1635  * configuration or alt setting is installed in the device, so there's no need
1636  * for mutual exclusion to protect the xhci->devs[slot_id] structure.
1637  */
1638 int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1639                 struct usb_host_endpoint *ep)
1640 {
1641         struct xhci_hcd *xhci;
1642         struct xhci_container_ctx *in_ctx, *out_ctx;
1643         unsigned int ep_index;
1644         struct xhci_ep_ctx *ep_ctx;
1645         struct xhci_slot_ctx *slot_ctx;
1646         struct xhci_input_control_ctx *ctrl_ctx;
1647         u32 added_ctxs;
1648         unsigned int last_ctx;
1649         u32 new_add_flags, new_drop_flags, new_slot_info;
1650         struct xhci_virt_device *virt_dev;
1651         int ret = 0;
1652
1653         ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1654         if (ret <= 0) {
1655                 /* So we won't queue a reset ep command for a root hub */
1656                 ep->hcpriv = NULL;
1657                 return ret;
1658         }
1659         xhci = hcd_to_xhci(hcd);
1660         if (xhci->xhc_state & XHCI_STATE_DYING)
1661                 return -ENODEV;
1662
1663         added_ctxs = xhci_get_endpoint_flag(&ep->desc);
1664         last_ctx = xhci_last_valid_endpoint(added_ctxs);
1665         if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
1666                 /* FIXME when we have to issue an evaluate endpoint command to
1667                  * deal with ep0 max packet size changing once we get the
1668                  * descriptors
1669                  */
1670                 xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",
1671                                 __func__, added_ctxs);
1672                 return 0;
1673         }
1674
1675         virt_dev = xhci->devs[udev->slot_id];
1676         in_ctx = virt_dev->in_ctx;
1677         out_ctx = virt_dev->out_ctx;
1678         ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
1679         ep_index = xhci_get_endpoint_index(&ep->desc);
1680         ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1681
1682         /* If this endpoint is already in use, and the upper layers are trying
1683          * to add it again without dropping it, reject the addition.
1684          */
1685         if (virt_dev->eps[ep_index].ring &&
1686                         !(le32_to_cpu(ctrl_ctx->drop_flags) &
1687                                 xhci_get_endpoint_flag(&ep->desc))) {
1688                 xhci_warn(xhci, "Trying to add endpoint 0x%x "
1689                                 "without dropping it.\n",
1690                                 (unsigned int) ep->desc.bEndpointAddress);
1691                 return -EINVAL;
1692         }
1693
1694         /* If the HCD has already noted the endpoint is enabled,
1695          * ignore this request.
1696          */
1697         if (le32_to_cpu(ctrl_ctx->add_flags) &
1698             xhci_get_endpoint_flag(&ep->desc)) {
1699                 xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
1700                                 __func__, ep);
1701                 return 0;
1702         }
1703
1704         /*
1705          * Configuration and alternate setting changes must be done in
1706          * process context, not interrupt context (or so documenation
1707          * for usb_set_interface() and usb_set_configuration() claim).
1708          */
1709         if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {
1710                 dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
1711                                 __func__, ep->desc.bEndpointAddress);
1712                 return -ENOMEM;
1713         }
1714
1715         ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs);
1716         new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1717
1718         /* If xhci_endpoint_disable() was called for this endpoint, but the
1719          * xHC hasn't been notified yet through the check_bandwidth() call,
1720          * this re-adds a new state for the endpoint from the new endpoint
1721          * descriptors.  We must drop and re-add this endpoint, so we leave the
1722          * drop flags alone.
1723          */
1724         new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1725
1726         slot_ctx = xhci_get_slot_ctx(xhci, in_ctx);
1727         /* Update the last valid endpoint context, if we just added one past */
1728         if ((le32_to_cpu(slot_ctx->dev_info) & LAST_CTX_MASK) <
1729             LAST_CTX(last_ctx)) {
1730                 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
1731                 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(last_ctx));
1732         }
1733         new_slot_info = le32_to_cpu(slot_ctx->dev_info);
1734
1735         /* Store the usb_device pointer for later use */
1736         ep->hcpriv = udev;
1737
1738         xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x, new slot info = %#x\n",
1739                         (unsigned int) ep->desc.bEndpointAddress,
1740                         udev->slot_id,
1741                         (unsigned int) new_drop_flags,
1742                         (unsigned int) new_add_flags,
1743                         (unsigned int) new_slot_info);
1744         return 0;
1745 }
1746
1747 static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
1748 {
1749         struct xhci_input_control_ctx *ctrl_ctx;
1750         struct xhci_ep_ctx *ep_ctx;
1751         struct xhci_slot_ctx *slot_ctx;
1752         int i;
1753
1754         /* When a device's add flag and drop flag are zero, any subsequent
1755          * configure endpoint command will leave that endpoint's state
1756          * untouched.  Make sure we don't leave any old state in the input
1757          * endpoint contexts.
1758          */
1759         ctrl_ctx = xhci_get_input_control_ctx(xhci, virt_dev->in_ctx);
1760         ctrl_ctx->drop_flags = 0;
1761         ctrl_ctx->add_flags = 0;
1762         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
1763         slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
1764         /* Endpoint 0 is always valid */
1765         slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
1766         for (i = 1; i < 31; ++i) {
1767                 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
1768                 ep_ctx->ep_info = 0;
1769                 ep_ctx->ep_info2 = 0;
1770                 ep_ctx->deq = 0;
1771                 ep_ctx->tx_info = 0;
1772         }
1773 }
1774
1775 static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,
1776                 struct usb_device *udev, u32 *cmd_status)
1777 {
1778         int ret;
1779
1780         switch (*cmd_status) {
1781         case COMP_ENOMEM:
1782                 dev_warn(&udev->dev, "Not enough host controller resources "
1783                                 "for new device state.\n");
1784                 ret = -ENOMEM;
1785                 /* FIXME: can we allocate more resources for the HC? */
1786                 break;
1787         case COMP_BW_ERR:
1788         case COMP_2ND_BW_ERR:
1789                 dev_warn(&udev->dev, "Not enough bandwidth "
1790                                 "for new device state.\n");
1791                 ret = -ENOSPC;
1792                 /* FIXME: can we go back to the old state? */
1793                 break;
1794         case COMP_TRB_ERR:
1795                 /* the HCD set up something wrong */
1796                 dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "
1797                                 "add flag = 1, "
1798                                 "and endpoint is not disabled.\n");
1799                 ret = -EINVAL;
1800                 break;
1801         case COMP_DEV_ERR:
1802                 dev_warn(&udev->dev, "ERROR: Incompatible device for endpoint "
1803                                 "configure command.\n");
1804                 ret = -ENODEV;
1805                 break;
1806         case COMP_SUCCESS:
1807                 dev_dbg(&udev->dev, "Successful Endpoint Configure command\n");
1808                 ret = 0;
1809                 break;
1810         default:
1811                 xhci_err(xhci, "ERROR: unexpected command completion "
1812                                 "code 0x%x.\n", *cmd_status);
1813                 ret = -EINVAL;
1814                 break;
1815         }
1816         return ret;
1817 }
1818
1819 static int xhci_evaluate_context_result(struct xhci_hcd *xhci,
1820                 struct usb_device *udev, u32 *cmd_status)
1821 {
1822         int ret;
1823         struct xhci_virt_device *virt_dev = xhci->devs[udev->slot_id];
1824
1825         switch (*cmd_status) {
1826         case COMP_EINVAL:
1827                 dev_warn(&udev->dev, "WARN: xHCI driver setup invalid evaluate "
1828                                 "context command.\n");
1829                 ret = -EINVAL;
1830                 break;
1831         case COMP_EBADSLT:
1832                 dev_warn(&udev->dev, "WARN: slot not enabled for"
1833                                 "evaluate context command.\n");
1834         case COMP_CTX_STATE:
1835                 dev_warn(&udev->dev, "WARN: invalid context state for "
1836                                 "evaluate context command.\n");
1837                 xhci_dbg_ctx(xhci, virt_dev->out_ctx, 1);
1838                 ret = -EINVAL;
1839                 break;
1840         case COMP_DEV_ERR:
1841                 dev_warn(&udev->dev, "ERROR: Incompatible device for evaluate "
1842                                 "context command.\n");
1843                 ret = -ENODEV;
1844                 break;
1845         case COMP_MEL_ERR:
1846                 /* Max Exit Latency too large error */
1847                 dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n");
1848                 ret = -EINVAL;
1849                 break;
1850         case COMP_SUCCESS:
1851                 dev_dbg(&udev->dev, "Successful evaluate context command\n");
1852                 ret = 0;
1853                 break;
1854         default:
1855                 xhci_err(xhci, "ERROR: unexpected command completion "
1856                                 "code 0x%x.\n", *cmd_status);
1857                 ret = -EINVAL;
1858                 break;
1859         }
1860         return ret;
1861 }
1862
1863 static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,
1864                 struct xhci_container_ctx *in_ctx)
1865 {
1866         struct xhci_input_control_ctx *ctrl_ctx;
1867         u32 valid_add_flags;
1868         u32 valid_drop_flags;
1869
1870         ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
1871         /* Ignore the slot flag (bit 0), and the default control endpoint flag
1872          * (bit 1).  The default control endpoint is added during the Address
1873          * Device command and is never removed until the slot is disabled.
1874          */
1875         valid_add_flags = ctrl_ctx->add_flags >> 2;
1876         valid_drop_flags = ctrl_ctx->drop_flags >> 2;
1877
1878         /* Use hweight32 to count the number of ones in the add flags, or
1879          * number of endpoints added.  Don't count endpoints that are changed
1880          * (both added and dropped).
1881          */
1882         return hweight32(valid_add_flags) -
1883                 hweight32(valid_add_flags & valid_drop_flags);
1884 }
1885
1886 static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,
1887                 struct xhci_container_ctx *in_ctx)
1888 {
1889         struct xhci_input_control_ctx *ctrl_ctx;
1890         u32 valid_add_flags;
1891         u32 valid_drop_flags;
1892
1893         ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
1894         valid_add_flags = ctrl_ctx->add_flags >> 2;
1895         valid_drop_flags = ctrl_ctx->drop_flags >> 2;
1896
1897         return hweight32(valid_drop_flags) -
1898                 hweight32(valid_add_flags & valid_drop_flags);
1899 }
1900
1901 /*
1902  * We need to reserve the new number of endpoints before the configure endpoint
1903  * command completes.  We can't subtract the dropped endpoints from the number
1904  * of active endpoints until the command completes because we can oversubscribe
1905  * the host in this case:
1906  *
1907  *  - the first configure endpoint command drops more endpoints than it adds
1908  *  - a second configure endpoint command that adds more endpoints is queued
1909  *  - the first configure endpoint command fails, so the config is unchanged
1910  *  - the second command may succeed, even though there isn't enough resources
1911  *
1912  * Must be called with xhci->lock held.
1913  */
1914 static int xhci_reserve_host_resources(struct xhci_hcd *xhci,
1915                 struct xhci_container_ctx *in_ctx)
1916 {
1917         u32 added_eps;
1918
1919         added_eps = xhci_count_num_new_endpoints(xhci, in_ctx);
1920         if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) {
1921                 xhci_dbg(xhci, "Not enough ep ctxs: "
1922                                 "%u active, need to add %u, limit is %u.\n",
1923                                 xhci->num_active_eps, added_eps,
1924                                 xhci->limit_active_eps);
1925                 return -ENOMEM;
1926         }
1927         xhci->num_active_eps += added_eps;
1928         xhci_dbg(xhci, "Adding %u ep ctxs, %u now active.\n", added_eps,
1929                         xhci->num_active_eps);
1930         return 0;
1931 }
1932
1933 /*
1934  * The configure endpoint was failed by the xHC for some other reason, so we
1935  * need to revert the resources that failed configuration would have used.
1936  *
1937  * Must be called with xhci->lock held.
1938  */
1939 static void xhci_free_host_resources(struct xhci_hcd *xhci,
1940                 struct xhci_container_ctx *in_ctx)
1941 {
1942         u32 num_failed_eps;
1943
1944         num_failed_eps = xhci_count_num_new_endpoints(xhci, in_ctx);
1945         xhci->num_active_eps -= num_failed_eps;
1946         xhci_dbg(xhci, "Removing %u failed ep ctxs, %u now active.\n",
1947                         num_failed_eps,
1948                         xhci->num_active_eps);
1949 }
1950
1951 /*
1952  * Now that the command has completed, clean up the active endpoint count by
1953  * subtracting out the endpoints that were dropped (but not changed).
1954  *
1955  * Must be called with xhci->lock held.
1956  */
1957 static void xhci_finish_resource_reservation(struct xhci_hcd *xhci,
1958                 struct xhci_container_ctx *in_ctx)
1959 {
1960         u32 num_dropped_eps;
1961
1962         num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, in_ctx);
1963         xhci->num_active_eps -= num_dropped_eps;
1964         if (num_dropped_eps)
1965                 xhci_dbg(xhci, "Removing %u dropped ep ctxs, %u now active.\n",
1966                                 num_dropped_eps,
1967                                 xhci->num_active_eps);
1968 }
1969
1970 unsigned int xhci_get_block_size(struct usb_device *udev)
1971 {
1972         switch (udev->speed) {
1973         case USB_SPEED_LOW:
1974         case USB_SPEED_FULL:
1975                 return FS_BLOCK;
1976         case USB_SPEED_HIGH:
1977                 return HS_BLOCK;
1978         case USB_SPEED_SUPER:
1979                 return SS_BLOCK;
1980         case USB_SPEED_UNKNOWN:
1981         case USB_SPEED_WIRELESS:
1982         default:
1983                 /* Should never happen */
1984                 return 1;
1985         }
1986 }
1987
1988 unsigned int xhci_get_largest_overhead(struct xhci_interval_bw *interval_bw)
1989 {
1990         if (interval_bw->overhead[LS_OVERHEAD_TYPE])
1991                 return LS_OVERHEAD;
1992         if (interval_bw->overhead[FS_OVERHEAD_TYPE])
1993                 return FS_OVERHEAD;
1994         return HS_OVERHEAD;
1995 }
1996
1997 /* If we are changing a LS/FS device under a HS hub,
1998  * make sure (if we are activating a new TT) that the HS bus has enough
1999  * bandwidth for this new TT.
2000  */
2001 static int xhci_check_tt_bw_table(struct xhci_hcd *xhci,
2002                 struct xhci_virt_device *virt_dev,
2003                 int old_active_eps)
2004 {
2005         struct xhci_interval_bw_table *bw_table;
2006         struct xhci_tt_bw_info *tt_info;
2007
2008         /* Find the bandwidth table for the root port this TT is attached to. */
2009         bw_table = &xhci->rh_bw[virt_dev->real_port - 1].bw_table;
2010         tt_info = virt_dev->tt_info;
2011         /* If this TT already had active endpoints, the bandwidth for this TT
2012          * has already been added.  Removing all periodic endpoints (and thus
2013          * making the TT enactive) will only decrease the bandwidth used.
2014          */
2015         if (old_active_eps)
2016                 return 0;
2017         if (old_active_eps == 0 && tt_info->active_eps != 0) {
2018                 if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT)
2019                         return -ENOMEM;
2020                 return 0;
2021         }
2022         /* Not sure why we would have no new active endpoints...
2023          *
2024          * Maybe because of an Evaluate Context change for a hub update or a
2025          * control endpoint 0 max packet size change?
2026          * FIXME: skip the bandwidth calculation in that case.
2027          */
2028         return 0;
2029 }
2030
2031 static int xhci_check_ss_bw(struct xhci_hcd *xhci,
2032                 struct xhci_virt_device *virt_dev)
2033 {
2034         unsigned int bw_reserved;
2035
2036         bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_IN, 100);
2037         if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved))
2038                 return -ENOMEM;
2039
2040         bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_OUT, 100);
2041         if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved))
2042                 return -ENOMEM;
2043
2044         return 0;
2045 }
2046
2047 /*
2048  * This algorithm is a very conservative estimate of the worst-case scheduling
2049  * scenario for any one interval.  The hardware dynamically schedules the
2050  * packets, so we can't tell which microframe could be the limiting factor in
2051  * the bandwidth scheduling.  This only takes into account periodic endpoints.
2052  *
2053  * Obviously, we can't solve an NP complete problem to find the minimum worst
2054  * case scenario.  Instead, we come up with an estimate that is no less than
2055  * the worst case bandwidth used for any one microframe, but may be an
2056  * over-estimate.
2057  *
2058  * We walk the requirements for each endpoint by interval, starting with the
2059  * smallest interval, and place packets in the schedule where there is only one
2060  * possible way to schedule packets for that interval.  In order to simplify
2061  * this algorithm, we record the largest max packet size for each interval, and
2062  * assume all packets will be that size.
2063  *
2064  * For interval 0, we obviously must schedule all packets for each interval.
2065  * The bandwidth for interval 0 is just the amount of data to be transmitted
2066  * (the sum of all max ESIT payload sizes, plus any overhead per packet times
2067  * the number of packets).
2068  *
2069  * For interval 1, we have two possible microframes to schedule those packets
2070  * in.  For this algorithm, if we can schedule the same number of packets for
2071  * each possible scheduling opportunity (each microframe), we will do so.  The
2072  * remaining number of packets will be saved to be transmitted in the gaps in
2073  * the next interval's scheduling sequence.
2074  *
2075  * As we move those remaining packets to be scheduled with interval 2 packets,
2076  * we have to double the number of remaining packets to transmit.  This is
2077  * because the intervals are actually powers of 2, and we would be transmitting
2078  * the previous interval's packets twice in this interval.  We also have to be
2079  * sure that when we look at the largest max packet size for this interval, we
2080  * also look at the largest max packet size for the remaining packets and take
2081  * the greater of the two.
2082  *
2083  * The algorithm continues to evenly distribute packets in each scheduling
2084  * opportunity, and push the remaining packets out, until we get to the last
2085  * interval.  Then those packets and their associated overhead are just added
2086  * to the bandwidth used.
2087  */
2088 static int xhci_check_bw_table(struct xhci_hcd *xhci,
2089                 struct xhci_virt_device *virt_dev,
2090                 int old_active_eps)
2091 {
2092         unsigned int bw_reserved;
2093         unsigned int max_bandwidth;
2094         unsigned int bw_used;
2095         unsigned int block_size;
2096         struct xhci_interval_bw_table *bw_table;
2097         unsigned int packet_size = 0;
2098         unsigned int overhead = 0;
2099         unsigned int packets_transmitted = 0;
2100         unsigned int packets_remaining = 0;
2101         unsigned int i;
2102
2103         if (virt_dev->udev->speed == USB_SPEED_SUPER)
2104                 return xhci_check_ss_bw(xhci, virt_dev);
2105
2106         if (virt_dev->udev->speed == USB_SPEED_HIGH) {
2107                 max_bandwidth = HS_BW_LIMIT;
2108                 /* Convert percent of bus BW reserved to blocks reserved */
2109                 bw_reserved = DIV_ROUND_UP(HS_BW_RESERVED * max_bandwidth, 100);
2110         } else {
2111                 max_bandwidth = FS_BW_LIMIT;
2112                 bw_reserved = DIV_ROUND_UP(FS_BW_RESERVED * max_bandwidth, 100);
2113         }
2114
2115         bw_table = virt_dev->bw_table;
2116         /* We need to translate the max packet size and max ESIT payloads into
2117          * the units the hardware uses.
2118          */
2119         block_size = xhci_get_block_size(virt_dev->udev);
2120
2121         /* If we are manipulating a LS/FS device under a HS hub, double check
2122          * that the HS bus has enough bandwidth if we are activing a new TT.
2123          */
2124         if (virt_dev->tt_info) {
2125                 xhci_dbg(xhci, "Recalculating BW for rootport %u\n",
2126                                 virt_dev->real_port);
2127                 if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) {
2128                         xhci_warn(xhci, "Not enough bandwidth on HS bus for "
2129                                         "newly activated TT.\n");
2130                         return -ENOMEM;
2131                 }
2132                 xhci_dbg(xhci, "Recalculating BW for TT slot %u port %u\n",
2133                                 virt_dev->tt_info->slot_id,
2134                                 virt_dev->tt_info->ttport);
2135         } else {
2136                 xhci_dbg(xhci, "Recalculating BW for rootport %u\n",
2137                                 virt_dev->real_port);
2138         }
2139
2140         /* Add in how much bandwidth will be used for interval zero, or the
2141          * rounded max ESIT payload + number of packets * largest overhead.
2142          */
2143         bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) +
2144                 bw_table->interval_bw[0].num_packets *
2145                 xhci_get_largest_overhead(&bw_table->interval_bw[0]);
2146
2147         for (i = 1; i < XHCI_MAX_INTERVAL; i++) {
2148                 unsigned int bw_added;
2149                 unsigned int largest_mps;
2150                 unsigned int interval_overhead;
2151
2152                 /*
2153                  * How many packets could we transmit in this interval?
2154                  * If packets didn't fit in the previous interval, we will need
2155                  * to transmit that many packets twice within this interval.
2156                  */
2157                 packets_remaining = 2 * packets_remaining +
2158                         bw_table->interval_bw[i].num_packets;
2159
2160                 /* Find the largest max packet size of this or the previous
2161                  * interval.
2162                  */
2163                 if (list_empty(&bw_table->interval_bw[i].endpoints))
2164                         largest_mps = 0;
2165                 else {
2166                         struct xhci_virt_ep *virt_ep;
2167                         struct list_head *ep_entry;
2168
2169                         ep_entry = bw_table->interval_bw[i].endpoints.next;
2170                         virt_ep = list_entry(ep_entry,
2171                                         struct xhci_virt_ep, bw_endpoint_list);
2172                         /* Convert to blocks, rounding up */
2173                         largest_mps = DIV_ROUND_UP(
2174                                         virt_ep->bw_info.max_packet_size,
2175                                         block_size);
2176                 }
2177                 if (largest_mps > packet_size)
2178                         packet_size = largest_mps;
2179
2180                 /* Use the larger overhead of this or the previous interval. */
2181                 interval_overhead = xhci_get_largest_overhead(
2182                                 &bw_table->interval_bw[i]);
2183                 if (interval_overhead > overhead)
2184                         overhead = interval_overhead;
2185
2186                 /* How many packets can we evenly distribute across
2187                  * (1 << (i + 1)) possible scheduling opportunities?
2188                  */
2189                 packets_transmitted = packets_remaining >> (i + 1);
2190
2191                 /* Add in the bandwidth used for those scheduled packets */
2192                 bw_added = packets_transmitted * (overhead + packet_size);
2193
2194                 /* How many packets do we have remaining to transmit? */
2195                 packets_remaining = packets_remaining % (1 << (i + 1));
2196
2197                 /* What largest max packet size should those packets have? */
2198                 /* If we've transmitted all packets, don't carry over the
2199                  * largest packet size.
2200                  */
2201                 if (packets_remaining == 0) {
2202                         packet_size = 0;
2203                         overhead = 0;
2204                 } else if (packets_transmitted > 0) {
2205                         /* Otherwise if we do have remaining packets, and we've
2206                          * scheduled some packets in this interval, take the
2207                          * largest max packet size from endpoints with this
2208                          * interval.
2209                          */
2210                         packet_size = largest_mps;
2211                         overhead = interval_overhead;
2212                 }
2213                 /* Otherwise carry over packet_size and overhead from the last
2214                  * time we had a remainder.
2215                  */
2216                 bw_used += bw_added;
2217                 if (bw_used > max_bandwidth) {
2218                         xhci_warn(xhci, "Not enough bandwidth. "
2219                                         "Proposed: %u, Max: %u\n",
2220                                 bw_used, max_bandwidth);
2221                         return -ENOMEM;
2222                 }
2223         }
2224         /*
2225          * Ok, we know we have some packets left over after even-handedly
2226          * scheduling interval 15.  We don't know which microframes they will
2227          * fit into, so we over-schedule and say they will be scheduled every
2228          * microframe.
2229          */
2230         if (packets_remaining > 0)
2231                 bw_used += overhead + packet_size;
2232
2233         if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) {
2234                 unsigned int port_index = virt_dev->real_port - 1;
2235
2236                 /* OK, we're manipulating a HS device attached to a
2237                  * root port bandwidth domain.  Include the number of active TTs
2238                  * in the bandwidth used.
2239                  */
2240                 bw_used += TT_HS_OVERHEAD *
2241                         xhci->rh_bw[port_index].num_active_tts;
2242         }
2243
2244         xhci_dbg(xhci, "Final bandwidth: %u, Limit: %u, Reserved: %u, "
2245                 "Available: %u " "percent\n",
2246                 bw_used, max_bandwidth, bw_reserved,
2247                 (max_bandwidth - bw_used - bw_reserved) * 100 /
2248                 max_bandwidth);
2249
2250         bw_used += bw_reserved;
2251         if (bw_used > max_bandwidth) {
2252                 xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n",
2253                                 bw_used, max_bandwidth);
2254                 return -ENOMEM;
2255         }
2256
2257         bw_table->bw_used = bw_used;
2258         return 0;
2259 }
2260
2261 static bool xhci_is_async_ep(unsigned int ep_type)
2262 {
2263         return (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
2264                                         ep_type != ISOC_IN_EP &&
2265                                         ep_type != INT_IN_EP);
2266 }
2267
2268 static bool xhci_is_sync_in_ep(unsigned int ep_type)
2269 {
2270         return (ep_type == ISOC_IN_EP || ep_type == INT_IN_EP);
2271 }
2272
2273 static unsigned int xhci_get_ss_bw_consumed(struct xhci_bw_info *ep_bw)
2274 {
2275         unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK);
2276
2277         if (ep_bw->ep_interval == 0)
2278                 return SS_OVERHEAD_BURST +
2279                         (ep_bw->mult * ep_bw->num_packets *
2280                                         (SS_OVERHEAD + mps));
2281         return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets *
2282                                 (SS_OVERHEAD + mps + SS_OVERHEAD_BURST),
2283                                 1 << ep_bw->ep_interval);
2284
2285 }
2286
2287 void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci,
2288                 struct xhci_bw_info *ep_bw,
2289                 struct xhci_interval_bw_table *bw_table,
2290                 struct usb_device *udev,
2291                 struct xhci_virt_ep *virt_ep,
2292                 struct xhci_tt_bw_info *tt_info)
2293 {
2294         struct xhci_interval_bw *interval_bw;
2295         int normalized_interval;
2296
2297         if (xhci_is_async_ep(ep_bw->type))
2298                 return;
2299
2300         if (udev->speed == USB_SPEED_SUPER) {
2301                 if (xhci_is_sync_in_ep(ep_bw->type))
2302                         xhci->devs[udev->slot_id]->bw_table->ss_bw_in -=
2303                                 xhci_get_ss_bw_consumed(ep_bw);
2304                 else
2305                         xhci->devs[udev->slot_id]->bw_table->ss_bw_out -=
2306                                 xhci_get_ss_bw_consumed(ep_bw);
2307                 return;
2308         }
2309
2310         /* SuperSpeed endpoints never get added to intervals in the table, so
2311          * this check is only valid for HS/FS/LS devices.
2312          */
2313         if (list_empty(&virt_ep->bw_endpoint_list))
2314                 return;
2315         /* For LS/FS devices, we need to translate the interval expressed in
2316          * microframes to frames.
2317          */
2318         if (udev->speed == USB_SPEED_HIGH)
2319                 normalized_interval = ep_bw->ep_interval;
2320         else
2321                 normalized_interval = ep_bw->ep_interval - 3;
2322
2323         if (normalized_interval == 0)
2324                 bw_table->interval0_esit_payload -= ep_bw->max_esit_payload;
2325         interval_bw = &bw_table->interval_bw[normalized_interval];
2326         interval_bw->num_packets -= ep_bw->num_packets;
2327         switch (udev->speed) {
2328         case USB_SPEED_LOW:
2329                 interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1;
2330                 break;
2331         case USB_SPEED_FULL:
2332                 interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1;
2333                 break;
2334         case USB_SPEED_HIGH:
2335                 interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1;
2336                 break;
2337         case USB_SPEED_SUPER:
2338         case USB_SPEED_UNKNOWN:
2339         case USB_SPEED_WIRELESS:
2340                 /* Should never happen because only LS/FS/HS endpoints will get
2341                  * added to the endpoint list.
2342                  */
2343                 return;
2344         }
2345         if (tt_info)
2346                 tt_info->active_eps -= 1;
2347         list_del_init(&virt_ep->bw_endpoint_list);
2348 }
2349
2350 static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci,
2351                 struct xhci_bw_info *ep_bw,
2352                 struct xhci_interval_bw_table *bw_table,
2353                 struct usb_device *udev,
2354                 struct xhci_virt_ep *virt_ep,
2355                 struct xhci_tt_bw_info *tt_info)
2356 {
2357         struct xhci_interval_bw *interval_bw;
2358         struct xhci_virt_ep *smaller_ep;
2359         int normalized_interval;
2360
2361         if (xhci_is_async_ep(ep_bw->type))
2362                 return;
2363
2364         if (udev->speed == USB_SPEED_SUPER) {
2365                 if (xhci_is_sync_in_ep(ep_bw->type))
2366                         xhci->devs[udev->slot_id]->bw_table->ss_bw_in +=
2367                                 xhci_get_ss_bw_consumed(ep_bw);
2368                 else
2369                         xhci->devs[udev->slot_id]->bw_table->ss_bw_out +=
2370                                 xhci_get_ss_bw_consumed(ep_bw);
2371                 return;
2372         }
2373
2374         /* For LS/FS devices, we need to translate the interval expressed in
2375          * microframes to frames.
2376          */
2377         if (udev->speed == USB_SPEED_HIGH)
2378                 normalized_interval = ep_bw->ep_interval;
2379         else
2380                 normalized_interval = ep_bw->ep_interval - 3;
2381
2382         if (normalized_interval == 0)
2383                 bw_table->interval0_esit_payload += ep_bw->max_esit_payload;
2384         interval_bw = &bw_table->interval_bw[normalized_interval];
2385         interval_bw->num_packets += ep_bw->num_packets;
2386         switch (udev->speed) {
2387         case USB_SPEED_LOW:
2388                 interval_bw->overhead[LS_OVERHEAD_TYPE] += 1;
2389                 break;
2390         case USB_SPEED_FULL:
2391                 interval_bw->overhead[FS_OVERHEAD_TYPE] += 1;
2392                 break;
2393         case USB_SPEED_HIGH:
2394                 interval_bw->overhead[HS_OVERHEAD_TYPE] += 1;
2395                 break;
2396         case USB_SPEED_SUPER:
2397         case USB_SPEED_UNKNOWN:
2398         case USB_SPEED_WIRELESS:
2399                 /* Should never happen because only LS/FS/HS endpoints will get
2400                  * added to the endpoint list.
2401                  */
2402                 return;
2403         }
2404
2405         if (tt_info)
2406                 tt_info->active_eps += 1;
2407         /* Insert the endpoint into the list, largest max packet size first. */
2408         list_for_each_entry(smaller_ep, &interval_bw->endpoints,
2409                         bw_endpoint_list) {
2410                 if (ep_bw->max_packet_size >=
2411                                 smaller_ep->bw_info.max_packet_size) {
2412                         /* Add the new ep before the smaller endpoint */
2413                         list_add_tail(&virt_ep->bw_endpoint_list,
2414                                         &smaller_ep->bw_endpoint_list);
2415                         return;
2416                 }
2417         }
2418         /* Add the new endpoint at the end of the list. */
2419         list_add_tail(&virt_ep->bw_endpoint_list,
2420                         &interval_bw->endpoints);
2421 }
2422
2423 void xhci_update_tt_active_eps(struct xhci_hcd *xhci,
2424                 struct xhci_virt_device *virt_dev,
2425                 int old_active_eps)
2426 {
2427         struct xhci_root_port_bw_info *rh_bw_info;
2428         if (!virt_dev->tt_info)
2429                 return;
2430
2431         rh_bw_info = &xhci->rh_bw[virt_dev->real_port - 1];
2432         if (old_active_eps == 0 &&
2433                                 virt_dev->tt_info->active_eps != 0) {
2434                 rh_bw_info->num_active_tts += 1;
2435                 rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD;
2436         } else if (old_active_eps != 0 &&
2437                                 virt_dev->tt_info->active_eps == 0) {
2438                 rh_bw_info->num_active_tts -= 1;
2439                 rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD;
2440         }
2441 }
2442
2443 static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
2444                 struct xhci_virt_device *virt_dev,
2445                 struct xhci_container_ctx *in_ctx)
2446 {
2447         struct xhci_bw_info ep_bw_info[31];
2448         int i;
2449         struct xhci_input_control_ctx *ctrl_ctx;
2450         int old_active_eps = 0;
2451
2452         if (virt_dev->tt_info)
2453                 old_active_eps = virt_dev->tt_info->active_eps;
2454
2455         ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
2456
2457         for (i = 0; i < 31; i++) {
2458                 if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2459                         continue;
2460
2461                 /* Make a copy of the BW info in case we need to revert this */
2462                 memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,
2463                                 sizeof(ep_bw_info[i]));
2464                 /* Drop the endpoint from the interval table if the endpoint is
2465                  * being dropped or changed.
2466                  */
2467                 if (EP_IS_DROPPED(ctrl_ctx, i))
2468                         xhci_drop_ep_from_interval_table(xhci,
2469                                         &virt_dev->eps[i].bw_info,
2470                                         virt_dev->bw_table,
2471                                         virt_dev->udev,
2472                                         &virt_dev->eps[i],
2473                                         virt_dev->tt_info);
2474         }
2475         /* Overwrite the information stored in the endpoints' bw_info */
2476         xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);
2477         for (i = 0; i < 31; i++) {
2478                 /* Add any changed or added endpoints to the interval table */
2479                 if (EP_IS_ADDED(ctrl_ctx, i))
2480                         xhci_add_ep_to_interval_table(xhci,
2481                                         &virt_dev->eps[i].bw_info,
2482                                         virt_dev->bw_table,
2483                                         virt_dev->udev,
2484                                         &virt_dev->eps[i],
2485                                         virt_dev->tt_info);
2486         }
2487
2488         if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {
2489                 /* Ok, this fits in the bandwidth we have.
2490                  * Update the number of active TTs.
2491                  */
2492                 xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
2493                 return 0;
2494         }
2495
2496         /* We don't have enough bandwidth for this, revert the stored info. */
2497         for (i = 0; i < 31; i++) {
2498                 if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2499                         continue;
2500
2501                 /* Drop the new copies of any added or changed endpoints from
2502                  * the interval table.
2503                  */
2504                 if (EP_IS_ADDED(ctrl_ctx, i)) {
2505                         xhci_drop_ep_from_interval_table(xhci,
2506                                         &virt_dev->eps[i].bw_info,
2507                                         virt_dev->bw_table,
2508                                         virt_dev->udev,
2509                                         &virt_dev->eps[i],
2510                                         virt_dev->tt_info);
2511                 }
2512                 /* Revert the endpoint back to its old information */
2513                 memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],
2514                                 sizeof(ep_bw_info[i]));
2515                 /* Add any changed or dropped endpoints back into the table */
2516                 if (EP_IS_DROPPED(ctrl_ctx, i))
2517                         xhci_add_ep_to_interval_table(xhci,
2518                                         &virt_dev->eps[i].bw_info,
2519                                         virt_dev->bw_table,
2520                                         virt_dev->udev,
2521                                         &virt_dev->eps[i],
2522                                         virt_dev->tt_info);
2523         }
2524         return -ENOMEM;
2525 }
2526
2527
2528 /* Issue a configure endpoint command or evaluate context command
2529  * and wait for it to finish.
2530  */
2531 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
2532                 struct usb_device *udev,
2533                 struct xhci_command *command,
2534                 bool ctx_change, bool must_succeed)
2535 {
2536         int ret;
2537         int timeleft;
2538         unsigned long flags;
2539         struct xhci_container_ctx *in_ctx;
2540         struct completion *cmd_completion;
2541         u32 *cmd_status;
2542         struct xhci_virt_device *virt_dev;
2543         union xhci_trb *cmd_trb;
2544
2545         spin_lock_irqsave(&xhci->lock, flags);
2546         virt_dev = xhci->devs[udev->slot_id];
2547
2548         if (command)
2549                 in_ctx = command->in_ctx;
2550         else
2551                 in_ctx = virt_dev->in_ctx;
2552
2553         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) &&
2554                         xhci_reserve_host_resources(xhci, in_ctx)) {
2555                 spin_unlock_irqrestore(&xhci->lock, flags);
2556                 xhci_warn(xhci, "Not enough host resources, "
2557                                 "active endpoint contexts = %u\n",
2558                                 xhci->num_active_eps);
2559                 return -ENOMEM;
2560         }
2561         if ((xhci->quirks & XHCI_SW_BW_CHECKING) &&
2562                         xhci_reserve_bandwidth(xhci, virt_dev, in_ctx)) {
2563                 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2564                         xhci_free_host_resources(xhci, in_ctx);
2565                 spin_unlock_irqrestore(&xhci->lock, flags);
2566                 xhci_warn(xhci, "Not enough bandwidth\n");
2567                 return -ENOMEM;
2568         }
2569
2570         if (command) {
2571                 cmd_completion = command->completion;
2572                 cmd_status = &command->status;
2573                 command->command_trb = xhci->cmd_ring->enqueue;
2574
2575                 /* Enqueue pointer can be left pointing to the link TRB,
2576                  * we must handle that
2577                  */
2578                 if (TRB_TYPE_LINK_LE32(command->command_trb->link.control))
2579                         command->command_trb =
2580                                 xhci->cmd_ring->enq_seg->next->trbs;
2581
2582                 list_add_tail(&command->cmd_list, &virt_dev->cmd_list);
2583         } else {
2584                 cmd_completion = &virt_dev->cmd_completion;
2585                 cmd_status = &virt_dev->cmd_status;
2586         }
2587         init_completion(cmd_completion);
2588
2589         cmd_trb = xhci->cmd_ring->dequeue;
2590         if (!ctx_change)
2591                 ret = xhci_queue_configure_endpoint(xhci, in_ctx->dma,
2592                                 udev->slot_id, must_succeed);
2593         else
2594                 ret = xhci_queue_evaluate_context(xhci, in_ctx->dma,
2595                                 udev->slot_id);
2596         if (ret < 0) {
2597                 if (command)
2598                         list_del(&command->cmd_list);
2599                 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2600                         xhci_free_host_resources(xhci, in_ctx);
2601                 spin_unlock_irqrestore(&xhci->lock, flags);
2602                 xhci_dbg(xhci, "FIXME allocate a new ring segment\n");
2603                 return -ENOMEM;
2604         }
2605         xhci_ring_cmd_db(xhci);
2606         spin_unlock_irqrestore(&xhci->lock, flags);
2607
2608         /* Wait for the configure endpoint command to complete */
2609         timeleft = wait_for_completion_interruptible_timeout(
2610                         cmd_completion,
2611                         XHCI_CMD_DEFAULT_TIMEOUT);
2612         if (timeleft <= 0) {
2613                 xhci_warn(xhci, "%s while waiting for %s command\n",
2614                                 timeleft == 0 ? "Timeout" : "Signal",
2615                                 ctx_change == 0 ?
2616                                         "configure endpoint" :
2617                                         "evaluate context");
2618                 /* cancel the configure endpoint command */
2619                 ret = xhci_cancel_cmd(xhci, command, cmd_trb);
2620                 if (ret < 0)
2621                         return ret;
2622                 return -ETIME;
2623         }
2624
2625         if (!ctx_change)
2626                 ret = xhci_configure_endpoint_result(xhci, udev, cmd_status);
2627         else
2628                 ret = xhci_evaluate_context_result(xhci, udev, cmd_status);
2629
2630         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
2631                 spin_lock_irqsave(&xhci->lock, flags);
2632                 /* If the command failed, remove the reserved resources.
2633                  * Otherwise, clean up the estimate to include dropped eps.
2634                  */
2635                 if (ret)
2636                         xhci_free_host_resources(xhci, in_ctx);
2637                 else
2638                         xhci_finish_resource_reservation(xhci, in_ctx);
2639                 spin_unlock_irqrestore(&xhci->lock, flags);
2640         }
2641         return ret;
2642 }
2643
2644 /* Called after one or more calls to xhci_add_endpoint() or
2645  * xhci_drop_endpoint().  If this call fails, the USB core is expected
2646  * to call xhci_reset_bandwidth().
2647  *
2648  * Since we are in the middle of changing either configuration or
2649  * installing a new alt setting, the USB core won't allow URBs to be
2650  * enqueued for any endpoint on the old config or interface.  Nothing
2651  * else should be touching the xhci->devs[slot_id] structure, so we
2652  * don't need to take the xhci->lock for manipulating that.
2653  */
2654 int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
2655 {
2656         int i;
2657         int ret = 0;
2658         struct xhci_hcd *xhci;
2659         struct xhci_virt_device *virt_dev;
2660         struct xhci_input_control_ctx *ctrl_ctx;
2661         struct xhci_slot_ctx *slot_ctx;
2662
2663         ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2664         if (ret <= 0)
2665                 return ret;
2666         xhci = hcd_to_xhci(hcd);
2667         if (xhci->xhc_state & XHCI_STATE_DYING)
2668                 return -ENODEV;
2669
2670         xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
2671         virt_dev = xhci->devs[udev->slot_id];
2672
2673         /* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */
2674         ctrl_ctx = xhci_get_input_control_ctx(xhci, virt_dev->in_ctx);
2675         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
2676         ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
2677         ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
2678
2679         /* Don't issue the command if there's no endpoints to update. */
2680         if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&
2681                         ctrl_ctx->drop_flags == 0)
2682                 return 0;
2683
2684         xhci_dbg(xhci, "New Input Control Context:\n");
2685         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
2686         xhci_dbg_ctx(xhci, virt_dev->in_ctx,
2687                      LAST_CTX_TO_EP_NUM(le32_to_cpu(slot_ctx->dev_info)));
2688
2689         ret = xhci_configure_endpoint(xhci, udev, NULL,
2690                         false, false);
2691         if (ret) {
2692                 /* Callee should call reset_bandwidth() */
2693                 return ret;
2694         }
2695
2696         xhci_dbg(xhci, "Output context after successful config ep cmd:\n");
2697         xhci_dbg_ctx(xhci, virt_dev->out_ctx,
2698                      LAST_CTX_TO_EP_NUM(le32_to_cpu(slot_ctx->dev_info)));
2699
2700         /* Free any rings that were dropped, but not changed. */
2701         for (i = 1; i < 31; ++i) {
2702                 if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&
2703                     !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1))))
2704                         xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
2705         }
2706         xhci_zero_in_ctx(xhci, virt_dev);
2707         /*
2708          * Install any rings for completely new endpoints or changed endpoints,
2709          * and free or cache any old rings from changed endpoints.
2710          */
2711         for (i = 1; i < 31; ++i) {
2712                 if (!virt_dev->eps[i].new_ring)
2713                         continue;
2714                 /* Only cache or free the old ring if it exists.
2715                  * It may not if this is the first add of an endpoint.
2716                  */
2717                 if (virt_dev->eps[i].ring) {
2718                         xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
2719                 }
2720                 virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;
2721                 virt_dev->eps[i].new_ring = NULL;
2722         }
2723
2724         return ret;
2725 }
2726
2727 void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
2728 {
2729         struct xhci_hcd *xhci;
2730         struct xhci_virt_device *virt_dev;
2731         int i, ret;
2732
2733         ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2734         if (ret <= 0)
2735                 return;
2736         xhci = hcd_to_xhci(hcd);
2737
2738         xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
2739         virt_dev = xhci->devs[udev->slot_id];
2740         /* Free any rings allocated for added endpoints */
2741         for (i = 0; i < 31; ++i) {
2742                 if (virt_dev->eps[i].new_ring) {
2743                         xhci_ring_free(xhci, virt_dev->eps[i].new_ring);
2744                         virt_dev->eps[i].new_ring = NULL;
2745                 }
2746         }
2747         xhci_zero_in_ctx(xhci, virt_dev);
2748 }
2749
2750 static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
2751                 struct xhci_container_ctx *in_ctx,
2752                 struct xhci_container_ctx *out_ctx,
2753                 u32 add_flags, u32 drop_flags)
2754 {
2755         struct xhci_input_control_ctx *ctrl_ctx;
2756         ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
2757         ctrl_ctx->add_flags = cpu_to_le32(add_flags);
2758         ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);
2759         xhci_slot_copy(xhci, in_ctx, out_ctx);
2760         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
2761
2762         xhci_dbg(xhci, "Input Context:\n");
2763         xhci_dbg_ctx(xhci, in_ctx, xhci_last_valid_endpoint(add_flags));
2764 }
2765
2766 static void xhci_setup_input_ctx_for_quirk(struct xhci_hcd *xhci,
2767                 unsigned int slot_id, unsigned int ep_index,
2768                 struct xhci_dequeue_state *deq_state)
2769 {
2770         struct xhci_container_ctx *in_ctx;
2771         struct xhci_ep_ctx *ep_ctx;
2772         u32 added_ctxs;
2773         dma_addr_t addr;
2774
2775         xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
2776                         xhci->devs[slot_id]->out_ctx, ep_index);
2777         in_ctx = xhci->devs[slot_id]->in_ctx;
2778         ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
2779         addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
2780                         deq_state->new_deq_ptr);
2781         if (addr == 0) {
2782                 xhci_warn(xhci, "WARN Cannot submit config ep after "
2783                                 "reset ep command\n");
2784                 xhci_warn(xhci, "WARN deq seg = %p, deq ptr = %p\n",
2785                                 deq_state->new_deq_seg,
2786                                 deq_state->new_deq_ptr);
2787                 return;
2788         }
2789         ep_ctx->deq = cpu_to_le64(addr | deq_state->new_cycle_state);
2790
2791         added_ctxs = xhci_get_endpoint_flag_from_index(ep_index);
2792         xhci_setup_input_ctx_for_config_ep(xhci, xhci->devs[slot_id]->in_ctx,
2793                         xhci->devs[slot_id]->out_ctx, added_ctxs, added_ctxs);
2794 }
2795
2796 void xhci_cleanup_stalled_ring(struct xhci_hcd *xhci,
2797                 struct usb_device *udev, unsigned int ep_index)
2798 {
2799         struct xhci_dequeue_state deq_state;
2800         struct xhci_virt_ep *ep;
2801
2802         xhci_dbg(xhci, "Cleaning up stalled endpoint ring\n");
2803         ep = &xhci->devs[udev->slot_id]->eps[ep_index];
2804         /* We need to move the HW's dequeue pointer past this TD,
2805          * or it will attempt to resend it on the next doorbell ring.
2806          */
2807         xhci_find_new_dequeue_state(xhci, udev->slot_id,
2808                         ep_index, ep->stopped_stream, ep->stopped_td,
2809                         &deq_state);
2810
2811         /* HW with the reset endpoint quirk will use the saved dequeue state to
2812          * issue a configure endpoint command later.
2813          */
2814         if (!(xhci->quirks & XHCI_RESET_EP_QUIRK)) {
2815                 xhci_dbg(xhci, "Queueing new dequeue state\n");
2816                 xhci_queue_new_dequeue_state(xhci, udev->slot_id,
2817                                 ep_index, ep->stopped_stream, &deq_state);
2818         } else {
2819                 /* Better hope no one uses the input context between now and the
2820                  * reset endpoint completion!
2821                  * XXX: No idea how this hardware will react when stream rings
2822                  * are enabled.
2823                  */
2824                 xhci_dbg(xhci, "Setting up input context for "
2825                                 "configure endpoint command\n");
2826                 xhci_setup_input_ctx_for_quirk(xhci, udev->slot_id,
2827                                 ep_index, &deq_state);
2828         }
2829 }
2830
2831 /* Deal with stalled endpoints.  The core should have sent the control message
2832  * to clear the halt condition.  However, we need to make the xHCI hardware
2833  * reset its sequence number, since a device will expect a sequence number of
2834  * zero after the halt condition is cleared.
2835  * Context: in_interrupt
2836  */
2837 void xhci_endpoint_reset(struct usb_hcd *hcd,
2838                 struct usb_host_endpoint *ep)
2839 {
2840         struct xhci_hcd *xhci;
2841         struct usb_device *udev;
2842         unsigned int ep_index;
2843         unsigned long flags;
2844         int ret;
2845         struct xhci_virt_ep *virt_ep;
2846
2847         xhci = hcd_to_xhci(hcd);
2848         udev = (struct usb_device *) ep->hcpriv;
2849         /* Called with a root hub endpoint (or an endpoint that wasn't added
2850          * with xhci_add_endpoint()
2851          */
2852         if (!ep->hcpriv)
2853                 return;
2854         ep_index = xhci_get_endpoint_index(&ep->desc);
2855         virt_ep = &xhci->devs[udev->slot_id]->eps[ep_index];
2856         if (!virt_ep->stopped_td) {
2857                 xhci_dbg(xhci, "Endpoint 0x%x not halted, refusing to reset.\n",
2858                                 ep->desc.bEndpointAddress);
2859                 return;
2860         }
2861         if (usb_endpoint_xfer_control(&ep->desc)) {
2862                 xhci_dbg(xhci, "Control endpoint stall already handled.\n");
2863                 return;
2864         }
2865
2866         xhci_dbg(xhci, "Queueing reset endpoint command\n");
2867         spin_lock_irqsave(&xhci->lock, flags);
2868         ret = xhci_queue_reset_ep(xhci, udev->slot_id, ep_index);
2869         /*
2870          * Can't change the ring dequeue pointer until it's transitioned to the
2871          * stopped state, which is only upon a successful reset endpoint
2872          * command.  Better hope that last command worked!
2873          */
2874         if (!ret) {
2875                 xhci_cleanup_stalled_ring(xhci, udev, ep_index);
2876                 kfree(virt_ep->stopped_td);
2877                 xhci_ring_cmd_db(xhci);
2878         }
2879         virt_ep->stopped_td = NULL;
2880         virt_ep->stopped_trb = NULL;
2881         virt_ep->stopped_stream = 0;
2882         spin_unlock_irqrestore(&xhci->lock, flags);
2883
2884         if (ret)
2885                 xhci_warn(xhci, "FIXME allocate a new ring segment\n");
2886 }
2887
2888 static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
2889                 struct usb_device *udev, struct usb_host_endpoint *ep,
2890                 unsigned int slot_id)
2891 {
2892         int ret;
2893         unsigned int ep_index;
2894         unsigned int ep_state;
2895
2896         if (!ep)
2897                 return -EINVAL;
2898         ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
2899         if (ret <= 0)
2900                 return -EINVAL;
2901         if (ep->ss_ep_comp.bmAttributes == 0) {
2902                 xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
2903                                 " descriptor for ep 0x%x does not support streams\n",
2904                                 ep->desc.bEndpointAddress);
2905                 return -EINVAL;
2906         }
2907
2908         ep_index = xhci_get_endpoint_index(&ep->desc);
2909         ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
2910         if (ep_state & EP_HAS_STREAMS ||
2911                         ep_state & EP_GETTING_STREAMS) {
2912                 xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "
2913                                 "already has streams set up.\n",
2914                                 ep->desc.bEndpointAddress);
2915                 xhci_warn(xhci, "Send email to xHCI maintainer and ask for "
2916                                 "dynamic stream context array reallocation.\n");
2917                 return -EINVAL;
2918         }
2919         if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {
2920                 xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "
2921                                 "endpoint 0x%x; URBs are pending.\n",
2922                                 ep->desc.bEndpointAddress);
2923                 return -EINVAL;
2924         }
2925         return 0;
2926 }
2927
2928 static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,
2929                 unsigned int *num_streams, unsigned int *num_stream_ctxs)
2930 {
2931         unsigned int max_streams;
2932
2933         /* The stream context array size must be a power of two */
2934         *num_stream_ctxs = roundup_pow_of_two(*num_streams);
2935         /*
2936          * Find out how many primary stream array entries the host controller
2937          * supports.  Later we may use secondary stream arrays (similar to 2nd
2938          * level page entries), but that's an optional feature for xHCI host
2939          * controllers. xHCs must support at least 4 stream IDs.
2940          */
2941         max_streams = HCC_MAX_PSA(xhci->hcc_params);
2942         if (*num_stream_ctxs > max_streams) {
2943                 xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",
2944                                 max_streams);
2945                 *num_stream_ctxs = max_streams;
2946                 *num_streams = max_streams;
2947         }
2948 }
2949
2950 /* Returns an error code if one of the endpoint already has streams.
2951  * This does not change any data structures, it only checks and gathers
2952  * information.
2953  */
2954 static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,
2955                 struct usb_device *udev,
2956                 struct usb_host_endpoint **eps, unsigned int num_eps,
2957                 unsigned int *num_streams, u32 *changed_ep_bitmask)
2958 {
2959         unsigned int max_streams;
2960         unsigned int endpoint_flag;
2961         int i;
2962         int ret;
2963
2964         for (i = 0; i < num_eps; i++) {
2965                 ret = xhci_check_streams_endpoint(xhci, udev,
2966                                 eps[i], udev->slot_id);
2967                 if (ret < 0)
2968                         return ret;
2969
2970                 max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp);
2971                 if (max_streams < (*num_streams - 1)) {
2972                         xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",
2973                                         eps[i]->desc.bEndpointAddress,
2974                                         max_streams);
2975                         *num_streams = max_streams+1;
2976                 }
2977
2978                 endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);
2979                 if (*changed_ep_bitmask & endpoint_flag)
2980                         return -EINVAL;
2981                 *changed_ep_bitmask |= endpoint_flag;
2982         }
2983         return 0;
2984 }
2985
2986 static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,
2987                 struct usb_device *udev,
2988                 struct usb_host_endpoint **eps, unsigned int num_eps)
2989 {
2990         u32 changed_ep_bitmask = 0;
2991         unsigned int slot_id;
2992         unsigned int ep_index;
2993         unsigned int ep_state;
2994         int i;
2995
2996         slot_id = udev->slot_id;
2997         if (!xhci->devs[slot_id])
2998                 return 0;
2999
3000         for (i = 0; i < num_eps; i++) {
3001                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3002                 ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3003                 /* Are streams already being freed for the endpoint? */
3004                 if (ep_state & EP_GETTING_NO_STREAMS) {
3005                         xhci_warn(xhci, "WARN Can't disable streams for "
3006                                         "endpoint 0x%x\n, "
3007                                         "streams are being disabled already.",
3008                                         eps[i]->desc.bEndpointAddress);
3009                         return 0;
3010                 }
3011                 /* Are there actually any streams to free? */
3012                 if (!(ep_state & EP_HAS_STREAMS) &&
3013                                 !(ep_state & EP_GETTING_STREAMS)) {
3014                         xhci_warn(xhci, "WARN Can't disable streams for "
3015                                         "endpoint 0x%x\n, "
3016                                         "streams are already disabled!",
3017                                         eps[i]->desc.bEndpointAddress);
3018                         xhci_warn(xhci, "WARN xhci_free_streams() called "
3019                                         "with non-streams endpoint\n");
3020                         return 0;
3021                 }
3022                 changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);
3023         }
3024         return changed_ep_bitmask;
3025 }
3026
3027 /*
3028  * The USB device drivers use this function (though the HCD interface in USB
3029  * core) to prepare a set of bulk endpoints to use streams.  Streams are used to
3030  * coordinate mass storage command queueing across multiple endpoints (basically
3031  * a stream ID == a task ID).
3032  *
3033  * Setting up streams involves allocating the same size stream context array
3034  * for each endpoint and issuing a configure endpoint command for all endpoints.
3035  *
3036  * Don't allow the call to succeed if one endpoint only supports one stream
3037  * (which means it doesn't support streams at all).
3038  *
3039  * Drivers may get less stream IDs than they asked for, if the host controller
3040  * hardware or endpoints claim they can't support the number of requested
3041  * stream IDs.
3042  */
3043 int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
3044                 struct usb_host_endpoint **eps, unsigned int num_eps,
3045                 unsigned int num_streams, gfp_t mem_flags)
3046 {
3047         int i, ret;
3048         struct xhci_hcd *xhci;
3049         struct xhci_virt_device *vdev;
3050         struct xhci_command *config_cmd;
3051         unsigned int ep_index;
3052         unsigned int num_stream_ctxs;
3053         unsigned long flags;
3054         u32 changed_ep_bitmask = 0;
3055
3056         if (!eps)
3057                 return -EINVAL;
3058
3059         /* Add one to the number of streams requested to account for
3060          * stream 0 that is reserved for xHCI usage.
3061          */
3062         num_streams += 1;
3063         xhci = hcd_to_xhci(hcd);
3064         xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",
3065                         num_streams);
3066
3067         config_cmd = xhci_alloc_command(xhci, true, true, mem_flags);
3068         if (!config_cmd) {
3069                 xhci_dbg(xhci, "Could not allocate xHCI command structure.\n");
3070                 return -ENOMEM;
3071         }
3072
3073         /* Check to make sure all endpoints are not already configured for
3074          * streams.  While we're at it, find the maximum number of streams that
3075          * all the endpoints will support and check for duplicate endpoints.
3076          */
3077         spin_lock_irqsave(&xhci->lock, flags);
3078         ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,
3079                         num_eps, &num_streams, &changed_ep_bitmask);
3080         if (ret < 0) {
3081                 xhci_free_command(xhci, config_cmd);
3082                 spin_unlock_irqrestore(&xhci->lock, flags);
3083                 return ret;
3084         }
3085         if (num_streams <= 1) {
3086                 xhci_warn(xhci, "WARN: endpoints can't handle "
3087                                 "more than one stream.\n");
3088                 xhci_free_command(xhci, config_cmd);
3089                 spin_unlock_irqrestore(&xhci->lock, flags);
3090                 return -EINVAL;
3091         }
3092         vdev = xhci->devs[udev->slot_id];
3093         /* Mark each endpoint as being in transition, so
3094          * xhci_urb_enqueue() will reject all URBs.
3095          */
3096         for (i = 0; i < num_eps; i++) {
3097                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3098                 vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;
3099         }
3100         spin_unlock_irqrestore(&xhci->lock, flags);
3101
3102         /* Setup internal data structures and allocate HW data structures for
3103          * streams (but don't install the HW structures in the input context
3104          * until we're sure all memory allocation succeeded).
3105          */
3106         xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);
3107         xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",
3108                         num_stream_ctxs, num_streams);
3109
3110         for (i = 0; i < num_eps; i++) {
3111                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3112                 vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
3113                                 num_stream_ctxs,
3114                                 num_streams, mem_flags);
3115                 if (!vdev->eps[ep_index].stream_info)
3116                         goto cleanup;
3117                 /* Set maxPstreams in endpoint context and update deq ptr to
3118                  * point to stream context array. FIXME
3119                  */
3120         }
3121
3122         /* Set up the input context for a configure endpoint command. */
3123         for (i = 0; i < num_eps; i++) {
3124                 struct xhci_ep_ctx *ep_ctx;
3125
3126                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3127                 ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);
3128
3129                 xhci_endpoint_copy(xhci, config_cmd->in_ctx,
3130                                 vdev->out_ctx, ep_index);
3131                 xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,
3132                                 vdev->eps[ep_index].stream_info);
3133         }
3134         /* Tell the HW to drop its old copy of the endpoint context info
3135          * and add the updated copy from the input context.
3136          */
3137         xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,
3138                         vdev->out_ctx, changed_ep_bitmask, changed_ep_bitmask);
3139
3140         /* Issue and wait for the configure endpoint command */
3141         ret = xhci_configure_endpoint(xhci, udev, config_cmd,
3142                         false, false);
3143
3144         /* xHC rejected the configure endpoint command for some reason, so we
3145          * leave the old ring intact and free our internal streams data
3146          * structure.
3147          */
3148         if (ret < 0)
3149                 goto cleanup;
3150
3151         spin_lock_irqsave(&xhci->lock, flags);
3152         for (i = 0; i < num_eps; i++) {
3153                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3154                 vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3155                 xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",
3156                          udev->slot_id, ep_index);
3157                 vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;
3158         }
3159         xhci_free_command(xhci, config_cmd);
3160         spin_unlock_irqrestore(&xhci->lock, flags);
3161
3162         /* Subtract 1 for stream 0, which drivers can't use */
3163         return num_streams - 1;
3164
3165 cleanup:
3166         /* If it didn't work, free the streams! */
3167         for (i = 0; i < num_eps; i++) {
3168                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3169                 xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3170                 vdev->eps[ep_index].stream_info = NULL;
3171                 /* FIXME Unset maxPstreams in endpoint context and
3172                  * update deq ptr to point to normal string ring.
3173                  */
3174                 vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3175                 vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3176                 xhci_endpoint_zero(xhci, vdev, eps[i]);
3177         }
3178         xhci_free_command(xhci, config_cmd);
3179         return -ENOMEM;
3180 }
3181
3182 /* Transition the endpoint from using streams to being a "normal" endpoint
3183  * without streams.
3184  *
3185  * Modify the endpoint context state, submit a configure endpoint command,
3186  * and free all endpoint rings for streams if that completes successfully.
3187  */
3188 int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
3189                 struct usb_host_endpoint **eps, unsigned int num_eps,
3190                 gfp_t mem_flags)
3191 {
3192         int i, ret;
3193         struct xhci_hcd *xhci;
3194         struct xhci_virt_device *vdev;
3195         struct xhci_command *command;
3196         unsigned int ep_index;
3197         unsigned long flags;
3198         u32 changed_ep_bitmask;
3199
3200         xhci = hcd_to_xhci(hcd);
3201         vdev = xhci->devs[udev->slot_id];
3202
3203         /* Set up a configure endpoint command to remove the streams rings */
3204         spin_lock_irqsave(&xhci->lock, flags);
3205         changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,
3206                         udev, eps, num_eps);
3207         if (changed_ep_bitmask == 0) {
3208                 spin_unlock_irqrestore(&xhci->lock, flags);
3209                 return -EINVAL;
3210         }
3211
3212         /* Use the xhci_command structure from the first endpoint.  We may have
3213          * allocated too many, but the driver may call xhci_free_streams() for
3214          * each endpoint it grouped into one call to xhci_alloc_streams().
3215          */
3216         ep_index = xhci_get_endpoint_index(&eps[0]->desc);
3217         command = vdev->eps[ep_index].stream_info->free_streams_command;
3218         for (i = 0; i < num_eps; i++) {
3219                 struct xhci_ep_ctx *ep_ctx;
3220
3221                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3222                 ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
3223                 xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=
3224                         EP_GETTING_NO_STREAMS;
3225
3226                 xhci_endpoint_copy(xhci, command->in_ctx,
3227                                 vdev->out_ctx, ep_index);
3228                 xhci_setup_no_streams_ep_input_ctx(xhci, ep_ctx,
3229                                 &vdev->eps[ep_index]);
3230         }
3231         xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,
3232                         vdev->out_ctx, changed_ep_bitmask, changed_ep_bitmask);
3233         spin_unlock_irqrestore(&xhci->lock, flags);
3234
3235         /* Issue and wait for the configure endpoint command,
3236          * which must succeed.
3237          */
3238         ret = xhci_configure_endpoint(xhci, udev, command,
3239                         false, true);
3240
3241         /* xHC rejected the configure endpoint command for some reason, so we
3242          * leave the streams rings intact.
3243          */
3244         if (ret < 0)
3245                 return ret;
3246
3247         spin_lock_irqsave(&xhci->lock, flags);
3248         for (i = 0; i < num_eps; i++) {
3249                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3250                 xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3251                 vdev->eps[ep_index].stream_info = NULL;
3252                 /* FIXME Unset maxPstreams in endpoint context and
3253                  * update deq ptr to point to normal string ring.
3254                  */
3255                 vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;
3256                 vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3257         }
3258         spin_unlock_irqrestore(&xhci->lock, flags);
3259
3260         return 0;
3261 }
3262
3263 /*
3264  * Deletes endpoint resources for endpoints that were active before a Reset
3265  * Device command, or a Disable Slot command.  The Reset Device command leaves
3266  * the control endpoint intact, whereas the Disable Slot command deletes it.
3267  *
3268  * Must be called with xhci->lock held.
3269  */
3270 void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,
3271         struct xhci_virt_device *virt_dev, bool drop_control_ep)
3272 {
3273         int i;
3274         unsigned int num_dropped_eps = 0;
3275         unsigned int drop_flags = 0;
3276
3277         for (i = (drop_control_ep ? 0 : 1); i < 31; i++) {
3278                 if (virt_dev->eps[i].ring) {
3279                         drop_flags |= 1 << i;
3280                         num_dropped_eps++;
3281                 }
3282         }
3283         xhci->num_active_eps -= num_dropped_eps;
3284         if (num_dropped_eps)
3285                 xhci_dbg(xhci, "Dropped %u ep ctxs, flags = 0x%x, "
3286                                 "%u now active.\n",
3287                                 num_dropped_eps, drop_flags,
3288                                 xhci->num_active_eps);
3289 }
3290
3291 /*
3292  * This submits a Reset Device Command, which will set the device state to 0,
3293  * set the device address to 0, and disable all the endpoints except the default
3294  * control endpoint.  The USB core should come back and call
3295  * xhci_address_device(), and then re-set up the configuration.  If this is
3296  * called because of a usb_reset_and_verify_device(), then the old alternate
3297  * settings will be re-installed through the normal bandwidth allocation
3298  * functions.
3299  *
3300  * Wait for the Reset Device command to finish.  Remove all structures
3301  * associated with the endpoints that were disabled.  Clear the input device
3302  * structure?  Cache the rings?  Reset the control endpoint 0 max packet size?
3303  *
3304  * If the virt_dev to be reset does not exist or does not match the udev,
3305  * it means the device is lost, possibly due to the xHC restore error and
3306  * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to
3307  * re-allocate the device.
3308  */
3309 int xhci_discover_or_reset_device(struct usb_hcd *hcd, struct usb_device *udev)
3310 {
3311         int ret, i;
3312         unsigned long flags;
3313         struct xhci_hcd *xhci;
3314         unsigned int slot_id;
3315         struct xhci_virt_device *virt_dev;
3316         struct xhci_command *reset_device_cmd;
3317         int timeleft;
3318         int last_freed_endpoint;
3319         struct xhci_slot_ctx *slot_ctx;
3320         int old_active_eps = 0;
3321
3322         ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__);
3323         if (ret <= 0)
3324                 return ret;
3325         xhci = hcd_to_xhci(hcd);
3326         slot_id = udev->slot_id;
3327         virt_dev = xhci->devs[slot_id];
3328         if (!virt_dev) {
3329                 xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3330                                 "not exist. Re-allocate the device\n", slot_id);
3331                 ret = xhci_alloc_dev(hcd, udev);
3332                 if (ret == 1)
3333                         return 0;
3334                 else
3335                         return -EINVAL;
3336         }
3337
3338         if (virt_dev->udev != udev) {
3339                 /* If the virt_dev and the udev does not match, this virt_dev
3340                  * may belong to another udev.
3341                  * Re-allocate the device.
3342                  */
3343                 xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3344                                 "not match the udev. Re-allocate the device\n",
3345                                 slot_id);
3346                 ret = xhci_alloc_dev(hcd, udev);
3347                 if (ret == 1)
3348                         return 0;
3349                 else
3350                         return -EINVAL;
3351         }
3352
3353         /* If device is not setup, there is no point in resetting it */
3354         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3355         if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
3356                                                 SLOT_STATE_DISABLED)
3357                 return 0;
3358
3359         xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
3360         /* Allocate the command structure that holds the struct completion.
3361          * Assume we're in process context, since the normal device reset
3362          * process has to wait for the device anyway.  Storage devices are
3363          * reset as part of error handling, so use GFP_NOIO instead of
3364          * GFP_KERNEL.
3365          */
3366         reset_device_cmd = xhci_alloc_command(xhci, false, true, GFP_NOIO);
3367         if (!reset_device_cmd) {
3368                 xhci_dbg(xhci, "Couldn't allocate command structure.\n");
3369                 return -ENOMEM;
3370         }
3371
3372         /* Attempt to submit the Reset Device command to the command ring */
3373         spin_lock_irqsave(&xhci->lock, flags);
3374         reset_device_cmd->command_trb = xhci->cmd_ring->enqueue;
3375
3376         /* Enqueue pointer can be left pointing to the link TRB,
3377          * we must handle that
3378          */
3379         if (TRB_TYPE_LINK_LE32(reset_device_cmd->command_trb->link.control))
3380                 reset_device_cmd->command_trb =
3381                         xhci->cmd_ring->enq_seg->next->trbs;
3382
3383         list_add_tail(&reset_device_cmd->cmd_list, &virt_dev->cmd_list);
3384         ret = xhci_queue_reset_device(xhci, slot_id);
3385         if (ret) {
3386                 xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3387                 list_del(&reset_device_cmd->cmd_list);
3388                 spin_unlock_irqrestore(&xhci->lock, flags);
3389                 goto command_cleanup;
3390         }
3391         xhci_ring_cmd_db(xhci);
3392         spin_unlock_irqrestore(&xhci->lock, flags);
3393
3394         /* Wait for the Reset Device command to finish */
3395         timeleft = wait_for_completion_interruptible_timeout(
3396                         reset_device_cmd->completion,
3397                         USB_CTRL_SET_TIMEOUT);
3398         if (timeleft <= 0) {
3399                 xhci_warn(xhci, "%s while waiting for reset device command\n",
3400                                 timeleft == 0 ? "Timeout" : "Signal");
3401                 spin_lock_irqsave(&xhci->lock, flags);
3402                 /* The timeout might have raced with the event ring handler, so
3403                  * only delete from the list if the item isn't poisoned.
3404                  */
3405                 if (reset_device_cmd->cmd_list.next != LIST_POISON1)
3406                         list_del(&reset_device_cmd->cmd_list);
3407                 spin_unlock_irqrestore(&xhci->lock, flags);
3408                 ret = -ETIME;
3409                 goto command_cleanup;
3410         }
3411
3412         /* The Reset Device command can't fail, according to the 0.95/0.96 spec,
3413          * unless we tried to reset a slot ID that wasn't enabled,
3414          * or the device wasn't in the addressed or configured state.
3415          */
3416         ret = reset_device_cmd->status;
3417         switch (ret) {
3418         case COMP_EBADSLT: /* 0.95 completion code for bad slot ID */
3419         case COMP_CTX_STATE: /* 0.96 completion code for same thing */
3420                 xhci_info(xhci, "Can't reset device (slot ID %u) in %s state\n",
3421                                 slot_id,
3422                                 xhci_get_slot_state(xhci, virt_dev->out_ctx));
3423                 xhci_info(xhci, "Not freeing device rings.\n");
3424                 /* Don't treat this as an error.  May change my mind later. */
3425                 ret = 0;
3426                 goto command_cleanup;
3427         case COMP_SUCCESS:
3428                 xhci_dbg(xhci, "Successful reset device command.\n");
3429                 break;
3430         default:
3431                 if (xhci_is_vendor_info_code(xhci, ret))
3432                         break;
3433                 xhci_warn(xhci, "Unknown completion code %u for "
3434                                 "reset device command.\n", ret);
3435                 ret = -EINVAL;
3436                 goto command_cleanup;
3437         }
3438
3439         /* Free up host controller endpoint resources */
3440         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
3441                 spin_lock_irqsave(&xhci->lock, flags);
3442                 /* Don't delete the default control endpoint resources */
3443                 xhci_free_device_endpoint_resources(xhci, virt_dev, false);
3444                 spin_unlock_irqrestore(&xhci->lock, flags);
3445         }
3446
3447         /* Everything but endpoint 0 is disabled, so free or cache the rings. */
3448         last_freed_endpoint = 1;
3449         for (i = 1; i < 31; ++i) {
3450                 struct xhci_virt_ep *ep = &virt_dev->eps[i];
3451
3452                 if (ep->ep_state & EP_HAS_STREAMS) {
3453                         xhci_free_stream_info(xhci, ep->stream_info);
3454                         ep->stream_info = NULL;
3455                         ep->ep_state &= ~EP_HAS_STREAMS;
3456                 }
3457
3458                 if (ep->ring) {
3459                         xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
3460                         last_freed_endpoint = i;
3461                 }
3462                 if (!list_empty(&virt_dev->eps[i].bw_endpoint_list))
3463                         xhci_drop_ep_from_interval_table(xhci,
3464                                         &virt_dev->eps[i].bw_info,
3465                                         virt_dev->bw_table,
3466                                         udev,
3467                                         &virt_dev->eps[i],
3468                                         virt_dev->tt_info);
3469                 xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info);
3470         }
3471         /* If necessary, update the number of active TTs on this root port */
3472         xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
3473
3474         xhci_dbg(xhci, "Output context after successful reset device cmd:\n");
3475         xhci_dbg_ctx(xhci, virt_dev->out_ctx, last_freed_endpoint);
3476         ret = 0;
3477
3478 command_cleanup:
3479         xhci_free_command(xhci, reset_device_cmd);
3480         return ret;
3481 }
3482
3483 /*
3484  * At this point, the struct usb_device is about to go away, the device has
3485  * disconnected, and all traffic has been stopped and the endpoints have been
3486  * disabled.  Free any HC data structures associated with that device.
3487  */
3488 void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
3489 {
3490         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3491         struct xhci_virt_device *virt_dev;
3492         struct device *dev = hcd->self.controller;
3493         unsigned long flags;
3494         u32 state;
3495         int i, ret;
3496
3497 #ifndef CONFIG_USB_DEFAULT_PERSIST
3498         /*
3499          * We called pm_runtime_get_noresume when the device was attached.
3500          * Decrement the counter here to allow controller to runtime suspend
3501          * if no devices remain.
3502          */
3503         if (xhci->quirks & XHCI_RESET_ON_RESUME)
3504                 pm_runtime_put_noidle(dev);
3505 #endif
3506
3507         ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3508         /* If the host is halted due to driver unload, we still need to free the
3509          * device.
3510          */
3511         if (ret <= 0 && ret != -ENODEV)
3512                 return;
3513
3514         virt_dev = xhci->devs[udev->slot_id];
3515
3516         /* Stop any wayward timer functions (which may grab the lock) */
3517         for (i = 0; i < 31; ++i) {
3518                 virt_dev->eps[i].ep_state &= ~EP_HALT_PENDING;
3519                 del_timer_sync(&virt_dev->eps[i].stop_cmd_timer);
3520         }
3521
3522         if (udev->usb2_hw_lpm_enabled) {
3523                 xhci_set_usb2_hardware_lpm(hcd, udev, 0);
3524                 udev->usb2_hw_lpm_enabled = 0;
3525         }
3526
3527         spin_lock_irqsave(&xhci->lock, flags);
3528         /* Don't disable the slot if the host controller is dead. */
3529         state = xhci_readl(xhci, &xhci->op_regs->status);
3530         if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||
3531                         (xhci->xhc_state & XHCI_STATE_HALTED)) {
3532                 xhci_free_virt_device(xhci, udev->slot_id);
3533                 spin_unlock_irqrestore(&xhci->lock, flags);
3534                 return;
3535         }
3536
3537         if (xhci_queue_slot_control(xhci, TRB_DISABLE_SLOT, udev->slot_id)) {