drivers/usb/dwc3/gadget.c

   1 /**
   2  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   3  *
   4  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   5  *
   6  * Authors: Felipe Balbi <balbi@ti.com>,
   7  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. The names of the above-listed copyright holders may not be used
  19  *    to endorse or promote products derived from this software without
  20  *    specific prior written permission.
  21  *
  22  * ALTERNATIVELY, this software may be distributed under the terms of the
  23  * GNU General Public License ("GPL") version 2, as published by the Free
  24  * Software Foundation.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  27  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  28  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  29  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  30  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  31  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  32  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  33  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  34  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  35  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  36  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 #include <linux/kernel.h>
  40 #include <linux/delay.h>
  41 #include <linux/slab.h>
  42 #include <linux/spinlock.h>
  43 #include <linux/platform_device.h>
  44 #include <linux/pm_runtime.h>
  45 #include <linux/interrupt.h>
  46 #include <linux/io.h>
  47 #include <linux/list.h>
  48 #include <linux/dma-mapping.h>
  49
  50 #include <linux/usb/ch9.h>
  51 #include <linux/usb/gadget.h>
  52
  53 #include "core.h"
  54 #include "gadget.h"
  55 #include "io.h"
  56
  57 #define DMA_ADDR_INVALID        (~(dma_addr_t)0)
  58
  59 void dwc3_map_buffer_to_dma(struct dwc3_request *req)
  60 {
  61         struct dwc3                     *dwc = req->dep->dwc;
  62
  63         if (req->request.length == 0) {
  64                 /* req->request.dma = dwc->setup_buf_addr; */
  65                 return;
  66         }
  67
  68         if (req->request.num_sgs) {
  69                 int     mapped;
  70
  71                 mapped = dma_map_sg(dwc->dev, req->request.sg,
  72                                 req->request.num_sgs,
  73                                 req->direction ? DMA_TO_DEVICE
  74                                 : DMA_FROM_DEVICE);
  75                 if (mapped < 0) {
  76                         dev_err(dwc->dev, "failed to map SGs\n");
  77                         return;
  78                 }
  79
  80                 req->request.num_mapped_sgs = mapped;
  81                 return;
  82         }
  83
  84         if (req->request.dma == DMA_ADDR_INVALID) {
  85                 req->request.dma = dma_map_single(dwc->dev, req->request.buf,
  86                                 req->request.length, req->direction
  87                                 ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
  88                 req->mapped = true;
  89         }
  90 }
  91
  92 void dwc3_unmap_buffer_from_dma(struct dwc3_request *req)
  93 {
  94         struct dwc3                     *dwc = req->dep->dwc;
  95
  96         if (req->request.length == 0) {
  97                 req->request.dma = DMA_ADDR_INVALID;
  98                 return;
  99         }
 100
 101         if (req->request.num_mapped_sgs) {
 102                 req->request.dma = DMA_ADDR_INVALID;
 103                 dma_unmap_sg(dwc->dev, req->request.sg,
 104                                 req->request.num_sgs,
 105                                 req->direction ? DMA_TO_DEVICE
 106                                 : DMA_FROM_DEVICE);
 107
 108                 req->request.num_mapped_sgs = 0;
 109                 return;
 110         }
 111
 112         if (req->mapped) {
 113                 dma_unmap_single(dwc->dev, req->request.dma,
 114                                 req->request.length, req->direction
 115                                 ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 116                 req->mapped = 0;
 117                 req->request.dma = DMA_ADDR_INVALID;
 118         }
 119 }
 120
 121 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 122                 int status)
 123 {
 124         struct dwc3                     *dwc = dep->dwc;
 125
 126         if (req->queued) {
 127                 if (req->request.num_mapped_sgs)
 128                         dep->busy_slot += req->request.num_mapped_sgs;
 129                 else
 130                         dep->busy_slot++;
 131
 132                 /*
 133                  * Skip LINK TRB. We can't use req->trb and check for
 134                  * DWC3_TRBCTL_LINK_TRB because it points the TRB we just
 135                  * completed (not the LINK TRB).
 136                  */
 137                 if (((dep->busy_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
 138                                 usb_endpoint_xfer_isoc(dep->desc))
 139                         dep->busy_slot++;
 140         }
 141         list_del(&req->list);
 142         req->trb = NULL;
 143
 144         if (req->request.status == -EINPROGRESS)
 145                 req->request.status = status;
 146
 147         dwc3_unmap_buffer_from_dma(req);
 148
 149         dev_dbg(dwc->dev, "request %p from %s completed %d/%d ===> %d\n",
 150                         req, dep->name, req->request.actual,
 151                         req->request.length, status);
 152
 153         spin_unlock(&dwc->lock);
 154         req->request.complete(&req->dep->endpoint, &req->request);
 155         spin_lock(&dwc->lock);
 156 }
 157
 158 static const char *dwc3_gadget_ep_cmd_string(u8 cmd)
 159 {
 160         switch (cmd) {
 161         case DWC3_DEPCMD_DEPSTARTCFG:
 162                 return "Start New Configuration";
 163         case DWC3_DEPCMD_ENDTRANSFER:
 164                 return "End Transfer";
 165         case DWC3_DEPCMD_UPDATETRANSFER:
 166                 return "Update Transfer";
 167         case DWC3_DEPCMD_STARTTRANSFER:
 168                 return "Start Transfer";
 169         case DWC3_DEPCMD_CLEARSTALL:
 170                 return "Clear Stall";
 171         case DWC3_DEPCMD_SETSTALL:
 172                 return "Set Stall";
 173         case DWC3_DEPCMD_GETSEQNUMBER:
 174                 return "Get Data Sequence Number";
 175         case DWC3_DEPCMD_SETTRANSFRESOURCE:
 176                 return "Set Endpoint Transfer Resource";
 177         case DWC3_DEPCMD_SETEPCONFIG:
 178                 return "Set Endpoint Configuration";
 179         default:
 180                 return "UNKNOWN command";
 181         }
 182 }
 183
 184 int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
 185                 unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
 186 {
 187         struct dwc3_ep          *dep = dwc->eps[ep];
 188         u32                     timeout = 500;
 189         u32                     reg;
 190
 191         dev_vdbg(dwc->dev, "%s: cmd '%s' params %08x %08x %08x\n",
 192                         dep->name,
 193                         dwc3_gadget_ep_cmd_string(cmd), params->param0,
 194                         params->param1, params->param2);
 195
 196         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0);
 197         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1);
 198         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2);
 199
 200         dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
 201         do {
 202                 reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
 203                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 204                         dev_vdbg(dwc->dev, "Command Complete --> %d\n",
 205                                         DWC3_DEPCMD_STATUS(reg));
 206                         return 0;
 207                 }
 208
 209                 /*
 210                  * We can't sleep here, because it is also called from
 211                  * interrupt context.
 212                  */
 213                 timeout--;
 214                 if (!timeout)
 215                         return -ETIMEDOUT;
 216
 217                 udelay(1);
 218         } while (1);
 219 }
 220
 221 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 222                 struct dwc3_trb_hw *trb)
 223 {
 224         u32             offset = (char *) trb - (char *) dep->trb_pool;
 225
 226         return dep->trb_pool_dma + offset;
 227 }
 228
 229 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 230 {
 231         struct dwc3             *dwc = dep->dwc;
 232
 233         if (dep->trb_pool)
 234                 return 0;
 235
 236         if (dep->number == 0 || dep->number == 1)
 237                 return 0;
 238
 239         dep->trb_pool = dma_alloc_coherent(dwc->dev,
 240                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 241                         &dep->trb_pool_dma, GFP_KERNEL);
 242         if (!dep->trb_pool) {
 243                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 244                                 dep->name);
 245                 return -ENOMEM;
 246         }
 247
 248         return 0;
 249 }
 250
 251 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 252 {
 253         struct dwc3             *dwc = dep->dwc;
 254
 255         dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 256                         dep->trb_pool, dep->trb_pool_dma);
 257
 258         dep->trb_pool = NULL;
 259         dep->trb_pool_dma = 0;
 260 }
 261
 262 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 263 {
 264         struct dwc3_gadget_ep_cmd_params params;
 265         u32                     cmd;
 266
 267         memset(&params, 0x00, sizeof(params));
 268
 269         if (dep->number != 1) {
 270                 cmd = DWC3_DEPCMD_DEPSTARTCFG;
 271                 /* XferRscIdx == 0 for ep0 and 2 for the remaining */
 272                 if (dep->number > 1) {
 273                         if (dwc->start_config_issued)
 274                                 return 0;
 275                         dwc->start_config_issued = true;
 276                         cmd |= DWC3_DEPCMD_PARAM(2);
 277                 }
 278
 279                 return dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
 280         }
 281
 282         return 0;
 283 }
 284
 285 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 286                 const struct usb_endpoint_descriptor *desc,
 287                 const struct usb_ss_ep_comp_descriptor *comp_desc)
 288 {
 289         struct dwc3_gadget_ep_cmd_params params;
 290
 291         memset(&params, 0x00, sizeof(params));
 292
 293         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 294                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc))
 295                 | DWC3_DEPCFG_BURST_SIZE(dep->endpoint.maxburst);
 296
 297         params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
 298                 | DWC3_DEPCFG_XFER_NOT_READY_EN;
 299
 300         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 301                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 302                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 303                 dep->stream_capable = true;
 304         }
 305
 306         if (usb_endpoint_xfer_isoc(desc))
 307                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 308
 309         /*
 310          * We are doing 1:1 mapping for endpoints, meaning
 311          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 312          * so on. We consider the direction bit as part of the physical
 313          * endpoint number. So USB endpoint 0x81 is 0x03.
 314          */
 315         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 316
 317         /*
 318          * We must use the lower 16 TX FIFOs even though
 319          * HW might have more
 320          */
 321         if (dep->direction)
 322                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 323
 324         if (desc->bInterval) {
 325                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 326                 dep->interval = 1 << (desc->bInterval - 1);
 327         }
 328
 329         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 330                         DWC3_DEPCMD_SETEPCONFIG, &params);
 331 }
 332
 333 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
 334 {
 335         struct dwc3_gadget_ep_cmd_params params;
 336
 337         memset(&params, 0x00, sizeof(params));
 338
 339         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 340
 341         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 342                         DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
 343 }
 344
 345 /**
 346  * __dwc3_gadget_ep_enable - Initializes a HW endpoint
 347  * @dep: endpoint to be initialized
 348  * @desc: USB Endpoint Descriptor
 349  *
 350  * Caller should take care of locking
 351  */
 352 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 353                 const struct usb_endpoint_descriptor *desc,
 354                 const struct usb_ss_ep_comp_descriptor *comp_desc)
 355 {
 356         struct dwc3             *dwc = dep->dwc;
 357         u32                     reg;
 358         int                     ret = -ENOMEM;
 359
 360         if (!(dep->flags & DWC3_EP_ENABLED)) {
 361                 ret = dwc3_gadget_start_config(dwc, dep);
 362                 if (ret)
 363                         return ret;
 364         }
 365
 366         ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc);
 367         if (ret)
 368                 return ret;
 369
 370         if (!(dep->flags & DWC3_EP_ENABLED)) {
 371                 struct dwc3_trb_hw      *trb_st_hw;
 372                 struct dwc3_trb_hw      *trb_link_hw;
 373                 struct dwc3_trb         trb_link;
 374
 375                 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 376                 if (ret)
 377                         return ret;
 378
 379                 dep->desc = desc;
 380                 dep->comp_desc = comp_desc;
 381                 dep->type = usb_endpoint_type(desc);
 382                 dep->flags |= DWC3_EP_ENABLED;
 383
 384                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 385                 reg |= DWC3_DALEPENA_EP(dep->number);
 386                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 387
 388                 if (!usb_endpoint_xfer_isoc(desc))
 389                         return 0;
 390
 391                 memset(&trb_link, 0, sizeof(trb_link));
 392
 393                 /* Link TRB for ISOC. The HWO but is never reset */
 394                 trb_st_hw = &dep->trb_pool[0];
 395
 396                 trb_link.bplh = dwc3_trb_dma_offset(dep, trb_st_hw);
 397                 trb_link.trbctl = DWC3_TRBCTL_LINK_TRB;
 398                 trb_link.hwo = true;
 399
 400                 trb_link_hw = &dep->trb_pool[DWC3_TRB_NUM - 1];
 401                 dwc3_trb_to_hw(&trb_link, trb_link_hw);
 402         }
 403
 404         return 0;
 405 }
 406
 407 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum);
 408 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 409 {
 410         struct dwc3_request             *req;
 411
 412         if (!list_empty(&dep->req_queued))
 413                 dwc3_stop_active_transfer(dwc, dep->number);
 414
 415         while (!list_empty(&dep->request_list)) {
 416                 req = next_request(&dep->request_list);
 417
 418                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 419         }
 420 }
 421
 422 /**
 423  * __dwc3_gadget_ep_disable - Disables a HW endpoint
 424  * @dep: the endpoint to disable
 425  *
 426  * This function also removes requests which are currently processed ny the
 427  * hardware and those which are not yet scheduled.
 428  * Caller should take care of locking.
 429  */
 430 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 431 {
 432         struct dwc3             *dwc = dep->dwc;
 433         u32                     reg;
 434
 435         dwc3_remove_requests(dwc, dep);
 436
 437         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 438         reg &= ~DWC3_DALEPENA_EP(dep->number);
 439         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 440
 441         dep->stream_capable = false;
 442         dep->desc = NULL;
 443         dep->comp_desc = NULL;
 444         dep->type = 0;
 445         dep->flags = 0;
 446
 447         return 0;
 448 }
 449
 450 /* -------------------------------------------------------------------------- */
 451
 452 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 453                 const struct usb_endpoint_descriptor *desc)
 454 {
 455         return -EINVAL;
 456 }
 457
 458 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 459 {
 460         return -EINVAL;
 461 }
 462
 463 /* -------------------------------------------------------------------------- */
 464
 465 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 466                 const struct usb_endpoint_descriptor *desc)
 467 {
 468         struct dwc3_ep                  *dep;
 469         struct dwc3                     *dwc;
 470         unsigned long                   flags;
 471         int                             ret;
 472
 473         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 474                 pr_debug("dwc3: invalid parameters\n");
 475                 return -EINVAL;
 476         }
 477
 478         if (!desc->wMaxPacketSize) {
 479                 pr_debug("dwc3: missing wMaxPacketSize\n");
 480                 return -EINVAL;
 481         }
 482
 483         dep = to_dwc3_ep(ep);
 484         dwc = dep->dwc;
 485
 486         switch (usb_endpoint_type(desc)) {
 487         case USB_ENDPOINT_XFER_CONTROL:
 488                 strncat(dep->name, "-control", sizeof(dep->name));
 489                 break;
 490         case USB_ENDPOINT_XFER_ISOC:
 491                 strncat(dep->name, "-isoc", sizeof(dep->name));
 492                 break;
 493         case USB_ENDPOINT_XFER_BULK:
 494                 strncat(dep->name, "-bulk", sizeof(dep->name));
 495                 break;
 496         case USB_ENDPOINT_XFER_INT:
 497                 strncat(dep->name, "-int", sizeof(dep->name));
 498                 break;
 499         default:
 500                 dev_err(dwc->dev, "invalid endpoint transfer type\n");
 501         }
 502
 503         if (dep->flags & DWC3_EP_ENABLED) {
 504                 dev_WARN_ONCE(dwc->dev, true, "%s is already enabled\n",
 505                                 dep->name);
 506                 return 0;
 507         }
 508
 509         dev_vdbg(dwc->dev, "Enabling %s\n", dep->name);
 510
 511         spin_lock_irqsave(&dwc->lock, flags);
 512         ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc);
 513         spin_unlock_irqrestore(&dwc->lock, flags);
 514
 515         return ret;
 516 }
 517
 518 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 519 {
 520         struct dwc3_ep                  *dep;
 521         struct dwc3                     *dwc;
 522         unsigned long                   flags;
 523         int                             ret;
 524
 525         if (!ep) {
 526                 pr_debug("dwc3: invalid parameters\n");
 527                 return -EINVAL;
 528         }
 529
 530         dep = to_dwc3_ep(ep);
 531         dwc = dep->dwc;
 532
 533         if (!(dep->flags & DWC3_EP_ENABLED)) {
 534                 dev_WARN_ONCE(dwc->dev, true, "%s is already disabled\n",
 535                                 dep->name);
 536                 return 0;
 537         }
 538
 539         snprintf(dep->name, sizeof(dep->name), "ep%d%s",
 540                         dep->number >> 1,
 541                         (dep->number & 1) ? "in" : "out");
 542
 543         spin_lock_irqsave(&dwc->lock, flags);
 544         ret = __dwc3_gadget_ep_disable(dep);
 545         spin_unlock_irqrestore(&dwc->lock, flags);
 546
 547         return ret;
 548 }
 549
 550 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 551         gfp_t gfp_flags)
 552 {
 553         struct dwc3_request             *req;
 554         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 555         struct dwc3                     *dwc = dep->dwc;
 556
 557         req = kzalloc(sizeof(*req), gfp_flags);
 558         if (!req) {
 559                 dev_err(dwc->dev, "not enough memory\n");
 560                 return NULL;
 561         }
 562
 563         req->epnum      = dep->number;
 564         req->dep        = dep;
 565         req->request.dma = DMA_ADDR_INVALID;
 566
 567         return &req->request;
 568 }
 569
 570 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 571                 struct usb_request *request)
 572 {
 573         struct dwc3_request             *req = to_dwc3_request(request);
 574
 575         kfree(req);
 576 }
 577
 578 /**
 579  * dwc3_prepare_one_trb - setup one TRB from one request
 580  * @dep: endpoint for which this request is prepared
 581  * @req: dwc3_request pointer
 582  */
 583 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 584                 struct dwc3_request *req, dma_addr_t dma,
 585                 unsigned length, unsigned last, unsigned chain)
 586 {
 587         struct dwc3             *dwc = dep->dwc;
 588         struct dwc3_trb_hw      *trb_hw;
 589         struct dwc3_trb         trb;
 590
 591         unsigned int            cur_slot;
 592
 593         dev_vdbg(dwc->dev, "%s: req %p dma %08llx length %d%s%s\n",
 594                         dep->name, req, (unsigned long long) dma,
 595                         length, last ? " last" : "",
 596                         chain ? " chain" : "");
 597
 598         trb_hw = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
 599         cur_slot = dep->free_slot;
 600         dep->free_slot++;
 601
 602         /* Skip the LINK-TRB on ISOC */
 603         if (((cur_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
 604                         usb_endpoint_xfer_isoc(dep->desc))
 605                 return;
 606
 607         memset(&trb, 0, sizeof(trb));
 608         if (!req->trb) {
 609                 dwc3_gadget_move_request_queued(req);
 610                 req->trb = trb_hw;
 611                 req->trb_dma = dwc3_trb_dma_offset(dep, trb_hw);
 612         }
 613
 614         if (usb_endpoint_xfer_isoc(dep->desc)) {
 615                 trb.isp_imi = true;
 616                 trb.csp = true;
 617         } else {
 618                 trb.chn = chain;
 619                 trb.lst = last;
 620         }
 621
 622         if (usb_endpoint_xfer_bulk(dep->desc) && dep->stream_capable)
 623                 trb.sid_sofn = req->request.stream_id;
 624
 625         switch (usb_endpoint_type(dep->desc)) {
 626         case USB_ENDPOINT_XFER_CONTROL:
 627                 trb.trbctl = DWC3_TRBCTL_CONTROL_SETUP;
 628                 break;
 629
 630         case USB_ENDPOINT_XFER_ISOC:
 631                 trb.trbctl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 632
 633                 /* IOC every DWC3_TRB_NUM / 4 so we can refill */
 634                 if (!(cur_slot % (DWC3_TRB_NUM / 4)))
 635                         trb.ioc = last;
 636                 break;
 637
 638         case USB_ENDPOINT_XFER_BULK:
 639         case USB_ENDPOINT_XFER_INT:
 640                 trb.trbctl = DWC3_TRBCTL_NORMAL;
 641                 break;
 642         default:
 643                 /*
 644                  * This is only possible with faulty memory because we
 645                  * checked it already :)
 646                  */
 647                 BUG();
 648         }
 649
 650         trb.length      = length;
 651         trb.bplh        = dma;
 652         trb.hwo         = true;
 653
 654         dwc3_trb_to_hw(&trb, trb_hw);
 655 }
 656
 657 /*
 658  * dwc3_prepare_trbs - setup TRBs from requests
 659  * @dep: endpoint for which requests are being prepared
 660  * @starting: true if the endpoint is idle and no requests are queued.
 661  *
 662  * The functions goes through the requests list and setups TRBs for the
 663  * transfers. The functions returns once there are not more TRBs available or
 664  * it run out of requests.
 665  */
 666 static void dwc3_prepare_trbs(struct dwc3_ep *dep, bool starting)
 667 {
 668         struct dwc3_request     *req, *n;
 669         u32                     trbs_left;
 670         unsigned int            last_one = 0;
 671
 672         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
 673
 674         /* the first request must not be queued */
 675         trbs_left = (dep->busy_slot - dep->free_slot) & DWC3_TRB_MASK;
 676
 677         /*
 678          * if busy & slot are equal than it is either full or empty. If we are
 679          * starting to proceed requests then we are empty. Otherwise we ar
 680          * full and don't do anything
 681          */
 682         if (!trbs_left) {
 683                 if (!starting)
 684                         return;
 685                 trbs_left = DWC3_TRB_NUM;
 686                 /*
 687                  * In case we start from scratch, we queue the ISOC requests
 688                  * starting from slot 1. This is done because we use ring
 689                  * buffer and have no LST bit to stop us. Instead, we place
 690                  * IOC bit TRB_NUM/4. We try to avoid to having an interrupt
 691                  * after the first request so we start at slot 1 and have
 692                  * 7 requests proceed before we hit the first IOC.
 693                  * Other transfer types don't use the ring buffer and are
 694                  * processed from the first TRB until the last one. Since we
 695                  * don't wrap around we have to start at the beginning.
 696                  */
 697                 if (usb_endpoint_xfer_isoc(dep->desc)) {
 698                         dep->busy_slot = 1;
 699                         dep->free_slot = 1;
 700                 } else {
 701                         dep->busy_slot = 0;
 702                         dep->free_slot = 0;
 703                 }
 704         }
 705
 706         /* The last TRB is a link TRB, not used for xfer */
 707         if ((trbs_left <= 1) && usb_endpoint_xfer_isoc(dep->desc))
 708                 return;
 709
 710         list_for_each_entry_safe(req, n, &dep->request_list, list) {
 711                 unsigned        length;
 712                 dma_addr_t      dma;
 713
 714                 if (req->request.num_mapped_sgs > 0) {
 715                         struct usb_request *request = &req->request;
 716                         struct scatterlist *sg = request->sg;
 717                         struct scatterlist *s;
 718                         int             i;
 719
 720                         for_each_sg(sg, s, request->num_mapped_sgs, i) {
 721                                 unsigned chain = true;
 722
 723                                 length = sg_dma_len(s);
 724                                 dma = sg_dma_address(s);
 725
 726                                 if (i == (request->num_mapped_sgs - 1)
 727                                                 || sg_is_last(s)) {
 728                                         last_one = true;
 729                                         chain = false;
 730                                 }
 731
 732                                 trbs_left--;
 733                                 if (!trbs_left)
 734                                         last_one = true;
 735
 736                                 if (last_one)
 737                                         chain = false;
 738
 739                                 dwc3_prepare_one_trb(dep, req, dma, length,
 740                                                 last_one, chain);
 741
 742                                 if (last_one)
 743                                         break;
 744                         }
 745                 } else {
 746                         dma = req->request.dma;
 747                         length = req->request.length;
 748                         trbs_left--;
 749
 750                         if (!trbs_left)
 751                                 last_one = 1;
 752
 753                         /* Is this the last request? */
 754                         if (list_is_last(&req->list, &dep->request_list))
 755                                 last_one = 1;
 756
 757                         dwc3_prepare_one_trb(dep, req, dma, length,
 758                                         last_one, false);
 759
 760                         if (last_one)
 761                                 break;
 762                 }
 763         }
 764 }
 765
 766 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,
 767                 int start_new)
 768 {
 769         struct dwc3_gadget_ep_cmd_params params;
 770         struct dwc3_request             *req;
 771         struct dwc3                     *dwc = dep->dwc;
 772         int                             ret;
 773         u32                             cmd;
 774
 775         if (start_new && (dep->flags & DWC3_EP_BUSY)) {
 776                 dev_vdbg(dwc->dev, "%s: endpoint busy\n", dep->name);
 777                 return -EBUSY;
 778         }
 779         dep->flags &= ~DWC3_EP_PENDING_REQUEST;
 780
 781         /*
 782          * If we are getting here after a short-out-packet we don't enqueue any
 783          * new requests as we try to set the IOC bit only on the last request.
 784          */
 785         if (start_new) {
 786                 if (list_empty(&dep->req_queued))
 787                         dwc3_prepare_trbs(dep, start_new);
 788
 789                 /* req points to the first request which will be sent */
 790                 req = next_request(&dep->req_queued);
 791         } else {
 792                 dwc3_prepare_trbs(dep, start_new);
 793
 794                 /*
 795                  * req points to the first request where HWO changed
 796                  * from 0 to 1
 797                  */
 798                 req = next_request(&dep->req_queued);
 799         }
 800         if (!req) {
 801                 dep->flags |= DWC3_EP_PENDING_REQUEST;
 802                 return 0;
 803         }
 804
 805         memset(&params, 0, sizeof(params));
 806         params.param0 = upper_32_bits(req->trb_dma);
 807         params.param1 = lower_32_bits(req->trb_dma);
 808
 809         if (start_new)
 810                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 811         else
 812                 cmd = DWC3_DEPCMD_UPDATETRANSFER;
 813
 814         cmd |= DWC3_DEPCMD_PARAM(cmd_param);
 815         ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
 816         if (ret < 0) {
 817                 dev_dbg(dwc->dev, "failed to send STARTTRANSFER command\n");
 818
 819                 /*
 820                  * FIXME we need to iterate over the list of requests
 821                  * here and stop, unmap, free and del each of the linked
 822                  * requests instead of we do now.
 823                  */
 824                 dwc3_unmap_buffer_from_dma(req);
 825                 list_del(&req->list);
 826                 return ret;
 827         }
 828
 829         dep->flags |= DWC3_EP_BUSY;
 830         dep->res_trans_idx = dwc3_gadget_ep_get_transfer_index(dwc,
 831                         dep->number);
 832
 833         WARN_ON_ONCE(!dep->res_trans_idx);
 834
 835         return 0;
 836 }
 837
 838 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
 839 {
 840         req->request.actual     = 0;
 841         req->request.status     = -EINPROGRESS;
 842         req->direction          = dep->direction;
 843         req->epnum              = dep->number;
 844
 845         /*
 846          * We only add to our list of requests now and
 847          * start consuming the list once we get XferNotReady
 848          * IRQ.
 849          *
 850          * That way, we avoid doing anything that we don't need
 851          * to do now and defer it until the point we receive a
 852          * particular token from the Host side.
 853          *
 854          * This will also avoid Host cancelling URBs due to too
 855          * many NACKs.
 856          */
 857         dwc3_map_buffer_to_dma(req);
 858         list_add_tail(&req->list, &dep->request_list);
 859
 860         /*
 861          * There is one special case: XferNotReady with
 862          * empty list of requests. We need to kick the
 863          * transfer here in that situation, otherwise
 864          * we will be NAKing forever.
 865          *
 866          * If we get XferNotReady before gadget driver
 867          * has a chance to queue a request, we will ACK
 868          * the IRQ but won't be able to receive the data
 869          * until the next request is queued. The following
 870          * code is handling exactly that.
 871          */
 872         if (dep->flags & DWC3_EP_PENDING_REQUEST) {
 873                 int ret;
 874                 int start_trans;
 875
 876                 start_trans = 1;
 877                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
 878                                 dep->flags & DWC3_EP_BUSY)
 879                         start_trans = 0;
 880
 881                 ret =  __dwc3_gadget_kick_transfer(dep, 0, start_trans);
 882                 if (ret && ret != -EBUSY) {
 883                         struct dwc3     *dwc = dep->dwc;
 884
 885                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
 886                                         dep->name);
 887                 }
 888         };
 889
 890         return 0;
 891 }
 892
 893 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
 894         gfp_t gfp_flags)
 895 {
 896         struct dwc3_request             *req = to_dwc3_request(request);
 897         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 898         struct dwc3                     *dwc = dep->dwc;
 899
 900         unsigned long                   flags;
 901
 902         int                             ret;
 903
 904         if (!dep->desc) {
 905                 dev_dbg(dwc->dev, "trying to queue request %p to disabled %s\n",
 906                                 request, ep->name);
 907                 return -ESHUTDOWN;
 908         }
 909
 910         dev_vdbg(dwc->dev, "queing request %p to %s length %d\n",
 911                         request, ep->name, request->length);
 912
 913         spin_lock_irqsave(&dwc->lock, flags);
 914         ret = __dwc3_gadget_ep_queue(dep, req);
 915         spin_unlock_irqrestore(&dwc->lock, flags);
 916
 917         return ret;
 918 }
 919
 920 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
 921                 struct usb_request *request)
 922 {
 923         struct dwc3_request             *req = to_dwc3_request(request);
 924         struct dwc3_request             *r = NULL;
 925
 926         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 927         struct dwc3                     *dwc = dep->dwc;
 928
 929         unsigned long                   flags;
 930         int                             ret = 0;
 931
 932         spin_lock_irqsave(&dwc->lock, flags);
 933
 934         list_for_each_entry(r, &dep->request_list, list) {
 935                 if (r == req)
 936                         break;
 937         }
 938
 939         if (r != req) {
 940                 list_for_each_entry(r, &dep->req_queued, list) {
 941                         if (r == req)
 942                                 break;
 943                 }
 944                 if (r == req) {
 945                         /* wait until it is processed */
 946                         dwc3_stop_active_transfer(dwc, dep->number);
 947                         goto out0;
 948                 }
 949                 dev_err(dwc->dev, "request %p was not queued to %s\n",
 950                                 request, ep->name);
 951                 ret = -EINVAL;
 952                 goto out0;
 953         }
 954
 955         /* giveback the request */
 956         dwc3_gadget_giveback(dep, req, -ECONNRESET);
 957
 958 out0:
 959         spin_unlock_irqrestore(&dwc->lock, flags);
 960
 961         return ret;
 962 }
 963
 964 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value)
 965 {
 966         struct dwc3_gadget_ep_cmd_params        params;
 967         struct dwc3                             *dwc = dep->dwc;
 968         int                                     ret;
 969
 970         memset(&params, 0x00, sizeof(params));
 971
 972         if (value) {
 973                 if (dep->number == 0 || dep->number == 1) {
 974                         /*
 975                          * Whenever EP0 is stalled, we will restart
 976                          * the state machine, thus moving back to
 977                          * Setup Phase
 978                          */
 979                         dwc->ep0state = EP0_SETUP_PHASE;
 980                 }
 981
 982                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
 983                         DWC3_DEPCMD_SETSTALL, &params);
 984                 if (ret)
 985                         dev_err(dwc->dev, "failed to %s STALL on %s\n",
 986                                         value ? "set" : "clear",
 987                                         dep->name);
 988                 else
 989                         dep->flags |= DWC3_EP_STALL;
 990         } else {
 991                 if (dep->flags & DWC3_EP_WEDGE)
 992                         return 0;
 993
 994                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
 995                         DWC3_DEPCMD_CLEARSTALL, &params);
 996                 if (ret)
 997                         dev_err(dwc->dev, "failed to %s STALL on %s\n",
 998                                         value ? "set" : "clear",
 999                                         dep->name);
1000                 else
1001                         dep->flags &= ~DWC3_EP_STALL;
1002         }
1003
1004         return ret;
1005 }
1006
1007 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1008 {
1009         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1010         struct dwc3                     *dwc = dep->dwc;
1011
1012         unsigned long                   flags;
1013
1014         int                             ret;
1015
1016         spin_lock_irqsave(&dwc->lock, flags);
1017
1018         if (usb_endpoint_xfer_isoc(dep->desc)) {
1019                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1020                 ret = -EINVAL;
1021                 goto out;
1022         }
1023
1024         ret = __dwc3_gadget_ep_set_halt(dep, value);
1025 out:
1026         spin_unlock_irqrestore(&dwc->lock, flags);
1027
1028         return ret;
1029 }
1030
1031 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1032 {
1033         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1034
1035         dep->flags |= DWC3_EP_WEDGE;
1036
1037         return dwc3_gadget_ep_set_halt(ep, 1);
1038 }
1039
1040 /* -------------------------------------------------------------------------- */
1041
1042 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1043         .bLength        = USB_DT_ENDPOINT_SIZE,
1044         .bDescriptorType = USB_DT_ENDPOINT,
1045         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1046 };
1047
1048 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1049         .enable         = dwc3_gadget_ep0_enable,
1050         .disable        = dwc3_gadget_ep0_disable,
1051         .alloc_request  = dwc3_gadget_ep_alloc_request,
1052         .free_request   = dwc3_gadget_ep_free_request,
1053         .queue          = dwc3_gadget_ep0_queue,
1054         .dequeue        = dwc3_gadget_ep_dequeue,
1055         .set_halt       = dwc3_gadget_ep_set_halt,
1056         .set_wedge      = dwc3_gadget_ep_set_wedge,
1057 };
1058
1059 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1060         .enable         = dwc3_gadget_ep_enable,
1061         .disable        = dwc3_gadget_ep_disable,
1062         .alloc_request  = dwc3_gadget_ep_alloc_request,
1063         .free_request   = dwc3_gadget_ep_free_request,
1064         .queue          = dwc3_gadget_ep_queue,
1065         .dequeue        = dwc3_gadget_ep_dequeue,
1066         .set_halt       = dwc3_gadget_ep_set_halt,
1067         .set_wedge      = dwc3_gadget_ep_set_wedge,
1068 };
1069
1070 /* -------------------------------------------------------------------------- */
1071
1072 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1073 {
1074         struct dwc3             *dwc = gadget_to_dwc(g);
1075         u32                     reg;
1076
1077         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1078         return DWC3_DSTS_SOFFN(reg);
1079 }
1080
1081 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1082 {
1083         struct dwc3             *dwc = gadget_to_dwc(g);
1084
1085         unsigned long           timeout;
1086         unsigned long           flags;
1087
1088         u32                     reg;
1089
1090         int                     ret = 0;
1091
1092         u8                      link_state;
1093         u8                      speed;
1094
1095         spin_lock_irqsave(&dwc->lock, flags);
1096
1097         /*
1098          * According to the Databook Remote wakeup request should
1099          * be issued only when the device is in early suspend state.
1100          *
1101          * We can check that via USB Link State bits in DSTS register.
1102          */
1103         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1104
1105         speed = reg & DWC3_DSTS_CONNECTSPD;
1106         if (speed == DWC3_DSTS_SUPERSPEED) {
1107                 dev_dbg(dwc->dev, "no wakeup on SuperSpeed\n");
1108                 ret = -EINVAL;
1109                 goto out;
1110         }
1111
1112         link_state = DWC3_DSTS_USBLNKST(reg);
1113
1114         switch (link_state) {
1115         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1116         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1117                 break;
1118         default:
1119                 dev_dbg(dwc->dev, "can't wakeup from link state %d\n",
1120                                 link_state);
1121                 ret = -EINVAL;
1122                 goto out;
1123         }
1124
1125         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1126
1127         /*
1128          * Switch link state to Recovery. In HS/FS/LS this means
1129          * RemoteWakeup Request
1130          */
1131         reg |= DWC3_DCTL_ULSTCHNG_RECOVERY;
1132         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1133
1134         /* wait for at least 2000us */
1135         usleep_range(2000, 2500);
1136
1137         /* write zeroes to Link Change Request */
1138         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1139         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1140
1141         /* pool until Link State change to ON */
1142         timeout = jiffies + msecs_to_jiffies(100);
1143
1144         while (!(time_after(jiffies, timeout))) {
1145                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1146
1147                 /* in HS, means ON */
1148                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1149                         break;
1150         }
1151
1152         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1153                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1154                 ret = -EINVAL;
1155         }
1156
1157 out:
1158         spin_unlock_irqrestore(&dwc->lock, flags);
1159
1160         return ret;
1161 }
1162
1163 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1164                 int is_selfpowered)
1165 {
1166         struct dwc3             *dwc = gadget_to_dwc(g);
1167
1168         dwc->is_selfpowered = !!is_selfpowered;
1169
1170         return 0;
1171 }
1172
1173 static void dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
1174 {
1175         u32                     reg;
1176         u32                     timeout = 500;
1177
1178         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1179         if (is_on)
1180                 reg |= DWC3_DCTL_RUN_STOP;
1181         else
1182                 reg &= ~DWC3_DCTL_RUN_STOP;
1183
1184         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1185
1186         do {
1187                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1188                 if (is_on) {
1189                         if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1190                                 break;
1191                 } else {
1192                         if (reg & DWC3_DSTS_DEVCTRLHLT)
1193                                 break;
1194                 }
1195                 timeout--;
1196                 if (!timeout)
1197                         break;
1198                 udelay(1);
1199         } while (1);
1200
1201         dev_vdbg(dwc->dev, "gadget %s data soft-%s\n",
1202                         dwc->gadget_driver
1203                         ? dwc->gadget_driver->function : "no-function",
1204                         is_on ? "connect" : "disconnect");
1205 }
1206
1207 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1208 {
1209         struct dwc3             *dwc = gadget_to_dwc(g);
1210         unsigned long           flags;
1211
1212         is_on = !!is_on;
1213
1214         spin_lock_irqsave(&dwc->lock, flags);
1215         dwc3_gadget_run_stop(dwc, is_on);
1216         spin_unlock_irqrestore(&dwc->lock, flags);
1217
1218         return 0;
1219 }
1220
1221 static int dwc3_gadget_start(struct usb_gadget *g,
1222                 struct usb_gadget_driver *driver)
1223 {
1224         struct dwc3             *dwc = gadget_to_dwc(g);
1225         struct dwc3_ep          *dep;
1226         unsigned long           flags;
1227         int                     ret = 0;
1228         u32                     reg;
1229
1230         spin_lock_irqsave(&dwc->lock, flags);
1231
1232         if (dwc->gadget_driver) {
1233                 dev_err(dwc->dev, "%s is already bound to %s\n",
1234                                 dwc->gadget.name,
1235                                 dwc->gadget_driver->driver.name);
1236                 ret = -EBUSY;
1237                 goto err0;
1238         }
1239
1240         dwc->gadget_driver      = driver;
1241         dwc->gadget.dev.driver  = &driver->driver;
1242
1243         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1244         reg &= ~(DWC3_DCFG_SPEED_MASK);
1245         reg |= dwc->maximum_speed;
1246         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1247
1248         dwc->start_config_issued = false;
1249
1250         /* Start with SuperSpeed Default */
1251         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1252
1253         dep = dwc->eps[0];
1254         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL);
1255         if (ret) {
1256                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1257                 goto err0;
1258         }
1259
1260         dep = dwc->eps[1];
1261         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL);
1262         if (ret) {
1263                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1264                 goto err1;
1265         }
1266
1267         /* begin to receive SETUP packets */
1268         dwc->ep0state = EP0_SETUP_PHASE;
1269         dwc3_ep0_out_start(dwc);
1270
1271         spin_unlock_irqrestore(&dwc->lock, flags);
1272
1273         return 0;
1274
1275 err1:
1276         __dwc3_gadget_ep_disable(dwc->eps[0]);
1277
1278 err0:
1279         spin_unlock_irqrestore(&dwc->lock, flags);
1280
1281         return ret;
1282 }
1283
1284 static int dwc3_gadget_stop(struct usb_gadget *g,
1285                 struct usb_gadget_driver *driver)
1286 {
1287         struct dwc3             *dwc = gadget_to_dwc(g);
1288         unsigned long           flags;
1289
1290         spin_lock_irqsave(&dwc->lock, flags);
1291
1292         __dwc3_gadget_ep_disable(dwc->eps[0]);
1293         __dwc3_gadget_ep_disable(dwc->eps[1]);
1294
1295         dwc->gadget_driver      = NULL;
1296         dwc->gadget.dev.driver  = NULL;
1297
1298         spin_unlock_irqrestore(&dwc->lock, flags);
1299
1300         return 0;
1301 }
1302 static const struct usb_gadget_ops dwc3_gadget_ops = {
1303         .get_frame              = dwc3_gadget_get_frame,
1304         .wakeup                 = dwc3_gadget_wakeup,
1305         .set_selfpowered        = dwc3_gadget_set_selfpowered,
1306         .pullup                 = dwc3_gadget_pullup,
1307         .udc_start              = dwc3_gadget_start,
1308         .udc_stop               = dwc3_gadget_stop,
1309 };
1310
1311 /* -------------------------------------------------------------------------- */
1312
1313 static int __devinit dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1314 {
1315         struct dwc3_ep                  *dep;
1316         u8                              epnum;
1317
1318         INIT_LIST_HEAD(&dwc->gadget.ep_list);
1319
1320         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1321                 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1322                 if (!dep) {
1323                         dev_err(dwc->dev, "can't allocate endpoint %d\n",
1324                                         epnum);
1325                         return -ENOMEM;
1326                 }
1327
1328                 dep->dwc = dwc;
1329                 dep->number = epnum;
1330                 dwc->eps[epnum] = dep;
1331
1332                 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1333                                 (epnum & 1) ? "in" : "out");
1334                 dep->endpoint.name = dep->name;
1335                 dep->direction = (epnum & 1);
1336
1337                 if (epnum == 0 || epnum == 1) {
1338                         dep->endpoint.maxpacket = 512;
1339                         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1340                         if (!epnum)
1341                                 dwc->gadget.ep0 = &dep->endpoint;
1342                 } else {
1343                         int             ret;
1344
1345                         dep->endpoint.maxpacket = 1024;
1346                         dep->endpoint.max_streams = 15;
1347                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
1348                         list_add_tail(&dep->endpoint.ep_list,
1349                                         &dwc->gadget.ep_list);
1350
1351                         ret = dwc3_alloc_trb_pool(dep);
1352                         if (ret)
1353                                 return ret;
1354                 }
1355
1356                 INIT_LIST_HEAD(&dep->request_list);
1357                 INIT_LIST_HEAD(&dep->req_queued);
1358         }
1359
1360         return 0;
1361 }
1362
1363 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1364 {
1365         struct dwc3_ep                  *dep;
1366         u8                              epnum;
1367
1368         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1369                 dep = dwc->eps[epnum];
1370                 dwc3_free_trb_pool(dep);
1371
1372                 if (epnum != 0 && epnum != 1)
1373                         list_del(&dep->endpoint.ep_list);
1374
1375                 kfree(dep);
1376         }
1377 }
1378
1379 static void dwc3_gadget_release(struct device *dev)
1380 {
1381         dev_dbg(dev, "%s\n", __func__);
1382 }
1383
1384 /* -------------------------------------------------------------------------- */
1385 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1386                 const struct dwc3_event_depevt *event, int status)
1387 {
1388         struct dwc3_request     *req;
1389         struct dwc3_trb         trb;
1390         unsigned int            count;
1391         unsigned int            s_pkt = 0;
1392
1393         do {
1394                 req = next_request(&dep->req_queued);
1395                 if (!req) {
1396                         WARN_ON_ONCE(1);
1397                         return 1;
1398                 }
1399
1400                 dwc3_trb_to_nat(req->trb, &trb);
1401
1402                 if (trb.hwo && status != -ESHUTDOWN)
1403                         /*
1404                          * We continue despite the error. There is not much we
1405                          * can do. If we don't clean in up we loop for ever. If
1406                          * we skip the TRB than it gets overwritten reused after
1407                          * a while since we use them in a ring buffer. a BUG()
1408                          * would help. Lets hope that if this occures, someone
1409                          * fixes the root cause instead of looking away :)
1410                          */
1411                         dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1412                                         dep->name, req->trb);
1413                 count = trb.length;
1414
1415                 if (dep->direction) {
1416                         if (count) {
1417                                 dev_err(dwc->dev, "incomplete IN transfer %s\n",
1418                                                 dep->name);
1419                                 status = -ECONNRESET;
1420                         }
1421                 } else {
1422                         if (count && (event->status & DEPEVT_STATUS_SHORT))
1423                                 s_pkt = 1;
1424                 }
1425
1426                 /*
1427                  * We assume here we will always receive the entire data block
1428                  * which we should receive. Meaning, if we program RX to
1429                  * receive 4K but we receive only 2K, we assume that's all we
1430                  * should receive and we simply bounce the request back to the
1431                  * gadget driver for further processing.
1432                  */
1433                 req->request.actual += req->request.length - count;
1434                 dwc3_gadget_giveback(dep, req, status);
1435                 if (s_pkt)
1436                         break;
1437                 if ((event->status & DEPEVT_STATUS_LST) && trb.lst)
1438                         break;
1439                 if ((event->status & DEPEVT_STATUS_IOC) && trb.ioc)
1440                         break;
1441         } while (1);
1442
1443         if ((event->status & DEPEVT_STATUS_IOC) && trb.ioc)
1444                 return 0;
1445         return 1;
1446 }
1447
1448 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
1449                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event,
1450                 int start_new)
1451 {
1452         unsigned                status = 0;
1453         int                     clean_busy;
1454
1455         if (event->status & DEPEVT_STATUS_BUSERR)
1456                 status = -ECONNRESET;
1457
1458         clean_busy =  dwc3_cleanup_done_reqs(dwc, dep, event, status);
1459         if (clean_busy) {
1460                 dep->flags &= ~DWC3_EP_BUSY;
1461                 dep->res_trans_idx = 0;
1462         }
1463
1464         /*
1465          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
1466          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
1467          */
1468         if (dwc->revision < DWC3_REVISION_183A) {
1469                 u32             reg;
1470                 int             i;
1471
1472                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
1473                         struct dwc3_ep  *dep = dwc->eps[i];
1474
1475                         if (!(dep->flags & DWC3_EP_ENABLED))
1476                                 continue;
1477
1478                         if (!list_empty(&dep->req_queued))
1479                                 return;
1480                 }
1481
1482                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1483                 reg |= dwc->u1u2;
1484                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1485
1486                 dwc->u1u2 = 0;
1487         }
1488 }
1489
1490 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1491                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1492 {
1493         u32 uf;
1494
1495         if (list_empty(&dep->request_list)) {
1496                 dev_vdbg(dwc->dev, "ISOC ep %s run out for requests.\n",
1497                         dep->name);
1498                 return;
1499         }
1500
1501         if (event->parameters) {
1502                 u32 mask;
1503
1504                 mask = ~(dep->interval - 1);
1505                 uf = event->parameters & mask;
1506                 /* 4 micro frames in the future */
1507                 uf += dep->interval * 4;
1508         } else {
1509                 uf = 0;
1510         }
1511
1512         __dwc3_gadget_kick_transfer(dep, uf, 1);
1513 }
1514
1515 static void dwc3_process_ep_cmd_complete(struct dwc3_ep *dep,
1516                 const struct dwc3_event_depevt *event)
1517 {
1518         struct dwc3 *dwc = dep->dwc;
1519         struct dwc3_event_depevt mod_ev = *event;
1520
1521         /*
1522          * We were asked to remove one requests. It is possible that this
1523          * request and a few other were started together and have the same
1524          * transfer index. Since we stopped the complete endpoint we don't
1525          * know how many requests were already completed (and not yet)
1526          * reported and how could be done (later). We purge them all until
1527          * the end of the list.
1528          */
1529         mod_ev.status = DEPEVT_STATUS_LST;
1530         dwc3_cleanup_done_reqs(dwc, dep, &mod_ev, -ESHUTDOWN);
1531         dep->flags &= ~DWC3_EP_BUSY;
1532         /* pending requets are ignored and are queued on XferNotReady */
1533 }
1534
1535 static void dwc3_ep_cmd_compl(struct dwc3_ep *dep,
1536                 const struct dwc3_event_depevt *event)
1537 {
1538         u32 param = event->parameters;
1539         u32 cmd_type = (param >> 8) & ((1 << 5) - 1);
1540
1541         switch (cmd_type) {
1542         case DWC3_DEPCMD_ENDTRANSFER:
1543                 dwc3_process_ep_cmd_complete(dep, event);
1544                 break;
1545         case DWC3_DEPCMD_STARTTRANSFER:
1546                 dep->res_trans_idx = param & 0x7f;
1547                 break;
1548         default:
1549                 printk(KERN_ERR "%s() unknown /unexpected type: %d\n",
1550                                 __func__, cmd_type);
1551                 break;
1552         };
1553 }
1554
1555 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
1556                 const struct dwc3_event_depevt *event)
1557 {
1558         struct dwc3_ep          *dep;
1559         u8                      epnum = event->endpoint_number;
1560
1561         dep = dwc->eps[epnum];
1562
1563         dev_vdbg(dwc->dev, "%s: %s\n", dep->name,
1564                         dwc3_ep_event_string(event->endpoint_event));
1565
1566         if (epnum == 0 || epnum == 1) {
1567                 dwc3_ep0_interrupt(dwc, event);
1568                 return;
1569         }
1570
1571         switch (event->endpoint_event) {
1572         case DWC3_DEPEVT_XFERCOMPLETE:
1573                 if (usb_endpoint_xfer_isoc(dep->desc)) {
1574                         dev_dbg(dwc->dev, "%s is an Isochronous endpoint\n",
1575                                         dep->name);
1576                         return;
1577                 }
1578
1579                 dwc3_endpoint_transfer_complete(dwc, dep, event, 1);
1580                 break;
1581         case DWC3_DEPEVT_XFERINPROGRESS:
1582                 if (!usb_endpoint_xfer_isoc(dep->desc)) {
1583                         dev_dbg(dwc->dev, "%s is not an Isochronous endpoint\n",
1584                                         dep->name);
1585                         return;
1586                 }
1587
1588                 dwc3_endpoint_transfer_complete(dwc, dep, event, 0);
1589                 break;
1590         case DWC3_DEPEVT_XFERNOTREADY:
1591                 if (usb_endpoint_xfer_isoc(dep->desc)) {
1592                         dwc3_gadget_start_isoc(dwc, dep, event);
1593                 } else {
1594                         int ret;
1595
1596                         dev_vdbg(dwc->dev, "%s: reason %s\n",
1597                                         dep->name, event->status
1598                                         ? "Transfer Active"
1599                                         : "Transfer Not Active");
1600
1601                         ret = __dwc3_gadget_kick_transfer(dep, 0, 1);
1602                         if (!ret || ret == -EBUSY)
1603                                 return;
1604
1605                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1606                                         dep->name);
1607                 }
1608
1609                 break;
1610         case DWC3_DEPEVT_STREAMEVT:
1611                 if (!usb_endpoint_xfer_bulk(dep->desc)) {
1612                         dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
1613                                         dep->name);
1614                         return;
1615                 }
1616
1617                 switch (event->status) {
1618                 case DEPEVT_STREAMEVT_FOUND:
1619                         dev_vdbg(dwc->dev, "Stream %d found and started\n",
1620                                         event->parameters);
1621
1622                         break;
1623                 case DEPEVT_STREAMEVT_NOTFOUND:
1624                         /* FALLTHROUGH */
1625                 default:
1626                         dev_dbg(dwc->dev, "Couldn't find suitable stream\n");
1627                 }
1628                 break;
1629         case DWC3_DEPEVT_RXTXFIFOEVT:
1630                 dev_dbg(dwc->dev, "%s FIFO Overrun\n", dep->name);
1631                 break;
1632         case DWC3_DEPEVT_EPCMDCMPLT:
1633                 dwc3_ep_cmd_compl(dep, event);
1634                 break;
1635         }
1636 }
1637
1638 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
1639 {
1640         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
1641                 spin_unlock(&dwc->lock);
1642                 dwc->gadget_driver->disconnect(&dwc->gadget);
1643                 spin_lock(&dwc->lock);
1644         }
1645 }
1646
1647 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum)
1648 {
1649         struct dwc3_ep *dep;
1650         struct dwc3_gadget_ep_cmd_params params;
1651         u32 cmd;
1652         int ret;
1653
1654         dep = dwc->eps[epnum];
1655
1656         WARN_ON(!dep->res_trans_idx);
1657         if (dep->res_trans_idx) {
1658                 cmd = DWC3_DEPCMD_ENDTRANSFER;
1659                 cmd |= DWC3_DEPCMD_HIPRI_FORCERM | DWC3_DEPCMD_CMDIOC;
1660                 cmd |= DWC3_DEPCMD_PARAM(dep->res_trans_idx);
1661                 memset(&params, 0, sizeof(params));
1662                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
1663                 WARN_ON_ONCE(ret);
1664                 dep->res_trans_idx = 0;
1665         }
1666 }
1667
1668 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
1669 {
1670         u32 epnum;
1671
1672         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1673                 struct dwc3_ep *dep;
1674
1675                 dep = dwc->eps[epnum];
1676                 if (!(dep->flags & DWC3_EP_ENABLED))
1677                         continue;
1678
1679                 dwc3_remove_requests(dwc, dep);
1680         }
1681 }
1682
1683 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
1684 {
1685         u32 epnum;
1686
1687         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1688                 struct dwc3_ep *dep;
1689                 struct dwc3_gadget_ep_cmd_params params;
1690                 int ret;
1691
1692                 dep = dwc->eps[epnum];
1693
1694                 if (!(dep->flags & DWC3_EP_STALL))
1695                         continue;
1696
1697                 dep->flags &= ~DWC3_EP_STALL;
1698
1699                 memset(&params, 0, sizeof(params));
1700                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1701                                 DWC3_DEPCMD_CLEARSTALL, &params);
1702                 WARN_ON_ONCE(ret);
1703         }
1704 }
1705
1706 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
1707 {
1708         dev_vdbg(dwc->dev, "%s\n", __func__);
1709 #if 0
1710         XXX
1711         U1/U2 is powersave optimization. Skip it for now. Anyway we need to
1712         enable it before we can disable it.
1713
1714         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1715         reg &= ~DWC3_DCTL_INITU1ENA;
1716         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1717
1718         reg &= ~DWC3_DCTL_INITU2ENA;
1719         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1720 #endif
1721
1722         dwc3_stop_active_transfers(dwc);
1723         dwc3_disconnect_gadget(dwc);
1724         dwc->start_config_issued = false;
1725
1726         dwc->gadget.speed = USB_SPEED_UNKNOWN;
1727         dwc->setup_packet_pending = false;
1728 }
1729
1730 static void dwc3_gadget_usb3_phy_power(struct dwc3 *dwc, int on)
1731 {
1732         u32                     reg;
1733
1734         reg = dwc3_readl(dwc->regs, DWC3_GUSB3PIPECTL(0));
1735
1736         if (on)
1737                 reg &= ~DWC3_GUSB3PIPECTL_SUSPHY;
1738         else
1739                 reg |= DWC3_GUSB3PIPECTL_SUSPHY;
1740
1741         dwc3_writel(dwc->regs, DWC3_GUSB3PIPECTL(0), reg);
1742 }
1743
1744 static void dwc3_gadget_usb2_phy_power(struct dwc3 *dwc, int on)
1745 {
1746         u32                     reg;
1747
1748         reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
1749
1750         if (on)
1751                 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
1752         else
1753                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
1754
1755         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
1756 }
1757
1758 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
1759 {
1760         u32                     reg;
1761
1762         dev_vdbg(dwc->dev, "%s\n", __func__);
1763
1764         /*
1765          * WORKAROUND: DWC3 revisions <1.88a have an issue which
1766          * would cause a missing Disconnect Event if there's a
1767          * pending Setup Packet in the FIFO.
1768          *
1769          * There's no suggested workaround on the official Bug
1770          * report, which states that "unless the driver/application
1771          * is doing any special handling of a disconnect event,
1772          * there is no functional issue".
1773          *
1774          * Unfortunately, it turns out that we _do_ some special
1775          * handling of a disconnect event, namely complete all
1776          * pending transfers, notify gadget driver of the
1777          * disconnection, and so on.
1778          *
1779          * Our suggested workaround is to follow the Disconnect
1780          * Event steps here, instead, based on a setup_packet_pending
1781          * flag. Such flag gets set whenever we have a XferNotReady
1782          * event on EP0 and gets cleared on XferComplete for the
1783          * same endpoint.
1784          *
1785          * Refers to:
1786          *
1787          * STAR#9000466709: RTL: Device : Disconnect event not
1788          * generated if setup packet pending in FIFO
1789          */
1790         if (dwc->revision < DWC3_REVISION_188A) {
1791                 if (dwc->setup_packet_pending)
1792                         dwc3_gadget_disconnect_interrupt(dwc);
1793         }
1794
1795         /* after reset -> Default State */
1796         dwc->dev_state = DWC3_DEFAULT_STATE;
1797
1798         /* Enable PHYs */
1799         dwc3_gadget_usb2_phy_power(dwc, true);
1800         dwc3_gadget_usb3_phy_power(dwc, true);
1801
1802         if (dwc->gadget.speed != USB_SPEED_UNKNOWN)
1803                 dwc3_disconnect_gadget(dwc);
1804
1805         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1806         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
1807         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1808
1809         dwc3_stop_active_transfers(dwc);
1810         dwc3_clear_stall_all_ep(dwc);
1811         dwc->start_config_issued = false;
1812
1813         /* Reset device address to zero */
1814         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1815         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
1816         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1817 }
1818
1819 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
1820 {
1821         u32 reg;
1822         u32 usb30_clock = DWC3_GCTL_CLK_BUS;
1823
1824         /*
1825          * We change the clock only at SS but I dunno why I would want to do
1826          * this. Maybe it becomes part of the power saving plan.
1827          */
1828
1829         if (speed != DWC3_DSTS_SUPERSPEED)
1830                 return;
1831
1832         /*
1833          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
1834          * each time on Connect Done.
1835          */
1836         if (!usb30_clock)
1837                 return;
1838
1839         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
1840         reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
1841         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
1842 }
1843
1844 static void dwc3_gadget_disable_phy(struct dwc3 *dwc, u8 speed)
1845 {
1846         switch (speed) {
1847         case USB_SPEED_SUPER:
1848                 dwc3_gadget_usb2_phy_power(dwc, false);
1849                 break;
1850         case USB_SPEED_HIGH:
1851         case USB_SPEED_FULL:
1852         case USB_SPEED_LOW:
1853                 dwc3_gadget_usb3_phy_power(dwc, false);
1854                 break;
1855         }
1856 }
1857
1858 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
1859 {
1860         struct dwc3_gadget_ep_cmd_params params;
1861         struct dwc3_ep          *dep;
1862         int                     ret;
1863         u32                     reg;
1864         u8                      speed;
1865
1866         dev_vdbg(dwc->dev, "%s\n", __func__);
1867
1868         memset(&params, 0x00, sizeof(params));
1869
1870         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1871         speed = reg & DWC3_DSTS_CONNECTSPD;
1872         dwc->speed = speed;
1873
1874         dwc3_update_ram_clk_sel(dwc, speed);
1875
1876         switch (speed) {
1877         case DWC3_DCFG_SUPERSPEED:
1878                 /*
1879                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
1880                  * would cause a missing USB3 Reset event.
1881                  *
1882                  * In such situations, we should force a USB3 Reset
1883                  * event by calling our dwc3_gadget_reset_interrupt()
1884                  * routine.
1885                  *
1886                  * Refers to:
1887                  *
1888                  * STAR#9000483510: RTL: SS : USB3 reset event may
1889                  * not be generated always when the link enters poll
1890                  */
1891                 if (dwc->revision < DWC3_REVISION_190A)
1892                         dwc3_gadget_reset_interrupt(dwc);
1893
1894                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1895                 dwc->gadget.ep0->maxpacket = 512;
1896                 dwc->gadget.speed = USB_SPEED_SUPER;
1897                 break;
1898         case DWC3_DCFG_HIGHSPEED:
1899                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1900                 dwc->gadget.ep0->maxpacket = 64;
1901                 dwc->gadget.speed = USB_SPEED_HIGH;
1902                 break;
1903         case DWC3_DCFG_FULLSPEED2:
1904         case DWC3_DCFG_FULLSPEED1:
1905                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1906                 dwc->gadget.ep0->maxpacket = 64;
1907                 dwc->gadget.speed = USB_SPEED_FULL;
1908                 break;
1909         case DWC3_DCFG_LOWSPEED:
1910                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
1911                 dwc->gadget.ep0->maxpacket = 8;
1912                 dwc->gadget.speed = USB_SPEED_LOW;
1913                 break;
1914         }
1915
1916         /* Disable unneded PHY */
1917         dwc3_gadget_disable_phy(dwc, dwc->gadget.speed);
1918
1919         dep = dwc->eps[0];
1920         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL);
1921         if (ret) {
1922                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1923                 return;
1924         }
1925
1926         dep = dwc->eps[1];
1927         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL);
1928         if (ret) {
1929                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1930                 return;
1931         }
1932
1933         /*
1934          * Configure PHY via GUSB3PIPECTLn if required.
1935          *
1936          * Update GTXFIFOSIZn
1937          *
1938          * In both cases reset values should be sufficient.
1939          */
1940 }
1941
1942 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
1943 {
1944         dev_vdbg(dwc->dev, "%s\n", __func__);
1945
1946         /*
1947          * TODO take core out of low power mode when that's
1948          * implemented.
1949          */
1950
1951         dwc->gadget_driver->resume(&dwc->gadget);
1952 }
1953
1954 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
1955                 unsigned int evtinfo)
1956 {
1957         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
1958
1959         /*
1960          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
1961          * on the link partner, the USB session might do multiple entry/exit
1962          * of low power states before a transfer takes place.
1963          *
1964          * Due to this problem, we might experience lower throughput. The
1965          * suggested workaround is to disable DCTL[12:9] bits if we're
1966          * transitioning from U1/U2 to U0 and enable those bits again
1967          * after a transfer completes and there are no pending transfers
1968          * on any of the enabled endpoints.
1969          *
1970          * This is the first half of that workaround.
1971          *
1972          * Refers to:
1973          *
1974          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
1975          * core send LGO_Ux entering U0
1976          */
1977         if (dwc->revision < DWC3_REVISION_183A) {
1978                 if (next == DWC3_LINK_STATE_U0) {
1979                         u32     u1u2;
1980                         u32     reg;
1981
1982                         switch (dwc->link_state) {
1983                         case DWC3_LINK_STATE_U1:
1984                         case DWC3_LINK_STATE_U2:
1985                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1986                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
1987                                                 | DWC3_DCTL_ACCEPTU2ENA
1988                                                 | DWC3_DCTL_INITU1ENA
1989                                                 | DWC3_DCTL_ACCEPTU1ENA);
1990
1991                                 if (!dwc->u1u2)
1992                                         dwc->u1u2 = reg & u1u2;
1993
1994                                 reg &= ~u1u2;
1995
1996                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1997                                 break;
1998                         default:
1999                                 /* do nothing */
2000                                 break;
2001                         }
2002                 }
2003         }
2004
2005         dwc->link_state = next;
2006
2007         dev_vdbg(dwc->dev, "%s link %d\n", __func__, dwc->link_state);
2008 }
2009
2010 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2011                 const struct dwc3_event_devt *event)
2012 {
2013         switch (event->type) {
2014         case DWC3_DEVICE_EVENT_DISCONNECT:
2015                 dwc3_gadget_disconnect_interrupt(dwc);
2016                 break;
2017         case DWC3_DEVICE_EVENT_RESET:
2018                 dwc3_gadget_reset_interrupt(dwc);
2019                 break;
2020         case DWC3_DEVICE_EVENT_CONNECT_DONE:
2021                 dwc3_gadget_conndone_interrupt(dwc);
2022                 break;
2023         case DWC3_DEVICE_EVENT_WAKEUP:
2024                 dwc3_gadget_wakeup_interrupt(dwc);
2025                 break;
2026         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2027                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2028                 break;
2029         case DWC3_DEVICE_EVENT_EOPF:
2030                 dev_vdbg(dwc->dev, "End of Periodic Frame\n");
2031                 break;
2032         case DWC3_DEVICE_EVENT_SOF:
2033                 dev_vdbg(dwc->dev, "Start of Periodic Frame\n");
2034                 break;
2035         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2036                 dev_vdbg(dwc->dev, "Erratic Error\n");
2037                 break;
2038         case DWC3_DEVICE_EVENT_CMD_CMPL:
2039                 dev_vdbg(dwc->dev, "Command Complete\n");
2040                 break;
2041         case DWC3_DEVICE_EVENT_OVERFLOW:
2042                 dev_vdbg(dwc->dev, "Overflow\n");
2043                 break;
2044         default:
2045                 dev_dbg(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2046         }
2047 }
2048
2049 static void dwc3_process_event_entry(struct dwc3 *dwc,
2050                 const union dwc3_event *event)
2051 {
2052         /* Endpoint IRQ, handle it and return early */
2053         if (event->type.is_devspec == 0) {
2054                 /* depevt */
2055                 return dwc3_endpoint_interrupt(dwc, &event->depevt);
2056         }
2057
2058         switch (event->type.type) {
2059         case DWC3_EVENT_TYPE_DEV:
2060                 dwc3_gadget_interrupt(dwc, &event->devt);
2061                 break;
2062         /* REVISIT what to do with Carkit and I2C events ? */
2063         default:
2064                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2065         }
2066 }
2067
2068 static irqreturn_t dwc3_process_event_buf(struct dwc3 *dwc, u32 buf)
2069 {
2070         struct dwc3_event_buffer *evt;
2071         int left;
2072         u32 count;
2073
2074         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(buf));
2075         count &= DWC3_GEVNTCOUNT_MASK;
2076         if (!count)
2077                 return IRQ_NONE;
2078
2079         evt = dwc->ev_buffs[buf];
2080         left = count;
2081
2082         while (left > 0) {
2083                 union dwc3_event event;
2084
2085                 memcpy(&event.raw, (evt->buf + evt->lpos), sizeof(event.raw));
2086                 dwc3_process_event_entry(dwc, &event);
2087                 /*
2088                  * XXX we wrap around correctly to the next entry as almost all
2089                  * entries are 4 bytes in size. There is one entry which has 12
2090                  * bytes which is a regular entry followed by 8 bytes data. ATM
2091                  * I don't know how things are organized if were get next to the
2092                  * a boundary so I worry about that once we try to handle that.
2093                  */
2094                 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2095                 left -= 4;
2096
2097                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(buf), 4);
2098         }
2099
2100         return IRQ_HANDLED;
2101 }
2102
2103 static irqreturn_t dwc3_interrupt(int irq, void *_dwc)
2104 {
2105         struct dwc3                     *dwc = _dwc;
2106         int                             i;
2107         irqreturn_t                     ret = IRQ_NONE;
2108
2109         spin_lock(&dwc->lock);
2110
2111         for (i = 0; i < dwc->num_event_buffers; i++) {
2112                 irqreturn_t status;
2113
2114                 status = dwc3_process_event_buf(dwc, i);
2115                 if (status == IRQ_HANDLED)
2116                         ret = status;
2117         }
2118
2119         spin_unlock(&dwc->lock);
2120
2121         return ret;
2122 }
2123
2124 /**
2125  * dwc3_gadget_init - Initializes gadget related registers
2126  * @dwc: Pointer to out controller context structure
2127  *
2128  * Returns 0 on success otherwise negative errno.
2129  */
2130 int __devinit dwc3_gadget_init(struct dwc3 *dwc)
2131 {
2132         u32                                     reg;
2133         int                                     ret;
2134         int                                     irq;
2135
2136         dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2137                         &dwc->ctrl_req_addr, GFP_KERNEL);
2138         if (!dwc->ctrl_req) {
2139                 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2140                 ret = -ENOMEM;
2141                 goto err0;
2142         }
2143
2144         dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2145                         &dwc->ep0_trb_addr, GFP_KERNEL);
2146         if (!dwc->ep0_trb) {
2147                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2148                 ret = -ENOMEM;
2149                 goto err1;
2150         }
2151
2152         dwc->setup_buf = dma_alloc_coherent(dwc->dev,
2153                         sizeof(*dwc->setup_buf) * 2,
2154                         &dwc->setup_buf_addr, GFP_KERNEL);
2155         if (!dwc->setup_buf) {
2156                 dev_err(dwc->dev, "failed to allocate setup buffer\n");
2157                 ret = -ENOMEM;
2158                 goto err2;
2159         }
2160
2161         dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
2162                         512, &dwc->ep0_bounce_addr, GFP_KERNEL);
2163         if (!dwc->ep0_bounce) {
2164                 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2165                 ret = -ENOMEM;
2166                 goto err3;
2167         }
2168
2169         dev_set_name(&dwc->gadget.dev, "gadget");
2170
2171         dwc->gadget.ops                 = &dwc3_gadget_ops;
2172         dwc->gadget.max_speed           = USB_SPEED_SUPER;
2173         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
2174         dwc->gadget.dev.parent          = dwc->dev;
2175         dwc->gadget.sg_supported        = true;
2176
2177         dma_set_coherent_mask(&dwc->gadget.dev, dwc->dev->coherent_dma_mask);
2178
2179         dwc->gadget.dev.dma_parms       = dwc->dev->dma_parms;
2180         dwc->gadget.dev.dma_mask        = dwc->dev->dma_mask;
2181         dwc->gadget.dev.release         = dwc3_gadget_release;
2182         dwc->gadget.name                = "dwc3-gadget";
2183
2184         /*
2185          * REVISIT: Here we should clear all pending IRQs to be
2186          * sure we're starting from a well known location.
2187          */
2188
2189         ret = dwc3_gadget_init_endpoints(dwc);
2190         if (ret)
2191                 goto err4;
2192
2193         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
2194
2195         ret = request_irq(irq, dwc3_interrupt, IRQF_SHARED,
2196                         "dwc3", dwc);
2197         if (ret) {
2198                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2199                                 irq, ret);
2200                 goto err5;
2201         }
2202
2203         /* Enable all but Start and End of Frame IRQs */
2204         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
2205                         DWC3_DEVTEN_EVNTOVERFLOWEN |
2206                         DWC3_DEVTEN_CMDCMPLTEN |
2207                         DWC3_DEVTEN_ERRTICERREN |
2208                         DWC3_DEVTEN_WKUPEVTEN |
2209                         DWC3_DEVTEN_ULSTCNGEN |
2210                         DWC3_DEVTEN_CONNECTDONEEN |
2211                         DWC3_DEVTEN_USBRSTEN |
2212                         DWC3_DEVTEN_DISCONNEVTEN);
2213         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2214
2215         ret = device_register(&dwc->gadget.dev);
2216         if (ret) {
2217                 dev_err(dwc->dev, "failed to register gadget device\n");
2218                 put_device(&dwc->gadget.dev);
2219                 goto err6;
2220         }
2221
2222         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2223         if (ret) {
2224                 dev_err(dwc->dev, "failed to register udc\n");
2225                 goto err7;
2226         }
2227
2228         return 0;
2229
2230 err7:
2231         device_unregister(&dwc->gadget.dev);
2232
2233 err6:
2234         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2235         free_irq(irq, dwc);
2236
2237 err5:
2238         dwc3_gadget_free_endpoints(dwc);
2239
2240 err4:
2241         dma_free_coherent(dwc->dev, 512, dwc->ep0_bounce,
2242                         dwc->ep0_bounce_addr);
2243
2244 err3:
2245         dma_free_coherent(dwc->dev, sizeof(*dwc->setup_buf) * 2,
2246                         dwc->setup_buf, dwc->setup_buf_addr);
2247
2248 err2:
2249         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2250                         dwc->ep0_trb, dwc->ep0_trb_addr);
2251
2252 err1:
2253         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2254                         dwc->ctrl_req, dwc->ctrl_req_addr);
2255
2256 err0:
2257         return ret;
2258 }
2259
2260 void dwc3_gadget_exit(struct dwc3 *dwc)
2261 {
2262         int                     irq;
2263
2264         usb_del_gadget_udc(&dwc->gadget);
2265         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
2266
2267         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2268         free_irq(irq, dwc);
2269
2270         dwc3_gadget_free_endpoints(dwc);
2271
2272         dma_free_coherent(dwc->dev, 512, dwc->ep0_bounce,
2273                         dwc->ep0_bounce_addr);
2274
2275         dma_free_coherent(dwc->dev, sizeof(*dwc->setup_buf) * 2,
2276                         dwc->setup_buf, dwc->setup_buf_addr);
2277
2278         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2279                         dwc->ep0_trb, dwc->ep0_trb_addr);
2280
2281         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2282                         dwc->ctrl_req, dwc->ctrl_req_addr);
2283
2284         device_unregister(&dwc->gadget.dev);
2285 }