drivers/usb/dwc3/gadget.c

   1 /**
   2  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   3  *
   4  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   5  *
   6  * Authors: Felipe Balbi <balbi@ti.com>,
   7  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. The names of the above-listed copyright holders may not be used
  19  *    to endorse or promote products derived from this software without
  20  *    specific prior written permission.
  21  *
  22  * ALTERNATIVELY, this software may be distributed under the terms of the
  23  * GNU General Public License ("GPL") version 2, as published by the Free
  24  * Software Foundation.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  27  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  28  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  29  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  30  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  31  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  32  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  33  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  34  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  35  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  36  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 #include <linux/kernel.h>
  40 #include <linux/delay.h>
  41 #include <linux/slab.h>
  42 #include <linux/spinlock.h>
  43 #include <linux/platform_device.h>
  44 #include <linux/pm_runtime.h>
  45 #include <linux/interrupt.h>
  46 #include <linux/io.h>
  47 #include <linux/list.h>
  48 #include <linux/dma-mapping.h>
  49
  50 #include <linux/usb/ch9.h>
  51 #include <linux/usb/gadget.h>
  52
  53 #include "core.h"
  54 #include "gadget.h"
  55 #include "io.h"
  56
  57 #define DMA_ADDR_INVALID        (~(dma_addr_t)0)
  58
  59 void dwc3_map_buffer_to_dma(struct dwc3_request *req)
  60 {
  61         struct dwc3                     *dwc = req->dep->dwc;
  62
  63         if (req->request.length == 0) {
  64                 /* req->request.dma = dwc->setup_buf_addr; */
  65                 return;
  66         }
  67
  68         if (req->request.dma == DMA_ADDR_INVALID) {
  69                 req->request.dma = dma_map_single(dwc->dev, req->request.buf,
  70                                 req->request.length, req->direction
  71                                 ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
  72                 req->mapped = true;
  73         }
  74 }
  75
  76 void dwc3_unmap_buffer_from_dma(struct dwc3_request *req)
  77 {
  78         struct dwc3                     *dwc = req->dep->dwc;
  79
  80         if (req->request.length == 0) {
  81                 req->request.dma = DMA_ADDR_INVALID;
  82                 return;
  83         }
  84
  85         if (req->mapped) {
  86                 dma_unmap_single(dwc->dev, req->request.dma,
  87                                 req->request.length, req->direction
  88                                 ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
  89                 req->mapped = 0;
  90                 req->request.dma = DMA_ADDR_INVALID;
  91         }
  92 }
  93
  94 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
  95                 int status)
  96 {
  97         struct dwc3                     *dwc = dep->dwc;
  98
  99         if (req->queued) {
 100                 dep->busy_slot++;
 101                 /*
 102                  * Skip LINK TRB. We can't use req->trb and check for
 103                  * DWC3_TRBCTL_LINK_TRB because it points the TRB we just
 104                  * completed (not the LINK TRB).
 105                  */
 106                 if (((dep->busy_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
 107                                 usb_endpoint_xfer_isoc(dep->desc))
 108                         dep->busy_slot++;
 109         }
 110         list_del(&req->list);
 111
 112         if (req->request.status == -EINPROGRESS)
 113                 req->request.status = status;
 114
 115         dwc3_unmap_buffer_from_dma(req);
 116
 117         dev_dbg(dwc->dev, "request %p from %s completed %d/%d ===> %d\n",
 118                         req, dep->name, req->request.actual,
 119                         req->request.length, status);
 120
 121         spin_unlock(&dwc->lock);
 122         req->request.complete(&req->dep->endpoint, &req->request);
 123         spin_lock(&dwc->lock);
 124 }
 125
 126 static const char *dwc3_gadget_ep_cmd_string(u8 cmd)
 127 {
 128         switch (cmd) {
 129         case DWC3_DEPCMD_DEPSTARTCFG:
 130                 return "Start New Configuration";
 131         case DWC3_DEPCMD_ENDTRANSFER:
 132                 return "End Transfer";
 133         case DWC3_DEPCMD_UPDATETRANSFER:
 134                 return "Update Transfer";
 135         case DWC3_DEPCMD_STARTTRANSFER:
 136                 return "Start Transfer";
 137         case DWC3_DEPCMD_CLEARSTALL:
 138                 return "Clear Stall";
 139         case DWC3_DEPCMD_SETSTALL:
 140                 return "Set Stall";
 141         case DWC3_DEPCMD_GETSEQNUMBER:
 142                 return "Get Data Sequence Number";
 143         case DWC3_DEPCMD_SETTRANSFRESOURCE:
 144                 return "Set Endpoint Transfer Resource";
 145         case DWC3_DEPCMD_SETEPCONFIG:
 146                 return "Set Endpoint Configuration";
 147         default:
 148                 return "UNKNOWN command";
 149         }
 150 }
 151
 152 int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
 153                 unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
 154 {
 155         struct dwc3_ep          *dep = dwc->eps[ep];
 156         u32                     timeout = 500;
 157         u32                     reg;
 158
 159         dev_vdbg(dwc->dev, "%s: cmd '%s' params %08x %08x %08x\n",
 160                         dep->name,
 161                         dwc3_gadget_ep_cmd_string(cmd), params->param0,
 162                         params->param1, params->param2);
 163
 164         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0);
 165         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1);
 166         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2);
 167
 168         dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
 169         do {
 170                 reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
 171                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 172                         dev_vdbg(dwc->dev, "Command Complete --> %d\n",
 173                                         DWC3_DEPCMD_STATUS(reg));
 174                         if (DWC3_DEPCMD_STATUS(reg))
 175                                 return -EINVAL;
 176                         return 0;
 177                 }
 178
 179                 /*
 180                  * We can't sleep here, because it is also called from
 181                  * interrupt context.
 182                  */
 183                 timeout--;
 184                 if (!timeout)
 185                         return -ETIMEDOUT;
 186
 187                 udelay(1);
 188         } while (1);
 189 }
 190
 191 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 192                 struct dwc3_trb_hw *trb)
 193 {
 194         u32             offset = (char *) trb - (char *) dep->trb_pool;
 195
 196         return dep->trb_pool_dma + offset;
 197 }
 198
 199 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 200 {
 201         struct dwc3             *dwc = dep->dwc;
 202
 203         if (dep->trb_pool)
 204                 return 0;
 205
 206         if (dep->number == 0 || dep->number == 1)
 207                 return 0;
 208
 209         dep->trb_pool = dma_alloc_coherent(dwc->dev,
 210                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 211                         &dep->trb_pool_dma, GFP_KERNEL);
 212         if (!dep->trb_pool) {
 213                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 214                                 dep->name);
 215                 return -ENOMEM;
 216         }
 217
 218         return 0;
 219 }
 220
 221 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 222 {
 223         struct dwc3             *dwc = dep->dwc;
 224
 225         dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 226                         dep->trb_pool, dep->trb_pool_dma);
 227
 228         dep->trb_pool = NULL;
 229         dep->trb_pool_dma = 0;
 230 }
 231
 232 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
 233
 234 /**
 235  * dwc3_gadget_start_config - Configure EP resources
 236  * @dwc: pointer to our controller context structure
 237  * @dep: endpoint that is being enabled
 238  *
 239  * The assignment of transfer resources cannot perfectly follow the
 240  * data book due to the fact that the controller driver does not have
 241  * all knowledge of the configuration in advance. It is given this
 242  * information piecemeal by the composite gadget framework after every
 243  * SET_CONFIGURATION and SET_INTERFACE. Trying to follow the databook
 244  * programming model in this scenario can cause errors. For two
 245  * reasons:
 246  *
 247  * 1) The databook says to do DEPSTARTCFG for every SET_CONFIGURATION
 248  * and SET_INTERFACE (8.1.5). This is incorrect in the scenario of
 249  * multiple interfaces.
 250  *
 251  * 2) The databook does not mention doing more DEPXFERCFG for new
 252  * endpoint on alt setting (8.1.6).
 253  *
 254  * The following simplified method is used instead:
 255  *
 256  * All hardware endpoints can be assigned a transfer resource and this
 257  * setting will stay persistent until either a core reset or
 258  * hibernation. So whenever we do a DEPSTARTCFG(0) we can go ahead and
 259  * do DEPXFERCFG for every hardware endpoint as well. We are
 260  * guaranteed that there are as many transfer resources as endpoints.
 261  *
 262  * This function is called for each endpoint when it is being enabled
 263  * but is triggered only when called for EP0-out, which always happens
 264  * first, and which should only happen in one of the above conditions.
 265  */
 266 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 267 {
 268         struct dwc3_gadget_ep_cmd_params params;
 269         u32                     cmd;
 270         int                     i;
 271         int                     ret;
 272
 273         if (dep->number)
 274                 return 0;
 275
 276         memset(&params, 0x00, sizeof(params));
 277         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 278
 279         ret = dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
 280         if (ret)
 281                 return ret;
 282
 283         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 284                 struct dwc3_ep *dep = dwc->eps[i];
 285
 286                 if (!dep)
 287                         continue;
 288
 289                 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 290                 if (ret)
 291                         return ret;
 292         }
 293
 294         return 0;
 295 }
 296
 297 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 298                 const struct usb_endpoint_descriptor *desc)
 299 {
 300         struct dwc3_gadget_ep_cmd_params params;
 301
 302         memset(&params, 0x00, sizeof(params));
 303
 304         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 305                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc))
 306                 | DWC3_DEPCFG_BURST_SIZE(dep->endpoint.maxburst);
 307
 308         params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
 309                 | DWC3_DEPCFG_XFER_NOT_READY_EN;
 310
 311         if (usb_endpoint_xfer_bulk(desc) && dep->endpoint.max_streams) {
 312                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 313                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 314                 dep->stream_capable = true;
 315         }
 316
 317         if (usb_endpoint_xfer_isoc(desc))
 318                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 319
 320         /*
 321          * We are doing 1:1 mapping for endpoints, meaning
 322          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 323          * so on. We consider the direction bit as part of the physical
 324          * endpoint number. So USB endpoint 0x81 is 0x03.
 325          */
 326         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 327
 328         /*
 329          * We must use the lower 16 TX FIFOs even though
 330          * HW might have more
 331          */
 332         if (dep->direction)
 333                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 334
 335         if (desc->bInterval) {
 336                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 337                 dep->interval = 1 << (desc->bInterval - 1);
 338         }
 339
 340         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 341                         DWC3_DEPCMD_SETEPCONFIG, &params);
 342 }
 343
 344 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
 345 {
 346         struct dwc3_gadget_ep_cmd_params params;
 347
 348         memset(&params, 0x00, sizeof(params));
 349
 350         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 351
 352         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 353                         DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
 354 }
 355
 356 /**
 357  * __dwc3_gadget_ep_enable - Initializes a HW endpoint
 358  * @dep: endpoint to be initialized
 359  * @desc: USB Endpoint Descriptor
 360  *
 361  * Caller should take care of locking
 362  */
 363 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 364                 const struct usb_endpoint_descriptor *desc)
 365 {
 366         struct dwc3             *dwc = dep->dwc;
 367         u32                     reg;
 368         int                     ret = -ENOMEM;
 369
 370         if (!(dep->flags & DWC3_EP_ENABLED)) {
 371                 ret = dwc3_gadget_start_config(dwc, dep);
 372                 if (ret)
 373                         return ret;
 374         }
 375
 376         ret = dwc3_gadget_set_ep_config(dwc, dep, desc);
 377         if (ret)
 378                 return ret;
 379
 380         if (!(dep->flags & DWC3_EP_ENABLED)) {
 381                 struct dwc3_trb_hw      *trb_st_hw;
 382                 struct dwc3_trb_hw      *trb_link_hw;
 383                 struct dwc3_trb         trb_link;
 384
 385                 dep->desc = desc;
 386                 dep->type = usb_endpoint_type(desc);
 387                 dep->flags |= DWC3_EP_ENABLED;
 388
 389                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 390                 reg |= DWC3_DALEPENA_EP(dep->number);
 391                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 392
 393                 if (!usb_endpoint_xfer_isoc(desc))
 394                         return 0;
 395
 396                 memset(&trb_link, 0, sizeof(trb_link));
 397
 398                 /* Link TRB for ISOC. The HWO but is never reset */
 399                 trb_st_hw = &dep->trb_pool[0];
 400
 401                 trb_link.bplh = dwc3_trb_dma_offset(dep, trb_st_hw);
 402                 trb_link.trbctl = DWC3_TRBCTL_LINK_TRB;
 403                 trb_link.hwo = true;
 404
 405                 trb_link_hw = &dep->trb_pool[DWC3_TRB_NUM - 1];
 406                 dwc3_trb_to_hw(&trb_link, trb_link_hw);
 407         }
 408
 409         return 0;
 410 }
 411
 412 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum);
 413 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 414 {
 415         struct dwc3_request             *req;
 416
 417         if (!list_empty(&dep->req_queued))
 418                 dwc3_stop_active_transfer(dwc, dep->number);
 419
 420         while (!list_empty(&dep->request_list)) {
 421                 req = next_request(&dep->request_list);
 422
 423                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 424         }
 425 }
 426
 427 /**
 428  * __dwc3_gadget_ep_disable - Disables a HW endpoint
 429  * @dep: the endpoint to disable
 430  *
 431  * This function also removes requests which are currently processed ny the
 432  * hardware and those which are not yet scheduled.
 433  * Caller should take care of locking.
 434  */
 435 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 436 {
 437         struct dwc3             *dwc = dep->dwc;
 438         u32                     reg;
 439
 440         dwc3_remove_requests(dwc, dep);
 441
 442         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 443         reg &= ~DWC3_DALEPENA_EP(dep->number);
 444         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 445
 446         dep->stream_capable = false;
 447         dep->desc = NULL;
 448         dep->type = 0;
 449         dep->flags = 0;
 450
 451         return 0;
 452 }
 453
 454 /* -------------------------------------------------------------------------- */
 455
 456 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 457                 const struct usb_endpoint_descriptor *desc)
 458 {
 459         return -EINVAL;
 460 }
 461
 462 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 463 {
 464         return -EINVAL;
 465 }
 466
 467 /* -------------------------------------------------------------------------- */
 468
 469 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 470                 const struct usb_endpoint_descriptor *desc)
 471 {
 472         struct dwc3_ep                  *dep;
 473         struct dwc3                     *dwc;
 474         unsigned long                   flags;
 475         int                             ret;
 476
 477         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 478                 pr_debug("dwc3: invalid parameters\n");
 479                 return -EINVAL;
 480         }
 481
 482         if (!desc->wMaxPacketSize) {
 483                 pr_debug("dwc3: missing wMaxPacketSize\n");
 484                 return -EINVAL;
 485         }
 486
 487         dep = to_dwc3_ep(ep);
 488         dwc = dep->dwc;
 489
 490         switch (usb_endpoint_type(desc)) {
 491         case USB_ENDPOINT_XFER_CONTROL:
 492                 strlcat(dep->name, "-control", sizeof(dep->name));
 493                 break;
 494         case USB_ENDPOINT_XFER_ISOC:
 495                 strlcat(dep->name, "-isoc", sizeof(dep->name));
 496                 break;
 497         case USB_ENDPOINT_XFER_BULK:
 498                 strlcat(dep->name, "-bulk", sizeof(dep->name));
 499                 break;
 500         case USB_ENDPOINT_XFER_INT:
 501                 strlcat(dep->name, "-int", sizeof(dep->name));
 502                 break;
 503         default:
 504                 dev_err(dwc->dev, "invalid endpoint transfer type\n");
 505         }
 506
 507         if (dep->flags & DWC3_EP_ENABLED) {
 508                 dev_WARN_ONCE(dwc->dev, true, "%s is already enabled\n",
 509                                 dep->name);
 510                 return 0;
 511         }
 512
 513         dev_vdbg(dwc->dev, "Enabling %s\n", dep->name);
 514
 515         spin_lock_irqsave(&dwc->lock, flags);
 516         ret = __dwc3_gadget_ep_enable(dep, desc);
 517         spin_unlock_irqrestore(&dwc->lock, flags);
 518
 519         return ret;
 520 }
 521
 522 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 523 {
 524         struct dwc3_ep                  *dep;
 525         struct dwc3                     *dwc;
 526         unsigned long                   flags;
 527         int                             ret;
 528
 529         if (!ep) {
 530                 pr_debug("dwc3: invalid parameters\n");
 531                 return -EINVAL;
 532         }
 533
 534         dep = to_dwc3_ep(ep);
 535         dwc = dep->dwc;
 536
 537         if (!(dep->flags & DWC3_EP_ENABLED)) {
 538                 dev_WARN_ONCE(dwc->dev, true, "%s is already disabled\n",
 539                                 dep->name);
 540                 return 0;
 541         }
 542
 543         snprintf(dep->name, sizeof(dep->name), "ep%d%s",
 544                         dep->number >> 1,
 545                         (dep->number & 1) ? "in" : "out");
 546
 547         spin_lock_irqsave(&dwc->lock, flags);
 548         ret = __dwc3_gadget_ep_disable(dep);
 549         spin_unlock_irqrestore(&dwc->lock, flags);
 550
 551         return ret;
 552 }
 553
 554 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 555         gfp_t gfp_flags)
 556 {
 557         struct dwc3_request             *req;
 558         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 559         struct dwc3                     *dwc = dep->dwc;
 560
 561         req = kzalloc(sizeof(*req), gfp_flags);
 562         if (!req) {
 563                 dev_err(dwc->dev, "not enough memory\n");
 564                 return NULL;
 565         }
 566
 567         req->epnum      = dep->number;
 568         req->dep        = dep;
 569         req->request.dma = DMA_ADDR_INVALID;
 570
 571         return &req->request;
 572 }
 573
 574 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 575                 struct usb_request *request)
 576 {
 577         struct dwc3_request             *req = to_dwc3_request(request);
 578
 579         kfree(req);
 580 }
 581
 582 /*
 583  * dwc3_prepare_trbs - setup TRBs from requests
 584  * @dep: endpoint for which requests are being prepared
 585  * @starting: true if the endpoint is idle and no requests are queued.
 586  *
 587  * The functions goes through the requests list and setups TRBs for the
 588  * transfers. The functions returns once there are not more TRBs available or
 589  * it run out of requests.
 590  */
 591 static struct dwc3_request *dwc3_prepare_trbs(struct dwc3_ep *dep,
 592                 bool starting)
 593 {
 594         struct dwc3_request     *req, *n, *ret = NULL;
 595         struct dwc3_trb_hw      *trb_hw;
 596         struct dwc3_trb         trb;
 597         u32                     trbs_left;
 598
 599         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
 600
 601         /* the first request must not be queued */
 602         trbs_left = (dep->busy_slot - dep->free_slot) & DWC3_TRB_MASK;
 603         /*
 604          * if busy & slot are equal than it is either full or empty. If we are
 605          * starting to proceed requests then we are empty. Otherwise we ar
 606          * full and don't do anything
 607          */
 608         if (!trbs_left) {
 609                 if (!starting)
 610                         return NULL;
 611                 trbs_left = DWC3_TRB_NUM;
 612                 /*
 613                  * In case we start from scratch, we queue the ISOC requests
 614                  * starting from slot 1. This is done because we use ring
 615                  * buffer and have no LST bit to stop us. Instead, we place
 616                  * IOC bit TRB_NUM/4. We try to avoid to having an interrupt
 617                  * after the first request so we start at slot 1 and have
 618                  * 7 requests proceed before we hit the first IOC.
 619                  * Other transfer types don't use the ring buffer and are
 620                  * processed from the first TRB until the last one. Since we
 621                  * don't wrap around we have to start at the beginning.
 622                  */
 623                 if (usb_endpoint_xfer_isoc(dep->desc)) {
 624                         dep->busy_slot = 1;
 625                         dep->free_slot = 1;
 626                 } else {
 627                         dep->busy_slot = 0;
 628                         dep->free_slot = 0;
 629                 }
 630         }
 631
 632         /* The last TRB is a link TRB, not used for xfer */
 633         if ((trbs_left <= 1) && usb_endpoint_xfer_isoc(dep->desc))
 634                 return NULL;
 635
 636         list_for_each_entry_safe(req, n, &dep->request_list, list) {
 637                 unsigned int last_one = 0;
 638                 unsigned int cur_slot;
 639
 640                 trb_hw = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
 641                 cur_slot = dep->free_slot;
 642                 dep->free_slot++;
 643
 644                 /* Skip the LINK-TRB on ISOC */
 645                 if (((cur_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
 646                                 usb_endpoint_xfer_isoc(dep->desc))
 647                         continue;
 648
 649                 dwc3_gadget_move_request_queued(req);
 650                 memset(&trb, 0, sizeof(trb));
 651                 trbs_left--;
 652
 653                 /* Is our TRB pool empty? */
 654                 if (!trbs_left)
 655                         last_one = 1;
 656                 /* Is this the last request? */
 657                 if (list_empty(&dep->request_list))
 658                         last_one = 1;
 659
 660                 /*
 661                  * FIXME we shouldn't need to set LST bit always but we are
 662                  * facing some weird problem with the Hardware where it doesn't
 663                  * complete even though it has been previously started.
 664                  *
 665                  * While we're debugging the problem, as a workaround to
 666                  * multiple TRBs handling, use only one TRB at a time.
 667                  */
 668                 last_one = 1;
 669
 670                 req->trb = trb_hw;
 671                 if (!ret)
 672                         ret = req;
 673
 674                 trb.bplh = req->request.dma;
 675
 676                 if (usb_endpoint_xfer_isoc(dep->desc)) {
 677                         trb.isp_imi = true;
 678                         trb.csp = true;
 679                 } else {
 680                         trb.lst = last_one;
 681                 }
 682
 683                 if (usb_endpoint_xfer_bulk(dep->desc) && dep->stream_capable)
 684                         trb.sid_sofn = req->request.stream_id;
 685
 686                 switch (usb_endpoint_type(dep->desc)) {
 687                 case USB_ENDPOINT_XFER_CONTROL:
 688                         trb.trbctl = DWC3_TRBCTL_CONTROL_SETUP;
 689                         break;
 690
 691                 case USB_ENDPOINT_XFER_ISOC:
 692                         trb.trbctl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 693
 694                         /* IOC every DWC3_TRB_NUM / 4 so we can refill */
 695                         if (!(cur_slot % (DWC3_TRB_NUM / 4)))
 696                                 trb.ioc = last_one;
 697                         break;
 698
 699                 case USB_ENDPOINT_XFER_BULK:
 700                 case USB_ENDPOINT_XFER_INT:
 701                         trb.trbctl = DWC3_TRBCTL_NORMAL;
 702                         break;
 703                 default:
 704                         /*
 705                          * This is only possible with faulty memory because we
 706                          * checked it already :)
 707                          */
 708                         BUG();
 709                 }
 710
 711                 trb.length      = req->request.length;
 712                 trb.hwo = true;
 713
 714                 dwc3_trb_to_hw(&trb, trb_hw);
 715                 req->trb_dma = dwc3_trb_dma_offset(dep, trb_hw);
 716
 717                 if (last_one)
 718                         break;
 719         }
 720
 721         return ret;
 722 }
 723
 724 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,
 725                 int start_new)
 726 {
 727         struct dwc3_gadget_ep_cmd_params params;
 728         struct dwc3_request             *req;
 729         struct dwc3                     *dwc = dep->dwc;
 730         int                             ret;
 731         u32                             cmd;
 732
 733         if (start_new && (dep->flags & DWC3_EP_BUSY)) {
 734                 dev_vdbg(dwc->dev, "%s: endpoint busy\n", dep->name);
 735                 return -EBUSY;
 736         }
 737         dep->flags &= ~DWC3_EP_PENDING_REQUEST;
 738
 739         /*
 740          * If we are getting here after a short-out-packet we don't enqueue any
 741          * new requests as we try to set the IOC bit only on the last request.
 742          */
 743         if (start_new) {
 744                 if (list_empty(&dep->req_queued))
 745                         dwc3_prepare_trbs(dep, start_new);
 746
 747                 /* req points to the first request which will be sent */
 748                 req = next_request(&dep->req_queued);
 749         } else {
 750                 /*
 751                  * req points to the first request where HWO changed
 752                  * from 0 to 1
 753                  */
 754                 req = dwc3_prepare_trbs(dep, start_new);
 755         }
 756         if (!req) {
 757                 dep->flags |= DWC3_EP_PENDING_REQUEST;
 758                 return 0;
 759         }
 760
 761         memset(&params, 0, sizeof(params));
 762         params.param0 = upper_32_bits(req->trb_dma);
 763         params.param1 = lower_32_bits(req->trb_dma);
 764
 765         if (start_new)
 766                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 767         else
 768                 cmd = DWC3_DEPCMD_UPDATETRANSFER;
 769
 770         cmd |= DWC3_DEPCMD_PARAM(cmd_param);
 771         ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
 772         if (ret < 0) {
 773                 dev_dbg(dwc->dev, "failed to send STARTTRANSFER command\n");
 774
 775                 /*
 776                  * FIXME we need to iterate over the list of requests
 777                  * here and stop, unmap, free and del each of the linked
 778                  * requests instead of we do now.
 779                  */
 780                 dwc3_unmap_buffer_from_dma(req);
 781                 list_del(&req->list);
 782                 return ret;
 783         }
 784
 785         dep->flags |= DWC3_EP_BUSY;
 786         dep->res_trans_idx = dwc3_gadget_ep_get_transfer_index(dwc,
 787                         dep->number);
 788         if (!dep->res_trans_idx)
 789                 printk_once(KERN_ERR "%s() res_trans_idx is invalid\n", __func__);
 790         return 0;
 791 }
 792
 793 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
 794 {
 795         req->request.actual     = 0;
 796         req->request.status     = -EINPROGRESS;
 797         req->direction          = dep->direction;
 798         req->epnum              = dep->number;
 799
 800         /*
 801          * We only add to our list of requests now and
 802          * start consuming the list once we get XferNotReady
 803          * IRQ.
 804          *
 805          * That way, we avoid doing anything that we don't need
 806          * to do now and defer it until the point we receive a
 807          * particular token from the Host side.
 808          *
 809          * This will also avoid Host cancelling URBs due to too
 810          * many NACKs.
 811          */
 812         dwc3_map_buffer_to_dma(req);
 813         list_add_tail(&req->list, &dep->request_list);
 814
 815         /*
 816          * There is one special case: XferNotReady with
 817          * empty list of requests. We need to kick the
 818          * transfer here in that situation, otherwise
 819          * we will be NAKing forever.
 820          *
 821          * If we get XferNotReady before gadget driver
 822          * has a chance to queue a request, we will ACK
 823          * the IRQ but won't be able to receive the data
 824          * until the next request is queued. The following
 825          * code is handling exactly that.
 826          */
 827         if (dep->flags & DWC3_EP_PENDING_REQUEST) {
 828                 int ret;
 829                 int start_trans;
 830
 831                 start_trans = 1;
 832                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
 833                                 dep->flags & DWC3_EP_BUSY)
 834                         start_trans = 0;
 835
 836                 ret =  __dwc3_gadget_kick_transfer(dep, 0, start_trans);
 837                 if (ret && ret != -EBUSY) {
 838                         struct dwc3     *dwc = dep->dwc;
 839
 840                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
 841                                         dep->name);
 842                 }
 843         };
 844
 845         return 0;
 846 }
 847
 848 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
 849         gfp_t gfp_flags)
 850 {
 851         struct dwc3_request             *req = to_dwc3_request(request);
 852         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 853         struct dwc3                     *dwc = dep->dwc;
 854
 855         unsigned long                   flags;
 856
 857         int                             ret;
 858
 859         if (!dep->desc) {
 860                 dev_dbg(dwc->dev, "trying to queue request %p to disabled %s\n",
 861                                 request, ep->name);
 862                 return -ESHUTDOWN;
 863         }
 864
 865         dev_vdbg(dwc->dev, "queing request %p to %s length %d\n",
 866                         request, ep->name, request->length);
 867
 868         spin_lock_irqsave(&dwc->lock, flags);
 869         ret = __dwc3_gadget_ep_queue(dep, req);
 870         spin_unlock_irqrestore(&dwc->lock, flags);
 871
 872         return ret;
 873 }
 874
 875 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
 876                 struct usb_request *request)
 877 {
 878         struct dwc3_request             *req = to_dwc3_request(request);
 879         struct dwc3_request             *r = NULL;
 880
 881         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 882         struct dwc3                     *dwc = dep->dwc;
 883
 884         unsigned long                   flags;
 885         int                             ret = 0;
 886
 887         spin_lock_irqsave(&dwc->lock, flags);
 888
 889         list_for_each_entry(r, &dep->request_list, list) {
 890                 if (r == req)
 891                         break;
 892         }
 893
 894         if (r != req) {
 895                 list_for_each_entry(r, &dep->req_queued, list) {
 896                         if (r == req)
 897                                 break;
 898                 }
 899                 if (r == req) {
 900                         /* wait until it is processed */
 901                         dwc3_stop_active_transfer(dwc, dep->number);
 902                         goto out0;
 903                 }
 904                 dev_err(dwc->dev, "request %p was not queued to %s\n",
 905                                 request, ep->name);
 906                 ret = -EINVAL;
 907                 goto out0;
 908         }
 909
 910         /* giveback the request */
 911         dwc3_gadget_giveback(dep, req, -ECONNRESET);
 912
 913 out0:
 914         spin_unlock_irqrestore(&dwc->lock, flags);
 915
 916         return ret;
 917 }
 918
 919 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value)
 920 {
 921         struct dwc3_gadget_ep_cmd_params        params;
 922         struct dwc3                             *dwc = dep->dwc;
 923         int                                     ret;
 924
 925         memset(&params, 0x00, sizeof(params));
 926
 927         if (value) {
 928                 if (dep->flags & DWC3_EP_STALL)
 929                         return 0;
 930
 931                 if (dep->number == 0 || dep->number == 1) {
 932                         /*
 933                          * Whenever EP0 is stalled, we will restart
 934                          * the state machine, thus moving back to
 935                          * Setup Phase
 936                          */
 937                         dwc->ep0state = EP0_SETUP_PHASE;
 938                 }
 939
 940                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
 941                         DWC3_DEPCMD_SETSTALL, &params);
 942                 if (ret)
 943                         dev_err(dwc->dev, "failed to %s STALL on %s\n",
 944                                         value ? "set" : "clear",
 945                                         dep->name);
 946                 else
 947                         dep->flags |= DWC3_EP_STALL;
 948         } else {
 949                 if (!(dep->flags & DWC3_EP_STALL))
 950                         return 0;
 951
 952                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
 953                         DWC3_DEPCMD_CLEARSTALL, &params);
 954                 if (ret)
 955                         dev_err(dwc->dev, "failed to %s STALL on %s\n",
 956                                         value ? "set" : "clear",
 957                                         dep->name);
 958                 else
 959                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
 960         }
 961
 962         return ret;
 963 }
 964
 965 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
 966 {
 967         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 968         struct dwc3                     *dwc = dep->dwc;
 969
 970         unsigned long                   flags;
 971
 972         int                             ret;
 973
 974         spin_lock_irqsave(&dwc->lock, flags);
 975
 976         if (usb_endpoint_xfer_isoc(dep->desc)) {
 977                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
 978                 ret = -EINVAL;
 979                 goto out;
 980         }
 981
 982         ret = __dwc3_gadget_ep_set_halt(dep, value);
 983 out:
 984         spin_unlock_irqrestore(&dwc->lock, flags);
 985
 986         return ret;
 987 }
 988
 989 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
 990 {
 991         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 992
 993         dep->flags |= DWC3_EP_WEDGE;
 994
 995         return dwc3_gadget_ep_set_halt(ep, 1);
 996 }
 997
 998 /* -------------------------------------------------------------------------- */
 999
1000 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1001         .bLength        = USB_DT_ENDPOINT_SIZE,
1002         .bDescriptorType = USB_DT_ENDPOINT,
1003         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1004 };
1005
1006 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1007         .enable         = dwc3_gadget_ep0_enable,
1008         .disable        = dwc3_gadget_ep0_disable,
1009         .alloc_request  = dwc3_gadget_ep_alloc_request,
1010         .free_request   = dwc3_gadget_ep_free_request,
1011         .queue          = dwc3_gadget_ep0_queue,
1012         .dequeue        = dwc3_gadget_ep_dequeue,
1013         .set_halt       = dwc3_gadget_ep_set_halt,
1014         .set_wedge      = dwc3_gadget_ep_set_wedge,
1015 };
1016
1017 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1018         .enable         = dwc3_gadget_ep_enable,
1019         .disable        = dwc3_gadget_ep_disable,
1020         .alloc_request  = dwc3_gadget_ep_alloc_request,
1021         .free_request   = dwc3_gadget_ep_free_request,
1022         .queue          = dwc3_gadget_ep_queue,
1023         .dequeue        = dwc3_gadget_ep_dequeue,
1024         .set_halt       = dwc3_gadget_ep_set_halt,
1025         .set_wedge      = dwc3_gadget_ep_set_wedge,
1026 };
1027
1028 /* -------------------------------------------------------------------------- */
1029
1030 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1031 {
1032         struct dwc3             *dwc = gadget_to_dwc(g);
1033         u32                     reg;
1034
1035         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1036         return DWC3_DSTS_SOFFN(reg);
1037 }
1038
1039 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1040 {
1041         struct dwc3             *dwc = gadget_to_dwc(g);
1042
1043         unsigned long           timeout;
1044         unsigned long           flags;
1045
1046         u32                     reg;
1047
1048         int                     ret = 0;
1049
1050         u8                      link_state;
1051         u8                      speed;
1052
1053         spin_lock_irqsave(&dwc->lock, flags);
1054
1055         /*
1056          * According to the Databook Remote wakeup request should
1057          * be issued only when the device is in early suspend state.
1058          *
1059          * We can check that via USB Link State bits in DSTS register.
1060          */
1061         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1062
1063         speed = reg & DWC3_DSTS_CONNECTSPD;
1064         if (speed == DWC3_DSTS_SUPERSPEED) {
1065                 dev_dbg(dwc->dev, "no wakeup on SuperSpeed\n");
1066                 ret = -EINVAL;
1067                 goto out;
1068         }
1069
1070         link_state = DWC3_DSTS_USBLNKST(reg);
1071
1072         switch (link_state) {
1073         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1074         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1075                 break;
1076         default:
1077                 dev_dbg(dwc->dev, "can't wakeup from link state %d\n",
1078                                 link_state);
1079                 ret = -EINVAL;
1080                 goto out;
1081         }
1082
1083         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1084
1085         /*
1086          * Switch link state to Recovery. In HS/FS/LS this means
1087          * RemoteWakeup Request
1088          */
1089         reg |= DWC3_DCTL_ULSTCHNG_RECOVERY;
1090         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1091
1092         /* wait for at least 2000us */
1093         usleep_range(2000, 2500);
1094
1095         /* write zeroes to Link Change Request */
1096         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1097         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1098
1099         /* pool until Link State change to ON */
1100         timeout = jiffies + msecs_to_jiffies(100);
1101
1102         while (!(time_after(jiffies, timeout))) {
1103                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1104
1105                 /* in HS, means ON */
1106                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1107                         break;
1108         }
1109
1110         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1111                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1112                 ret = -EINVAL;
1113         }
1114
1115 out:
1116         spin_unlock_irqrestore(&dwc->lock, flags);
1117
1118         return ret;
1119 }
1120
1121 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1122                 int is_selfpowered)
1123 {
1124         struct dwc3             *dwc = gadget_to_dwc(g);
1125
1126         dwc->is_selfpowered = !!is_selfpowered;
1127
1128         return 0;
1129 }
1130
1131 static void dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
1132 {
1133         u32                     reg;
1134         u32                     timeout = 500;
1135
1136         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1137         if (is_on)
1138                 reg |= DWC3_DCTL_RUN_STOP;
1139         else
1140                 reg &= ~DWC3_DCTL_RUN_STOP;
1141
1142         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1143
1144         do {
1145                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1146                 if (is_on) {
1147                         if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1148                                 break;
1149                 } else {
1150                         if (reg & DWC3_DSTS_DEVCTRLHLT)
1151                                 break;
1152                 }
1153                 timeout--;
1154                 if (!timeout)
1155                         break;
1156                 udelay(1);
1157         } while (1);
1158
1159         dev_vdbg(dwc->dev, "gadget %s data soft-%s\n",
1160                         dwc->gadget_driver
1161                         ? dwc->gadget_driver->function : "no-function",
1162                         is_on ? "connect" : "disconnect");
1163 }
1164
1165 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1166 {
1167         struct dwc3             *dwc = gadget_to_dwc(g);
1168         unsigned long           flags;
1169
1170         is_on = !!is_on;
1171
1172         spin_lock_irqsave(&dwc->lock, flags);
1173         dwc3_gadget_run_stop(dwc, is_on);
1174         spin_unlock_irqrestore(&dwc->lock, flags);
1175
1176         return 0;
1177 }
1178
1179 static int dwc3_gadget_start(struct usb_gadget *g,
1180                 struct usb_gadget_driver *driver)
1181 {
1182         struct dwc3             *dwc = gadget_to_dwc(g);
1183         struct dwc3_ep          *dep;
1184         unsigned long           flags;
1185         int                     ret = 0;
1186         u32                     reg;
1187
1188         spin_lock_irqsave(&dwc->lock, flags);
1189
1190         if (dwc->gadget_driver) {
1191                 dev_err(dwc->dev, "%s is already bound to %s\n",
1192                                 dwc->gadget.name,
1193                                 dwc->gadget_driver->driver.name);
1194                 ret = -EBUSY;
1195                 goto err0;
1196         }
1197
1198         dwc->gadget_driver      = driver;
1199         dwc->gadget.dev.driver  = &driver->driver;
1200
1201         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
1202
1203         reg &= ~DWC3_GCTL_SCALEDOWN(3);
1204         reg &= ~DWC3_GCTL_PRTCAPDIR(DWC3_GCTL_PRTCAP_OTG);
1205         reg &= ~DWC3_GCTL_DISSCRAMBLE;
1206         reg |= DWC3_GCTL_PRTCAPDIR(DWC3_GCTL_PRTCAP_DEVICE);
1207
1208         switch (DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams0)) {
1209         case DWC3_GHWPARAMS1_EN_PWROPT_CLK:
1210                 reg &= ~DWC3_GCTL_DSBLCLKGTNG;
1211                 break;
1212         default:
1213                 dev_dbg(dwc->dev, "No power optimization available\n");
1214         }
1215
1216         /*
1217          * WORKAROUND: DWC3 revisions <1.90a have a bug
1218          * when The device fails to connect at SuperSpeed
1219          * and falls back to high-speed mode which causes
1220          * the device to enter in a Connect/Disconnect loop
1221          */
1222         if (dwc->revision < DWC3_REVISION_190A)
1223                 reg |= DWC3_GCTL_U2RSTECN;
1224
1225         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
1226
1227         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1228         reg &= ~(DWC3_DCFG_SPEED_MASK);
1229         reg |= DWC3_DCFG_SUPERSPEED;
1230         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1231
1232         /* Start with SuperSpeed Default */
1233         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1234
1235         dep = dwc->eps[0];
1236         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc);
1237         if (ret) {
1238                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1239                 goto err0;
1240         }
1241
1242         dep = dwc->eps[1];
1243         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc);
1244         if (ret) {
1245                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1246                 goto err1;
1247         }
1248
1249         /* begin to receive SETUP packets */
1250         dwc->ep0state = EP0_SETUP_PHASE;
1251         dwc3_ep0_out_start(dwc);
1252
1253         spin_unlock_irqrestore(&dwc->lock, flags);
1254
1255         return 0;
1256
1257 err1:
1258         __dwc3_gadget_ep_disable(dwc->eps[0]);
1259
1260 err0:
1261         dwc->gadget_driver = NULL;
1262         spin_unlock_irqrestore(&dwc->lock, flags);
1263
1264         return ret;
1265 }
1266
1267 static int dwc3_gadget_stop(struct usb_gadget *g,
1268                 struct usb_gadget_driver *driver)
1269 {
1270         struct dwc3             *dwc = gadget_to_dwc(g);
1271         unsigned long           flags;
1272
1273         spin_lock_irqsave(&dwc->lock, flags);
1274
1275         __dwc3_gadget_ep_disable(dwc->eps[0]);
1276         __dwc3_gadget_ep_disable(dwc->eps[1]);
1277
1278         dwc->gadget_driver      = NULL;
1279         dwc->gadget.dev.driver  = NULL;
1280
1281         spin_unlock_irqrestore(&dwc->lock, flags);
1282
1283         return 0;
1284 }
1285 static const struct usb_gadget_ops dwc3_gadget_ops = {
1286         .get_frame              = dwc3_gadget_get_frame,
1287         .wakeup                 = dwc3_gadget_wakeup,
1288         .set_selfpowered        = dwc3_gadget_set_selfpowered,
1289         .pullup                 = dwc3_gadget_pullup,
1290         .udc_start              = dwc3_gadget_start,
1291         .udc_stop               = dwc3_gadget_stop,
1292 };
1293
1294 /* -------------------------------------------------------------------------- */
1295
1296 static int __devinit dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1297 {
1298         struct dwc3_ep                  *dep;
1299         u8                              epnum;
1300
1301         INIT_LIST_HEAD(&dwc->gadget.ep_list);
1302
1303         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1304                 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1305                 if (!dep) {
1306                         dev_err(dwc->dev, "can't allocate endpoint %d\n",
1307                                         epnum);
1308                         return -ENOMEM;
1309                 }
1310
1311                 dep->dwc = dwc;
1312                 dep->number = epnum;
1313                 dwc->eps[epnum] = dep;
1314
1315                 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1316                                 (epnum & 1) ? "in" : "out");
1317                 dep->endpoint.name = dep->name;
1318                 dep->direction = (epnum & 1);
1319
1320                 if (epnum == 0 || epnum == 1) {
1321                         dep->endpoint.maxpacket = 512;
1322                         dep->endpoint.maxburst = 1;
1323                         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1324                         if (!epnum)
1325                                 dwc->gadget.ep0 = &dep->endpoint;
1326                 } else {
1327                         int             ret;
1328
1329                         dep->endpoint.maxpacket = 1024;
1330                         dep->endpoint.max_streams = 15;
1331                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
1332                         list_add_tail(&dep->endpoint.ep_list,
1333                                         &dwc->gadget.ep_list);
1334
1335                         ret = dwc3_alloc_trb_pool(dep);
1336                         if (ret) {
1337                                 dev_err(dwc->dev, "%s: failed to allocate TRB pool\n", dep->name);
1338                                 return ret;
1339                         }
1340                 }
1341                 INIT_LIST_HEAD(&dep->request_list);
1342                 INIT_LIST_HEAD(&dep->req_queued);
1343         }
1344
1345         return 0;
1346 }
1347
1348 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1349 {
1350         struct dwc3_ep                  *dep;
1351         u8                              epnum;
1352
1353         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1354                 dep = dwc->eps[epnum];
1355                 /*
1356                  * Physical endpoints 0 and 1 are special; they form the
1357                  * bi-directional USB endpoint 0.
1358                  *
1359                  * For those two physical endpoints, we don't allocate a TRB
1360                  * pool nor do we add them the endpoints list. Due to that, we
1361                  * shouldn't do these two operations otherwise we would end up
1362                  * with all sorts of bugs when removing dwc3.ko.
1363                  */
1364                 if (epnum != 0 && epnum != 1) {
1365                         dwc3_free_trb_pool(dep);
1366                         list_del(&dep->endpoint.ep_list);
1367                 }
1368
1369                 kfree(dep);
1370         }
1371 }
1372
1373 static void dwc3_gadget_release(struct device *dev)
1374 {
1375         dev_dbg(dev, "%s\n", __func__);
1376 }
1377
1378 /* -------------------------------------------------------------------------- */
1379 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1380                 const struct dwc3_event_depevt *event, int status)
1381 {
1382         struct dwc3_request     *req;
1383         struct dwc3_trb         trb;
1384         unsigned int            count;
1385         unsigned int            s_pkt = 0;
1386
1387         do {
1388                 req = next_request(&dep->req_queued);
1389                 if (!req)
1390                         break;
1391
1392                 dwc3_trb_to_nat(req->trb, &trb);
1393
1394                 if (trb.hwo && status != -ESHUTDOWN)
1395                         /*
1396                          * We continue despite the error. There is not much we
1397                          * can do. If we don't clean in up we loop for ever. If
1398                          * we skip the TRB than it gets overwritten reused after
1399                          * a while since we use them in a ring buffer. a BUG()
1400                          * would help. Lets hope that if this occures, someone
1401                          * fixes the root cause instead of looking away :)
1402                          */
1403                         dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1404                                         dep->name, req->trb);
1405                 count = trb.length;
1406
1407                 if (dep->direction) {
1408                         if (count) {
1409                                 dev_err(dwc->dev, "incomplete IN transfer %s\n",
1410                                                 dep->name);
1411                                 status = -ECONNRESET;
1412                         }
1413                 } else {
1414                         if (count && (event->status & DEPEVT_STATUS_SHORT))
1415                                 s_pkt = 1;
1416                 }
1417
1418                 /*
1419                  * We assume here we will always receive the entire data block
1420                  * which we should receive. Meaning, if we program RX to
1421                  * receive 4K but we receive only 2K, we assume that's all we
1422                  * should receive and we simply bounce the request back to the
1423                  * gadget driver for further processing.
1424                  */
1425                 req->request.actual += req->request.length - count;
1426                 dwc3_gadget_giveback(dep, req, status);
1427                 if (s_pkt)
1428                         break;
1429                 if ((event->status & DEPEVT_STATUS_LST) && trb.lst)
1430                         break;
1431                 if ((event->status & DEPEVT_STATUS_IOC) && trb.ioc)
1432                         break;
1433         } while (1);
1434
1435         if ((event->status & DEPEVT_STATUS_IOC) && trb.ioc)
1436                 return 0;
1437         return 1;
1438 }
1439
1440 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
1441                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event,
1442                 int start_new)
1443 {
1444         unsigned                status = 0;
1445         int                     clean_busy;
1446
1447         if (event->status & DEPEVT_STATUS_BUSERR)
1448                 status = -ECONNRESET;
1449
1450         clean_busy =  dwc3_cleanup_done_reqs(dwc, dep, event, status);
1451         if (clean_busy) {
1452                 dep->flags &= ~DWC3_EP_BUSY;
1453                 dep->res_trans_idx = 0;
1454         }
1455 }
1456
1457 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1458                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1459 {
1460         u32 uf, mask;
1461
1462         if (list_empty(&dep->request_list)) {
1463                 dev_vdbg(dwc->dev, "ISOC ep %s run out for requests.\n",
1464                         dep->name);
1465                 return;
1466         }
1467
1468         mask = ~(dep->interval - 1);
1469         uf = event->parameters & mask;
1470         /* 4 micro frames in the future */
1471         uf += dep->interval * 4;
1472
1473         __dwc3_gadget_kick_transfer(dep, uf, 1);
1474 }
1475
1476 static void dwc3_process_ep_cmd_complete(struct dwc3_ep *dep,
1477                 const struct dwc3_event_depevt *event)
1478 {
1479         struct dwc3 *dwc = dep->dwc;
1480         struct dwc3_event_depevt mod_ev = *event;
1481
1482         /*
1483          * We were asked to remove one requests. It is possible that this
1484          * request and a few other were started together and have the same
1485          * transfer index. Since we stopped the complete endpoint we don't
1486          * know how many requests were already completed (and not yet)
1487          * reported and how could be done (later). We purge them all until
1488          * the end of the list.
1489          */
1490         mod_ev.status = DEPEVT_STATUS_LST;
1491         dwc3_cleanup_done_reqs(dwc, dep, &mod_ev, -ESHUTDOWN);
1492         dep->flags &= ~DWC3_EP_BUSY;
1493         /* pending requets are ignored and are queued on XferNotReady */
1494 }
1495
1496 static void dwc3_ep_cmd_compl(struct dwc3_ep *dep,
1497                 const struct dwc3_event_depevt *event)
1498 {
1499         u32 param = event->parameters;
1500         u32 cmd_type = (param >> 8) & ((1 << 5) - 1);
1501
1502         switch (cmd_type) {
1503         case DWC3_DEPCMD_ENDTRANSFER:
1504                 dwc3_process_ep_cmd_complete(dep, event);
1505                 break;
1506         case DWC3_DEPCMD_STARTTRANSFER:
1507                 dep->res_trans_idx = param & 0x7f;
1508                 break;
1509         default:
1510                 printk(KERN_ERR "%s() unknown /unexpected type: %d\n",
1511                                 __func__, cmd_type);
1512                 break;
1513         };
1514 }
1515
1516 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
1517                 const struct dwc3_event_depevt *event)
1518 {
1519         struct dwc3_ep          *dep;
1520         u8                      epnum = event->endpoint_number;
1521
1522         dep = dwc->eps[epnum];
1523
1524         dev_vdbg(dwc->dev, "%s: %s\n", dep->name,
1525                         dwc3_ep_event_string(event->endpoint_event));
1526
1527         if (epnum == 0 || epnum == 1) {
1528                 dwc3_ep0_interrupt(dwc, event);
1529                 return;
1530         }
1531
1532         switch (event->endpoint_event) {
1533         case DWC3_DEPEVT_XFERCOMPLETE:
1534                 if (usb_endpoint_xfer_isoc(dep->desc)) {
1535                         dev_dbg(dwc->dev, "%s is an Isochronous endpoint\n",
1536                                         dep->name);
1537                         return;
1538                 }
1539
1540                 dwc3_endpoint_transfer_complete(dwc, dep, event, 1);
1541                 break;
1542         case DWC3_DEPEVT_XFERINPROGRESS:
1543                 if (!usb_endpoint_xfer_isoc(dep->desc)) {
1544                         dev_dbg(dwc->dev, "%s is not an Isochronous endpoint\n",
1545                                         dep->name);
1546                         return;
1547                 }
1548
1549                 dwc3_endpoint_transfer_complete(dwc, dep, event, 0);
1550                 break;
1551         case DWC3_DEPEVT_XFERNOTREADY:
1552                 if (usb_endpoint_xfer_isoc(dep->desc)) {
1553                         dwc3_gadget_start_isoc(dwc, dep, event);
1554                 } else {
1555                         int ret;
1556
1557                         dev_vdbg(dwc->dev, "%s: reason %s\n",
1558                                         dep->name, event->status
1559                                         ? "Transfer Active"
1560                                         : "Transfer Not Active");
1561
1562                         ret = __dwc3_gadget_kick_transfer(dep, 0, 1);
1563                         if (!ret || ret == -EBUSY)
1564                                 return;
1565
1566                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1567                                         dep->name);
1568                 }
1569
1570                 break;
1571         case DWC3_DEPEVT_STREAMEVT:
1572                 if (!usb_endpoint_xfer_bulk(dep->desc)) {
1573                         dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
1574                                         dep->name);
1575                         return;
1576                 }
1577
1578                 switch (event->status) {
1579                 case DEPEVT_STREAMEVT_FOUND:
1580                         dev_vdbg(dwc->dev, "Stream %d found and started\n",
1581                                         event->parameters);
1582
1583                         break;
1584                 case DEPEVT_STREAMEVT_NOTFOUND:
1585                         /* FALLTHROUGH */
1586                 default:
1587                         dev_dbg(dwc->dev, "Couldn't find suitable stream\n");
1588                 }
1589                 break;
1590         case DWC3_DEPEVT_RXTXFIFOEVT:
1591                 dev_dbg(dwc->dev, "%s FIFO Overrun\n", dep->name);
1592                 break;
1593         case DWC3_DEPEVT_EPCMDCMPLT:
1594                 dwc3_ep_cmd_compl(dep, event);
1595                 break;
1596         }
1597 }
1598
1599 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
1600 {
1601         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
1602                 spin_unlock(&dwc->lock);
1603                 dwc->gadget_driver->disconnect(&dwc->gadget);
1604                 spin_lock(&dwc->lock);
1605         }
1606 }
1607
1608 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum)
1609 {
1610         struct dwc3_ep *dep;
1611         struct dwc3_gadget_ep_cmd_params params;
1612         u32 cmd;
1613         int ret;
1614
1615         dep = dwc->eps[epnum];
1616
1617         WARN_ON(!dep->res_trans_idx);
1618         if (dep->res_trans_idx) {
1619                 cmd = DWC3_DEPCMD_ENDTRANSFER;
1620                 cmd |= DWC3_DEPCMD_HIPRI_FORCERM | DWC3_DEPCMD_CMDIOC;
1621                 cmd |= DWC3_DEPCMD_PARAM(dep->res_trans_idx);
1622                 memset(&params, 0, sizeof(params));
1623                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
1624                 WARN_ON_ONCE(ret);
1625                 dep->res_trans_idx = 0;
1626         }
1627 }
1628
1629 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
1630 {
1631         u32 epnum;
1632
1633         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1634                 struct dwc3_ep *dep;
1635
1636                 dep = dwc->eps[epnum];
1637                 if (!(dep->flags & DWC3_EP_ENABLED))
1638                         continue;
1639
1640                 dwc3_remove_requests(dwc, dep);
1641         }
1642 }
1643
1644 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
1645 {
1646         u32 epnum;
1647
1648         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1649                 struct dwc3_ep *dep;
1650                 struct dwc3_gadget_ep_cmd_params params;
1651                 int ret;
1652
1653                 dep = dwc->eps[epnum];
1654
1655                 if (!(dep->flags & DWC3_EP_STALL))
1656                         continue;
1657
1658                 dep->flags &= ~DWC3_EP_STALL;
1659
1660                 memset(&params, 0, sizeof(params));
1661                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1662                                 DWC3_DEPCMD_CLEARSTALL, &params);
1663                 WARN_ON_ONCE(ret);
1664         }
1665 }
1666
1667 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
1668 {
1669         dev_vdbg(dwc->dev, "%s\n", __func__);
1670 #if 0
1671         XXX
1672         U1/U2 is powersave optimization. Skip it for now. Anyway we need to
1673         enable it before we can disable it.
1674
1675         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1676         reg &= ~DWC3_DCTL_INITU1ENA;
1677         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1678
1679         reg &= ~DWC3_DCTL_INITU2ENA;
1680         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1681 #endif
1682
1683         dwc3_stop_active_transfers(dwc);
1684         dwc3_disconnect_gadget(dwc);
1685
1686         dwc->gadget.speed = USB_SPEED_UNKNOWN;
1687 }
1688
1689 static void dwc3_gadget_usb3_phy_power(struct dwc3 *dwc, int on)
1690 {
1691         u32                     reg;
1692
1693         reg = dwc3_readl(dwc->regs, DWC3_GUSB3PIPECTL(0));
1694
1695         if (on)
1696                 reg &= ~DWC3_GUSB3PIPECTL_SUSPHY;
1697         else
1698                 reg |= DWC3_GUSB3PIPECTL_SUSPHY;
1699
1700         dwc3_writel(dwc->regs, DWC3_GUSB3PIPECTL(0), reg);
1701 }
1702
1703 static void dwc3_gadget_usb2_phy_power(struct dwc3 *dwc, int on)
1704 {
1705         u32                     reg;
1706
1707         reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
1708
1709         if (on)
1710                 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
1711         else
1712                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
1713
1714         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
1715 }
1716
1717 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
1718 {
1719         u32                     reg;
1720
1721         dev_vdbg(dwc->dev, "%s\n", __func__);
1722
1723         /* Enable PHYs */
1724         dwc3_gadget_usb2_phy_power(dwc, true);
1725         dwc3_gadget_usb3_phy_power(dwc, true);
1726
1727         if (dwc->gadget.speed != USB_SPEED_UNKNOWN)
1728                 dwc3_disconnect_gadget(dwc);
1729
1730         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1731         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
1732         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1733
1734         dwc3_stop_active_transfers(dwc);
1735         dwc3_clear_stall_all_ep(dwc);
1736
1737         /* Reset device address to zero */
1738         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1739         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
1740         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1741 }
1742
1743 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
1744 {
1745         u32 reg;
1746         u32 usb30_clock = DWC3_GCTL_CLK_BUS;
1747
1748         /*
1749          * We change the clock only at SS but I dunno why I would want to do
1750          * this. Maybe it becomes part of the power saving plan.
1751          */
1752
1753         if (speed != DWC3_DSTS_SUPERSPEED)
1754                 return;
1755
1756         /*
1757          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
1758          * each time on Connect Done.
1759          */
1760         if (!usb30_clock)
1761                 return;
1762
1763         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
1764         reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
1765         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
1766 }
1767
1768 static void dwc3_gadget_disable_phy(struct dwc3 *dwc, u8 speed)
1769 {
1770         switch (speed) {
1771         case USB_SPEED_SUPER:
1772                 dwc3_gadget_usb2_phy_power(dwc, false);
1773                 break;
1774         case USB_SPEED_HIGH:
1775         case USB_SPEED_FULL:
1776         case USB_SPEED_LOW:
1777                 dwc3_gadget_usb3_phy_power(dwc, false);
1778                 break;
1779         }
1780 }
1781
1782 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
1783 {
1784         struct dwc3_gadget_ep_cmd_params params;
1785         struct dwc3_ep          *dep;
1786         int                     ret;
1787         u32                     reg;
1788         u8                      speed;
1789
1790         dev_vdbg(dwc->dev, "%s\n", __func__);
1791
1792         memset(&params, 0x00, sizeof(params));
1793
1794         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1795         speed = reg & DWC3_DSTS_CONNECTSPD;
1796         dwc->speed = speed;
1797
1798         dwc3_update_ram_clk_sel(dwc, speed);
1799
1800         switch (speed) {
1801         case DWC3_DCFG_SUPERSPEED:
1802                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1803                 dwc->gadget.ep0->maxpacket = 512;
1804                 dwc->gadget.speed = USB_SPEED_SUPER;
1805                 break;
1806         case DWC3_DCFG_HIGHSPEED:
1807                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1808                 dwc->gadget.ep0->maxpacket = 64;
1809                 dwc->gadget.speed = USB_SPEED_HIGH;
1810                 break;
1811         case DWC3_DCFG_FULLSPEED2:
1812         case DWC3_DCFG_FULLSPEED1:
1813                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1814                 dwc->gadget.ep0->maxpacket = 64;
1815                 dwc->gadget.speed = USB_SPEED_FULL;
1816                 break;
1817         case DWC3_DCFG_LOWSPEED:
1818                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
1819                 dwc->gadget.ep0->maxpacket = 8;
1820                 dwc->gadget.speed = USB_SPEED_LOW;
1821                 break;
1822         }
1823
1824         /* Disable unneded PHY */
1825         dwc3_gadget_disable_phy(dwc, dwc->gadget.speed);
1826
1827         dep = dwc->eps[0];
1828         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc);
1829         if (ret) {
1830                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1831                 return;
1832         }
1833
1834         dep = dwc->eps[1];
1835         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc);
1836         if (ret) {
1837                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1838                 return;
1839         }
1840
1841         /*
1842          * Configure PHY via GUSB3PIPECTLn if required.
1843          *
1844          * Update GTXFIFOSIZn
1845          *
1846          * In both cases reset values should be sufficient.
1847          */
1848 }
1849
1850 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
1851 {
1852         dev_vdbg(dwc->dev, "%s\n", __func__);
1853
1854         /*
1855          * TODO take core out of low power mode when that's
1856          * implemented.
1857          */
1858
1859         dwc->gadget_driver->resume(&dwc->gadget);
1860 }
1861
1862 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
1863                 unsigned int evtinfo)
1864 {
1865         /*  The fith bit says SuperSpeed yes or no. */
1866         dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
1867
1868         dev_vdbg(dwc->dev, "%s link %d\n", __func__, dwc->link_state);
1869 }
1870
1871 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
1872                 const struct dwc3_event_devt *event)
1873 {
1874         switch (event->type) {
1875         case DWC3_DEVICE_EVENT_DISCONNECT:
1876                 dwc3_gadget_disconnect_interrupt(dwc);
1877                 break;
1878         case DWC3_DEVICE_EVENT_RESET:
1879                 dwc3_gadget_reset_interrupt(dwc);
1880                 break;
1881         case DWC3_DEVICE_EVENT_CONNECT_DONE:
1882                 dwc3_gadget_conndone_interrupt(dwc);
1883                 break;
1884         case DWC3_DEVICE_EVENT_WAKEUP:
1885                 dwc3_gadget_wakeup_interrupt(dwc);
1886                 break;
1887         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
1888                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
1889                 break;
1890         case DWC3_DEVICE_EVENT_EOPF:
1891                 dev_vdbg(dwc->dev, "End of Periodic Frame\n");
1892                 break;
1893         case DWC3_DEVICE_EVENT_SOF:
1894                 dev_vdbg(dwc->dev, "Start of Periodic Frame\n");
1895                 break;
1896         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
1897                 dev_vdbg(dwc->dev, "Erratic Error\n");
1898                 break;
1899         case DWC3_DEVICE_EVENT_CMD_CMPL:
1900                 dev_vdbg(dwc->dev, "Command Complete\n");
1901                 break;
1902         case DWC3_DEVICE_EVENT_OVERFLOW:
1903                 dev_vdbg(dwc->dev, "Overflow\n");
1904                 break;
1905         default:
1906                 dev_dbg(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
1907         }
1908 }
1909
1910 static void dwc3_process_event_entry(struct dwc3 *dwc,
1911                 const union dwc3_event *event)
1912 {
1913         /* Endpoint IRQ, handle it and return early */
1914         if (event->type.is_devspec == 0) {
1915                 /* depevt */
1916                 return dwc3_endpoint_interrupt(dwc, &event->depevt);
1917         }
1918
1919         switch (event->type.type) {
1920         case DWC3_EVENT_TYPE_DEV:
1921                 dwc3_gadget_interrupt(dwc, &event->devt);
1922                 break;
1923         /* REVISIT what to do with Carkit and I2C events ? */
1924         default:
1925                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
1926         }
1927 }
1928
1929 static irqreturn_t dwc3_process_event_buf(struct dwc3 *dwc, u32 buf)
1930 {
1931         struct dwc3_event_buffer *evt;
1932         int left;
1933         u32 count;
1934
1935         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(buf));
1936         count &= DWC3_GEVNTCOUNT_MASK;
1937         if (!count)
1938                 return IRQ_NONE;
1939
1940         evt = dwc->ev_buffs[buf];
1941         left = count;
1942
1943         while (left > 0) {
1944                 union dwc3_event event;
1945
1946                 memcpy(&event.raw, (evt->buf + evt->lpos), sizeof(event.raw));
1947                 dwc3_process_event_entry(dwc, &event);
1948                 /*
1949                  * XXX we wrap around correctly to the next entry as almost all
1950                  * entries are 4 bytes in size. There is one entry which has 12
1951                  * bytes which is a regular entry followed by 8 bytes data. ATM
1952                  * I don't know how things are organized if were get next to the
1953                  * a boundary so I worry about that once we try to handle that.
1954                  */
1955                 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
1956                 left -= 4;
1957
1958                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(buf), 4);
1959         }
1960
1961         return IRQ_HANDLED;
1962 }
1963
1964 static irqreturn_t dwc3_interrupt(int irq, void *_dwc)
1965 {
1966         struct dwc3                     *dwc = _dwc;
1967         int                             i;
1968         irqreturn_t                     ret = IRQ_NONE;
1969
1970         spin_lock(&dwc->lock);
1971
1972         for (i = 0; i < DWC3_EVENT_BUFFERS_NUM; i++) {
1973                 irqreturn_t status;
1974
1975                 status = dwc3_process_event_buf(dwc, i);
1976                 if (status == IRQ_HANDLED)
1977                         ret = status;
1978         }
1979
1980         spin_unlock(&dwc->lock);
1981
1982         return ret;
1983 }
1984
1985 /**
1986  * dwc3_gadget_init - Initializes gadget related registers
1987  * @dwc: Pointer to out controller context structure
1988  *
1989  * Returns 0 on success otherwise negative errno.
1990  */
1991 int __devinit dwc3_gadget_init(struct dwc3 *dwc)
1992 {
1993         u32                                     reg;
1994         int                                     ret;
1995         int                                     irq;
1996
1997         dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
1998                         &dwc->ctrl_req_addr, GFP_KERNEL);
1999         if (!dwc->ctrl_req) {
2000                 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2001                 ret = -ENOMEM;
2002                 goto err0;
2003         }
2004
2005         dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2006                         &dwc->ep0_trb_addr, GFP_KERNEL);
2007         if (!dwc->ep0_trb) {
2008                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2009                 ret = -ENOMEM;
2010                 goto err1;
2011         }
2012
2013         dwc->setup_buf = dma_alloc_coherent(dwc->dev,
2014                         sizeof(*dwc->setup_buf) * 2,
2015                         &dwc->setup_buf_addr, GFP_KERNEL);
2016         if (!dwc->setup_buf) {
2017                 dev_err(dwc->dev, "failed to allocate setup buffer\n");
2018                 ret = -ENOMEM;
2019                 goto err2;
2020         }
2021
2022         dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
2023                         512, &dwc->ep0_bounce_addr, GFP_KERNEL);
2024         if (!dwc->ep0_bounce) {
2025                 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2026                 ret = -ENOMEM;
2027                 goto err3;
2028         }
2029
2030         dev_set_name(&dwc->gadget.dev, "gadget");
2031
2032         dwc->gadget.ops                 = &dwc3_gadget_ops;
2033         dwc->gadget.is_dualspeed        = true;
2034         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
2035         dwc->gadget.dev.parent          = dwc->dev;
2036
2037         dma_set_coherent_mask(&dwc->gadget.dev, dwc->dev->coherent_dma_mask);
2038
2039         dwc->gadget.dev.dma_parms       = dwc->dev->dma_parms;
2040         dwc->gadget.dev.dma_mask        = dwc->dev->dma_mask;
2041         dwc->gadget.dev.release         = dwc3_gadget_release;
2042         dwc->gadget.name                = "dwc3-gadget";
2043
2044         /*
2045          * REVISIT: Here we should clear all pending IRQs to be
2046          * sure we're starting from a well known location.
2047          */
2048
2049         ret = dwc3_gadget_init_endpoints(dwc);
2050         if (ret)
2051                 goto err4;
2052
2053         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
2054
2055         ret = request_irq(irq, dwc3_interrupt, IRQF_SHARED,
2056                         "dwc3", dwc);
2057         if (ret) {
2058                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2059                                 irq, ret);
2060                 goto err5;
2061         }
2062
2063         /* Enable all but Start and End of Frame IRQs */
2064         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
2065                         DWC3_DEVTEN_EVNTOVERFLOWEN |
2066                         DWC3_DEVTEN_CMDCMPLTEN |
2067                         DWC3_DEVTEN_ERRTICERREN |
2068                         DWC3_DEVTEN_WKUPEVTEN |
2069                         DWC3_DEVTEN_ULSTCNGEN |
2070                         DWC3_DEVTEN_CONNECTDONEEN |
2071                         DWC3_DEVTEN_USBRSTEN |
2072                         DWC3_DEVTEN_DISCONNEVTEN);
2073         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2074
2075         ret = device_register(&dwc->gadget.dev);
2076         if (ret) {
2077                 dev_err(dwc->dev, "failed to register gadget device\n");
2078                 put_device(&dwc->gadget.dev);
2079                 goto err6;
2080         }
2081
2082         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2083         if (ret) {
2084                 dev_err(dwc->dev, "failed to register udc\n");
2085                 goto err7;
2086         }
2087
2088         return 0;
2089
2090 err7:
2091         device_unregister(&dwc->gadget.dev);
2092
2093 err6:
2094         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2095         free_irq(irq, dwc);
2096
2097 err5:
2098         dwc3_gadget_free_endpoints(dwc);
2099
2100 err4:
2101         dma_free_coherent(dwc->dev, 512, dwc->ep0_bounce,
2102                         dwc->ep0_bounce_addr);
2103
2104 err3:
2105         dma_free_coherent(dwc->dev, sizeof(*dwc->setup_buf) * 2,
2106                         dwc->setup_buf, dwc->setup_buf_addr);
2107
2108 err2:
2109         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2110                         dwc->ep0_trb, dwc->ep0_trb_addr);
2111
2112 err1:
2113         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2114                         dwc->ctrl_req, dwc->ctrl_req_addr);
2115
2116 err0:
2117         return ret;
2118 }
2119
2120 void dwc3_gadget_exit(struct dwc3 *dwc)
2121 {
2122         int                     irq;
2123         int                     i;
2124
2125         usb_del_gadget_udc(&dwc->gadget);
2126         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
2127
2128         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2129         free_irq(irq, dwc);
2130
2131         for (i = 0; i < ARRAY_SIZE(dwc->eps); i++)
2132                 __dwc3_gadget_ep_disable(dwc->eps[i]);
2133
2134         dwc3_gadget_free_endpoints(dwc);
2135
2136         dma_free_coherent(dwc->dev, 512, dwc->ep0_bounce,
2137                         dwc->ep0_bounce_addr);
2138
2139         dma_free_coherent(dwc->dev, sizeof(*dwc->setup_buf) * 2,
2140                         dwc->setup_buf, dwc->setup_buf_addr);
2141
2142         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2143                         dwc->ep0_trb, dwc->ep0_trb_addr);
2144
2145         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2146                         dwc->ctrl_req, dwc->ctrl_req_addr);
2147
2148         device_unregister(&dwc->gadget.dev);
2149 }