drivers/usb/gadget/goku_udc.c

   1 /*
   2  * Toshiba TC86C001 ("Goku-S") USB Device Controller driver
   3  *
   4  * Copyright (C) 2000-2002 Lineo
   5  *      by Stuart Lynne, Tom Rushworth, and Bruce Balden
   6  * Copyright (C) 2002 Toshiba Corporation
   7  * Copyright (C) 2003 MontaVista Software (source@mvista.com)
   8  *
   9  * This file is licensed under the terms of the GNU General Public
  10  * License version 2.  This program is licensed "as is" without any
  11  * warranty of any kind, whether express or implied.
  12  */
  13
  14 /*
  15  * This device has ep0 and three semi-configurable bulk/interrupt endpoints.
  16  *
  17  *  - Endpoint numbering is fixed: ep{1,2,3}-bulk
  18  *  - Gadget drivers can choose ep maxpacket (8/16/32/64)
  19  *  - Gadget drivers can choose direction (IN, OUT)
  20  *  - DMA works with ep1 (OUT transfers) and ep2 (IN transfers).
  21  */
  22
  23 // #define      VERBOSE         /* extra debug messages (success too) */
  24 // #define      USB_TRACE       /* packet-level success messages */
  25
  26 #include <linux/kernel.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29 #include <linux/delay.h>
  30 #include <linux/ioport.h>
  31 #include <linux/slab.h>
  32 #include <linux/errno.h>
  33 #include <linux/init.h>
  34 #include <linux/timer.h>
  35 #include <linux/list.h>
  36 #include <linux/interrupt.h>
  37 #include <linux/proc_fs.h>
  38 #include <linux/device.h>
  39 #include <linux/usb/ch9.h>
  40 #include <linux/usb/gadget.h>
  41 #include <linux/prefetch.h>
  42
  43 #include <asm/byteorder.h>
  44 #include <asm/io.h>
  45 #include <asm/irq.h>
  46 #include <asm/system.h>
  47 #include <asm/unaligned.h>
  48
  49
  50 #include "goku_udc.h"
  51
  52 #define DRIVER_DESC             "TC86C001 USB Device Controller"
  53 #define DRIVER_VERSION          "30-Oct 2003"
  54
  55 #define DMA_ADDR_INVALID        (~(dma_addr_t)0)
  56
  57 static const char driver_name [] = "goku_udc";
  58 static const char driver_desc [] = DRIVER_DESC;
  59
  60 MODULE_AUTHOR("source@mvista.com");
  61 MODULE_DESCRIPTION(DRIVER_DESC);
  62 MODULE_LICENSE("GPL");
  63
  64
  65 /*
  66  * IN dma behaves ok under testing, though the IN-dma abort paths don't
  67  * seem to behave quite as expected.  Used by default.
  68  *
  69  * OUT dma documents design problems handling the common "short packet"
  70  * transfer termination policy; it couldn't be enabled by default, even
  71  * if the OUT-dma abort problems had a resolution.
  72  */
  73 static unsigned use_dma = 1;
  74
  75 #if 0
  76 //#include <linux/moduleparam.h>
  77 /* "modprobe goku_udc use_dma=1" etc
  78  *      0 to disable dma
  79  *      1 to use IN dma only (normal operation)
  80  *      2 to use IN and OUT dma
  81  */
  82 module_param(use_dma, uint, S_IRUGO);
  83 #endif
  84
  85 /*-------------------------------------------------------------------------*/
  86
  87 static void nuke(struct goku_ep *, int status);
  88
  89 static inline void
  90 command(struct goku_udc_regs __iomem *regs, int command, unsigned epnum)
  91 {
  92         writel(COMMAND_EP(epnum) | command, &regs->Command);
  93         udelay(300);
  94 }
  95
  96 static int
  97 goku_ep_enable(struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
  98 {
  99         struct goku_udc *dev;
 100         struct goku_ep  *ep;
 101         u32             mode;
 102         u16             max;
 103         unsigned long   flags;
 104
 105         ep = container_of(_ep, struct goku_ep, ep);
 106         if (!_ep || !desc || ep->desc
 107                         || desc->bDescriptorType != USB_DT_ENDPOINT)
 108                 return -EINVAL;
 109         dev = ep->dev;
 110         if (ep == &dev->ep[0])
 111                 return -EINVAL;
 112         if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
 113                 return -ESHUTDOWN;
 114         if (ep->num != usb_endpoint_num(desc))
 115                 return -EINVAL;
 116
 117         switch (usb_endpoint_type(desc)) {
 118         case USB_ENDPOINT_XFER_BULK:
 119         case USB_ENDPOINT_XFER_INT:
 120                 break;
 121         default:
 122                 return -EINVAL;
 123         }
 124
 125         if ((readl(ep->reg_status) & EPxSTATUS_EP_MASK)
 126                         != EPxSTATUS_EP_INVALID)
 127                 return -EBUSY;
 128
 129         /* enabling the no-toggle interrupt mode would need an api hook */
 130         mode = 0;
 131         max = get_unaligned_le16(&desc->wMaxPacketSize);
 132         switch (max) {
 133         case 64:        mode++;
 134         case 32:        mode++;
 135         case 16:        mode++;
 136         case 8:         mode <<= 3;
 137                         break;
 138         default:
 139                 return -EINVAL;
 140         }
 141         mode |= 2 << 1;         /* bulk, or intr-with-toggle */
 142
 143         /* ep1/ep2 dma direction is chosen early; it works in the other
 144          * direction, with pio.  be cautious with out-dma.
 145          */
 146         ep->is_in = usb_endpoint_dir_in(desc);
 147         if (ep->is_in) {
 148                 mode |= 1;
 149                 ep->dma = (use_dma != 0) && (ep->num == UDC_MSTRD_ENDPOINT);
 150         } else {
 151                 ep->dma = (use_dma == 2) && (ep->num == UDC_MSTWR_ENDPOINT);
 152                 if (ep->dma)
 153                         DBG(dev, "%s out-dma hides short packets\n",
 154                                 ep->ep.name);
 155         }
 156
 157         spin_lock_irqsave(&ep->dev->lock, flags);
 158
 159         /* ep1 and ep2 can do double buffering and/or dma */
 160         if (ep->num < 3) {
 161                 struct goku_udc_regs __iomem    *regs = ep->dev->regs;
 162                 u32                             tmp;
 163
 164                 /* double buffer except (for now) with pio in */
 165                 tmp = ((ep->dma || !ep->is_in)
 166                                 ? 0x10  /* double buffered */
 167                                 : 0x11  /* single buffer */
 168                         ) << ep->num;
 169                 tmp |= readl(&regs->EPxSingle);
 170                 writel(tmp, &regs->EPxSingle);
 171
 172                 tmp = (ep->dma ? 0x10/*dma*/ : 0x11/*pio*/) << ep->num;
 173                 tmp |= readl(&regs->EPxBCS);
 174                 writel(tmp, &regs->EPxBCS);
 175         }
 176         writel(mode, ep->reg_mode);
 177         command(ep->dev->regs, COMMAND_RESET, ep->num);
 178         ep->ep.maxpacket = max;
 179         ep->stopped = 0;
 180         ep->desc = desc;
 181         spin_unlock_irqrestore(&ep->dev->lock, flags);
 182
 183         DBG(dev, "enable %s %s %s maxpacket %u\n", ep->ep.name,
 184                 ep->is_in ? "IN" : "OUT",
 185                 ep->dma ? "dma" : "pio",
 186                 max);
 187
 188         return 0;
 189 }
 190
 191 static void ep_reset(struct goku_udc_regs __iomem *regs, struct goku_ep *ep)
 192 {
 193         struct goku_udc         *dev = ep->dev;
 194
 195         if (regs) {
 196                 command(regs, COMMAND_INVALID, ep->num);
 197                 if (ep->num) {
 198                         if (ep->num == UDC_MSTWR_ENDPOINT)
 199                                 dev->int_enable &= ~(INT_MSTWREND
 200                                                         |INT_MSTWRTMOUT);
 201                         else if (ep->num == UDC_MSTRD_ENDPOINT)
 202                                 dev->int_enable &= ~INT_MSTRDEND;
 203                         dev->int_enable &= ~INT_EPxDATASET (ep->num);
 204                 } else
 205                         dev->int_enable &= ~INT_EP0;
 206                 writel(dev->int_enable, &regs->int_enable);
 207                 readl(&regs->int_enable);
 208                 if (ep->num < 3) {
 209                         struct goku_udc_regs __iomem    *r = ep->dev->regs;
 210                         u32                             tmp;
 211
 212                         tmp = readl(&r->EPxSingle);
 213                         tmp &= ~(0x11 << ep->num);
 214                         writel(tmp, &r->EPxSingle);
 215
 216                         tmp = readl(&r->EPxBCS);
 217                         tmp &= ~(0x11 << ep->num);
 218                         writel(tmp, &r->EPxBCS);
 219                 }
 220                 /* reset dma in case we're still using it */
 221                 if (ep->dma) {
 222                         u32     master;
 223
 224                         master = readl(&regs->dma_master) & MST_RW_BITS;
 225                         if (ep->num == UDC_MSTWR_ENDPOINT) {
 226                                 master &= ~MST_W_BITS;
 227                                 master |= MST_WR_RESET;
 228                         } else {
 229                                 master &= ~MST_R_BITS;
 230                                 master |= MST_RD_RESET;
 231                         }
 232                         writel(master, &regs->dma_master);
 233                 }
 234         }
 235
 236         ep->ep.maxpacket = MAX_FIFO_SIZE;
 237         ep->desc = NULL;
 238         ep->stopped = 1;
 239         ep->irqs = 0;
 240         ep->dma = 0;
 241 }
 242
 243 static int goku_ep_disable(struct usb_ep *_ep)
 244 {
 245         struct goku_ep  *ep;
 246         struct goku_udc *dev;
 247         unsigned long   flags;
 248
 249         ep = container_of(_ep, struct goku_ep, ep);
 250         if (!_ep || !ep->desc)
 251                 return -ENODEV;
 252         dev = ep->dev;
 253         if (dev->ep0state == EP0_SUSPEND)
 254                 return -EBUSY;
 255
 256         VDBG(dev, "disable %s\n", _ep->name);
 257
 258         spin_lock_irqsave(&dev->lock, flags);
 259         nuke(ep, -ESHUTDOWN);
 260         ep_reset(dev->regs, ep);
 261         spin_unlock_irqrestore(&dev->lock, flags);
 262
 263         return 0;
 264 }
 265
 266 /*-------------------------------------------------------------------------*/
 267
 268 static struct usb_request *
 269 goku_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
 270 {
 271         struct goku_request     *req;
 272
 273         if (!_ep)
 274                 return NULL;
 275         req = kzalloc(sizeof *req, gfp_flags);
 276         if (!req)
 277                 return NULL;
 278
 279         req->req.dma = DMA_ADDR_INVALID;
 280         INIT_LIST_HEAD(&req->queue);
 281         return &req->req;
 282 }
 283
 284 static void
 285 goku_free_request(struct usb_ep *_ep, struct usb_request *_req)
 286 {
 287         struct goku_request     *req;
 288
 289         if (!_ep || !_req)
 290                 return;
 291
 292         req = container_of(_req, struct goku_request, req);
 293         WARN_ON(!list_empty(&req->queue));
 294         kfree(req);
 295 }
 296
 297 /*-------------------------------------------------------------------------*/
 298
 299 static void
 300 done(struct goku_ep *ep, struct goku_request *req, int status)
 301 {
 302         struct goku_udc         *dev;
 303         unsigned                stopped = ep->stopped;
 304
 305         list_del_init(&req->queue);
 306
 307         if (likely(req->req.status == -EINPROGRESS))
 308                 req->req.status = status;
 309         else
 310                 status = req->req.status;
 311
 312         dev = ep->dev;
 313         if (req->mapped) {
 314                 pci_unmap_single(dev->pdev, req->req.dma, req->req.length,
 315                         ep->is_in ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE);
 316                 req->req.dma = DMA_ADDR_INVALID;
 317                 req->mapped = 0;
 318         }
 319
 320 #ifndef USB_TRACE
 321         if (status && status != -ESHUTDOWN)
 322 #endif
 323                 VDBG(dev, "complete %s req %p stat %d len %u/%u\n",
 324                         ep->ep.name, &req->req, status,
 325                         req->req.actual, req->req.length);
 326
 327         /* don't modify queue heads during completion callback */
 328         ep->stopped = 1;
 329         spin_unlock(&dev->lock);
 330         req->req.complete(&ep->ep, &req->req);
 331         spin_lock(&dev->lock);
 332         ep->stopped = stopped;
 333 }
 334
 335 /*-------------------------------------------------------------------------*/
 336
 337 static inline int
 338 write_packet(u32 __iomem *fifo, u8 *buf, struct goku_request *req, unsigned max)
 339 {
 340         unsigned        length, count;
 341
 342         length = min(req->req.length - req->req.actual, max);
 343         req->req.actual += length;
 344
 345         count = length;
 346         while (likely(count--))
 347                 writel(*buf++, fifo);
 348         return length;
 349 }
 350
 351 // return:  0 = still running, 1 = completed, negative = errno
 352 static int write_fifo(struct goku_ep *ep, struct goku_request *req)
 353 {
 354         struct goku_udc *dev = ep->dev;
 355         u32             tmp;
 356         u8              *buf;
 357         unsigned        count;
 358         int             is_last;
 359
 360         tmp = readl(&dev->regs->DataSet);
 361         buf = req->req.buf + req->req.actual;
 362         prefetch(buf);
 363
 364         dev = ep->dev;
 365         if (unlikely(ep->num == 0 && dev->ep0state != EP0_IN))
 366                 return -EL2HLT;
 367
 368         /* NOTE:  just single-buffered PIO-IN for now.  */
 369         if (unlikely((tmp & DATASET_A(ep->num)) != 0))
 370                 return 0;
 371
 372         /* clear our "packet available" irq */
 373         if (ep->num != 0)
 374                 writel(~INT_EPxDATASET(ep->num), &dev->regs->int_status);
 375
 376         count = write_packet(ep->reg_fifo, buf, req, ep->ep.maxpacket);
 377
 378         /* last packet often short (sometimes a zlp, especially on ep0) */
 379         if (unlikely(count != ep->ep.maxpacket)) {
 380                 writel(~(1<<ep->num), &dev->regs->EOP);
 381                 if (ep->num == 0) {
 382                         dev->ep[0].stopped = 1;
 383                         dev->ep0state = EP0_STATUS;
 384                 }
 385                 is_last = 1;
 386         } else {
 387                 if (likely(req->req.length != req->req.actual)
 388                                 || req->req.zero)
 389                         is_last = 0;
 390                 else
 391                         is_last = 1;
 392         }
 393 #if 0           /* printk seemed to trash is_last...*/
 394 //#ifdef USB_TRACE
 395         VDBG(dev, "wrote %s %u bytes%s IN %u left %p\n",
 396                 ep->ep.name, count, is_last ? "/last" : "",
 397                 req->req.length - req->req.actual, req);
 398 #endif
 399
 400         /* requests complete when all IN data is in the FIFO,
 401          * or sometimes later, if a zlp was needed.
 402          */
 403         if (is_last) {
 404                 done(ep, req, 0);
 405                 return 1;
 406         }
 407
 408         return 0;
 409 }
 410
 411 static int read_fifo(struct goku_ep *ep, struct goku_request *req)
 412 {
 413         struct goku_udc_regs __iomem    *regs;
 414         u32                             size, set;
 415         u8                              *buf;
 416         unsigned                        bufferspace, is_short, dbuff;
 417
 418         regs = ep->dev->regs;
 419 top:
 420         buf = req->req.buf + req->req.actual;
 421         prefetchw(buf);
 422
 423         if (unlikely(ep->num == 0 && ep->dev->ep0state != EP0_OUT))
 424                 return -EL2HLT;
 425
 426         dbuff = (ep->num == 1 || ep->num == 2);
 427         do {
 428                 /* ack dataset irq matching the status we'll handle */
 429                 if (ep->num != 0)
 430                         writel(~INT_EPxDATASET(ep->num), &regs->int_status);
 431
 432                 set = readl(&regs->DataSet) & DATASET_AB(ep->num);
 433                 size = readl(&regs->EPxSizeLA[ep->num]);
 434                 bufferspace = req->req.length - req->req.actual;
 435
 436                 /* usually do nothing without an OUT packet */
 437                 if (likely(ep->num != 0 || bufferspace != 0)) {
 438                         if (unlikely(set == 0))
 439                                 break;
 440                         /* use ep1/ep2 double-buffering for OUT */
 441                         if (!(size & PACKET_ACTIVE))
 442                                 size = readl(&regs->EPxSizeLB[ep->num]);
 443                         if (!(size & PACKET_ACTIVE))    /* "can't happen" */
 444                                 break;
 445                         size &= DATASIZE;       /* EPxSizeH == 0 */
 446
 447                 /* ep0out no-out-data case for set_config, etc */
 448                 } else
 449                         size = 0;
 450
 451                 /* read all bytes from this packet */
 452                 req->req.actual += size;
 453                 is_short = (size < ep->ep.maxpacket);
 454 #ifdef USB_TRACE
 455                 VDBG(ep->dev, "read %s %u bytes%s OUT req %p %u/%u\n",
 456                         ep->ep.name, size, is_short ? "/S" : "",
 457                         req, req->req.actual, req->req.length);
 458 #endif
 459                 while (likely(size-- != 0)) {
 460                         u8      byte = (u8) readl(ep->reg_fifo);
 461
 462                         if (unlikely(bufferspace == 0)) {
 463                                 /* this happens when the driver's buffer
 464                                  * is smaller than what the host sent.
 465                                  * discard the extra data in this packet.
 466                                  */
 467                                 if (req->req.status != -EOVERFLOW)
 468                                         DBG(ep->dev, "%s overflow %u\n",
 469                                                 ep->ep.name, size);
 470                                 req->req.status = -EOVERFLOW;
 471                         } else {
 472                                 *buf++ = byte;
 473                                 bufferspace--;
 474                         }
 475                 }
 476
 477                 /* completion */
 478                 if (unlikely(is_short || req->req.actual == req->req.length)) {
 479                         if (unlikely(ep->num == 0)) {
 480                                 /* non-control endpoints now usable? */
 481                                 if (ep->dev->req_config)
 482                                         writel(ep->dev->configured
 483                                                         ? USBSTATE_CONFIGURED
 484                                                         : 0,
 485                                                 &regs->UsbState);
 486                                 /* ep0out status stage */
 487                                 writel(~(1<<0), &regs->EOP);
 488                                 ep->stopped = 1;
 489                                 ep->dev->ep0state = EP0_STATUS;
 490                         }
 491                         done(ep, req, 0);
 492
 493                         /* empty the second buffer asap */
 494                         if (dbuff && !list_empty(&ep->queue)) {
 495                                 req = list_entry(ep->queue.next,
 496                                                 struct goku_request, queue);
 497                                 goto top;
 498                         }
 499                         return 1;
 500                 }
 501         } while (dbuff);
 502         return 0;
 503 }
 504
 505 static inline void
 506 pio_irq_enable(struct goku_udc *dev,
 507                 struct goku_udc_regs __iomem *regs, int epnum)
 508 {
 509         dev->int_enable |= INT_EPxDATASET (epnum);
 510         writel(dev->int_enable, &regs->int_enable);
 511         /* write may still be posted */
 512 }
 513
 514 static inline void
 515 pio_irq_disable(struct goku_udc *dev,
 516                 struct goku_udc_regs __iomem *regs, int epnum)
 517 {
 518         dev->int_enable &= ~INT_EPxDATASET (epnum);
 519         writel(dev->int_enable, &regs->int_enable);
 520         /* write may still be posted */
 521 }
 522
 523 static inline void
 524 pio_advance(struct goku_ep *ep)
 525 {
 526         struct goku_request     *req;
 527
 528         if (unlikely(list_empty (&ep->queue)))
 529                 return;
 530         req = list_entry(ep->queue.next, struct goku_request, queue);
 531         (ep->is_in ? write_fifo : read_fifo)(ep, req);
 532 }
 533
 534
 535 /*-------------------------------------------------------------------------*/
 536
 537 // return:  0 = q running, 1 = q stopped, negative = errno
 538 static int start_dma(struct goku_ep *ep, struct goku_request *req)
 539 {
 540         struct goku_udc_regs __iomem    *regs = ep->dev->regs;
 541         u32                             master;
 542         u32                             start = req->req.dma;
 543         u32                             end = start + req->req.length - 1;
 544
 545         master = readl(&regs->dma_master) & MST_RW_BITS;
 546
 547         /* re-init the bits affecting IN dma; careful with zlps */
 548         if (likely(ep->is_in)) {
 549                 if (unlikely(master & MST_RD_ENA)) {
 550                         DBG (ep->dev, "start, IN active dma %03x!!\n",
 551                                 master);
 552 //                      return -EL2HLT;
 553                 }
 554                 writel(end, &regs->in_dma_end);
 555                 writel(start, &regs->in_dma_start);
 556
 557                 master &= ~MST_R_BITS;
 558                 if (unlikely(req->req.length == 0))
 559                         master = MST_RD_ENA | MST_RD_EOPB;
 560                 else if ((req->req.length % ep->ep.maxpacket) != 0
 561                                         || req->req.zero)
 562                         master = MST_RD_ENA | MST_EOPB_ENA;
 563                 else
 564                         master = MST_RD_ENA | MST_EOPB_DIS;
 565
 566                 ep->dev->int_enable |= INT_MSTRDEND;
 567
 568         /* Goku DMA-OUT merges short packets, which plays poorly with
 569          * protocols where short packets mark the transfer boundaries.
 570          * The chip supports a nonstandard policy with INT_MSTWRTMOUT,
 571          * ending transfers after 3 SOFs; we don't turn it on.
 572          */
 573         } else {
 574                 if (unlikely(master & MST_WR_ENA)) {
 575                         DBG (ep->dev, "start, OUT active dma %03x!!\n",
 576                                 master);
 577 //                      return -EL2HLT;
 578                 }
 579                 writel(end, &regs->out_dma_end);
 580                 writel(start, &regs->out_dma_start);
 581
 582                 master &= ~MST_W_BITS;
 583                 master |= MST_WR_ENA | MST_TIMEOUT_DIS;
 584
 585                 ep->dev->int_enable |= INT_MSTWREND|INT_MSTWRTMOUT;
 586         }
 587
 588         writel(master, &regs->dma_master);
 589         writel(ep->dev->int_enable, &regs->int_enable);
 590         return 0;
 591 }
 592
 593 static void dma_advance(struct goku_udc *dev, struct goku_ep *ep)
 594 {
 595         struct goku_request             *req;
 596         struct goku_udc_regs __iomem    *regs = ep->dev->regs;
 597         u32                             master;
 598
 599         master = readl(&regs->dma_master);
 600
 601         if (unlikely(list_empty(&ep->queue))) {
 602 stop:
 603                 if (ep->is_in)
 604                         dev->int_enable &= ~INT_MSTRDEND;
 605                 else
 606                         dev->int_enable &= ~(INT_MSTWREND|INT_MSTWRTMOUT);
 607                 writel(dev->int_enable, &regs->int_enable);
 608                 return;
 609         }
 610         req = list_entry(ep->queue.next, struct goku_request, queue);
 611
 612         /* normal hw dma completion (not abort) */
 613         if (likely(ep->is_in)) {
 614                 if (unlikely(master & MST_RD_ENA))
 615                         return;
 616                 req->req.actual = readl(&regs->in_dma_current);
 617         } else {
 618                 if (unlikely(master & MST_WR_ENA))
 619                         return;
 620
 621                 /* hardware merges short packets, and also hides packet
 622                  * overruns.  a partial packet MAY be in the fifo here.
 623                  */
 624                 req->req.actual = readl(&regs->out_dma_current);
 625         }
 626         req->req.actual -= req->req.dma;
 627         req->req.actual++;
 628
 629 #ifdef USB_TRACE
 630         VDBG(dev, "done %s %s dma, %u/%u bytes, req %p\n",
 631                 ep->ep.name, ep->is_in ? "IN" : "OUT",
 632                 req->req.actual, req->req.length, req);
 633 #endif
 634         done(ep, req, 0);
 635         if (list_empty(&ep->queue))
 636                 goto stop;
 637         req = list_entry(ep->queue.next, struct goku_request, queue);
 638         (void) start_dma(ep, req);
 639 }
 640
 641 static void abort_dma(struct goku_ep *ep, int status)
 642 {
 643         struct goku_udc_regs __iomem    *regs = ep->dev->regs;
 644         struct goku_request             *req;
 645         u32                             curr, master;
 646
 647         /* NAK future host requests, hoping the implicit delay lets the
 648          * dma engine finish reading (or writing) its latest packet and
 649          * empty the dma buffer (up to 16 bytes).
 650          *
 651          * This avoids needing to clean up a partial packet in the fifo;
 652          * we can't do that for IN without side effects to HALT and TOGGLE.
 653          */
 654         command(regs, COMMAND_FIFO_DISABLE, ep->num);
 655         req = list_entry(ep->queue.next, struct goku_request, queue);
 656         master = readl(&regs->dma_master) & MST_RW_BITS;
 657
 658         /* FIXME using these resets isn't usably documented. this may
 659          * not work unless it's followed by disabling the endpoint.
 660          *
 661          * FIXME the OUT reset path doesn't even behave consistently.
 662          */
 663         if (ep->is_in) {
 664                 if (unlikely((readl(&regs->dma_master) & MST_RD_ENA) == 0))
 665                         goto finished;
 666                 curr = readl(&regs->in_dma_current);
 667
 668                 writel(curr, &regs->in_dma_end);
 669                 writel(curr, &regs->in_dma_start);
 670
 671                 master &= ~MST_R_BITS;
 672                 master |= MST_RD_RESET;
 673                 writel(master, &regs->dma_master);
 674
 675                 if (readl(&regs->dma_master) & MST_RD_ENA)
 676                         DBG(ep->dev, "IN dma active after reset!\n");
 677
 678         } else {
 679                 if (unlikely((readl(&regs->dma_master) & MST_WR_ENA) == 0))
 680                         goto finished;
 681                 curr = readl(&regs->out_dma_current);
 682
 683                 writel(curr, &regs->out_dma_end);
 684                 writel(curr, &regs->out_dma_start);
 685
 686                 master &= ~MST_W_BITS;
 687                 master |= MST_WR_RESET;
 688                 writel(master, &regs->dma_master);
 689
 690                 if (readl(&regs->dma_master) & MST_WR_ENA)
 691                         DBG(ep->dev, "OUT dma active after reset!\n");
 692         }
 693         req->req.actual = (curr - req->req.dma) + 1;
 694         req->req.status = status;
 695
 696         VDBG(ep->dev, "%s %s %s %d/%d\n", __func__, ep->ep.name,
 697                 ep->is_in ? "IN" : "OUT",
 698                 req->req.actual, req->req.length);
 699
 700         command(regs, COMMAND_FIFO_ENABLE, ep->num);
 701
 702         return;
 703
 704 finished:
 705         /* dma already completed; no abort needed */
 706         command(regs, COMMAND_FIFO_ENABLE, ep->num);
 707         req->req.actual = req->req.length;
 708         req->req.status = 0;
 709 }
 710
 711 /*-------------------------------------------------------------------------*/
 712
 713 static int
 714 goku_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
 715 {
 716         struct goku_request     *req;
 717         struct goku_ep          *ep;
 718         struct goku_udc         *dev;
 719         unsigned long           flags;
 720         int                     status;
 721
 722         /* always require a cpu-view buffer so pio works */
 723         req = container_of(_req, struct goku_request, req);
 724         if (unlikely(!_req || !_req->complete
 725                         || !_req->buf || !list_empty(&req->queue)))
 726                 return -EINVAL;
 727         ep = container_of(_ep, struct goku_ep, ep);
 728         if (unlikely(!_ep || (!ep->desc && ep->num != 0)))
 729                 return -EINVAL;
 730         dev = ep->dev;
 731         if (unlikely(!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN))
 732                 return -ESHUTDOWN;
 733
 734         /* can't touch registers when suspended */
 735         if (dev->ep0state == EP0_SUSPEND)
 736                 return -EBUSY;
 737
 738         /* set up dma mapping in case the caller didn't */
 739         if (ep->dma && _req->dma == DMA_ADDR_INVALID) {
 740                 _req->dma = pci_map_single(dev->pdev, _req->buf, _req->length,
 741                         ep->is_in ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE);
 742                 req->mapped = 1;
 743         }
 744
 745 #ifdef USB_TRACE
 746         VDBG(dev, "%s queue req %p, len %u buf %p\n",
 747                         _ep->name, _req, _req->length, _req->buf);
 748 #endif
 749
 750         spin_lock_irqsave(&dev->lock, flags);
 751
 752         _req->status = -EINPROGRESS;
 753         _req->actual = 0;
 754
 755         /* for ep0 IN without premature status, zlp is required and
 756          * writing EOP starts the status stage (OUT).
 757          */
 758         if (unlikely(ep->num == 0 && ep->is_in))
 759                 _req->zero = 1;
 760
 761         /* kickstart this i/o queue? */
 762         status = 0;
 763         if (list_empty(&ep->queue) && likely(!ep->stopped)) {
 764                 /* dma:  done after dma completion IRQ (or error)
 765                  * pio:  done after last fifo operation
 766                  */
 767                 if (ep->dma)
 768                         status = start_dma(ep, req);
 769                 else
 770                         status = (ep->is_in ? write_fifo : read_fifo)(ep, req);
 771
 772                 if (unlikely(status != 0)) {
 773                         if (status > 0)
 774                                 status = 0;
 775                         req = NULL;
 776                 }
 777
 778         } /* else pio or dma irq handler advances the queue. */
 779
 780         if (likely(req != 0))
 781                 list_add_tail(&req->queue, &ep->queue);
 782
 783         if (likely(!list_empty(&ep->queue))
 784                         && likely(ep->num != 0)
 785                         && !ep->dma
 786                         && !(dev->int_enable & INT_EPxDATASET (ep->num)))
 787                 pio_irq_enable(dev, dev->regs, ep->num);
 788
 789         spin_unlock_irqrestore(&dev->lock, flags);
 790
 791         /* pci writes may still be posted */
 792         return status;
 793 }
 794
 795 /* dequeue ALL requests */
 796 static void nuke(struct goku_ep *ep, int status)
 797 {
 798         struct goku_request     *req;
 799
 800         ep->stopped = 1;
 801         if (list_empty(&ep->queue))
 802                 return;
 803         if (ep->dma)
 804                 abort_dma(ep, status);
 805         while (!list_empty(&ep->queue)) {
 806                 req = list_entry(ep->queue.next, struct goku_request, queue);
 807                 done(ep, req, status);
 808         }
 809 }
 810
 811 /* dequeue JUST ONE request */
 812 static int goku_dequeue(struct usb_ep *_ep, struct usb_request *_req)
 813 {
 814         struct goku_request     *req;
 815         struct goku_ep          *ep;
 816         struct goku_udc         *dev;
 817         unsigned long           flags;
 818
 819         ep = container_of(_ep, struct goku_ep, ep);
 820         if (!_ep || !_req || (!ep->desc && ep->num != 0))
 821                 return -EINVAL;
 822         dev = ep->dev;
 823         if (!dev->driver)
 824                 return -ESHUTDOWN;
 825
 826         /* we can't touch (dma) registers when suspended */
 827         if (dev->ep0state == EP0_SUSPEND)
 828                 return -EBUSY;
 829
 830         VDBG(dev, "%s %s %s %s %p\n", __func__, _ep->name,
 831                 ep->is_in ? "IN" : "OUT",
 832                 ep->dma ? "dma" : "pio",
 833                 _req);
 834
 835         spin_lock_irqsave(&dev->lock, flags);
 836
 837         /* make sure it's actually queued on this endpoint */
 838         list_for_each_entry (req, &ep->queue, queue) {
 839                 if (&req->req == _req)
 840                         break;
 841         }
 842         if (&req->req != _req) {
 843                 spin_unlock_irqrestore (&dev->lock, flags);
 844                 return -EINVAL;
 845         }
 846
 847         if (ep->dma && ep->queue.next == &req->queue && !ep->stopped) {
 848                 abort_dma(ep, -ECONNRESET);
 849                 done(ep, req, -ECONNRESET);
 850                 dma_advance(dev, ep);
 851         } else if (!list_empty(&req->queue))
 852                 done(ep, req, -ECONNRESET);
 853         else
 854                 req = NULL;
 855         spin_unlock_irqrestore(&dev->lock, flags);
 856
 857         return req ? 0 : -EOPNOTSUPP;
 858 }
 859
 860 /*-------------------------------------------------------------------------*/
 861
 862 static void goku_clear_halt(struct goku_ep *ep)
 863 {
 864         // assert (ep->num !=0)
 865         VDBG(ep->dev, "%s clear halt\n", ep->ep.name);
 866         command(ep->dev->regs, COMMAND_SETDATA0, ep->num);
 867         command(ep->dev->regs, COMMAND_STALL_CLEAR, ep->num);
 868         if (ep->stopped) {
 869                 ep->stopped = 0;
 870                 if (ep->dma) {
 871                         struct goku_request     *req;
 872
 873                         if (list_empty(&ep->queue))
 874                                 return;
 875                         req = list_entry(ep->queue.next, struct goku_request,
 876                                                 queue);
 877                         (void) start_dma(ep, req);
 878                 } else
 879                         pio_advance(ep);
 880         }
 881 }
 882
 883 static int goku_set_halt(struct usb_ep *_ep, int value)
 884 {
 885         struct goku_ep  *ep;
 886         unsigned long   flags;
 887         int             retval = 0;
 888
 889         if (!_ep)
 890                 return -ENODEV;
 891         ep = container_of (_ep, struct goku_ep, ep);
 892
 893         if (ep->num == 0) {
 894                 if (value) {
 895                         ep->dev->ep0state = EP0_STALL;
 896                         ep->dev->ep[0].stopped = 1;
 897                 } else
 898                         return -EINVAL;
 899
 900         /* don't change EPxSTATUS_EP_INVALID to READY */
 901         } else if (!ep->desc) {
 902                 DBG(ep->dev, "%s %s inactive?\n", __func__, ep->ep.name);
 903                 return -EINVAL;
 904         }
 905
 906         spin_lock_irqsave(&ep->dev->lock, flags);
 907         if (!list_empty(&ep->queue))
 908                 retval = -EAGAIN;
 909         else if (ep->is_in && value
 910                         /* data in (either) packet buffer? */
 911                         && (readl(&ep->dev->regs->DataSet)
 912                                         & DATASET_AB(ep->num)))
 913                 retval = -EAGAIN;
 914         else if (!value)
 915                 goku_clear_halt(ep);
 916         else {
 917                 ep->stopped = 1;
 918                 VDBG(ep->dev, "%s set halt\n", ep->ep.name);
 919                 command(ep->dev->regs, COMMAND_STALL, ep->num);
 920                 readl(ep->reg_status);
 921         }
 922         spin_unlock_irqrestore(&ep->dev->lock, flags);
 923         return retval;
 924 }
 925
 926 static int goku_fifo_status(struct usb_ep *_ep)
 927 {
 928         struct goku_ep                  *ep;
 929         struct goku_udc_regs __iomem    *regs;
 930         u32                             size;
 931
 932         if (!_ep)
 933                 return -ENODEV;
 934         ep = container_of(_ep, struct goku_ep, ep);
 935
 936         /* size is only reported sanely for OUT */
 937         if (ep->is_in)
 938                 return -EOPNOTSUPP;
 939
 940         /* ignores 16-byte dma buffer; SizeH == 0 */
 941         regs = ep->dev->regs;
 942         size = readl(&regs->EPxSizeLA[ep->num]) & DATASIZE;
 943         size += readl(&regs->EPxSizeLB[ep->num]) & DATASIZE;
 944         VDBG(ep->dev, "%s %s %u\n", __func__, ep->ep.name, size);
 945         return size;
 946 }
 947
 948 static void goku_fifo_flush(struct usb_ep *_ep)
 949 {
 950         struct goku_ep                  *ep;
 951         struct goku_udc_regs __iomem    *regs;
 952         u32                             size;
 953
 954         if (!_ep)
 955                 return;
 956         ep = container_of(_ep, struct goku_ep, ep);
 957         VDBG(ep->dev, "%s %s\n", __func__, ep->ep.name);
 958
 959         /* don't change EPxSTATUS_EP_INVALID to READY */
 960         if (!ep->desc && ep->num != 0) {
 961                 DBG(ep->dev, "%s %s inactive?\n", __func__, ep->ep.name);
 962                 return;
 963         }
 964
 965         regs = ep->dev->regs;
 966         size = readl(&regs->EPxSizeLA[ep->num]);
 967         size &= DATASIZE;
 968
 969         /* Non-desirable behavior:  FIFO_CLEAR also clears the
 970          * endpoint halt feature.  For OUT, we _could_ just read
 971          * the bytes out (PIO, if !ep->dma); for in, no choice.
 972          */
 973         if (size)
 974                 command(regs, COMMAND_FIFO_CLEAR, ep->num);
 975 }
 976
 977 static struct usb_ep_ops goku_ep_ops = {
 978         .enable         = goku_ep_enable,
 979         .disable        = goku_ep_disable,
 980
 981         .alloc_request  = goku_alloc_request,
 982         .free_request   = goku_free_request,
 983
 984         .queue          = goku_queue,
 985         .dequeue        = goku_dequeue,
 986
 987         .set_halt       = goku_set_halt,
 988         .fifo_status    = goku_fifo_status,
 989         .fifo_flush     = goku_fifo_flush,
 990 };
 991
 992 /*-------------------------------------------------------------------------*/
 993
 994 static int goku_get_frame(struct usb_gadget *_gadget)
 995 {
 996         return -EOPNOTSUPP;
 997 }
 998
 999 static int goku_start(struct usb_gadget_driver *driver,
1000                 int (*bind)(struct usb_gadget *));
1001 static int goku_stop(struct usb_gadget_driver *driver);
1002
1003 static const struct usb_gadget_ops goku_ops = {
1004         .get_frame      = goku_get_frame,
1005         .start          = goku_start,
1006         .stop           = goku_stop,
1007         // no remote wakeup
1008         // not selfpowered
1009 };
1010
1011 /*-------------------------------------------------------------------------*/
1012
1013 static inline char *dmastr(void)
1014 {
1015         if (use_dma == 0)
1016                 return "(dma disabled)";
1017         else if (use_dma == 2)
1018                 return "(dma IN and OUT)";
1019         else
1020                 return "(dma IN)";
1021 }
1022
1023 #ifdef CONFIG_USB_GADGET_DEBUG_FILES
1024
1025 static const char proc_node_name [] = "driver/udc";
1026
1027 #define FOURBITS "%s%s%s%s"
1028 #define EIGHTBITS FOURBITS FOURBITS
1029
1030 static void
1031 dump_intmask(const char *label, u32 mask, char **next, unsigned *size)
1032 {
1033         int t;
1034
1035         /* int_status is the same format ... */
1036         t = scnprintf(*next, *size,
1037                 "%s %05X =" FOURBITS EIGHTBITS EIGHTBITS "\n",
1038                 label, mask,
1039                 (mask & INT_PWRDETECT) ? " power" : "",
1040                 (mask & INT_SYSERROR) ? " sys" : "",
1041                 (mask & INT_MSTRDEND) ? " in-dma" : "",
1042                 (mask & INT_MSTWRTMOUT) ? " wrtmo" : "",
1043
1044                 (mask & INT_MSTWREND) ? " out-dma" : "",
1045                 (mask & INT_MSTWRSET) ? " wrset" : "",
1046                 (mask & INT_ERR) ? " err" : "",
1047                 (mask & INT_SOF) ? " sof" : "",
1048
1049                 (mask & INT_EP3NAK) ? " ep3nak" : "",
1050                 (mask & INT_EP2NAK) ? " ep2nak" : "",
1051                 (mask & INT_EP1NAK) ? " ep1nak" : "",
1052                 (mask & INT_EP3DATASET) ? " ep3" : "",
1053
1054                 (mask & INT_EP2DATASET) ? " ep2" : "",
1055                 (mask & INT_EP1DATASET) ? " ep1" : "",
1056                 (mask & INT_STATUSNAK) ? " ep0snak" : "",
1057                 (mask & INT_STATUS) ? " ep0status" : "",
1058
1059                 (mask & INT_SETUP) ? " setup" : "",
1060                 (mask & INT_ENDPOINT0) ? " ep0" : "",
1061                 (mask & INT_USBRESET) ? " reset" : "",
1062                 (mask & INT_SUSPEND) ? " suspend" : "");
1063         *size -= t;
1064         *next += t;
1065 }
1066
1067
1068 static int
1069 udc_proc_read(char *buffer, char **start, off_t off, int count,
1070                 int *eof, void *_dev)
1071 {
1072         char                            *buf = buffer;
1073         struct goku_udc                 *dev = _dev;
1074         struct goku_udc_regs __iomem    *regs = dev->regs;
1075         char                            *next = buf;
1076         unsigned                        size = count;
1077         unsigned long                   flags;
1078         int                             i, t, is_usb_connected;
1079         u32                             tmp;
1080
1081         if (off != 0)
1082                 return 0;
1083
1084         local_irq_save(flags);
1085
1086         /* basic device status */
1087         tmp = readl(&regs->power_detect);
1088         is_usb_connected = tmp & PW_DETECT;
1089         t = scnprintf(next, size,
1090                 "%s - %s\n"
1091                 "%s version: %s %s\n"
1092                 "Gadget driver: %s\n"
1093                 "Host %s, %s\n"
1094                 "\n",
1095                 pci_name(dev->pdev), driver_desc,
1096                 driver_name, DRIVER_VERSION, dmastr(),
1097                 dev->driver ? dev->driver->driver.name : "(none)",
1098                 is_usb_connected
1099                         ? ((tmp & PW_PULLUP) ? "full speed" : "powered")
1100                         : "disconnected",
1101                 ({char *state;
1102                 switch(dev->ep0state){
1103                 case EP0_DISCONNECT:    state = "ep0_disconnect"; break;
1104                 case EP0_IDLE:          state = "ep0_idle"; break;
1105                 case EP0_IN:            state = "ep0_in"; break;
1106                 case EP0_OUT:           state = "ep0_out"; break;
1107                 case EP0_STATUS:        state = "ep0_status"; break;
1108                 case EP0_STALL:         state = "ep0_stall"; break;
1109                 case EP0_SUSPEND:       state = "ep0_suspend"; break;
1110                 default:                state = "ep0_?"; break;
1111                 } state; })
1112                 );
1113         size -= t;
1114         next += t;
1115
1116         dump_intmask("int_status", readl(&regs->int_status), &next, &size);
1117         dump_intmask("int_enable", readl(&regs->int_enable), &next, &size);
1118
1119         if (!is_usb_connected || !dev->driver || (tmp & PW_PULLUP) == 0)
1120                 goto done;
1121
1122         /* registers for (active) device and ep0 */
1123         t = scnprintf(next, size, "\nirqs %lu\ndataset %02x "
1124                         "single.bcs %02x.%02x state %x addr %u\n",
1125                         dev->irqs, readl(&regs->DataSet),
1126                         readl(&regs->EPxSingle), readl(&regs->EPxBCS),
1127                         readl(&regs->UsbState),
1128                         readl(&regs->address));
1129         size -= t;
1130         next += t;
1131
1132         tmp = readl(&regs->dma_master);
1133         t = scnprintf(next, size,
1134                 "dma %03X =" EIGHTBITS "%s %s\n", tmp,
1135                 (tmp & MST_EOPB_DIS) ? " eopb-" : "",
1136                 (tmp & MST_EOPB_ENA) ? " eopb+" : "",
1137                 (tmp & MST_TIMEOUT_DIS) ? " tmo-" : "",
1138                 (tmp & MST_TIMEOUT_ENA) ? " tmo+" : "",
1139
1140                 (tmp & MST_RD_EOPB) ? " eopb" : "",
1141                 (tmp & MST_RD_RESET) ? " in_reset" : "",
1142                 (tmp & MST_WR_RESET) ? " out_reset" : "",
1143                 (tmp & MST_RD_ENA) ? " IN" : "",
1144
1145                 (tmp & MST_WR_ENA) ? " OUT" : "",
1146                 (tmp & MST_CONNECTION)
1147                         ? "ep1in/ep2out"
1148                         : "ep1out/ep2in");
1149         size -= t;
1150         next += t;
1151
1152         /* dump endpoint queues */
1153         for (i = 0; i < 4; i++) {
1154                 struct goku_ep          *ep = &dev->ep [i];
1155                 struct goku_request     *req;
1156
1157                 if (i && !ep->desc)
1158                         continue;
1159
1160                 tmp = readl(ep->reg_status);
1161                 t = scnprintf(next, size,
1162                         "%s %s max %u %s, irqs %lu, "
1163                         "status %02x (%s) " FOURBITS "\n",
1164                         ep->ep.name,
1165                         ep->is_in ? "in" : "out",
1166                         ep->ep.maxpacket,
1167                         ep->dma ? "dma" : "pio",
1168                         ep->irqs,
1169                         tmp, ({ char *s;
1170                         switch (tmp & EPxSTATUS_EP_MASK) {
1171                         case EPxSTATUS_EP_READY:
1172                                 s = "ready"; break;
1173                         case EPxSTATUS_EP_DATAIN:
1174                                 s = "packet"; break;
1175                         case EPxSTATUS_EP_FULL:
1176                                 s = "full"; break;
1177                         case EPxSTATUS_EP_TX_ERR:       // host will retry
1178                                 s = "tx_err"; break;
1179                         case EPxSTATUS_EP_RX_ERR:
1180                                 s = "rx_err"; break;
1181                         case EPxSTATUS_EP_BUSY:         /* ep0 only */
1182                                 s = "busy"; break;
1183                         case EPxSTATUS_EP_STALL:
1184                                 s = "stall"; break;
1185                         case EPxSTATUS_EP_INVALID:      // these "can't happen"
1186                                 s = "invalid"; break;
1187                         default:
1188                                 s = "?"; break;
1189                         }; s; }),
1190                         (tmp & EPxSTATUS_TOGGLE) ? "data1" : "data0",
1191                         (tmp & EPxSTATUS_SUSPEND) ? " suspend" : "",
1192                         (tmp & EPxSTATUS_FIFO_DISABLE) ? " disable" : "",
1193                         (tmp & EPxSTATUS_STAGE_ERROR) ? " ep0stat" : ""
1194                         );
1195                 if (t <= 0 || t > size)
1196                         goto done;
1197                 size -= t;
1198                 next += t;
1199
1200                 if (list_empty(&ep->queue)) {
1201                         t = scnprintf(next, size, "\t(nothing queued)\n");
1202                         if (t <= 0 || t > size)
1203                                 goto done;
1204                         size -= t;
1205                         next += t;
1206                         continue;
1207                 }
1208                 list_for_each_entry(req, &ep->queue, queue) {
1209                         if (ep->dma && req->queue.prev == &ep->queue) {
1210                                 if (i == UDC_MSTRD_ENDPOINT)
1211                                         tmp = readl(&regs->in_dma_current);
1212                                 else
1213                                         tmp = readl(&regs->out_dma_current);
1214                                 tmp -= req->req.dma;
1215                                 tmp++;
1216                         } else
1217                                 tmp = req->req.actual;
1218
1219                         t = scnprintf(next, size,
1220                                 "\treq %p len %u/%u buf %p\n",
1221                                 &req->req, tmp, req->req.length,
1222                                 req->req.buf);
1223                         if (t <= 0 || t > size)
1224                                 goto done;
1225                         size -= t;
1226                         next += t;
1227                 }
1228         }
1229
1230 done:
1231         local_irq_restore(flags);
1232         *eof = 1;
1233         return count - size;
1234 }
1235
1236 #endif  /* CONFIG_USB_GADGET_DEBUG_FILES */
1237
1238 /*-------------------------------------------------------------------------*/
1239
1240 static void udc_reinit (struct goku_udc *dev)
1241 {
1242         static char *names [] = { "ep0", "ep1-bulk", "ep2-bulk", "ep3-bulk" };
1243
1244         unsigned i;
1245
1246         INIT_LIST_HEAD (&dev->gadget.ep_list);
1247         dev->gadget.ep0 = &dev->ep [0].ep;
1248         dev->gadget.speed = USB_SPEED_UNKNOWN;
1249         dev->ep0state = EP0_DISCONNECT;
1250         dev->irqs = 0;
1251
1252         for (i = 0; i < 4; i++) {
1253                 struct goku_ep  *ep = &dev->ep[i];
1254
1255                 ep->num = i;
1256                 ep->ep.name = names[i];
1257                 ep->reg_fifo = &dev->regs->ep_fifo [i];
1258                 ep->reg_status = &dev->regs->ep_status [i];
1259                 ep->reg_mode = &dev->regs->ep_mode[i];
1260
1261                 ep->ep.ops = &goku_ep_ops;
1262                 list_add_tail (&ep->ep.ep_list, &dev->gadget.ep_list);
1263                 ep->dev = dev;
1264                 INIT_LIST_HEAD (&ep->queue);
1265
1266                 ep_reset(NULL, ep);
1267         }
1268
1269         dev->ep[0].reg_mode = NULL;
1270         dev->ep[0].ep.maxpacket = MAX_EP0_SIZE;
1271         list_del_init (&dev->ep[0].ep.ep_list);
1272 }
1273
1274 static void udc_reset(struct goku_udc *dev)
1275 {
1276         struct goku_udc_regs __iomem    *regs = dev->regs;
1277
1278         writel(0, &regs->power_detect);
1279         writel(0, &regs->int_enable);
1280         readl(&regs->int_enable);
1281         dev->int_enable = 0;
1282
1283         /* deassert reset, leave USB D+ at hi-Z (no pullup)
1284          * don't let INT_PWRDETECT sequence begin
1285          */
1286         udelay(250);
1287         writel(PW_RESETB, &regs->power_detect);
1288         readl(&regs->int_enable);
1289 }
1290
1291 static void ep0_start(struct goku_udc *dev)
1292 {
1293         struct goku_udc_regs __iomem    *regs = dev->regs;
1294         unsigned                        i;
1295
1296         VDBG(dev, "%s\n", __func__);
1297
1298         udc_reset(dev);
1299         udc_reinit (dev);
1300         //writel(MST_EOPB_ENA | MST_TIMEOUT_ENA, &regs->dma_master);
1301
1302         /* hw handles set_address, set_feature, get_status; maybe more */
1303         writel(   G_REQMODE_SET_INTF | G_REQMODE_GET_INTF
1304                 | G_REQMODE_SET_CONF | G_REQMODE_GET_CONF
1305                 | G_REQMODE_GET_DESC
1306                 | G_REQMODE_CLEAR_FEAT
1307                 , &regs->reqmode);
1308
1309         for (i = 0; i < 4; i++)
1310                 dev->ep[i].irqs = 0;
1311
1312         /* can't modify descriptors after writing UsbReady */
1313         for (i = 0; i < DESC_LEN; i++)
1314                 writel(0, &regs->descriptors[i]);
1315         writel(0, &regs->UsbReady);
1316
1317         /* expect ep0 requests when the host drops reset */
1318         writel(PW_RESETB | PW_PULLUP, &regs->power_detect);
1319         dev->int_enable = INT_DEVWIDE | INT_EP0;
1320         writel(dev->int_enable, &dev->regs->int_enable);
1321         readl(&regs->int_enable);
1322         dev->gadget.speed = USB_SPEED_FULL;
1323         dev->ep0state = EP0_IDLE;
1324 }
1325
1326 static void udc_enable(struct goku_udc *dev)
1327 {
1328         /* start enumeration now, or after power detect irq */
1329         if (readl(&dev->regs->power_detect) & PW_DETECT)
1330                 ep0_start(dev);
1331         else {
1332                 DBG(dev, "%s\n", __func__);
1333                 dev->int_enable = INT_PWRDETECT;
1334                 writel(dev->int_enable, &dev->regs->int_enable);
1335         }
1336 }
1337
1338 /*-------------------------------------------------------------------------*/
1339
1340 /* keeping it simple:
1341  * - one bus driver, initted first;
1342  * - one function driver, initted second
1343  */
1344
1345 static struct goku_udc  *the_controller;
1346
1347 /* when a driver is successfully registered, it will receive
1348  * control requests including set_configuration(), which enables
1349  * non-control requests.  then usb traffic follows until a
1350  * disconnect is reported.  then a host may connect again, or
1351  * the driver might get unbound.
1352  */
1353 static int goku_start(struct usb_gadget_driver *driver,
1354                 int (*bind)(struct usb_gadget *))
1355 {
1356         struct goku_udc *dev = the_controller;
1357         int                     retval;
1358
1359         if (!driver
1360                         || driver->speed < USB_SPEED_FULL
1361                         || !bind
1362                         || !driver->disconnect
1363                         || !driver->setup)
1364                 return -EINVAL;
1365         if (!dev)
1366                 return -ENODEV;
1367         if (dev->driver)
1368                 return -EBUSY;
1369
1370         /* hook up the driver */
1371         driver->driver.bus = NULL;
1372         dev->driver = driver;
1373         dev->gadget.dev.driver = &driver->driver;
1374         retval = bind(&dev->gadget);
1375         if (retval) {
1376                 DBG(dev, "bind to driver %s --> error %d\n",
1377                                 driver->driver.name, retval);
1378                 dev->driver = NULL;
1379                 dev->gadget.dev.driver = NULL;
1380                 return retval;
1381         }
1382
1383         /* then enable host detection and ep0; and we're ready
1384          * for set_configuration as well as eventual disconnect.
1385          */
1386         udc_enable(dev);
1387
1388         DBG(dev, "registered gadget driver '%s'\n", driver->driver.name);
1389         return 0;
1390 }
1391
1392 static void
1393 stop_activity(struct goku_udc *dev, struct usb_gadget_driver *driver)
1394 {
1395         unsigned        i;
1396
1397         DBG (dev, "%s\n", __func__);
1398
1399         if (dev->gadget.speed == USB_SPEED_UNKNOWN)
1400                 driver = NULL;
1401
1402         /* disconnect gadget driver after quiesceing hw and the driver */
1403         udc_reset (dev);
1404         for (i = 0; i < 4; i++)
1405                 nuke(&dev->ep [i], -ESHUTDOWN);
1406         if (driver) {
1407                 spin_unlock(&dev->lock);
1408                 driver->disconnect(&dev->gadget);
1409                 spin_lock(&dev->lock);
1410         }
1411
1412         if (dev->driver)
1413                 udc_enable(dev);
1414 }
1415
1416 static int goku_stop(struct usb_gadget_driver *driver)
1417 {
1418         struct goku_udc *dev = the_controller;
1419         unsigned long   flags;
1420
1421         if (!dev)
1422                 return -ENODEV;
1423         if (!driver || driver != dev->driver || !driver->unbind)
1424                 return -EINVAL;
1425
1426         spin_lock_irqsave(&dev->lock, flags);
1427         dev->driver = NULL;
1428         stop_activity(dev, driver);
1429         spin_unlock_irqrestore(&dev->lock, flags);
1430
1431         driver->unbind(&dev->gadget);
1432         dev->gadget.dev.driver = NULL;
1433
1434         DBG(dev, "unregistered driver '%s'\n", driver->driver.name);
1435         return 0;
1436 }
1437
1438 /*-------------------------------------------------------------------------*/
1439
1440 static void ep0_setup(struct goku_udc *dev)
1441 {
1442         struct goku_udc_regs __iomem    *regs = dev->regs;
1443         struct usb_ctrlrequest          ctrl;
1444         int                             tmp;
1445
1446         /* read SETUP packet and enter DATA stage */
1447         ctrl.bRequestType = readl(&regs->bRequestType);
1448         ctrl.bRequest = readl(&regs->bRequest);
1449         ctrl.wValue  = cpu_to_le16((readl(&regs->wValueH)  << 8)
1450                                         | readl(&regs->wValueL));
1451         ctrl.wIndex  = cpu_to_le16((readl(&regs->wIndexH)  << 8)
1452                                         | readl(&regs->wIndexL));
1453         ctrl.wLength = cpu_to_le16((readl(&regs->wLengthH) << 8)
1454                                         | readl(&regs->wLengthL));
1455         writel(0, &regs->SetupRecv);
1456
1457         nuke(&dev->ep[0], 0);
1458         dev->ep[0].stopped = 0;
1459         if (likely(ctrl.bRequestType & USB_DIR_IN)) {
1460                 dev->ep[0].is_in = 1;
1461                 dev->ep0state = EP0_IN;
1462                 /* detect early status stages */
1463                 writel(ICONTROL_STATUSNAK, &dev->regs->IntControl);
1464         } else {
1465                 dev->ep[0].is_in = 0;
1466                 dev->ep0state = EP0_OUT;
1467
1468                 /* NOTE:  CLEAR_FEATURE is done in software so that we can
1469                  * synchronize transfer restarts after bulk IN stalls.  data
1470                  * won't even enter the fifo until the halt is cleared.
1471                  */
1472                 switch (ctrl.bRequest) {
1473                 case USB_REQ_CLEAR_FEATURE:
1474                         switch (ctrl.bRequestType) {
1475                         case USB_RECIP_ENDPOINT:
1476                                 tmp = le16_to_cpu(ctrl.wIndex) & 0x0f;
1477                                 /* active endpoint */
1478                                 if (tmp > 3 || (!dev->ep[tmp].desc && tmp != 0))
1479                                         goto stall;
1480                                 if (ctrl.wIndex & cpu_to_le16(
1481                                                 USB_DIR_IN)) {
1482                                         if (!dev->ep[tmp].is_in)
1483                                                 goto stall;
1484                                 } else {
1485                                         if (dev->ep[tmp].is_in)
1486                                                 goto stall;
1487                                 }
1488                                 if (ctrl.wValue != cpu_to_le16(
1489                                                 USB_ENDPOINT_HALT))
1490                                         goto stall;
1491                                 if (tmp)
1492                                         goku_clear_halt(&dev->ep[tmp]);
1493 succeed:
1494                                 /* start ep0out status stage */
1495                                 writel(~(1<<0), &regs->EOP);
1496                                 dev->ep[0].stopped = 1;
1497                                 dev->ep0state = EP0_STATUS;
1498                                 return;
1499                         case USB_RECIP_DEVICE:
1500                                 /* device remote wakeup: always clear */
1501                                 if (ctrl.wValue != cpu_to_le16(1))
1502                                         goto stall;
1503                                 VDBG(dev, "clear dev remote wakeup\n");
1504                                 goto succeed;
1505                         case USB_RECIP_INTERFACE:
1506                                 goto stall;
1507                         default:                /* pass to gadget driver */
1508                                 break;
1509                         }
1510                         break;
1511                 default:
1512                         break;
1513                 }
1514         }
1515
1516 #ifdef USB_TRACE
1517         VDBG(dev, "SETUP %02x.%02x v%04x i%04x l%04x\n",
1518                 ctrl.bRequestType, ctrl.bRequest,
1519                 le16_to_cpu(ctrl.wValue), le16_to_cpu(ctrl.wIndex),
1520                 le16_to_cpu(ctrl.wLength));
1521 #endif
1522
1523         /* hw wants to know when we're configured (or not) */
1524         dev->req_config = (ctrl.bRequest == USB_REQ_SET_CONFIGURATION
1525                                 && ctrl.bRequestType == USB_RECIP_DEVICE);
1526         if (unlikely(dev->req_config))
1527                 dev->configured = (ctrl.wValue != cpu_to_le16(0));
1528
1529         /* delegate everything to the gadget driver.
1530          * it may respond after this irq handler returns.
1531          */
1532         spin_unlock (&dev->lock);
1533         tmp = dev->driver->setup(&dev->gadget, &ctrl);
1534         spin_lock (&dev->lock);
1535         if (unlikely(tmp < 0)) {
1536 stall:
1537 #ifdef USB_TRACE
1538                 VDBG(dev, "req %02x.%02x protocol STALL; err %d\n",
1539                                 ctrl.bRequestType, ctrl.bRequest, tmp);
1540 #endif
1541                 command(regs, COMMAND_STALL, 0);
1542                 dev->ep[0].stopped = 1;
1543                 dev->ep0state = EP0_STALL;
1544         }
1545
1546         /* expect at least one data or status stage irq */
1547 }
1548
1549 #define ACK(irqbit) { \
1550                 stat &= ~irqbit; \
1551                 writel(~irqbit, &regs->int_status); \
1552                 handled = 1; \
1553                 }
1554
1555 static irqreturn_t goku_irq(int irq, void *_dev)
1556 {
1557         struct goku_udc                 *dev = _dev;
1558         struct goku_udc_regs __iomem    *regs = dev->regs;
1559         struct goku_ep                  *ep;
1560         u32                             stat, handled = 0;
1561         unsigned                        i, rescans = 5;
1562
1563         spin_lock(&dev->lock);
1564
1565 rescan:
1566         stat = readl(&regs->int_status) & dev->int_enable;
1567         if (!stat)
1568                 goto done;
1569         dev->irqs++;
1570
1571         /* device-wide irqs */
1572         if (unlikely(stat & INT_DEVWIDE)) {
1573                 if (stat & INT_SYSERROR) {
1574                         ERROR(dev, "system error\n");
1575                         stop_activity(dev, dev->driver);
1576                         stat = 0;
1577                         handled = 1;
1578                         // FIXME have a neater way to prevent re-enumeration
1579                         dev->driver = NULL;
1580                         goto done;
1581                 }
1582                 if (stat & INT_PWRDETECT) {
1583                         writel(~stat, &regs->int_status);
1584                         if (readl(&dev->regs->power_detect) & PW_DETECT) {
1585                                 VDBG(dev, "connect\n");
1586                                 ep0_start(dev);
1587                         } else {
1588                                 DBG(dev, "disconnect\n");
1589                                 if (dev->gadget.speed == USB_SPEED_FULL)
1590                                         stop_activity(dev, dev->driver);
1591                                 dev->ep0state = EP0_DISCONNECT;
1592                                 dev->int_enable = INT_DEVWIDE;
1593                                 writel(dev->int_enable, &dev->regs->int_enable);
1594                         }
1595                         stat = 0;
1596                         handled = 1;
1597                         goto done;
1598                 }
1599                 if (stat & INT_SUSPEND) {
1600                         ACK(INT_SUSPEND);
1601                         if (readl(&regs->ep_status[0]) & EPxSTATUS_SUSPEND) {
1602                                 switch (dev->ep0state) {
1603                                 case EP0_DISCONNECT:
1604                                 case EP0_SUSPEND:
1605                                         goto pm_next;
1606                                 default:
1607                                         break;
1608                                 }
1609                                 DBG(dev, "USB suspend\n");
1610                                 dev->ep0state = EP0_SUSPEND;
1611                                 if (dev->gadget.speed != USB_SPEED_UNKNOWN
1612                                                 && dev->driver
1613                                                 && dev->driver->suspend) {
1614                                         spin_unlock(&dev->lock);
1615                                         dev->driver->suspend(&dev->gadget);
1616                                         spin_lock(&dev->lock);
1617                                 }
1618                         } else {
1619                                 if (dev->ep0state != EP0_SUSPEND) {
1620                                         DBG(dev, "bogus USB resume %d\n",
1621                                                 dev->ep0state);
1622                                         goto pm_next;
1623                                 }
1624                                 DBG(dev, "USB resume\n");
1625                                 dev->ep0state = EP0_IDLE;
1626                                 if (dev->gadget.speed != USB_SPEED_UNKNOWN
1627                                                 && dev->driver
1628                                                 && dev->driver->resume) {
1629                                         spin_unlock(&dev->lock);
1630                                         dev->driver->resume(&dev->gadget);
1631                                         spin_lock(&dev->lock);
1632                                 }
1633                         }
1634                 }
1635 pm_next:
1636                 if (stat & INT_USBRESET) {              /* hub reset done */
1637                         ACK(INT_USBRESET);
1638                         INFO(dev, "USB reset done, gadget %s\n",
1639                                 dev->driver->driver.name);
1640                 }
1641                 // and INT_ERR on some endpoint's crc/bitstuff/... problem
1642         }
1643
1644         /* progress ep0 setup, data, or status stages.
1645          * no transition {EP0_STATUS, EP0_STALL} --> EP0_IDLE; saves irqs
1646          */
1647         if (stat & INT_SETUP) {
1648                 ACK(INT_SETUP);
1649                 dev->ep[0].irqs++;
1650                 ep0_setup(dev);
1651         }
1652         if (stat & INT_STATUSNAK) {
1653                 ACK(INT_STATUSNAK|INT_ENDPOINT0);
1654                 if (dev->ep0state == EP0_IN) {
1655                         ep = &dev->ep[0];
1656                         ep->irqs++;
1657                         nuke(ep, 0);
1658                         writel(~(1<<0), &regs->EOP);
1659                         dev->ep0state = EP0_STATUS;
1660                 }
1661         }
1662         if (stat & INT_ENDPOINT0) {
1663                 ACK(INT_ENDPOINT0);
1664                 ep = &dev->ep[0];
1665                 ep->irqs++;
1666                 pio_advance(ep);
1667         }
1668
1669         /* dma completion */
1670         if (stat & INT_MSTRDEND) {      /* IN */
1671                 ACK(INT_MSTRDEND);
1672                 ep = &dev->ep[UDC_MSTRD_ENDPOINT];
1673                 ep->irqs++;
1674                 dma_advance(dev, ep);
1675         }
1676         if (stat & INT_MSTWREND) {      /* OUT */
1677                 ACK(INT_MSTWREND);
1678                 ep = &dev->ep[UDC_MSTWR_ENDPOINT];
1679                 ep->irqs++;
1680                 dma_advance(dev, ep);
1681         }
1682         if (stat & INT_MSTWRTMOUT) {    /* OUT */
1683                 ACK(INT_MSTWRTMOUT);
1684                 ep = &dev->ep[UDC_MSTWR_ENDPOINT];
1685                 ep->irqs++;
1686                 ERROR(dev, "%s write timeout ?\n", ep->ep.name);
1687                 // reset dma? then dma_advance()
1688         }
1689
1690         /* pio */
1691         for (i = 1; i < 4; i++) {
1692                 u32             tmp = INT_EPxDATASET(i);
1693
1694                 if (!(stat & tmp))
1695                         continue;
1696                 ep = &dev->ep[i];
1697                 pio_advance(ep);
1698                 if (list_empty (&ep->queue))
1699                         pio_irq_disable(dev, regs, i);
1700                 stat &= ~tmp;
1701                 handled = 1;
1702                 ep->irqs++;
1703         }
1704
1705         if (rescans--)
1706                 goto rescan;
1707
1708 done:
1709         (void)readl(&regs->int_enable);
1710         spin_unlock(&dev->lock);
1711         if (stat)
1712                 DBG(dev, "unhandled irq status: %05x (%05x, %05x)\n", stat,
1713                                 readl(&regs->int_status), dev->int_enable);
1714         return IRQ_RETVAL(handled);
1715 }
1716
1717 #undef ACK
1718
1719 /*-------------------------------------------------------------------------*/
1720
1721 static void gadget_release(struct device *_dev)
1722 {
1723         struct goku_udc *dev = dev_get_drvdata(_dev);
1724
1725         kfree(dev);
1726 }
1727
1728 /* tear down the binding between this driver and the pci device */
1729
1730 static void goku_remove(struct pci_dev *pdev)
1731 {
1732         struct goku_udc         *dev = pci_get_drvdata(pdev);
1733
1734         DBG(dev, "%s\n", __func__);
1735
1736         usb_del_gadget_udc(&dev->gadget);
1737
1738         BUG_ON(dev->driver);
1739
1740 #ifdef CONFIG_USB_GADGET_DEBUG_FILES
1741         remove_proc_entry(proc_node_name, NULL);
1742 #endif
1743         if (dev->regs)
1744                 udc_reset(dev);
1745         if (dev->got_irq)
1746                 free_irq(pdev->irq, dev);
1747         if (dev->regs)
1748                 iounmap(dev->regs);
1749         if (dev->got_region)
1750                 release_mem_region(pci_resource_start (pdev, 0),
1751                                 pci_resource_len (pdev, 0));
1752         if (dev->enabled)
1753                 pci_disable_device(pdev);
1754         if (dev->registered)
1755                 device_unregister(&dev->gadget.dev);
1756
1757         pci_set_drvdata(pdev, NULL);
1758         dev->regs = NULL;
1759         the_controller = NULL;
1760
1761         INFO(dev, "unbind\n");
1762 }
1763
1764 /* wrap this driver around the specified pci device, but
1765  * don't respond over USB until a gadget driver binds to us.
1766  */
1767
1768 static int goku_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1769 {
1770         struct goku_udc         *dev = NULL;
1771         unsigned long           resource, len;
1772         void __iomem            *base = NULL;
1773         int                     retval;
1774
1775         /* if you want to support more than one controller in a system,
1776          * usb_gadget_driver_{register,unregister}() must change.
1777          */
1778         if (the_controller) {
1779                 pr_warning("ignoring %s\n", pci_name(pdev));
1780                 return -EBUSY;
1781         }
1782         if (!pdev->irq) {
1783                 printk(KERN_ERR "Check PCI %s IRQ setup!\n", pci_name(pdev));
1784                 retval = -ENODEV;
1785                 goto err;
1786         }
1787
1788         /* alloc, and start init */
1789         dev = kzalloc (sizeof *dev, GFP_KERNEL);
1790         if (dev == NULL){
1791                 pr_debug("enomem %s\n", pci_name(pdev));
1792                 retval = -ENOMEM;
1793                 goto err;
1794         }
1795
1796         spin_lock_init(&dev->lock);
1797         dev->pdev = pdev;
1798         dev->gadget.ops = &goku_ops;
1799
1800         /* the "gadget" abstracts/virtualizes the controller */
1801         dev_set_name(&dev->gadget.dev, "gadget");
1802         dev->gadget.dev.parent = &pdev->dev;
1803         dev->gadget.dev.dma_mask = pdev->dev.dma_mask;
1804         dev->gadget.dev.release = gadget_release;
1805         dev->gadget.name = driver_name;
1806
1807         /* now all the pci goodies ... */
1808         retval = pci_enable_device(pdev);
1809         if (retval < 0) {
1810                 DBG(dev, "can't enable, %d\n", retval);
1811                 goto err;
1812         }
1813         dev->enabled = 1;
1814
1815         resource = pci_resource_start(pdev, 0);
1816         len = pci_resource_len(pdev, 0);
1817         if (!request_mem_region(resource, len, driver_name)) {
1818                 DBG(dev, "controller already in use\n");
1819                 retval = -EBUSY;
1820                 goto err;
1821         }
1822         dev->got_region = 1;
1823
1824         base = ioremap_nocache(resource, len);
1825         if (base == NULL) {
1826                 DBG(dev, "can't map memory\n");
1827                 retval = -EFAULT;
1828                 goto err;
1829         }
1830         dev->regs = (struct goku_udc_regs __iomem *) base;
1831
1832         pci_set_drvdata(pdev, dev);
1833         INFO(dev, "%s\n", driver_desc);
1834         INFO(dev, "version: " DRIVER_VERSION " %s\n", dmastr());
1835         INFO(dev, "irq %d, pci mem %p\n", pdev->irq, base);
1836
1837         /* init to known state, then setup irqs */
1838         udc_reset(dev);
1839         udc_reinit (dev);
1840         if (request_irq(pdev->irq, goku_irq, IRQF_SHARED/*|IRQF_SAMPLE_RANDOM*/,
1841                         driver_name, dev) != 0) {
1842                 DBG(dev, "request interrupt %d failed\n", pdev->irq);
1843                 retval = -EBUSY;
1844                 goto err;
1845         }
1846         dev->got_irq = 1;
1847         if (use_dma)
1848                 pci_set_master(pdev);
1849
1850
1851 #ifdef CONFIG_USB_GADGET_DEBUG_FILES
1852         create_proc_read_entry(proc_node_name, 0, NULL, udc_proc_read, dev);
1853 #endif
1854
1855         the_controller = dev;
1856         retval = device_register(&dev->gadget.dev);
1857         if (retval) {
1858                 put_device(&dev->gadget.dev);
1859                 goto err;
1860         }
1861         dev->registered = 1;
1862         retval = usb_add_gadget_udc(&pdev->dev, &dev->gadget);
1863         if (retval)
1864                 goto err;
1865
1866         return 0;
1867
1868 err:
1869         if (dev)
1870                 goku_remove (pdev);
1871         return retval;
1872 }
1873
1874
1875 /*-------------------------------------------------------------------------*/
1876
1877 static const struct pci_device_id pci_ids[] = { {
1878         .class =        ((PCI_CLASS_SERIAL_USB << 8) | 0xfe),
1879         .class_mask =   ~0,
1880         .vendor =       0x102f,         /* Toshiba */
1881         .device =       0x0107,         /* this UDC */
1882         .subvendor =    PCI_ANY_ID,
1883         .subdevice =    PCI_ANY_ID,
1884
1885 }, { /* end: all zeroes */ }
1886 };
1887 MODULE_DEVICE_TABLE (pci, pci_ids);
1888
1889 static struct pci_driver goku_pci_driver = {
1890         .name =         (char *) driver_name,
1891         .id_table =     pci_ids,
1892
1893         .probe =        goku_probe,
1894         .remove =       goku_remove,
1895
1896         /* FIXME add power management support */
1897 };
1898
1899 static int __init init (void)
1900 {
1901         return pci_register_driver (&goku_pci_driver);
1902 }
1903 module_init (init);
1904
1905 static void __exit cleanup (void)
1906 {
1907         pci_unregister_driver (&goku_pci_driver);
1908 }
1909 module_exit (cleanup);