1 #include <linux/device.h>
2 #include <linux/dma-mapping.h>
3 #include <linux/dmaengine.h>
4 #include <linux/sizes.h>
5 #include <linux/platform_device.h>
10 #define RNDIS_REG(x) (0x80 + ((x - 1) * 4))
12 #define EP_MODE_AUTOREG_NONE 0
13 #define EP_MODE_AUTOREG_ALL_NEOP 1
14 #define EP_MODE_AUTOREG_ALWAYS 3
16 #define EP_MODE_DMA_TRANSPARENT 0
17 #define EP_MODE_DMA_RNDIS 1
18 #define EP_MODE_DMA_GEN_RNDIS 3
20 #define USB_CTRL_TX_MODE 0x70
21 #define USB_CTRL_RX_MODE 0x74
22 #define USB_CTRL_AUTOREQ 0xd0
23 #define USB_TDOWN 0xd8
25 struct cppi41_dma_channel {
26 struct dma_channel channel;
27 struct cppi41_dma_controller *controller;
28 struct musb_hw_ep *hw_ep;
41 struct list_head tx_check;
42 struct work_struct dma_completion;
45 #define MUSB_DMA_NUM_CHANNELS 15
47 struct cppi41_dma_controller {
48 struct dma_controller controller;
49 struct cppi41_dma_channel rx_channel[MUSB_DMA_NUM_CHANNELS];
50 struct cppi41_dma_channel tx_channel[MUSB_DMA_NUM_CHANNELS];
52 struct hrtimer early_tx;
53 struct list_head early_tx_list;
59 static void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
64 if (cppi41_channel->is_tx)
66 if (!is_host_active(cppi41_channel->controller->musb))
69 csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);
70 toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
72 cppi41_channel->usb_toggle = toggle;
75 static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
80 if (cppi41_channel->is_tx)
82 if (!is_host_active(cppi41_channel->controller->musb))
85 csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);
86 toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
89 * AM335x Advisory 1.0.13: Due to internal synchronisation error the
90 * data toggle may reset from DATA1 to DATA0 during receiving data from
91 * more than one endpoint.
93 if (!toggle && toggle == cppi41_channel->usb_toggle) {
94 csr |= MUSB_RXCSR_H_DATATOGGLE | MUSB_RXCSR_H_WR_DATATOGGLE;
95 musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr);
96 dev_dbg(cppi41_channel->controller->musb->controller,
97 "Restoring DATA1 toggle.\n");
100 cppi41_channel->usb_toggle = toggle;
103 static bool musb_is_tx_fifo_empty(struct musb_hw_ep *hw_ep)
105 u8 epnum = hw_ep->epnum;
106 struct musb *musb = hw_ep->musb;
107 void __iomem *epio = musb->endpoints[epnum].regs;
110 csr = musb_readw(epio, MUSB_TXCSR);
111 if (csr & MUSB_TXCSR_TXPKTRDY)
116 static bool is_isoc(struct musb_hw_ep *hw_ep, bool in)
118 if (in && hw_ep->in_qh) {
119 if (hw_ep->in_qh->type == USB_ENDPOINT_XFER_ISOC)
121 } else if (hw_ep->out_qh) {
122 if (hw_ep->out_qh->type == USB_ENDPOINT_XFER_ISOC)
128 static void cppi41_dma_callback(void *private_data);
130 static void cppi41_trans_done(struct cppi41_dma_channel *cppi41_channel)
132 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
133 struct musb *musb = hw_ep->musb;
135 if (!cppi41_channel->prog_len) {
138 cppi41_channel->channel.actual_len =
139 cppi41_channel->transferred;
140 cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
141 musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx);
143 /* next iteration, reload */
144 struct dma_chan *dc = cppi41_channel->dc;
145 struct dma_async_tx_descriptor *dma_desc;
146 enum dma_transfer_direction direction;
149 void __iomem *epio = cppi41_channel->hw_ep->regs;
151 cppi41_channel->buf_addr += cppi41_channel->packet_sz;
153 remain_bytes = cppi41_channel->total_len;
154 remain_bytes -= cppi41_channel->transferred;
155 remain_bytes = min(remain_bytes, cppi41_channel->packet_sz);
156 cppi41_channel->prog_len = remain_bytes;
158 direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV
160 dma_desc = dmaengine_prep_slave_single(dc,
161 cppi41_channel->buf_addr,
164 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
165 if (WARN_ON(!dma_desc))
168 dma_desc->callback = cppi41_dma_callback;
169 dma_desc->callback_param = &cppi41_channel->channel;
170 cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
171 dma_async_issue_pending(dc);
173 if (!cppi41_channel->is_tx) {
174 csr = musb_readw(epio, MUSB_RXCSR);
175 csr |= MUSB_RXCSR_H_REQPKT;
176 musb_writew(epio, MUSB_RXCSR, csr);
181 static void cppi_trans_done_work(struct work_struct *work)
184 struct cppi41_dma_channel *cppi41_channel =
185 container_of(work, struct cppi41_dma_channel, dma_completion);
186 struct cppi41_dma_controller *controller = cppi41_channel->controller;
187 struct musb *musb = controller->musb;
188 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
191 if (!cppi41_channel->is_tx && is_isoc(hw_ep, 1)) {
192 spin_lock_irqsave(&musb->lock, flags);
193 cppi41_trans_done(cppi41_channel);
194 spin_unlock_irqrestore(&musb->lock, flags);
196 empty = musb_is_tx_fifo_empty(hw_ep);
198 spin_lock_irqsave(&musb->lock, flags);
199 cppi41_trans_done(cppi41_channel);
200 spin_unlock_irqrestore(&musb->lock, flags);
202 schedule_work(&cppi41_channel->dma_completion);
207 static enum hrtimer_restart cppi41_recheck_tx_req(struct hrtimer *timer)
209 struct cppi41_dma_controller *controller;
210 struct cppi41_dma_channel *cppi41_channel, *n;
213 enum hrtimer_restart ret = HRTIMER_NORESTART;
215 controller = container_of(timer, struct cppi41_dma_controller,
217 musb = controller->musb;
219 spin_lock_irqsave(&musb->lock, flags);
220 list_for_each_entry_safe(cppi41_channel, n, &controller->early_tx_list,
223 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
225 empty = musb_is_tx_fifo_empty(hw_ep);
227 list_del_init(&cppi41_channel->tx_check);
228 cppi41_trans_done(cppi41_channel);
232 if (!list_empty(&controller->early_tx_list)) {
233 ret = HRTIMER_RESTART;
234 hrtimer_forward_now(&controller->early_tx,
235 ktime_set(0, 150 * NSEC_PER_USEC));
238 spin_unlock_irqrestore(&musb->lock, flags);
242 static void cppi41_dma_callback(void *private_data)
244 struct dma_channel *channel = private_data;
245 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
246 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
247 struct musb *musb = hw_ep->musb;
249 struct dma_tx_state txstate;
253 spin_lock_irqsave(&musb->lock, flags);
255 dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie,
257 transferred = cppi41_channel->prog_len - txstate.residue;
258 cppi41_channel->transferred += transferred;
260 dev_dbg(musb->controller, "DMA transfer done on hw_ep=%d bytes=%d/%d\n",
261 hw_ep->epnum, cppi41_channel->transferred,
262 cppi41_channel->total_len);
264 update_rx_toggle(cppi41_channel);
266 if (cppi41_channel->transferred == cppi41_channel->total_len ||
267 transferred < cppi41_channel->packet_sz)
268 cppi41_channel->prog_len = 0;
270 if (!cppi41_channel->is_tx) {
271 if (is_isoc(hw_ep, 1))
272 schedule_work(&cppi41_channel->dma_completion);
274 cppi41_trans_done(cppi41_channel);
278 empty = musb_is_tx_fifo_empty(hw_ep);
280 cppi41_trans_done(cppi41_channel);
282 struct cppi41_dma_controller *controller;
284 * On AM335x it has been observed that the TX interrupt fires
285 * too early that means the TXFIFO is not yet empty but the DMA
286 * engine says that it is done with the transfer. We don't
287 * receive a FIFO empty interrupt so the only thing we can do is
288 * to poll for the bit. On HS it usually takes 2us, on FS around
289 * 110us - 150us depending on the transfer size.
290 * We spin on HS (no longer than than 25us and setup a timer on
291 * FS to check for the bit and complete the transfer.
293 controller = cppi41_channel->controller;
295 if (musb->g.speed == USB_SPEED_HIGH) {
299 empty = musb_is_tx_fifo_empty(hw_ep);
308 empty = musb_is_tx_fifo_empty(hw_ep);
310 cppi41_trans_done(cppi41_channel);
314 if (is_isoc(hw_ep, 0)) {
315 schedule_work(&cppi41_channel->dma_completion);
318 list_add_tail(&cppi41_channel->tx_check,
319 &controller->early_tx_list);
320 if (!hrtimer_active(&controller->early_tx)) {
321 hrtimer_start_range_ns(&controller->early_tx,
322 ktime_set(0, 140 * NSEC_PER_USEC),
328 spin_unlock_irqrestore(&musb->lock, flags);
331 static u32 update_ep_mode(unsigned ep, unsigned mode, u32 old)
335 shift = (ep - 1) * 2;
336 old &= ~(3 << shift);
337 old |= mode << shift;
341 static void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,
344 struct cppi41_dma_controller *controller = cppi41_channel->controller;
349 if (cppi41_channel->is_tx)
350 old_mode = controller->tx_mode;
352 old_mode = controller->rx_mode;
353 port = cppi41_channel->port_num;
354 new_mode = update_ep_mode(port, mode, old_mode);
356 if (new_mode == old_mode)
358 if (cppi41_channel->is_tx) {
359 controller->tx_mode = new_mode;
360 musb_writel(controller->musb->ctrl_base, USB_CTRL_TX_MODE,
363 controller->rx_mode = new_mode;
364 musb_writel(controller->musb->ctrl_base, USB_CTRL_RX_MODE,
369 static void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel,
372 struct cppi41_dma_controller *controller = cppi41_channel->controller;
377 old_mode = controller->auto_req;
378 port = cppi41_channel->port_num;
379 new_mode = update_ep_mode(port, mode, old_mode);
381 if (new_mode == old_mode)
383 controller->auto_req = new_mode;
384 musb_writel(controller->musb->ctrl_base, USB_CTRL_AUTOREQ, new_mode);
387 static bool cppi41_configure_channel(struct dma_channel *channel,
388 u16 packet_sz, u8 mode,
389 dma_addr_t dma_addr, u32 len)
391 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
392 struct dma_chan *dc = cppi41_channel->dc;
393 struct dma_async_tx_descriptor *dma_desc;
394 enum dma_transfer_direction direction;
395 struct musb *musb = cppi41_channel->controller->musb;
396 unsigned use_gen_rndis = 0;
398 dev_dbg(musb->controller,
399 "configure ep%d/%x packet_sz=%d, mode=%d, dma_addr=0x%llx, len=%d is_tx=%d\n",
400 cppi41_channel->port_num, RNDIS_REG(cppi41_channel->port_num),
401 packet_sz, mode, (unsigned long long) dma_addr,
402 len, cppi41_channel->is_tx);
404 cppi41_channel->buf_addr = dma_addr;
405 cppi41_channel->total_len = len;
406 cppi41_channel->transferred = 0;
407 cppi41_channel->packet_sz = packet_sz;
410 * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more
411 * than max packet size at a time.
413 if (cppi41_channel->is_tx)
418 if (len > packet_sz) {
419 musb_writel(musb->ctrl_base,
420 RNDIS_REG(cppi41_channel->port_num), len);
422 cppi41_set_dma_mode(cppi41_channel,
423 EP_MODE_DMA_GEN_RNDIS);
426 cppi41_set_autoreq_mode(cppi41_channel,
427 EP_MODE_AUTOREG_ALL_NEOP);
429 musb_writel(musb->ctrl_base,
430 RNDIS_REG(cppi41_channel->port_num), 0);
431 cppi41_set_dma_mode(cppi41_channel,
432 EP_MODE_DMA_TRANSPARENT);
433 cppi41_set_autoreq_mode(cppi41_channel,
434 EP_MODE_AUTOREG_NONE);
438 cppi41_set_dma_mode(cppi41_channel, EP_MODE_DMA_TRANSPARENT);
439 cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREG_NONE);
440 len = min_t(u32, packet_sz, len);
442 cppi41_channel->prog_len = len;
443 direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
444 dma_desc = dmaengine_prep_slave_single(dc, dma_addr, len, direction,
445 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
449 dma_desc->callback = cppi41_dma_callback;
450 dma_desc->callback_param = channel;
451 cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
453 save_rx_toggle(cppi41_channel);
454 dma_async_issue_pending(dc);
458 static struct dma_channel *cppi41_dma_channel_allocate(struct dma_controller *c,
459 struct musb_hw_ep *hw_ep, u8 is_tx)
461 struct cppi41_dma_controller *controller = container_of(c,
462 struct cppi41_dma_controller, controller);
463 struct cppi41_dma_channel *cppi41_channel = NULL;
464 u8 ch_num = hw_ep->epnum - 1;
466 if (ch_num >= MUSB_DMA_NUM_CHANNELS)
470 cppi41_channel = &controller->tx_channel[ch_num];
472 cppi41_channel = &controller->rx_channel[ch_num];
474 if (!cppi41_channel->dc)
477 if (cppi41_channel->is_allocated)
480 cppi41_channel->hw_ep = hw_ep;
481 cppi41_channel->is_allocated = 1;
483 return &cppi41_channel->channel;
486 static void cppi41_dma_channel_release(struct dma_channel *channel)
488 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
490 if (cppi41_channel->is_allocated) {
491 cppi41_channel->is_allocated = 0;
492 channel->status = MUSB_DMA_STATUS_FREE;
493 channel->actual_len = 0;
497 static int cppi41_dma_channel_program(struct dma_channel *channel,
498 u16 packet_sz, u8 mode,
499 dma_addr_t dma_addr, u32 len)
502 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
505 BUG_ON(channel->status == MUSB_DMA_STATUS_UNKNOWN ||
506 channel->status == MUSB_DMA_STATUS_BUSY);
508 if (is_host_active(cppi41_channel->controller->musb)) {
509 if (cppi41_channel->is_tx)
510 hb_mult = cppi41_channel->hw_ep->out_qh->hb_mult;
512 hb_mult = cppi41_channel->hw_ep->in_qh->hb_mult;
515 channel->status = MUSB_DMA_STATUS_BUSY;
516 channel->actual_len = 0;
519 packet_sz = hb_mult * (packet_sz & 0x7FF);
521 ret = cppi41_configure_channel(channel, packet_sz, mode, dma_addr, len);
523 channel->status = MUSB_DMA_STATUS_FREE;
528 static int cppi41_is_compatible(struct dma_channel *channel, u16 maxpacket,
529 void *buf, u32 length)
531 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
532 struct cppi41_dma_controller *controller = cppi41_channel->controller;
533 struct musb *musb = controller->musb;
535 if (is_host_active(musb)) {
539 if (cppi41_channel->hw_ep->ep_in.type != USB_ENDPOINT_XFER_BULK)
541 if (cppi41_channel->is_tx)
543 /* AM335x Advisory 1.0.13. No workaround for device RX mode */
547 static int cppi41_dma_channel_abort(struct dma_channel *channel)
549 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
550 struct cppi41_dma_controller *controller = cppi41_channel->controller;
551 struct musb *musb = controller->musb;
552 void __iomem *epio = cppi41_channel->hw_ep->regs;
558 is_tx = cppi41_channel->is_tx;
559 dev_dbg(musb->controller, "abort channel=%d, is_tx=%d\n",
560 cppi41_channel->port_num, is_tx);
562 if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)
565 list_del_init(&cppi41_channel->tx_check);
567 csr = musb_readw(epio, MUSB_TXCSR);
568 csr &= ~MUSB_TXCSR_DMAENAB;
569 musb_writew(epio, MUSB_TXCSR, csr);
571 csr = musb_readw(epio, MUSB_RXCSR);
572 csr &= ~(MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_DMAENAB);
573 musb_writew(epio, MUSB_RXCSR, csr);
575 csr = musb_readw(epio, MUSB_RXCSR);
576 if (csr & MUSB_RXCSR_RXPKTRDY) {
577 csr |= MUSB_RXCSR_FLUSHFIFO;
578 musb_writew(epio, MUSB_RXCSR, csr);
579 musb_writew(epio, MUSB_RXCSR, csr);
583 tdbit = 1 << cppi41_channel->port_num;
588 musb_writel(musb->ctrl_base, USB_TDOWN, tdbit);
589 ret = dmaengine_terminate_all(cppi41_channel->dc);
590 } while (ret == -EAGAIN);
592 musb_writel(musb->ctrl_base, USB_TDOWN, tdbit);
595 csr = musb_readw(epio, MUSB_TXCSR);
596 if (csr & MUSB_TXCSR_TXPKTRDY) {
597 csr |= MUSB_TXCSR_FLUSHFIFO;
598 musb_writew(epio, MUSB_TXCSR, csr);
602 cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
606 static void cppi41_release_all_dma_chans(struct cppi41_dma_controller *ctrl)
611 for (i = 0; i < MUSB_DMA_NUM_CHANNELS; i++) {
612 dc = ctrl->tx_channel[i].dc;
614 dma_release_channel(dc);
615 dc = ctrl->rx_channel[i].dc;
617 dma_release_channel(dc);
621 static void cppi41_dma_controller_stop(struct cppi41_dma_controller *controller)
623 cppi41_release_all_dma_chans(controller);
626 static int cppi41_dma_controller_start(struct cppi41_dma_controller *controller)
628 struct musb *musb = controller->musb;
629 struct device *dev = musb->controller;
630 struct device_node *np = dev->of_node;
631 struct cppi41_dma_channel *cppi41_channel;
636 count = of_property_count_strings(np, "dma-names");
640 for (i = 0; i < count; i++) {
642 struct dma_channel *musb_dma;
647 ret = of_property_read_string_index(np, "dma-names", i, &str);
650 if (!strncmp(str, "tx", 2))
652 else if (!strncmp(str, "rx", 2))
655 dev_err(dev, "Wrong dmatype %s\n", str);
658 ret = kstrtouint(str + 2, 0, &port);
663 if (port > MUSB_DMA_NUM_CHANNELS || !port)
666 cppi41_channel = &controller->tx_channel[port - 1];
668 cppi41_channel = &controller->rx_channel[port - 1];
670 cppi41_channel->controller = controller;
671 cppi41_channel->port_num = port;
672 cppi41_channel->is_tx = is_tx;
673 INIT_LIST_HEAD(&cppi41_channel->tx_check);
674 INIT_WORK(&cppi41_channel->dma_completion,
675 cppi_trans_done_work);
677 musb_dma = &cppi41_channel->channel;
678 musb_dma->private_data = cppi41_channel;
679 musb_dma->status = MUSB_DMA_STATUS_FREE;
680 musb_dma->max_len = SZ_4M;
682 dc = dma_request_slave_channel(dev, str);
684 dev_err(dev, "Failed to request %s.\n", str);
688 cppi41_channel->dc = dc;
692 cppi41_release_all_dma_chans(controller);
696 void dma_controller_destroy(struct dma_controller *c)
698 struct cppi41_dma_controller *controller = container_of(c,
699 struct cppi41_dma_controller, controller);
701 hrtimer_cancel(&controller->early_tx);
702 cppi41_dma_controller_stop(controller);
706 struct dma_controller *dma_controller_create(struct musb *musb,
709 struct cppi41_dma_controller *controller;
712 if (!musb->controller->of_node) {
713 dev_err(musb->controller, "Need DT for the DMA engine.\n");
717 controller = kzalloc(sizeof(*controller), GFP_KERNEL);
721 hrtimer_init(&controller->early_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
722 controller->early_tx.function = cppi41_recheck_tx_req;
723 INIT_LIST_HEAD(&controller->early_tx_list);
724 controller->musb = musb;
726 controller->controller.channel_alloc = cppi41_dma_channel_allocate;
727 controller->controller.channel_release = cppi41_dma_channel_release;
728 controller->controller.channel_program = cppi41_dma_channel_program;
729 controller->controller.channel_abort = cppi41_dma_channel_abort;
730 controller->controller.is_compatible = cppi41_is_compatible;
732 ret = cppi41_dma_controller_start(controller);
735 return &controller->controller;
740 if (ret == -EPROBE_DEFER)