Lines Matching refs:priv_ep
76 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
79 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
116 struct cdns3_endpoint *priv_ep)
120 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
187 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
190 u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
192 return priv_ep->trb_pool_dma + offset;
195 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
197 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
199 if (priv_ep->trb_pool) {
201 priv_ep->trb_pool, priv_ep->trb_pool_dma);
202 priv_ep->trb_pool = NULL;
208 * @priv_ep: endpoint object
212 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
214 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
219 if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
220 cdns3_free_trb_pool(priv_ep);
222 if (!priv_ep->trb_pool) {
223 priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool,
225 &priv_ep->trb_pool_dma);
227 if (!priv_ep->trb_pool)
230 priv_ep->alloc_ring_size = ring_size;
233 memset(priv_ep->trb_pool, 0, ring_size);
235 priv_ep->num_trbs = num_trbs;
237 if (!priv_ep->num)
241 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
243 if (priv_ep->use_streams) {
250 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
258 * @priv_ep: endpoint object
262 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
264 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
267 trace_cdns3_halt(priv_ep, 1, 1);
275 priv_ep->flags |= EP_STALLED;
276 priv_ep->flags &= ~EP_STALL_PENDING;
323 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
325 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
327 priv_ep->free_trbs--;
328 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
333 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
335 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
337 priv_ep->free_trbs++;
338 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
379 * @priv_ep: The endpoint for whom request will be started.
385 struct cdns3_endpoint *priv_ep)
389 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
399 request = cdns3_next_request(&priv_ep->pending_req_list);
402 (priv_ep->flags & EP_TDLCHK_EN) ||
403 priv_ep->use_streams) {
409 while (!list_empty(&priv_ep->deferred_req_list)) {
410 request = cdns3_next_request(&priv_ep->deferred_req_list);
412 if (!priv_ep->use_streams) {
413 ret = cdns3_ep_run_transfer(priv_ep, request);
415 priv_ep->stream_sg_idx = 0;
416 ret = cdns3_ep_run_stream_transfer(priv_ep, request);
421 list_move_tail(&request->list, &priv_ep->pending_req_list);
422 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
426 priv_ep->flags &= ~EP_RING_FULL;
436 #define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
437 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
438 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
475 * @priv_ep: extended endpoint object
478 static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
484 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
488 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
500 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
501 --priv_ep->wa2_counter;
509 struct cdns3_endpoint *priv_ep,
512 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
516 req = cdns3_next_request(&priv_ep->deferred_req_list);
518 priv_ep->descmis_req = NULL;
525 priv_ep->dir);
527 cdns3_wa2_descmiss_copy_data(priv_ep, req);
528 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
533 usb_endpoint_dir_in(priv_ep->endpoint.desc));
541 cdns3_start_all_request(priv_dev, priv_ep);
549 struct cdns3_endpoint *priv_ep,
559 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
562 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
563 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
566 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
570 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
571 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
572 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
581 cdns3_wa2_descmiss_copy_data(priv_ep,
584 trace_cdns3_wa2(priv_ep, "get internal stored data");
587 &priv_ep->pending_req_list);
588 cdns3_gadget_giveback(priv_ep, priv_req,
604 trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
610 &priv_ep->wa2_descmiss_req_list);
616 static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
620 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
623 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
626 trace_cdns3_wa2(priv_ep, "removes eldest request");
630 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
632 --priv_ep->wa2_counter;
641 * @priv_ep: extended gadget object
646 static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
650 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
654 trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
658 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
659 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
660 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
663 trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
665 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
666 trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
667 cdns3_wa2_remove_old_request(priv_ep);
670 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
684 if (priv_ep->descmis_req)
685 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
689 priv_ep->wa2_counter++;
692 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
697 priv_ep->descmis_req = priv_req;
699 __cdns3_gadget_ep_queue(&priv_ep->endpoint,
700 &priv_ep->descmis_req->request,
706 dev_err(priv_ep->cdns3_dev->dev,
784 * @priv_ep: The endpoint to whom the request belongs to
792 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
796 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
806 priv_ep->dir);
809 priv_ep->dir == USB_DIR_OUT && !request->status) {
825 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
837 usb_gadget_giveback_request(&priv_ep->endpoint,
843 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
846 static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
849 if (priv_ep->wa1_set) {
850 trace_cdns3_wa1(priv_ep, "restore cycle bit");
852 priv_ep->wa1_set = 0;
853 priv_ep->wa1_trb_index = 0xFFFF;
854 if (priv_ep->wa1_cycle_bit) {
855 priv_ep->wa1_trb->control =
856 priv_ep->wa1_trb->control | cpu_to_le32(0x1);
858 priv_ep->wa1_trb->control =
859 priv_ep->wa1_trb->control & cpu_to_le32(~0x1);
895 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
896 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
911 buf->dir = usb_endpoint_dir_in(priv_ep->endpoint.desc) ?
938 if (priv_ep->dir == USB_DIR_IN) {
956 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
959 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
961 if (!priv_ep->wa1_set) {
967 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
968 priv_ep->wa1_set = 1;
969 priv_ep->wa1_trb = trb;
970 priv_ep->wa1_trb_index = priv_ep->enqueue;
971 trace_cdns3_wa1(priv_ep, "set guard");
979 struct cdns3_endpoint *priv_ep)
985 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
987 if (!doorbell || dma_index != priv_ep->wa1_trb_index)
988 cdns3_wa1_restore_cycle_bit(priv_ep);
991 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
994 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1002 unsigned int sg_idx = priv_ep->stream_sg_idx;
1005 address = priv_ep->endpoint.desc->bEndpointAddress;
1007 priv_ep->flags |= EP_PENDING_REQUEST;
1016 trb = priv_ep->trb_pool + priv_ep->enqueue;
1017 priv_req->start_trb = priv_ep->enqueue;
1021 cdns3_select_ep(priv_ep->cdns3_dev, address);
1034 tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1051 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1060 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1063 if (!(priv_ep->flags & EP_STALLED)) {
1064 trace_cdns3_ring(priv_ep);
1068 priv_ep->prime_flag = false;
1081 priv_ep->last_stream_id = priv_req->request.stream_id;
1086 trace_cdns3_doorbell_epx(priv_ep->name,
1096 static void cdns3_rearm_drdy_if_needed(struct cdns3_endpoint *priv_ep)
1098 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1111 * @priv_ep: endpoint object
1116 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1119 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1139 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1140 num_trb = priv_ep->interval * num_trb_req;
1145 address = priv_ep->endpoint.desc->bEndpointAddress;
1147 priv_ep->flags |= EP_PENDING_REQUEST;
1155 trb = priv_ep->trb_pool + priv_ep->enqueue;
1156 priv_req->start_trb = priv_ep->enqueue;
1159 cdns3_select_ep(priv_ep->cdns3_dev, address);
1162 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
1167 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1170 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1171 priv_ep->flags |= EP_DEFERRED_DRDY;
1176 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1185 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1189 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
1192 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1197 while (priv_ep->enqueue) {
1199 trace_cdns3_prepare_trb(priv_ep, trb);
1201 cdns3_ep_inc_enq(priv_ep);
1202 trb = priv_ep->trb_pool + priv_ep->enqueue;
1208 if (num_trb > priv_ep->free_trbs) {
1209 priv_ep->flags |= EP_RING_FULL;
1214 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1217 control = priv_ep->pcs ? 0 : TRB_CYCLE;
1223 priv_ep->endpoint.maxpacket);
1246 if (priv_ep->flags & EP_TDLCHK_EN)
1248 priv_ep->endpoint.maxpacket);
1250 trb_burst = priv_ep->trb_burst_size;
1274 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_dev->dev_ver <= DEV_VER_V2)
1281 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1290 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
1314 priv_req->end_trb = priv_ep->enqueue;
1315 cdns3_ep_inc_enq(priv_ep);
1316 trb = priv_ep->trb_pool + priv_ep->enqueue;
1329 (priv_ep->flags & EP_TDLCHK_EN)) {
1335 priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1355 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1361 trace_cdns3_prepare_trb(priv_ep, trb + i);
1363 trb = priv_ep->trb_pool;
1371 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1384 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1390 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
1391 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1392 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1397 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1401 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1404 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1405 trace_cdns3_ring(priv_ep);
1409 cdns3_rearm_drdy_if_needed(priv_ep);
1410 trace_cdns3_doorbell_epx(priv_ep->name,
1422 struct cdns3_endpoint *priv_ep;
1437 priv_ep = ep_to_cdns3_ep(ep);
1438 cdns3_start_all_request(priv_dev, priv_ep);
1448 * @priv_ep: extended endpoint object.
1457 * DQ = priv_ep->dequeue - dequeue position
1458 * EQ = priv_ep->enqueue - enqueue position
1464 * Then, we check if cycle bit for index priv_ep->dequeue
1468 * 1. priv_ep->dequeue never equals to current_index.
1469 * 2 priv_ep->enqueue never exceed priv_ep->dequeue
1470 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1471 * and priv_ep->free_trbs is zero.
1475 * Case 1 - priv_ep->dequeue < current_index
1479 * Case 2 - priv_ep->dequeue > current_index
1484 static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep,
1487 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1493 current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1498 if (priv_ep->dequeue > priv_req->end_trb)
1501 if (priv_ep->dequeue < priv_req->start_trb)
1506 (priv_ep->dequeue > priv_req->end_trb) &&
1507 (priv_ep->dequeue < priv_req->start_trb))
1511 (priv_ep->dequeue != priv_req->end_trb))
1514 trb = &priv_ep->trb_pool[priv_ep->dequeue];
1516 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs)
1519 if (doorbell == 1 && current_index == priv_ep->dequeue)
1523 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1528 if (priv_ep->enqueue == priv_ep->dequeue &&
1529 priv_ep->free_trbs == 0) {
1531 } else if (priv_ep->dequeue < current_index) {
1532 if ((current_index == (priv_ep->num_trbs - 1)) &&
1533 !priv_ep->dequeue)
1537 } else if (priv_ep->dequeue > current_index) {
1548 struct cdns3_endpoint *priv_ep)
1556 while (!list_empty(&priv_ep->pending_req_list)) {
1557 request = cdns3_next_request(&priv_ep->pending_req_list);
1560 trb = priv_ep->trb_pool + priv_ep->dequeue;
1566 if (priv_ep->dequeue == cdns3_get_dma_pos(priv_dev, priv_ep) &&
1567 priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1570 trace_cdns3_complete_trb(priv_ep, trb);
1571 cdns3_ep_inc_deq(priv_ep);
1572 trb = priv_ep->trb_pool + priv_ep->dequeue;
1579 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1581 while (cdns3_trb_handled(priv_ep, priv_req)) {
1586 trb = priv_ep->trb_pool + priv_ep->dequeue;
1587 trace_cdns3_complete_trb(priv_ep, trb);
1598 cdns3_ep_inc_deq(priv_ep);
1602 /* TRBs are duplicated by priv_ep->interval time for ISO IN */
1603 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_ep->dir)
1604 request->actual /= priv_ep->interval;
1606 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1613 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1620 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1622 trb = priv_ep->trb_pool;
1623 trace_cdns3_complete_trb(priv_ep, trb);
1633 (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1634 priv_ep->stream_sg_idx = 0;
1635 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1637 priv_ep->stream_sg_idx++;
1638 cdns3_ep_run_stream_transfer(priv_ep, request);
1643 priv_ep->flags &= ~EP_PENDING_REQUEST;
1646 if (!(priv_ep->flags & EP_STALLED) &&
1647 !(priv_ep->flags & EP_STALL_PENDING))
1648 cdns3_start_all_request(priv_dev, priv_ep);
1651 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1653 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1655 cdns3_wa1_restore_cycle_bit(priv_ep);
1658 trace_cdns3_ring(priv_ep);
1666 trace_cdns3_doorbell_epx(priv_ep->name,
1671 static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1673 u16 tdl = priv_ep->pending_tdl;
1674 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1678 priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1680 priv_ep->pending_tdl = 0;
1688 * @priv_ep: endpoint object
1692 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1694 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1700 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1702 trace_cdns3_epx_irq(priv_dev, priv_ep);
1707 if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1722 EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1726 priv_ep->prime_flag = true;
1728 pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1729 deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1732 cdns3_start_all_request(priv_dev, priv_ep);
1738 if (priv_ep->flags & EP_STALL_PENDING &&
1741 cdns3_ep_stall_flush(priv_ep);
1751 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1752 !priv_ep->wa1_set) {
1753 if (!priv_ep->dir) {
1758 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1759 priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
1761 cdns3_transfer_completed(priv_dev, priv_ep);
1762 } else if (!(priv_ep->flags & EP_STALLED) &&
1763 !(priv_ep->flags & EP_STALL_PENDING)) {
1764 if (priv_ep->flags & EP_DEFERRED_DRDY) {
1765 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1766 cdns3_start_all_request(priv_dev, priv_ep);
1768 cdns3_rearm_transfer(priv_ep,
1769 priv_ep->wa1_set);
1776 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1778 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1780 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1783 if (!priv_ep->use_streams) {
1786 cdns3_transfer_completed(priv_dev, priv_ep);
1787 } else if ((priv_ep->flags & EP_TDLCHK_EN) &
1788 priv_ep->pending_tdl) {
1790 cdns3_reprogram_tdl(priv_ep);
1792 } else if (priv_ep->dir == USB_DIR_OUT) {
1793 priv_ep->ep_sts_pending |= ep_sts_reg;
1795 cdns3_transfer_completed(priv_dev, priv_ep);
1803 if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1804 (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1805 priv_ep->ep_sts_pending = 0;
1806 cdns3_transfer_completed(priv_dev, priv_ep);
1811 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1812 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1816 !(priv_ep->flags & EP_STALLED))
1817 cdns3_wa2_descmissing_packet(priv_ep);
2058 struct cdns3_endpoint *priv_ep)
2069 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
2072 if (priv_ep->dir)
2073 mask = BIT(priv_ep->num + 16);
2075 mask = BIT(priv_ep->num);
2077 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
2084 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2093 * @priv_ep: extended endpoint object
2096 int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable)
2098 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
2099 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2100 u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
2101 u32 max_packet_size = priv_ep->wMaxPacketSize;
2102 u8 maxburst = priv_ep->bMaxBurst;
2109 cdns3_configure_dmult(priv_dev, priv_ep);
2111 switch (priv_ep->type) {
2115 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2121 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2126 buffering = (priv_ep->bMaxBurst + 1) * (priv_ep->mult + 1) - 1;
2137 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
2148 priv_ep->trb_burst_size = 128;
2150 priv_ep->trb_burst_size = 64;
2152 priv_ep->trb_burst_size = 16;
2165 priv_ep->trb_burst_size = 16;
2173 !!priv_ep->dir);
2183 if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2185 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
2200 EP_CFG_MULT(priv_ep->mult) | /* must match EP setting */
2206 priv_ep->flags |= EP_CONFIGURED;
2209 priv_ep->name, ep_cfg);
2216 struct cdns3_endpoint *priv_ep)
2218 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2219 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2227 struct cdns3_endpoint *priv_ep;
2239 priv_ep = ep_to_cdns3_ep(ep);
2240 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2241 if (!(priv_ep->flags & EP_CLAIMED)) {
2242 priv_ep->num = num;
2243 return priv_ep;
2273 struct cdns3_endpoint *priv_ep;
2276 priv_ep = cdns3_find_available_ep(priv_dev, desc);
2277 if (IS_ERR(priv_ep)) {
2282 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2285 priv_ep->endpoint.desc = desc;
2286 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2287 priv_ep->type = usb_endpoint_type(desc);
2288 priv_ep->flags |= EP_CLAIMED;
2289 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2290 priv_ep->wMaxPacketSize = usb_endpoint_maxp(desc);
2291 priv_ep->mult = USB_EP_MAXP_MULT(priv_ep->wMaxPacketSize);
2292 priv_ep->wMaxPacketSize &= USB_ENDPOINT_MAXP_MASK;
2293 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && comp_desc) {
2294 priv_ep->mult = USB_SS_MULT(comp_desc->bmAttributes) - 1;
2295 priv_ep->bMaxBurst = comp_desc->bMaxBurst;
2299 return &priv_ep->endpoint;
2312 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2319 priv_req->priv_ep = priv_ep;
2352 struct cdns3_endpoint *priv_ep;
2367 priv_ep = ep_to_cdns3_ep(ep);
2368 priv_dev = priv_ep->cdns3_dev;
2369 comp_desc = priv_ep->endpoint.comp_desc;
2381 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2382 "%s is already enabled\n", priv_ep->name))
2387 priv_ep->endpoint.desc = desc;
2388 priv_ep->type = usb_endpoint_type(desc);
2389 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2391 if (priv_ep->interval > ISO_MAX_INTERVAL &&
2392 priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2400 bEndpointAddress = priv_ep->num | priv_ep->dir;
2411 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2423 priv_ep->use_streams = true;
2424 ret = cdns3_ep_config(priv_ep, enable);
2428 ret = cdns3_ep_config(priv_ep, enable);
2434 ret = cdns3_allocate_trb_pool(priv_ep);
2438 bEndpointAddress = priv_ep->num | priv_ep->dir;
2441 trace_cdns3_gadget_ep_enable(priv_ep);
2450 cdns3_free_trb_pool(priv_ep);
2460 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2465 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2467 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2468 priv_ep->wa1_set = 0;
2469 priv_ep->enqueue = 0;
2470 priv_ep->dequeue = 0;
2472 priv_ep->pcs = !!EP_STS_CCS(reg);
2473 priv_ep->ccs = !!EP_STS_CCS(reg);
2475 priv_ep->free_trbs = priv_ep->num_trbs - 1;
2490 struct cdns3_endpoint *priv_ep;
2504 priv_ep = ep_to_cdns3_ep(ep);
2505 priv_dev = priv_ep->cdns3_dev;
2507 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2508 "%s is already disabled\n", priv_ep->name))
2513 trace_cdns3_gadget_ep_disable(priv_ep);
2535 priv_ep->name);
2537 while (!list_empty(&priv_ep->pending_req_list)) {
2538 request = cdns3_next_request(&priv_ep->pending_req_list);
2540 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2544 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2545 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2549 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2551 --priv_ep->wa2_counter;
2554 while (!list_empty(&priv_ep->deferred_req_list)) {
2555 request = cdns3_next_request(&priv_ep->deferred_req_list);
2557 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2561 priv_ep->descmis_req = NULL;
2564 priv_ep->flags &= ~EP_ENABLED;
2565 priv_ep->use_streams = false;
2584 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2585 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2595 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2613 list_add_tail(&request->list, &priv_ep->deferred_req_list);
2624 !(priv_ep->flags & EP_STALLED) &&
2625 !(priv_ep->flags & EP_STALL_PENDING))
2626 cdns3_start_all_request(priv_dev, priv_ep);
2628 if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2629 cdns3_start_all_request(priv_dev, priv_ep);
2639 struct cdns3_endpoint *priv_ep;
2647 priv_ep = ep_to_cdns3_ep(ep);
2648 priv_dev = priv_ep->cdns3_dev;
2666 priv_ep->name);
2684 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2697 priv_dev = priv_ep->cdns3_dev;
2707 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2715 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2735 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma +
2740 if (priv_ep->wa1_trb == priv_req->trb)
2741 cdns3_wa1_restore_cycle_bit(priv_ep);
2744 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2746 req = cdns3_next_request(&priv_ep->pending_req_list);
2748 cdns3_rearm_transfer(priv_ep, 1);
2758 * @priv_ep: endpoint object to set stall on.
2760 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2762 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2764 trace_cdns3_halt(priv_ep, 1, 0);
2766 if (!(priv_ep->flags & EP_STALLED)) {
2770 cdns3_ep_stall_flush(priv_ep);
2772 priv_ep->flags |= EP_STALL_PENDING;
2779 * @priv_ep: endpoint object to clear stall on
2781 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2783 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2791 trace_cdns3_halt(priv_ep, 0, 0);
2793 request = cdns3_next_request(&priv_ep->pending_req_list);
2811 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2817 cdns3_rearm_transfer(priv_ep, 1);
2820 cdns3_start_all_request(priv_dev, priv_ep);
2833 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2834 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2838 if (!(priv_ep->flags & EP_ENABLED))
2846 priv_ep->flags &= ~EP_WEDGE;
2847 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2849 __cdns3_gadget_ep_set_halt(priv_ep);
3034 struct cdns3_endpoint *priv_ep;
3046 priv_ep = ep_to_cdns3_ep(ep);
3047 bEndpointAddress = priv_ep->num | priv_ep->dir;
3053 priv_ep->flags &= ~EP_CLAIMED;
3077 struct cdns3_endpoint *priv_ep;
3086 priv_ep = ep_to_cdns3_ep(ep);
3087 if (!(priv_ep->flags & EP_CLAIMED))
3090 n = (priv_ep->mult + 1) * (priv_ep->bMaxBurst + 1);
3097 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
3102 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
3152 struct cdns3_endpoint *priv_ep;
3177 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
3179 if (!priv_ep)
3183 priv_ep->cdns3_dev = priv_dev;
3184 priv_dev->eps[i] = priv_ep;
3185 priv_ep->num = ep_number;
3186 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
3189 ret = cdns3_init_ep0(priv_dev, priv_ep);
3195 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
3197 priv_ep->endpoint.name = priv_ep->name;
3199 usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
3201 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
3202 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
3204 priv_ep->endpoint.caps.dir_in = 1;
3206 priv_ep->endpoint.caps.dir_out = 1;
3209 priv_ep->endpoint.caps.type_iso = 1;
3211 priv_ep->endpoint.caps.type_bulk = 1;
3212 priv_ep->endpoint.caps.type_int = 1;
3214 list_add_tail(&priv_ep->endpoint.ep_list,
3218 priv_ep->flags = 0;
3221 priv_ep->name,
3222 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
3223 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
3225 INIT_LIST_HEAD(&priv_ep->pending_req_list);
3226 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
3227 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);