Lines Matching refs:priv_ep
74 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
77 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
114 struct cdns3_endpoint *priv_ep)
118 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
185 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
188 u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
190 return priv_ep->trb_pool_dma + offset;
193 static int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
195 switch (priv_ep->type) {
201 if (priv_ep->use_streams)
208 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
210 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
212 if (priv_ep->trb_pool) {
214 cdns3_ring_size(priv_ep),
215 priv_ep->trb_pool, priv_ep->trb_pool_dma);
216 priv_ep->trb_pool = NULL;
222 * @priv_ep: endpoint object
226 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
228 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
229 int ring_size = cdns3_ring_size(priv_ep);
233 if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
234 cdns3_free_trb_pool(priv_ep);
236 if (!priv_ep->trb_pool) {
237 priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
239 &priv_ep->trb_pool_dma,
241 if (!priv_ep->trb_pool)
244 priv_ep->alloc_ring_size = ring_size;
247 memset(priv_ep->trb_pool, 0, ring_size);
249 priv_ep->num_trbs = num_trbs;
251 if (!priv_ep->num)
255 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
257 if (priv_ep->use_streams) {
264 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
272 * @priv_ep: endpoint object
276 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
278 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
281 trace_cdns3_halt(priv_ep, 1, 1);
289 priv_ep->flags |= EP_STALLED;
290 priv_ep->flags &= ~EP_STALL_PENDING;
337 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
339 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
341 priv_ep->free_trbs--;
342 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
347 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
349 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
351 priv_ep->free_trbs++;
352 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
393 * @priv_ep: The endpoint for whom request will be started.
399 struct cdns3_endpoint *priv_ep)
403 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
413 request = cdns3_next_request(&priv_ep->pending_req_list);
416 (priv_ep->flags & EP_TDLCHK_EN) ||
417 priv_ep->use_streams) {
423 while (!list_empty(&priv_ep->deferred_req_list)) {
424 request = cdns3_next_request(&priv_ep->deferred_req_list);
426 if (!priv_ep->use_streams) {
427 ret = cdns3_ep_run_transfer(priv_ep, request);
429 priv_ep->stream_sg_idx = 0;
430 ret = cdns3_ep_run_stream_transfer(priv_ep, request);
437 &priv_ep->pending_req_list);
438 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
442 priv_ep->flags &= ~EP_RING_FULL;
452 #define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
453 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
454 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
491 * @priv_ep: extended endpoint object
494 static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
500 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
504 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
516 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
517 --priv_ep->wa2_counter;
525 struct cdns3_endpoint *priv_ep,
528 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
532 req = cdns3_next_request(&priv_ep->deferred_req_list);
534 priv_ep->descmis_req = NULL;
541 priv_ep->dir);
543 cdns3_wa2_descmiss_copy_data(priv_ep, req);
544 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
549 usb_endpoint_dir_in(priv_ep->endpoint.desc));
557 cdns3_start_all_request(priv_dev, priv_ep);
565 struct cdns3_endpoint *priv_ep,
575 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
578 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
579 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
582 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
586 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
587 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
588 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
597 cdns3_wa2_descmiss_copy_data(priv_ep,
600 trace_cdns3_wa2(priv_ep, "get internal stored data");
603 &priv_ep->pending_req_list);
604 cdns3_gadget_giveback(priv_ep, priv_req,
620 trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
626 &priv_ep->wa2_descmiss_req_list);
632 static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
636 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
639 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
642 trace_cdns3_wa2(priv_ep, "removes eldest request");
646 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
648 --priv_ep->wa2_counter;
657 * @priv_ep: extended gadget object
662 static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
666 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
670 trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
674 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
675 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
676 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
679 trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
681 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
682 trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
683 cdns3_wa2_remove_old_request(priv_ep);
686 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
700 if (priv_ep->descmis_req)
701 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
705 priv_ep->wa2_counter++;
708 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
713 priv_ep->descmis_req = priv_req;
715 __cdns3_gadget_ep_queue(&priv_ep->endpoint,
716 &priv_ep->descmis_req->request,
722 dev_err(priv_ep->cdns3_dev->dev,
800 * @priv_ep: The endpoint to whom the request belongs to
808 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
812 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
821 priv_ep->dir);
824 priv_ep->dir == USB_DIR_OUT && !request->status)
834 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
842 usb_gadget_giveback_request(&priv_ep->endpoint,
848 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
851 static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
854 if (priv_ep->wa1_set) {
855 trace_cdns3_wa1(priv_ep, "restore cycle bit");
857 priv_ep->wa1_set = 0;
858 priv_ep->wa1_trb_index = 0xFFFF;
859 if (priv_ep->wa1_cycle_bit) {
860 priv_ep->wa1_trb->control =
861 priv_ep->wa1_trb->control | cpu_to_le32(0x1);
863 priv_ep->wa1_trb->control =
864 priv_ep->wa1_trb->control & cpu_to_le32(~0x1);
900 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
901 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
940 if (priv_ep->dir == USB_DIR_IN) {
951 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
954 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
956 if (!priv_ep->wa1_set) {
962 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
963 priv_ep->wa1_set = 1;
964 priv_ep->wa1_trb = trb;
965 priv_ep->wa1_trb_index = priv_ep->enqueue;
966 trace_cdns3_wa1(priv_ep, "set guard");
974 struct cdns3_endpoint *priv_ep)
980 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
982 if (!doorbell || dma_index != priv_ep->wa1_trb_index)
983 cdns3_wa1_restore_cycle_bit(priv_ep);
986 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
989 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
997 unsigned int sg_idx = priv_ep->stream_sg_idx;
1000 address = priv_ep->endpoint.desc->bEndpointAddress;
1002 priv_ep->flags |= EP_PENDING_REQUEST;
1011 trb = priv_ep->trb_pool + priv_ep->enqueue;
1012 priv_req->start_trb = priv_ep->enqueue;
1016 cdns3_select_ep(priv_ep->cdns3_dev, address);
1029 tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1046 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1055 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1058 if (!(priv_ep->flags & EP_STALLED)) {
1059 trace_cdns3_ring(priv_ep);
1063 priv_ep->prime_flag = false;
1076 priv_ep->last_stream_id = priv_req->request.stream_id;
1081 trace_cdns3_doorbell_epx(priv_ep->name,
1091 static void cdns3_rearm_drdy_if_needed(struct cdns3_endpoint *priv_ep)
1093 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1106 * @priv_ep: endpoint object
1111 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1114 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1134 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1135 num_trb = priv_ep->interval * num_trb_req;
1140 address = priv_ep->endpoint.desc->bEndpointAddress;
1142 priv_ep->flags |= EP_PENDING_REQUEST;
1150 trb = priv_ep->trb_pool + priv_ep->enqueue;
1151 priv_req->start_trb = priv_ep->enqueue;
1154 cdns3_select_ep(priv_ep->cdns3_dev, address);
1157 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
1162 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1165 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1166 priv_ep->flags |= EP_DEFERRED_DRDY;
1171 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1180 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1184 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
1187 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1192 while (priv_ep->enqueue) {
1194 trace_cdns3_prepare_trb(priv_ep, trb);
1196 cdns3_ep_inc_enq(priv_ep);
1197 trb = priv_ep->trb_pool + priv_ep->enqueue;
1203 if (num_trb > priv_ep->free_trbs) {
1204 priv_ep->flags |= EP_RING_FULL;
1209 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1212 control = priv_ep->pcs ? 0 : TRB_CYCLE;
1218 priv_ep->endpoint.maxpacket);
1241 if (priv_ep->flags & EP_TDLCHK_EN)
1243 priv_ep->endpoint.maxpacket);
1245 trb_burst = priv_ep->trb_burst_size;
1269 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_dev->dev_ver <= DEV_VER_V2)
1276 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1285 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
1309 priv_req->end_trb = priv_ep->enqueue;
1310 cdns3_ep_inc_enq(priv_ep);
1311 trb = priv_ep->trb_pool + priv_ep->enqueue;
1324 (priv_ep->flags & EP_TDLCHK_EN)) {
1330 priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1350 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1356 trace_cdns3_prepare_trb(priv_ep, trb + i);
1358 trb = priv_ep->trb_pool;
1366 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1379 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1385 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
1386 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1387 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1392 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1396 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1399 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1400 trace_cdns3_ring(priv_ep);
1404 cdns3_rearm_drdy_if_needed(priv_ep);
1405 trace_cdns3_doorbell_epx(priv_ep->name,
1417 struct cdns3_endpoint *priv_ep;
1432 priv_ep = ep_to_cdns3_ep(ep);
1433 cdns3_start_all_request(priv_dev, priv_ep);
1443 * @priv_ep: extended endpoint object.
1452 * DQ = priv_ep->dequeue - dequeue position
1453 * EQ = priv_ep->enqueue - enqueue position
1459 * Then, we check if cycle bit for index priv_ep->dequeue
1463 * 1. priv_ep->dequeue never equals to current_index.
1464 * 2 priv_ep->enqueue never exceed priv_ep->dequeue
1465 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1466 * and priv_ep->free_trbs is zero.
1470 * Case 1 - priv_ep->dequeue < current_index
1474 * Case 2 - priv_ep->dequeue > current_index
1479 static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep,
1482 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1488 current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1493 if (priv_ep->dequeue > priv_req->end_trb)
1496 if (priv_ep->dequeue < priv_req->start_trb)
1501 (priv_ep->dequeue > priv_req->end_trb) &&
1502 (priv_ep->dequeue < priv_req->start_trb))
1506 (priv_ep->dequeue != priv_req->end_trb))
1509 trb = &priv_ep->trb_pool[priv_ep->dequeue];
1511 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs)
1514 if (doorbell == 1 && current_index == priv_ep->dequeue)
1518 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1523 if (priv_ep->enqueue == priv_ep->dequeue &&
1524 priv_ep->free_trbs == 0) {
1526 } else if (priv_ep->dequeue < current_index) {
1527 if ((current_index == (priv_ep->num_trbs - 1)) &&
1528 !priv_ep->dequeue)
1532 } else if (priv_ep->dequeue > current_index) {
1543 struct cdns3_endpoint *priv_ep)
1551 while (!list_empty(&priv_ep->pending_req_list)) {
1552 request = cdns3_next_request(&priv_ep->pending_req_list);
1555 trb = priv_ep->trb_pool + priv_ep->dequeue;
1561 if (priv_ep->dequeue == cdns3_get_dma_pos(priv_dev, priv_ep) &&
1562 priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1565 trace_cdns3_complete_trb(priv_ep, trb);
1566 cdns3_ep_inc_deq(priv_ep);
1567 trb = priv_ep->trb_pool + priv_ep->dequeue;
1574 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1576 while (cdns3_trb_handled(priv_ep, priv_req)) {
1581 trb = priv_ep->trb_pool + priv_ep->dequeue;
1582 trace_cdns3_complete_trb(priv_ep, trb);
1593 cdns3_ep_inc_deq(priv_ep);
1597 /* TRBs are duplicated by priv_ep->interval time for ISO IN */
1598 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_ep->dir)
1599 request->actual /= priv_ep->interval;
1601 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1608 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1615 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1617 trb = priv_ep->trb_pool;
1618 trace_cdns3_complete_trb(priv_ep, trb);
1628 (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1629 priv_ep->stream_sg_idx = 0;
1630 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1632 priv_ep->stream_sg_idx++;
1633 cdns3_ep_run_stream_transfer(priv_ep, request);
1638 priv_ep->flags &= ~EP_PENDING_REQUEST;
1641 if (!(priv_ep->flags & EP_STALLED) &&
1642 !(priv_ep->flags & EP_STALL_PENDING))
1643 cdns3_start_all_request(priv_dev, priv_ep);
1646 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1648 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1650 cdns3_wa1_restore_cycle_bit(priv_ep);
1653 trace_cdns3_ring(priv_ep);
1661 trace_cdns3_doorbell_epx(priv_ep->name,
1666 static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1668 u16 tdl = priv_ep->pending_tdl;
1669 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1673 priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1675 priv_ep->pending_tdl = 0;
1683 * @priv_ep: endpoint object
1687 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1689 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1695 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1697 trace_cdns3_epx_irq(priv_dev, priv_ep);
1702 if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1717 EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1721 priv_ep->prime_flag = true;
1723 pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1724 deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1727 cdns3_start_all_request(priv_dev, priv_ep);
1733 if (priv_ep->flags & EP_STALL_PENDING &&
1736 cdns3_ep_stall_flush(priv_ep);
1746 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1747 !priv_ep->wa1_set) {
1748 if (!priv_ep->dir) {
1753 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1754 priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
1756 cdns3_transfer_completed(priv_dev, priv_ep);
1757 } else if (!(priv_ep->flags & EP_STALLED) &&
1758 !(priv_ep->flags & EP_STALL_PENDING)) {
1759 if (priv_ep->flags & EP_DEFERRED_DRDY) {
1760 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1761 cdns3_start_all_request(priv_dev, priv_ep);
1763 cdns3_rearm_transfer(priv_ep,
1764 priv_ep->wa1_set);
1771 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1773 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1775 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1778 if (!priv_ep->use_streams) {
1781 cdns3_transfer_completed(priv_dev, priv_ep);
1782 } else if ((priv_ep->flags & EP_TDLCHK_EN) &
1783 priv_ep->pending_tdl) {
1785 cdns3_reprogram_tdl(priv_ep);
1787 } else if (priv_ep->dir == USB_DIR_OUT) {
1788 priv_ep->ep_sts_pending |= ep_sts_reg;
1790 cdns3_transfer_completed(priv_dev, priv_ep);
1798 if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1799 (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1800 priv_ep->ep_sts_pending = 0;
1801 cdns3_transfer_completed(priv_dev, priv_ep);
1806 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1807 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1811 !(priv_ep->flags & EP_STALLED))
1812 cdns3_wa2_descmissing_packet(priv_ep);
2053 struct cdns3_endpoint *priv_ep)
2064 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
2067 if (priv_ep->dir)
2068 mask = BIT(priv_ep->num + 16);
2070 mask = BIT(priv_ep->num);
2072 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
2079 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2088 * @priv_ep: extended endpoint object
2091 int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable)
2093 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
2094 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2095 u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
2096 u32 max_packet_size = priv_ep->wMaxPacketSize;
2097 u8 maxburst = priv_ep->bMaxBurst;
2104 cdns3_configure_dmult(priv_dev, priv_ep);
2106 switch (priv_ep->type) {
2110 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2116 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2121 buffering = (priv_ep->bMaxBurst + 1) * (priv_ep->mult + 1) - 1;
2132 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
2143 priv_ep->trb_burst_size = 128;
2145 priv_ep->trb_burst_size = 64;
2147 priv_ep->trb_burst_size = 16;
2160 priv_ep->trb_burst_size = 16;
2168 !!priv_ep->dir);
2178 if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2180 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
2195 EP_CFG_MULT(priv_ep->mult) | /* must match EP setting */
2201 priv_ep->flags |= EP_CONFIGURED;
2204 priv_ep->name, ep_cfg);
2211 struct cdns3_endpoint *priv_ep)
2213 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2214 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2222 struct cdns3_endpoint *priv_ep;
2234 priv_ep = ep_to_cdns3_ep(ep);
2235 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2236 if (!(priv_ep->flags & EP_CLAIMED)) {
2237 priv_ep->num = num;
2238 return priv_ep;
2268 struct cdns3_endpoint *priv_ep;
2271 priv_ep = cdns3_find_available_ep(priv_dev, desc);
2272 if (IS_ERR(priv_ep)) {
2277 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2280 priv_ep->endpoint.desc = desc;
2281 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2282 priv_ep->type = usb_endpoint_type(desc);
2283 priv_ep->flags |= EP_CLAIMED;
2284 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2285 priv_ep->wMaxPacketSize = usb_endpoint_maxp(desc);
2286 priv_ep->mult = USB_EP_MAXP_MULT(priv_ep->wMaxPacketSize);
2287 priv_ep->wMaxPacketSize &= USB_ENDPOINT_MAXP_MASK;
2288 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && comp_desc) {
2289 priv_ep->mult = USB_SS_MULT(comp_desc->bmAttributes) - 1;
2290 priv_ep->bMaxBurst = comp_desc->bMaxBurst;
2294 return &priv_ep->endpoint;
2307 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2314 priv_req->priv_ep = priv_ep;
2347 struct cdns3_endpoint *priv_ep;
2362 priv_ep = ep_to_cdns3_ep(ep);
2363 priv_dev = priv_ep->cdns3_dev;
2364 comp_desc = priv_ep->endpoint.comp_desc;
2376 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2377 "%s is already enabled\n", priv_ep->name))
2382 priv_ep->endpoint.desc = desc;
2383 priv_ep->type = usb_endpoint_type(desc);
2384 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2386 if (priv_ep->interval > ISO_MAX_INTERVAL &&
2387 priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2395 bEndpointAddress = priv_ep->num | priv_ep->dir;
2406 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2418 priv_ep->use_streams = true;
2419 ret = cdns3_ep_config(priv_ep, enable);
2423 ret = cdns3_ep_config(priv_ep, enable);
2429 ret = cdns3_allocate_trb_pool(priv_ep);
2433 bEndpointAddress = priv_ep->num | priv_ep->dir;
2436 trace_cdns3_gadget_ep_enable(priv_ep);
2445 cdns3_free_trb_pool(priv_ep);
2455 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2460 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2462 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2463 priv_ep->wa1_set = 0;
2464 priv_ep->enqueue = 0;
2465 priv_ep->dequeue = 0;
2467 priv_ep->pcs = !!EP_STS_CCS(reg);
2468 priv_ep->ccs = !!EP_STS_CCS(reg);
2470 priv_ep->free_trbs = priv_ep->num_trbs - 1;
2485 struct cdns3_endpoint *priv_ep;
2499 priv_ep = ep_to_cdns3_ep(ep);
2500 priv_dev = priv_ep->cdns3_dev;
2502 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2503 "%s is already disabled\n", priv_ep->name))
2508 trace_cdns3_gadget_ep_disable(priv_ep);
2530 priv_ep->name);
2532 while (!list_empty(&priv_ep->pending_req_list)) {
2533 request = cdns3_next_request(&priv_ep->pending_req_list);
2535 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2539 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2540 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2543 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2546 --priv_ep->wa2_counter;
2549 while (!list_empty(&priv_ep->deferred_req_list)) {
2550 request = cdns3_next_request(&priv_ep->deferred_req_list);
2552 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2556 priv_ep->descmis_req = NULL;
2559 priv_ep->flags &= ~EP_ENABLED;
2560 priv_ep->use_streams = false;
2579 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2580 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2590 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2606 list_add_tail(&request->list, &priv_ep->deferred_req_list);
2617 !(priv_ep->flags & EP_STALLED) &&
2618 !(priv_ep->flags & EP_STALL_PENDING))
2619 cdns3_start_all_request(priv_dev, priv_ep);
2621 if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2622 cdns3_start_all_request(priv_dev, priv_ep);
2632 struct cdns3_endpoint *priv_ep;
2640 priv_ep = ep_to_cdns3_ep(ep);
2641 priv_dev = priv_ep->cdns3_dev;
2659 priv_ep->name);
2677 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2689 priv_dev = priv_ep->cdns3_dev;
2699 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2707 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2720 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma +
2725 if (priv_ep->wa1_trb == priv_req->trb)
2726 cdns3_wa1_restore_cycle_bit(priv_ep);
2729 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2739 * @priv_ep: endpoint object to set stall on.
2741 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2743 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2745 trace_cdns3_halt(priv_ep, 1, 0);
2747 if (!(priv_ep->flags & EP_STALLED)) {
2751 cdns3_ep_stall_flush(priv_ep);
2753 priv_ep->flags |= EP_STALL_PENDING;
2760 * @priv_ep: endpoint object to clear stall on
2762 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2764 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2772 trace_cdns3_halt(priv_ep, 0, 0);
2774 request = cdns3_next_request(&priv_ep->pending_req_list);
2792 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2798 cdns3_rearm_transfer(priv_ep, 1);
2801 cdns3_start_all_request(priv_dev, priv_ep);
2814 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2815 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2819 if (!(priv_ep->flags & EP_ENABLED))
2827 priv_ep->flags &= ~EP_WEDGE;
2828 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2830 __cdns3_gadget_ep_set_halt(priv_ep);
3015 struct cdns3_endpoint *priv_ep;
3027 priv_ep = ep_to_cdns3_ep(ep);
3028 bEndpointAddress = priv_ep->num | priv_ep->dir;
3034 priv_ep->flags &= ~EP_CLAIMED;
3058 struct cdns3_endpoint *priv_ep;
3067 priv_ep = ep_to_cdns3_ep(ep);
3068 if (!(priv_ep->flags & EP_CLAIMED))
3071 n = (priv_ep->mult + 1) * (priv_ep->bMaxBurst + 1);
3078 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
3083 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
3133 struct cdns3_endpoint *priv_ep;
3158 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
3160 if (!priv_ep)
3164 priv_ep->cdns3_dev = priv_dev;
3165 priv_dev->eps[i] = priv_ep;
3166 priv_ep->num = ep_number;
3167 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
3170 ret = cdns3_init_ep0(priv_dev, priv_ep);
3176 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
3178 priv_ep->endpoint.name = priv_ep->name;
3180 usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
3182 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
3183 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
3185 priv_ep->endpoint.caps.dir_in = 1;
3187 priv_ep->endpoint.caps.dir_out = 1;
3190 priv_ep->endpoint.caps.type_iso = 1;
3192 priv_ep->endpoint.caps.type_bulk = 1;
3193 priv_ep->endpoint.caps.type_int = 1;
3195 list_add_tail(&priv_ep->endpoint.ep_list,
3199 priv_ep->flags = 0;
3202 priv_ep->name,
3203 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
3204 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
3206 INIT_LIST_HEAD(&priv_ep->pending_req_list);
3207 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
3208 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);