Lines Matching refs:dep
159 * @dep: The endpoint whose enqueue pointer we're incrementing
161 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
163 dwc3_ep_inc_trb(&dep->trb_enqueue);
168 * @dep: The endpoint whose enqueue pointer we're incrementing
170 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
172 dwc3_ep_inc_trb(&dep->trb_dequeue);
175 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
178 struct dwc3 *dwc = dep->dwc;
195 if (dep->number > 1)
201 * @dep: The endpoint to whom the request belongs to
209 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
212 struct dwc3 *dwc = dep->dwc;
214 dwc3_gadget_del_and_unmap_request(dep, req, status);
218 usb_gadget_giveback_request(&dep->endpoint, &req->request);
266 * @dep: the endpoint to which the command is going to be issued
271 * @params to @dep and wait for its completion.
273 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
276 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
277 struct dwc3 *dwc = dep->dwc;
336 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
337 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
338 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
361 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
363 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
373 dep->name);
403 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
407 dep->flags |= DWC3_EP_TRANSFER_STARTED;
410 dwc3_gadget_ep_get_transfer_index(dep);
422 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
424 struct dwc3 *dwc = dep->dwc;
436 if (dep->direction &&
443 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
446 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
449 u32 offset = (char *) trb - (char *) dep->trb_pool;
451 return dep->trb_pool_dma + offset;
454 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
456 struct dwc3 *dwc = dep->dwc;
458 if (dep->trb_pool)
461 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
463 &dep->trb_pool_dma, GFP_KERNEL);
464 if (!dep->trb_pool) {
465 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
466 dep->name);
473 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
475 struct dwc3 *dwc = dep->dwc;
478 dep->trb_pool, dep->trb_pool_dma);
480 dep->trb_pool = NULL;
481 dep->trb_pool_dma = 0;
484 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
492 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
498 * @dep: endpoint that is being enabled
500 * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
529 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
537 if (dep->number)
542 dwc = dep->dwc;
544 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
549 struct dwc3_ep *dep = dwc->eps[i];
551 if (!dep)
554 ret = dwc3_gadget_set_xfer_resource(dep);
562 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
567 struct dwc3 *dwc = dep->dwc;
569 comp_desc = dep->endpoint.comp_desc;
570 desc = dep->endpoint.desc;
579 u32 burst = dep->endpoint.maxburst;
586 params.param2 |= dep->saved_state;
591 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
598 dep->stream_capable = true;
610 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
616 if (dep->direction)
617 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
634 dep->interval = desc->bInterval;
636 dep->interval = 1 << (desc->bInterval - 1);
641 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms);
644 static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
649 * @dep: endpoint to be initialized
655 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
657 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
658 struct dwc3 *dwc = dep->dwc;
663 if (!(dep->flags & DWC3_EP_ENABLED)) {
664 ret = dwc3_gadget_start_config(dep);
669 ret = dwc3_gadget_set_ep_config(dep, action);
673 if (!(dep->flags & DWC3_EP_ENABLED)) {
677 dep->type = usb_endpoint_type(desc);
678 dep->flags |= DWC3_EP_ENABLED;
681 reg |= DWC3_DALEPENA_EP(dep->number);
688 dep->trb_dequeue = 0;
689 dep->trb_enqueue = 0;
690 memset(dep->trb_pool, 0,
694 trb_st_hw = &dep->trb_pool[0];
696 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
697 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
698 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
715 trb = &dep->trb_pool[0];
716 trb_dma = dwc3_trb_dma_offset(dep, trb);
723 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
727 if (dep->stream_capable) {
740 dwc3_stop_active_transfer(dep, true, true);
747 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
752 trace_dwc3_gadget_ep_enable(dep);
757 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status)
761 dwc3_stop_active_transfer(dep, true, false);
764 while (!list_empty(&dep->started_list)) {
765 req = next_request(&dep->started_list);
767 dwc3_gadget_giveback(dep, req, status);
770 while (!list_empty(&dep->pending_list)) {
771 req = next_request(&dep->pending_list);
773 dwc3_gadget_giveback(dep, req, status);
776 while (!list_empty(&dep->cancelled_list)) {
777 req = next_request(&dep->cancelled_list);
779 dwc3_gadget_giveback(dep, req, status);
785 * @dep: the endpoint to disable
793 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
795 struct dwc3 *dwc = dep->dwc;
798 trace_dwc3_gadget_ep_disable(dep);
801 if (dep->flags & DWC3_EP_STALL)
802 __dwc3_gadget_ep_set_halt(dep, 0, false);
805 reg &= ~DWC3_DALEPENA_EP(dep->number);
808 dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
810 dep->stream_capable = false;
811 dep->type = 0;
812 dep->flags = 0;
815 if (dep->number > 1) {
816 dep->endpoint.comp_desc = NULL;
817 dep->endpoint.desc = NULL;
841 struct dwc3_ep *dep;
856 dep = to_dwc3_ep(ep);
857 dwc = dep->dwc;
859 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
861 dep->name))
865 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
873 struct dwc3_ep *dep;
883 dep = to_dwc3_ep(ep);
884 dwc = dep->dwc;
886 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
888 dep->name))
892 ret = __dwc3_gadget_ep_disable(dep);
902 struct dwc3_ep *dep = to_dwc3_ep(ep);
908 req->direction = dep->direction;
909 req->epnum = dep->number;
910 req->dep = dep;
929 * @dep: The endpoint with the TRB ring
936 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
943 return &dep->trb_pool[tmp - 1];
946 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
955 if (dep->trb_enqueue == dep->trb_dequeue) {
960 if (!list_empty(&dep->started_list))
966 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
969 if (dep->trb_dequeue < dep->trb_enqueue)
977 * @dep: endpoint for which this request is prepared
985 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
996 struct dwc3 *dwc = dep->dwc;
1001 dma = dep->dwc->bounce_addr;
1007 trb = &dep->trb_pool[dep->trb_enqueue];
1012 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1021 switch (usb_endpoint_type(dep->endpoint.desc)) {
1053 struct usb_ep *ep = &dep->endpoint;
1083 usb_endpoint_type(dep->endpoint.desc));
1090 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1091 if (!dep->stream_capable)
1103 else if (dep->stream_capable && is_last)
1106 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1124 dwc3_ep_inc_enq(dep);
1126 trace_dwc3_prepare_trb(dep, trb);
1129 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1131 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1135 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1144 * @dep: The endpoint that the request belongs to
1151 static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1155 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1159 if (dwc3_needs_extra_trb(dep, req))
1162 if (dwc3_calc_trbs_left(dep) < num_trbs)
1169 dwc3_prepare_one_trb(dep, req, entry_length,
1174 dwc3_prepare_one_trb(dep, req,
1181 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1191 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1201 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1224 if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1238 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1272 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1275 return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1280 * @dep: endpoint for which requests are being prepared
1288 static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1305 list_for_each_entry(req, &dep->started_list, list) {
1307 ret = dwc3_prepare_trbs_sg(dep, req);
1312 if (!dwc3_calc_trbs_left(dep))
1320 if (dep->stream_capable && req->request.is_last)
1324 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1325 struct dwc3 *dwc = dep->dwc;
1328 dep->direction);
1338 ret = dwc3_prepare_trbs_sg(dep, req);
1342 ret = dwc3_prepare_trbs_linear(dep, req);
1345 if (!ret || !dwc3_calc_trbs_left(dep))
1353 if (dep->stream_capable && req->request.is_last)
1360 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1362 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1375 ret = dwc3_prepare_trbs(dep);
1379 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1388 req = next_request(&dep->started_list);
1390 dep->flags |= DWC3_EP_PENDING_REQUEST;
1401 if (dep->stream_capable)
1404 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1405 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1408 DWC3_DEPCMD_PARAM(dep->resource_index);
1411 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1418 dwc3_stop_active_transfer(dep, true, true);
1420 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1424 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1425 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1430 if (dep->stream_capable && req->request.is_last)
1431 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1446 * @dep: isoc endpoint
1455 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1457 struct dwc3 *dwc = dep->dwc;
1465 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1467 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1469 dep->resource_index = 0;
1474 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1476 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1484 * @dep: isoc endpoint
1524 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1530 while (dep->combo_num < 2) {
1537 * 4 uframes in the future with BIT[15:14] as dep->combo_num
1539 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1540 test_frame_number |= dep->combo_num << 14;
1541 test_frame_number += max_t(u32, 4, dep->interval);
1543 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1544 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1548 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1552 dep->start_cmd_status = 0;
1553 dep->combo_num = 0;
1558 if (dep->combo_num == 0)
1559 dep->start_cmd_status = cmd_status;
1561 dep->combo_num++;
1568 dwc3_stop_active_transfer(dep, true, true);
1574 test0 = (dep->start_cmd_status == 0);
1578 dep->combo_num = 1;
1580 dep->combo_num = 2;
1582 dep->combo_num = 3;
1584 dep->combo_num = 0;
1586 dep->frame_number &= DWC3_FRNUMBER_MASK;
1587 dep->frame_number |= dep->combo_num << 14;
1588 dep->frame_number += max_t(u32, 4, dep->interval);
1591 dep->start_cmd_status = 0;
1592 dep->combo_num = 0;
1594 return __dwc3_gadget_kick_transfer(dep);
1597 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1599 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1600 struct dwc3 *dwc = dep->dwc;
1604 if (list_empty(&dep->pending_list) &&
1605 list_empty(&dep->started_list)) {
1606 dep->flags |= DWC3_EP_PENDING_REQUEST;
1613 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1614 return dwc3_gadget_start_isoc_quirk(dep);
1621 (dep->frame_number & DWC3_FRNUMBER_MASK);
1632 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1635 dep->frame_number += BIT(14);
1639 dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1);
1641 ret = __dwc3_gadget_kick_transfer(dep);
1652 ret = __dwc3_stop_active_transfer(dep, false, true);
1657 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1659 struct dwc3 *dwc = dep->dwc;
1661 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1663 dep->name);
1667 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1668 &req->request, req->dep->name))
1673 dep->name, &req->request))
1683 list_add_tail(&req->list, &dep->pending_list);
1686 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1693 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1694 (dep->flags & DWC3_EP_WEDGE) ||
1695 (dep->flags & DWC3_EP_STALL)) {
1696 dep->flags |= DWC3_EP_DELAY_START;
1708 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1709 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1710 !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1713 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1714 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
1715 return __dwc3_gadget_start_isoc(dep);
1719 __dwc3_gadget_kick_transfer(dep);
1728 struct dwc3_ep *dep = to_dwc3_ep(ep);
1729 struct dwc3 *dwc = dep->dwc;
1736 ret = __dwc3_gadget_ep_queue(dep, req);
1742 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1763 trb = &dep->trb_pool[dep->trb_dequeue];
1765 dwc3_ep_inc_deq(dep);
1771 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
1776 list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) {
1777 dwc3_gadget_ep_skip_trbs(dep, req);
1778 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1788 struct dwc3_ep *dep = to_dwc3_ep(ep);
1789 struct dwc3 *dwc = dep->dwc;
1798 list_for_each_entry(r, &dep->cancelled_list, list) {
1803 list_for_each_entry(r, &dep->pending_list, list) {
1805 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1810 list_for_each_entry(r, &dep->started_list, list) {
1815 dwc3_stop_active_transfer(dep, true, true);
1821 list_for_each_entry_safe(r, t, &dep->started_list, list)
1824 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
1839 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1842 struct dwc3 *dwc = dep->dwc;
1847 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1848 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1860 if (dep->number > 1)
1861 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1863 trb = &dwc->ep0_trb[dep->trb_enqueue];
1866 started = !list_empty(&dep->started_list);
1868 if (!protocol && ((dep->direction && transfer_in_flight) ||
1869 (!dep->direction && started))) {
1873 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1877 dep->name);
1879 dep->flags |= DWC3_EP_STALL;
1886 if (dep->number <= 1) {
1887 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1891 dwc3_stop_active_transfer(dep, true, true);
1893 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1896 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) {
1897 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
1901 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1903 ret = dwc3_send_clear_stall_ep_cmd(dep);
1906 dep->name);
1910 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1912 if ((dep->flags & DWC3_EP_DELAY_START) &&
1913 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
1914 __dwc3_gadget_kick_transfer(dep);
1916 dep->flags &= ~DWC3_EP_DELAY_START;
1924 struct dwc3_ep *dep = to_dwc3_ep(ep);
1925 struct dwc3 *dwc = dep->dwc;
1932 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1940 struct dwc3_ep *dep = to_dwc3_ep(ep);
1941 struct dwc3 *dwc = dep->dwc;
1946 dep->flags |= DWC3_EP_WEDGE;
1948 if (dep->number == 0 || dep->number == 1)
1951 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
2091 struct dwc3_ep *dep;
2093 dep = dwc->eps[epnum];
2094 if (!dep)
2097 dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
2327 struct dwc3_ep *dep;
2362 dep = dwc->eps[0];
2363 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2365 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2369 dep = dwc->eps[1];
2370 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2372 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2566 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2568 struct dwc3 *dwc = dep->dwc;
2570 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2571 dep->endpoint.maxburst = 1;
2572 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2573 if (!dep->direction)
2574 dwc->gadget->ep0 = &dep->endpoint;
2576 dep->endpoint.caps.type_control = true;
2581 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2583 struct dwc3 *dwc = dep->dwc;
2594 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2615 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2617 dep->endpoint.max_streams = 16;
2618 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2619 list_add_tail(&dep->endpoint.ep_list,
2621 dep->endpoint.caps.type_iso = true;
2622 dep->endpoint.caps.type_bulk = true;
2623 dep->endpoint.caps.type_int = true;
2625 return dwc3_alloc_trb_pool(dep);
2628 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2630 struct dwc3 *dwc = dep->dwc;
2665 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2666 dep->endpoint.max_streams = 16;
2667 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2668 list_add_tail(&dep->endpoint.ep_list,
2670 dep->endpoint.caps.type_iso = true;
2671 dep->endpoint.caps.type_bulk = true;
2672 dep->endpoint.caps.type_int = true;
2674 return dwc3_alloc_trb_pool(dep);
2679 struct dwc3_ep *dep;
2684 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2685 if (!dep)
2688 dep->dwc = dwc;
2689 dep->number = epnum;
2690 dep->direction = direction;
2691 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2692 dwc->eps[epnum] = dep;
2693 dep->combo_num = 0;
2694 dep->start_cmd_status = 0;
2696 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2699 dep->endpoint.name = dep->name;
2701 if (!(dep->number > 1)) {
2702 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2703 dep->endpoint.comp_desc = NULL;
2707 ret = dwc3_gadget_init_control_endpoint(dep);
2709 ret = dwc3_gadget_init_in_endpoint(dep);
2711 ret = dwc3_gadget_init_out_endpoint(dep);
2716 dep->endpoint.caps.dir_in = direction;
2717 dep->endpoint.caps.dir_out = !direction;
2719 INIT_LIST_HEAD(&dep->pending_list);
2720 INIT_LIST_HEAD(&dep->started_list);
2721 INIT_LIST_HEAD(&dep->cancelled_list);
2723 dwc3_debugfs_create_endpoint_dir(dep);
2747 struct dwc3_ep *dep;
2751 dep = dwc->eps[epnum];
2752 if (!dep)
2764 dwc3_free_trb_pool(dep);
2765 list_del(&dep->endpoint.ep_list);
2768 debugfs_remove_recursive(debugfs_lookup(dep->name, dwc->root));
2769 kfree(dep);
2775 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2781 dwc3_ep_inc_deq(dep);
2783 trace_dwc3_complete_trb(dep, trb);
2803 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2808 frame_number &= ~(dep->interval - 1);
2817 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
2818 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
2843 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2847 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2855 trb = &dep->trb_pool[dep->trb_dequeue];
2860 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2869 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2873 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2875 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2884 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
2892 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2895 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2904 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2918 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
2937 dwc3_gadget_giveback(dep, req, request_status);
2943 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
2949 list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
2952 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2959 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
2963 if (!list_empty(&dep->pending_list))
2970 req = next_request(&dep->started_list);
2977 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2980 dep->frame_number = event->parameters;
2983 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
2986 struct dwc3 *dwc = dep->dwc;
2989 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2991 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
2994 if (!dep->endpoint.desc)
2997 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2998 list_empty(&dep->started_list) &&
2999 (list_empty(&dep->pending_list) || status == -EXDEV))
3000 dwc3_stop_active_transfer(dep, true, true);
3001 else if (dwc3_gadget_ep_should_continue(dep))
3002 if (__dwc3_gadget_kick_transfer(dep) == 0)
3015 dep = dwc->eps[i];
3017 if (!(dep->flags & DWC3_EP_ENABLED))
3020 if (!list_empty(&dep->started_list))
3034 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3039 if (!dep->endpoint.desc)
3042 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3043 dwc3_gadget_endpoint_frame_from_event(dep, event);
3051 dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3054 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3059 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3064 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3065 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3068 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3071 dwc3_gadget_endpoint_frame_from_event(dep, event);
3081 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3084 (void) __dwc3_gadget_start_isoc(dep);
3087 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3100 if (dep->stream_capable)
3101 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3103 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3104 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3105 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3107 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3108 struct dwc3 *dwc = dep->dwc;
3110 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3111 if (dwc3_send_clear_stall_ep_cmd(dep)) {
3114 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3120 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3125 if ((dep->flags & DWC3_EP_DELAY_START) &&
3126 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3127 __dwc3_gadget_kick_transfer(dep);
3129 dep->flags &= ~DWC3_EP_DELAY_START;
3132 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3135 struct dwc3 *dwc = dep->dwc;
3138 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3152 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3153 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3154 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3156 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3161 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3162 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3163 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))
3183 dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3185 dep->flags |= DWC3_EP_DELAY_START;
3186 dwc3_stop_active_transfer(dep, true, true);
3193 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3199 struct dwc3_ep *dep;
3202 dep = dwc->eps[epnum];
3204 if (!(dep->flags & DWC3_EP_ENABLED)) {
3205 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
3220 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3223 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3226 dwc3_gadget_endpoint_command_complete(dep, event);
3229 dwc3_gadget_endpoint_transfer_complete(dep, event);
3232 dwc3_gadget_endpoint_stream_event(dep, event);
3278 static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3281 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3282 (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3316 __dwc3_stop_active_transfer(dep, force, interrupt);
3324 struct dwc3_ep *dep;
3327 dep = dwc->eps[epnum];
3328 if (!dep)
3331 if (!(dep->flags & DWC3_EP_STALL))
3334 dep->flags &= ~DWC3_EP_STALL;
3336 ret = dwc3_send_clear_stall_ep_cmd(dep);
3429 struct dwc3_ep *dep;
3534 dep = dwc->eps[0];
3535 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3537 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3541 dep = dwc->eps[1];
3542 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3544 dev_err(dwc->dev, "failed to enable %s\n", dep->name);