Lines Matching refs:dep
177 * @dep: The endpoint whose enqueue pointer we're incrementing
179 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
181 dwc3_ep_inc_trb(&dep->trb_enqueue);
186 * @dep: The endpoint whose enqueue pointer we're incrementing
188 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
190 dwc3_ep_inc_trb(&dep->trb_dequeue);
193 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
196 struct dwc3 *dwc = dep->dwc;
213 if (dep->number > 1)
219 * @dep: The endpoint to whom the request belongs to
227 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
230 struct dwc3 *dwc = dep->dwc;
232 dwc3_gadget_del_and_unmap_request(dep, req, status);
236 usb_gadget_giveback_request(&dep->endpoint, &req->request);
284 * @dep: the endpoint to which the command is going to be issued
289 * @params to @dep and wait for its completion.
291 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
294 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
295 struct dwc3 *dwc = dep->dwc;
361 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
362 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
363 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
387 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
397 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
407 dep->name);
438 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
442 dep->flags |= DWC3_EP_TRANSFER_STARTED;
445 dwc3_gadget_ep_get_transfer_index(dep);
457 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
459 struct dwc3 *dwc = dep->dwc;
471 if (dep->direction &&
478 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
481 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
484 u32 offset = (char *) trb - (char *) dep->trb_pool;
486 return dep->trb_pool_dma + offset;
489 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
491 struct dwc3 *dwc = dep->dwc;
493 if (dep->trb_pool)
496 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
498 &dep->trb_pool_dma, GFP_KERNEL);
499 if (!dep->trb_pool) {
500 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
501 dep->name);
508 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
510 struct dwc3 *dwc = dep->dwc;
513 dep->trb_pool, dep->trb_pool_dma);
515 dep->trb_pool = NULL;
516 dep->trb_pool_dma = 0;
519 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
527 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
533 * @dep: endpoint that is being enabled
535 * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
564 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
572 if (dep->number)
577 dwc = dep->dwc;
579 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
584 struct dwc3_ep *dep = dwc->eps[i];
586 if (!dep)
589 ret = dwc3_gadget_set_xfer_resource(dep);
597 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
602 struct dwc3 *dwc = dep->dwc;
604 comp_desc = dep->endpoint.comp_desc;
605 desc = dep->endpoint.desc;
614 u32 burst = dep->endpoint.maxburst;
621 params.param2 |= dep->saved_state;
626 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
633 dep->stream_capable = true;
645 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
651 if (dep->direction)
652 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
669 dep->interval = desc->bInterval;
671 dep->interval = 1 << (desc->bInterval - 1);
676 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms);
723 struct dwc3_ep *dep;
732 dep = dwc->eps[1];
743 dep = dwc->eps[num];
750 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
776 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
778 struct dwc3 *dwc = dep->dwc;
793 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)
797 if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
802 if ((dep->endpoint.maxburst > 1 &&
803 usb_endpoint_xfer_bulk(dep->endpoint.desc)) ||
804 usb_endpoint_xfer_isoc(dep->endpoint.desc))
807 if (dep->endpoint.maxburst > 6 &&
808 (usb_endpoint_xfer_bulk(dep->endpoint.desc) ||
809 usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
850 dep->endpoint.name, fifo_size);
860 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
861 dep->flags |= DWC3_EP_TXFIFO_RESIZED;
869 * @dep: endpoint to be initialized
875 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
877 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
878 struct dwc3 *dwc = dep->dwc;
883 if (!(dep->flags & DWC3_EP_ENABLED)) {
884 ret = dwc3_gadget_resize_tx_fifos(dep);
888 ret = dwc3_gadget_start_config(dep);
893 ret = dwc3_gadget_set_ep_config(dep, action);
897 if (!(dep->flags & DWC3_EP_ENABLED)) {
901 dep->type = usb_endpoint_type(desc);
902 dep->flags |= DWC3_EP_ENABLED;
905 reg |= DWC3_DALEPENA_EP(dep->number);
908 dep->trb_dequeue = 0;
909 dep->trb_enqueue = 0;
915 memset(dep->trb_pool, 0,
919 trb_st_hw = &dep->trb_pool[0];
921 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
922 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
923 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
940 trb = &dep->trb_pool[0];
941 trb_dma = dwc3_trb_dma_offset(dep, trb);
948 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
952 if (dep->stream_capable) {
965 dwc3_stop_active_transfer(dep, true, true);
977 if (!dep->direction ||
980 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
985 trace_dwc3_gadget_ep_enable(dep);
990 void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status)
994 dwc3_stop_active_transfer(dep, true, false);
997 if (dep->flags & DWC3_EP_DELAY_STOP)
1001 while (!list_empty(&dep->started_list)) {
1002 req = next_request(&dep->started_list);
1004 dwc3_gadget_giveback(dep, req, status);
1007 while (!list_empty(&dep->pending_list)) {
1008 req = next_request(&dep->pending_list);
1010 dwc3_gadget_giveback(dep, req, status);
1013 while (!list_empty(&dep->cancelled_list)) {
1014 req = next_request(&dep->cancelled_list);
1016 dwc3_gadget_giveback(dep, req, status);
1022 * @dep: the endpoint to disable
1030 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1032 struct dwc3 *dwc = dep->dwc;
1036 trace_dwc3_gadget_ep_disable(dep);
1039 if (dep->flags & DWC3_EP_STALL)
1040 __dwc3_gadget_ep_set_halt(dep, 0, false);
1043 reg &= ~DWC3_DALEPENA_EP(dep->number);
1046 dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
1048 dep->stream_capable = false;
1049 dep->type = 0;
1056 if (dep->flags & DWC3_EP_DELAY_STOP)
1058 dep->flags &= mask;
1061 if (dep->number > 1) {
1062 dep->endpoint.comp_desc = NULL;
1063 dep->endpoint.desc = NULL;
1087 struct dwc3_ep *dep;
1102 dep = to_dwc3_ep(ep);
1103 dwc = dep->dwc;
1105 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1107 dep->name))
1111 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1119 struct dwc3_ep *dep;
1129 dep = to_dwc3_ep(ep);
1130 dwc = dep->dwc;
1132 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1134 dep->name))
1138 ret = __dwc3_gadget_ep_disable(dep);
1148 struct dwc3_ep *dep = to_dwc3_ep(ep);
1154 req->direction = dep->direction;
1155 req->epnum = dep->number;
1156 req->dep = dep;
1175 * @dep: The endpoint with the TRB ring
1182 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1189 return &dep->trb_pool[tmp - 1];
1192 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1201 if (dep->trb_enqueue == dep->trb_dequeue) {
1206 if (!list_empty(&dep->started_list))
1212 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1215 if (dep->trb_dequeue < dep->trb_enqueue)
1223 * @dep: endpoint for which this request is prepared
1231 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1242 struct dwc3 *dwc = dep->dwc;
1247 dma = dep->dwc->bounce_addr;
1253 trb = &dep->trb_pool[dep->trb_enqueue];
1258 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1267 switch (usb_endpoint_type(dep->endpoint.desc)) {
1299 struct usb_ep *ep = &dep->endpoint;
1329 usb_endpoint_type(dep->endpoint.desc));
1336 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1337 if (!dep->stream_capable)
1345 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1353 else if (dep->stream_capable && is_last &&
1357 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1375 dwc3_ep_inc_enq(dep);
1377 trace_dwc3_prepare_trb(dep, trb);
1380 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1382 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1386 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1395 * @dep: The endpoint that the request belongs to
1402 static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1406 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1410 if (dwc3_needs_extra_trb(dep, req))
1413 if (dwc3_calc_trbs_left(dep) < num_trbs)
1420 dwc3_prepare_one_trb(dep, req, entry_length,
1425 dwc3_prepare_one_trb(dep, req,
1432 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1442 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1452 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1475 if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1490 list_for_each_entry(r, &dep->started_list, list) {
1499 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1533 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1536 return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1541 * @dep: endpoint for which requests are being prepared
1549 static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1566 list_for_each_entry(req, &dep->started_list, list) {
1568 ret = dwc3_prepare_trbs_sg(dep, req);
1573 if (!dwc3_calc_trbs_left(dep))
1581 if (dep->stream_capable && req->request.is_last &&
1582 !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1586 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1587 struct dwc3 *dwc = dep->dwc;
1590 dep->direction);
1600 ret = dwc3_prepare_trbs_sg(dep, req);
1604 ret = dwc3_prepare_trbs_linear(dep, req);
1607 if (!ret || !dwc3_calc_trbs_left(dep))
1615 if (dep->stream_capable && req->request.is_last &&
1623 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1625 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1638 ret = dwc3_prepare_trbs(dep);
1642 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1651 req = next_request(&dep->started_list);
1653 dep->flags |= DWC3_EP_PENDING_REQUEST;
1664 if (dep->stream_capable)
1667 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1668 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1671 DWC3_DEPCMD_PARAM(dep->resource_index);
1674 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1681 dwc3_stop_active_transfer(dep, true, true);
1683 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1687 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1688 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1693 if (dep->stream_capable && req->request.is_last &&
1694 !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1695 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1710 * @dep: isoc endpoint
1719 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1721 struct dwc3 *dwc = dep->dwc;
1729 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1731 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1738 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
1739 dep->flags |= DWC3_EP_DELAY_STOP;
1743 dep->resource_index = 0;
1748 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1750 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1753 dep->flags &= ~DWC3_EP_DELAY_STOP;
1759 * @dep: isoc endpoint
1799 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1805 while (dep->combo_num < 2) {
1812 * 4 uframes in the future with BIT[15:14] as dep->combo_num
1814 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1815 test_frame_number |= dep->combo_num << 14;
1816 test_frame_number += max_t(u32, 4, dep->interval);
1818 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1819 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1823 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1827 dep->start_cmd_status = 0;
1828 dep->combo_num = 0;
1833 if (dep->combo_num == 0)
1834 dep->start_cmd_status = cmd_status;
1836 dep->combo_num++;
1843 dwc3_stop_active_transfer(dep, true, true);
1849 test0 = (dep->start_cmd_status == 0);
1853 dep->combo_num = 1;
1855 dep->combo_num = 2;
1857 dep->combo_num = 3;
1859 dep->combo_num = 0;
1861 dep->frame_number &= DWC3_FRNUMBER_MASK;
1862 dep->frame_number |= dep->combo_num << 14;
1863 dep->frame_number += max_t(u32, 4, dep->interval);
1866 dep->start_cmd_status = 0;
1867 dep->combo_num = 0;
1869 return __dwc3_gadget_kick_transfer(dep);
1872 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1874 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1875 struct dwc3 *dwc = dep->dwc;
1879 if (list_empty(&dep->pending_list) &&
1880 list_empty(&dep->started_list)) {
1881 dep->flags |= DWC3_EP_PENDING_REQUEST;
1888 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1889 return dwc3_gadget_start_isoc_quirk(dep);
1896 (dep->frame_number & DWC3_FRNUMBER_MASK);
1907 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1910 dep->frame_number += BIT(14);
1920 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1922 ret = __dwc3_gadget_kick_transfer(dep);
1933 ret = __dwc3_stop_active_transfer(dep, false, true);
1938 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1940 struct dwc3 *dwc = dep->dwc;
1942 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1944 dep->name);
1948 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1949 &req->request, req->dep->name))
1954 dep->name, &req->request))
1964 list_add_tail(&req->list, &dep->pending_list);
1967 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1974 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1975 (dep->flags & DWC3_EP_WEDGE) ||
1976 (dep->flags & DWC3_EP_DELAY_STOP) ||
1977 (dep->flags & DWC3_EP_STALL)) {
1978 dep->flags |= DWC3_EP_DELAY_START;
1990 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1991 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1992 if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1993 return __dwc3_gadget_start_isoc(dep);
1999 __dwc3_gadget_kick_transfer(dep);
2008 struct dwc3_ep *dep = to_dwc3_ep(ep);
2009 struct dwc3 *dwc = dep->dwc;
2016 ret = __dwc3_gadget_ep_queue(dep, req);
2022 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
2043 trb = &dep->trb_pool[dep->trb_dequeue];
2045 dwc3_ep_inc_deq(dep);
2051 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
2054 struct dwc3 *dwc = dep->dwc;
2056 while (!list_empty(&dep->cancelled_list)) {
2057 req = next_request(&dep->cancelled_list);
2058 dwc3_gadget_ep_skip_trbs(dep, req);
2061 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
2064 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2067 dwc3_gadget_giveback(dep, req, -EPIPE);
2071 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2078 if (!dep->endpoint.desc)
2089 struct dwc3_ep *dep = to_dwc3_ep(ep);
2090 struct dwc3 *dwc = dep->dwc;
2099 list_for_each_entry(r, &dep->cancelled_list, list) {
2104 list_for_each_entry(r, &dep->pending_list, list) {
2113 if (dep->number > 1)
2114 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2121 list_for_each_entry(r, &dep->started_list, list) {
2126 dwc3_stop_active_transfer(dep, true, true);
2132 list_for_each_entry_safe(r, t, &dep->started_list, list)
2136 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2151 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2154 struct dwc3 *dwc = dep->dwc;
2159 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2160 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2172 if (dep->number > 1)
2173 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
2175 trb = &dwc->ep0_trb[dep->trb_enqueue];
2178 started = !list_empty(&dep->started_list);
2180 if (!protocol && ((dep->direction && transfer_in_flight) ||
2181 (!dep->direction && started))) {
2185 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2189 dep->name);
2191 dep->flags |= DWC3_EP_STALL;
2198 if (dep->number <= 1) {
2199 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2203 dwc3_stop_active_transfer(dep, true, true);
2205 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2208 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2209 (dep->flags & DWC3_EP_DELAY_STOP)) {
2210 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2212 dwc->clear_stall_protocol = dep->number;
2217 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2219 ret = dwc3_send_clear_stall_ep_cmd(dep);
2222 dep->name);
2226 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2228 if ((dep->flags & DWC3_EP_DELAY_START) &&
2229 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2230 __dwc3_gadget_kick_transfer(dep);
2232 dep->flags &= ~DWC3_EP_DELAY_START;
2240 struct dwc3_ep *dep = to_dwc3_ep(ep);
2241 struct dwc3 *dwc = dep->dwc;
2248 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
2256 struct dwc3_ep *dep = to_dwc3_ep(ep);
2257 struct dwc3 *dwc = dep->dwc;
2262 dep->flags |= DWC3_EP_WEDGE;
2264 if (dep->number == 0 || dep->number == 1)
2267 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
2499 struct dwc3_ep *dep;
2501 dep = dwc->eps[epnum];
2502 if (!dep)
2505 dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
2863 struct dwc3_ep *dep;
2916 dep = dwc->eps[0];
2917 dep->flags = 0;
2918 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2920 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2924 dep = dwc->eps[1];
2925 dep->flags = 0;
2926 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2928 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3150 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
3152 struct dwc3 *dwc = dep->dwc;
3154 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
3155 dep->endpoint.maxburst = 1;
3156 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
3157 if (!dep->direction)
3158 dwc->gadget->ep0 = &dep->endpoint;
3160 dep->endpoint.caps.type_control = true;
3165 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
3167 struct dwc3 *dwc = dep->dwc;
3177 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
3201 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3203 dep->endpoint.max_streams = 16;
3204 dep->endpoint.ops = &dwc3_gadget_ep_ops;
3205 list_add_tail(&dep->endpoint.ep_list,
3207 dep->endpoint.caps.type_iso = true;
3208 dep->endpoint.caps.type_bulk = true;
3209 dep->endpoint.caps.type_int = true;
3211 return dwc3_alloc_trb_pool(dep);
3214 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3216 struct dwc3 *dwc = dep->dwc;
3249 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3250 dep->endpoint.max_streams = 16;
3251 dep->endpoint.ops = &dwc3_gadget_ep_ops;
3252 list_add_tail(&dep->endpoint.ep_list,
3254 dep->endpoint.caps.type_iso = true;
3255 dep->endpoint.caps.type_bulk = true;
3256 dep->endpoint.caps.type_int = true;
3258 return dwc3_alloc_trb_pool(dep);
3263 struct dwc3_ep *dep;
3268 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
3269 if (!dep)
3272 dep->dwc = dwc;
3273 dep->number = epnum;
3274 dep->direction = direction;
3275 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3276 dwc->eps[epnum] = dep;
3277 dep->combo_num = 0;
3278 dep->start_cmd_status = 0;
3280 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
3283 dep->endpoint.name = dep->name;
3285 if (!(dep->number > 1)) {
3286 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3287 dep->endpoint.comp_desc = NULL;
3291 ret = dwc3_gadget_init_control_endpoint(dep);
3293 ret = dwc3_gadget_init_in_endpoint(dep);
3295 ret = dwc3_gadget_init_out_endpoint(dep);
3300 dep->endpoint.caps.dir_in = direction;
3301 dep->endpoint.caps.dir_out = !direction;
3303 INIT_LIST_HEAD(&dep->pending_list);
3304 INIT_LIST_HEAD(&dep->started_list);
3305 INIT_LIST_HEAD(&dep->cancelled_list);
3307 dwc3_debugfs_create_endpoint_dir(dep);
3331 struct dwc3_ep *dep;
3335 dep = dwc->eps[epnum];
3336 if (!dep)
3348 dwc3_free_trb_pool(dep);
3349 list_del(&dep->endpoint.ep_list);
3352 dwc3_debugfs_remove_endpoint_dir(dep);
3353 kfree(dep);
3359 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3365 dwc3_ep_inc_deq(dep);
3367 trace_dwc3_complete_trb(dep, trb);
3387 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3392 frame_number &= ~(dep->interval - 1);
3401 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3402 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3427 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3431 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3439 trb = &dep->trb_pool[dep->trb_dequeue];
3444 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3453 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3457 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3459 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3468 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3476 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3479 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3488 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3502 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
3521 dwc3_gadget_giveback(dep, req, request_status);
3527 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3532 while (!list_empty(&dep->started_list)) {
3535 req = next_request(&dep->started_list);
3536 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3544 if (!dep->endpoint.desc)
3549 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3552 struct dwc3 *dwc = dep->dwc;
3554 if (!dep->endpoint.desc || !dwc->pullups_connected ||
3558 if (!list_empty(&dep->pending_list))
3565 req = next_request(&dep->started_list);
3572 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3575 dep->frame_number = event->parameters;
3578 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3581 struct dwc3 *dwc = dep->dwc;
3584 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3586 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3589 if (!dep->endpoint.desc)
3592 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3593 list_empty(&dep->started_list) &&
3594 (list_empty(&dep->pending_list) || status == -EXDEV))
3595 dwc3_stop_active_transfer(dep, true, true);
3596 else if (dwc3_gadget_ep_should_continue(dep))
3597 if (__dwc3_gadget_kick_transfer(dep) == 0)
3610 dep = dwc->eps[i];
3612 if (!(dep->flags & DWC3_EP_ENABLED))
3615 if (!list_empty(&dep->started_list))
3629 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3634 if (!dep->endpoint.desc)
3637 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3638 dwc3_gadget_endpoint_frame_from_event(dep, event);
3646 dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3649 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3654 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3659 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3660 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3663 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3666 dwc3_gadget_endpoint_frame_from_event(dep, event);
3676 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3679 (void) __dwc3_gadget_start_isoc(dep);
3682 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3695 if (dep->stream_capable)
3696 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3698 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3699 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3700 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3702 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3703 struct dwc3 *dwc = dep->dwc;
3705 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3706 if (dwc3_send_clear_stall_ep_cmd(dep)) {
3709 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3715 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3716 if (dwc->clear_stall_protocol == dep->number)
3720 if ((dep->flags & DWC3_EP_DELAY_START) &&
3721 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3722 __dwc3_gadget_kick_transfer(dep);
3724 dep->flags &= ~DWC3_EP_DELAY_START;
3727 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3730 struct dwc3 *dwc = dep->dwc;
3733 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3747 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3748 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3749 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3751 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3756 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3757 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3759 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3779 dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3781 dep->flags |= DWC3_EP_DELAY_START;
3782 dwc3_stop_active_transfer(dep, true, true);
3789 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3795 struct dwc3_ep *dep;
3798 dep = dwc->eps[epnum];
3800 if (!(dep->flags & DWC3_EP_ENABLED)) {
3801 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3817 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3820 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3823 dwc3_gadget_endpoint_command_complete(dep, event);
3826 dwc3_gadget_endpoint_transfer_complete(dep, event);
3829 dwc3_gadget_endpoint_stream_event(dep, event);
3878 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3881 struct dwc3 *dwc = dep->dwc;
3889 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
3892 if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))
3895 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3896 (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3907 dep->flags |= DWC3_EP_DELAY_STOP;
3942 __dwc3_stop_active_transfer(dep, force, interrupt);
3950 struct dwc3_ep *dep;
3953 dep = dwc->eps[epnum];
3954 if (!dep)
3957 if (!(dep->flags & DWC3_EP_STALL))
3960 dep->flags &= ~DWC3_EP_STALL;
3962 ret = dwc3_send_clear_stall_ep_cmd(dep);
4080 struct dwc3_ep *dep;
4199 dep = dwc->eps[0];
4200 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4202 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4206 dep = dwc->eps[1];
4207 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4209 dev_err(dwc->dev, "failed to enable %s\n", dep->name);