Lines Matching refs:xhci
58 #include "xhci.h"
59 #include "xhci-trace.h"
60 #include "xhci-mtk.h"
139 static void next_trb(struct xhci_hcd *xhci,
156 void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring)
204 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
234 (xhci->quirks & XHCI_AMD_0x96_HOST)) &&
235 !xhci_link_trb_quirk(xhci)) {
259 static inline int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring,
277 void xhci_ring_cmd_db(struct xhci_hcd *xhci)
279 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING))
282 xhci_dbg(xhci, "// Ding dong!\n");
286 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]);
288 readl(&xhci->dba->doorbell[0]);
291 static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay)
293 return mod_delayed_work(system_wq, &xhci->cmd_timer, delay);
296 static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci)
298 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command,
305 * This must be called with command ring stopped and xhci->lock held.
307 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci,
313 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) {
320 xhci_dbg(xhci, "Turn aborted command %p to no-op\n",
331 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
334 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) &&
335 !(xhci->xhc_state & XHCI_STATE_DYING)) {
336 xhci->current_cmd = cur_cmd;
337 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
338 xhci_ring_cmd_db(xhci);
342 /* Must be called with xhci->lock held, releases and aquires lock back */
343 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags)
345 struct xhci_segment *new_seg = xhci->cmd_ring->deq_seg;
346 union xhci_trb *new_deq = xhci->cmd_ring->dequeue;
350 xhci_dbg(xhci, "Abort command ring\n");
352 reinit_completion(&xhci->cmd_ring_stop_completion);
362 next_trb(xhci, NULL, &new_seg, &new_deq);
364 next_trb(xhci, NULL, &new_seg, &new_deq);
367 xhci_write_64(xhci, crcr | CMD_RING_ABORT, &xhci->op_regs->cmd_ring);
375 ret = xhci_handshake(&xhci->op_regs->cmd_ring,
378 xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret);
379 xhci_halt(xhci);
380 xhci_hc_died(xhci);
389 spin_unlock_irqrestore(&xhci->lock, flags);
390 ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion,
392 spin_lock_irqsave(&xhci->lock, flags);
394 xhci_dbg(xhci, "No stop event for abort, ring start fail?\n");
395 xhci_cleanup_command_queue(xhci);
397 xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci));
402 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci,
407 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id];
408 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
430 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
437 ep = &xhci->devs[slot_id]->eps[ep_index];
442 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0);
450 xhci_ring_ep_doorbell(xhci, slot_id, ep_index,
455 void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
459 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
462 static struct xhci_virt_ep *xhci_get_virt_ep(struct xhci_hcd *xhci,
467 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id);
471 xhci_warn(xhci, "Invalid endpoint index %u\n", ep_index);
474 if (!xhci->devs[slot_id]) {
475 xhci_warn(xhci, "No xhci virt device for slot_id %u\n", slot_id);
479 return &xhci->devs[slot_id]->eps[ep_index];
486 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci,
492 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
501 xhci_warn(xhci,
511 xhci_warn(xhci,
528 static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev,
541 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
563 void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
568 struct xhci_virt_device *dev = xhci->devs[slot_id];
581 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,
584 xhci_warn(xhci, "WARN can't find new dequeue state "
601 xhci_warn(xhci, "Can't find new dequeue state, missing cur_td\n");
607 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
610 hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id);
619 if (xhci->quirks & XHCI_EP_CTX_BROKEN_DCS &&
621 halted_seg = trb_in_td(xhci, cur_td->start_seg,
629 xhci_dbg(xhci, "Endpoint DCS = %d TRB index = %d cycle = %d\n",
657 next_trb(xhci, ep_ring, &new_seg, &new_deq);
661 xhci_err(xhci, "Error: Failed finding new dequeue state\n");
674 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
677 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
681 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
690 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
706 next_trb(xhci, ep_ring, &seg, &trb);
710 static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci,
719 * Must be called with xhci->lock held in interrupt context,
720 * releases and re-acquires xhci->lock
722 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
730 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
731 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
732 if (xhci->quirks & XHCI_AMD_PLL_FIX)
742 static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
745 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
766 xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n",
776 static int xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td,
785 xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);
793 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
809 xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
816 xhci_giveback_urb_in_irq(xhci, td, status);
822 static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id,
828 command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
834 ret = xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type);
837 xhci_err(xhci, "ERROR queuing reset endpoint for slot %d ep_index %d, %d\n",
842 static void xhci_handle_halted_endpoint(struct xhci_hcd *xhci,
859 err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type);
865 xhci_cleanup_stalled_ring(xhci, slot_id, ep->ep_index, stream_id,
868 xhci_ring_cmd_db(xhci);
881 struct xhci_hcd *xhci;
887 xhci = ep->xhci;
890 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
895 ring = xhci_urb_to_transfer_ring(xhci, td->urb);
897 xhci_warn(xhci, "WARN Cancelled URB %p has invalid stream ID %u.\n",
905 hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index,
909 if (trb_in_td(xhci, td->start_seg, td->first_trb,
911 xhci_find_new_dequeue_state(xhci, ep->vdev->slot_id,
916 td_to_noop(xhci, ring, td, false);
933 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
946 if (!xhci->devs[slot_id])
947 xhci_warn(xhci, "Stop endpoint command "
956 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
960 vdev = xhci->devs[slot_id];
961 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
968 xhci_stop_watchdog_timer_in_irq(xhci, ep);
969 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
975 xhci_stop_watchdog_timer_in_irq(xhci, ep);
979 xhci_queue_new_dequeue_state(xhci, slot_id, ep_index,
981 xhci_ring_cmd_db(xhci);
984 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1002 ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
1003 xhci_unmap_td_bounce_buffer(xhci, ep_ring, cur_td);
1006 xhci_giveback_urb_in_irq(xhci, cur_td, 0);
1011 if (xhci->xhc_state & XHCI_STATE_DYING)
1015 /* Return to the event handler with xhci->lock re-acquired */
1018 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring)
1029 xhci_unmap_td_bounce_buffer(xhci, ring, cur_td);
1033 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
1037 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci,
1045 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1059 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1062 xhci_kill_ring_urbs(xhci, ring);
1068 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1071 xhci_kill_ring_urbs(xhci, ring);
1080 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
1090 * Call with xhci->lock held.
1093 void xhci_hc_died(struct xhci_hcd *xhci)
1097 if (xhci->xhc_state & XHCI_STATE_DYING)
1100 xhci_err(xhci, "xHCI host controller not responding, assume dead\n");
1101 xhci->xhc_state |= XHCI_STATE_DYING;
1103 xhci_cleanup_command_queue(xhci);
1106 for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) {
1107 if (!xhci->devs[i])
1110 xhci_kill_endpoint_urbs(xhci, i, j);
1114 if (!(xhci->xhc_state & XHCI_STATE_REMOVING))
1115 usb_hc_died(xhci_to_hcd(xhci));
1122 * through xhci->state.
1138 struct xhci_hcd *xhci = ep->xhci;
1143 spin_lock_irqsave(&xhci->lock, flags);
1148 spin_unlock_irqrestore(&xhci->lock, flags);
1149 xhci_dbg(xhci, "Stop EP timer raced with cmd completion, exit");
1152 usbsts = readl(&xhci->op_regs->status);
1154 xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n");
1155 xhci_warn(xhci, "USBSTS:%s\n", xhci_decode_usbsts(str, usbsts));
1159 xhci_halt(xhci);
1166 xhci_hc_died(xhci);
1168 spin_unlock_irqrestore(&xhci->lock, flags);
1169 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1173 static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci,
1214 xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
1226 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
1239 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1243 dev = xhci->devs[slot_id];
1246 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n",
1252 ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
1253 slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx);
1263 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n");
1266 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n");
1270 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1275 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n",
1279 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n",
1299 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1306 update_ring_for_set_deq_completion(xhci, dev,
1309 xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n");
1310 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n",
1320 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1323 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
1332 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1336 vdev = xhci->devs[slot_id];
1337 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
1343 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
1350 if (xhci->quirks & XHCI_RESET_EP_QUIRK) {
1353 command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1357 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1359 xhci_queue_configure_endpoint(xhci, command,
1360 xhci->devs[slot_id]->in_ctx->dma, slot_id,
1362 xhci_ring_cmd_db(xhci);
1370 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1373 static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
1382 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id)
1387 virt_dev = xhci->devs[slot_id];
1391 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
1394 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
1396 xhci_free_device_endpoint_resources(xhci, virt_dev, true);
1399 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
1417 virt_dev = xhci->devs[slot_id];
1420 xhci_warn(xhci, "Could not get input context, bad type.\n");
1429 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index);
1438 if (xhci->quirks & XHCI_RESET_EP_QUIRK &&
1444 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1450 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1456 static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id)
1461 vdev = xhci->devs[slot_id];
1462 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1466 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id,
1472 vdev = xhci->devs[slot_id];
1473 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1476 xhci_dbg(xhci, "Completed reset device command.\n");
1477 if (!xhci->devs[slot_id])
1478 xhci_warn(xhci, "Reset device command completion "
1482 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
1485 if (!(xhci->quirks & XHCI_NEC_HOST)) {
1486 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n");
1489 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1507 void xhci_cleanup_command_queue(struct xhci_hcd *xhci)
1510 xhci->current_cmd = NULL;
1511 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list)
1517 struct xhci_hcd *xhci;
1521 xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer);
1523 spin_lock_irqsave(&xhci->lock, flags);
1529 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) {
1530 spin_unlock_irqrestore(&xhci->lock, flags);
1534 xhci->current_cmd->status = COMP_COMMAND_ABORTED;
1537 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
1539 xhci_hc_died(xhci);
1543 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) &&
1546 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED;
1547 xhci_dbg(xhci, "Command timeout\n");
1548 xhci_abort_cmd_ring(xhci, flags);
1553 if (xhci->xhc_state & XHCI_STATE_REMOVING) {
1554 xhci_dbg(xhci, "host removed, ring start fail?\n");
1555 xhci_cleanup_command_queue(xhci);
1561 xhci_dbg(xhci, "Command timeout on stopped ring\n");
1562 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd);
1565 spin_unlock_irqrestore(&xhci->lock, flags);
1569 static void handle_cmd_completion(struct xhci_hcd *xhci,
1581 cmd_trb = xhci->cmd_ring->dequeue;
1583 trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic);
1585 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
1592 xhci_warn(xhci,
1597 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list);
1599 cancel_delayed_work(&xhci->cmd_timer);
1605 complete_all(&xhci->cmd_ring_stop_completion);
1609 if (cmd->command_trb != xhci->cmd_ring->dequeue) {
1610 xhci_err(xhci,
1622 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
1624 if (xhci->current_cmd == cmd)
1625 xhci->current_cmd = NULL;
1633 xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code);
1636 xhci_handle_cmd_disable_slot(xhci, slot_id);
1640 xhci_handle_cmd_config_ep(xhci, slot_id, event,
1646 xhci_handle_cmd_addr_dev(xhci, slot_id);
1652 xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, event);
1657 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code);
1667 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code);
1671 * Use the SLOT_ID from the command TRB instead (xhci 4.6.11)
1675 xhci_handle_cmd_reset_dev(xhci, slot_id, event);
1678 xhci_handle_cmd_nec_get_fw(xhci, event);
1682 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type);
1687 if (!list_is_singular(&xhci->cmd_list)) {
1688 xhci->current_cmd = list_first_entry(&cmd->cmd_list,
1690 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
1691 } else if (xhci->current_cmd == cmd) {
1692 xhci->current_cmd = NULL;
1698 inc_deq(xhci, xhci->cmd_ring);
1701 static void handle_vendor_event(struct xhci_hcd *xhci,
1707 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
1708 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
1709 handle_cmd_completion(xhci, &event->event_cmd);
1712 static void handle_device_notification(struct xhci_hcd *xhci,
1719 if (!xhci->devs[slot_id]) {
1720 xhci_warn(xhci, "Device Notification event for "
1725 xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n",
1727 udev = xhci->devs[slot_id]->udev;
1744 static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci)
1746 struct usb_hcd *hcd = xhci_to_hcd(xhci);
1761 static void handle_port_status(struct xhci_hcd *xhci,
1776 xhci_warn(xhci,
1780 max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1783 xhci_warn(xhci, "Port change event with invalid port ID %d\n",
1785 inc_deq(xhci, xhci->event_ring);
1789 port = &xhci->hw_ports[port_id - 1];
1791 xhci_warn(xhci, "Port change event, no port for port ID %u\n",
1798 if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) {
1799 xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n");
1809 xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n",
1815 xhci_dbg(xhci, "resume root hub\n");
1821 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1);
1822 if (slot_id && xhci->devs[slot_id])
1823 xhci->devs[slot_id]->flags |= VDEV_PORT_ERROR;
1827 xhci_dbg(xhci, "port resume event for port %d\n", port_id);
1829 cmd_reg = readl(&xhci->op_regs->command);
1831 xhci_warn(xhci, "xHC is not running.\n");
1836 xhci_dbg(xhci, "remote wake SS port %d\n", port_id);
1842 xhci_test_and_clear_bit(xhci, port, PORT_PLC);
1844 xhci_set_link_state(xhci, port, XDEV_U0);
1851 xhci_dbg(xhci, "resume HS port %d\n", port_id);
1872 xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
1881 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1);
1882 if (slot_id && xhci->devs[slot_id])
1883 xhci_ring_device(xhci, slot_id);
1885 xhci_test_and_clear_bit(xhci, port, PORT_PLC);
1894 * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or
1906 xhci_test_and_clear_bit(xhci, port, PORT_PLC);
1907 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) &&
1909 xhci_cavium_reset_phy_quirk(xhci);
1914 inc_deq(xhci, xhci->event_ring);
1930 xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
1932 spin_unlock(&xhci->lock);
1935 spin_lock(&xhci->lock);
1944 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci,
1969 xhci_warn(xhci,
2005 static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td,
2013 (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) &&
2028 static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci,
2048 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code)
2054 xhci_dbg(xhci, "Vendor defined info completion code %u\n",
2056 xhci_dbg(xhci, "Treating code as success.\n");
2062 static int finish_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2068 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index);
2080 xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
2083 * xhci internal endpoint state will go to a "halt" state for
2093 xhci_clear_hub_tt_buffer(xhci, td, ep);
2095 xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,
2102 inc_deq(xhci, ep_ring);
2105 return xhci_td_cleanup(xhci, td, ep_ring, td->status);
2109 static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring,
2116 for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) {
2126 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2136 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index);
2144 xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n",
2158 xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n");
2173 xhci_warn(xhci, "WARN: unexpected TRB Type %d\n",
2180 if (!xhci_requires_manual_halt_cleanup(xhci,
2183 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n",
2207 xhci_dbg(xhci, "Waiting for status stage event\n");
2216 return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2222 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2252 if (xhci->quirks & XHCI_TRUST_TX_LENGTH)
2305 frame->actual_length = sum_trb_lengths(xhci, ep->ring, ep_trb) +
2315 xhci_dbg(xhci, "Error mid isoc TD, wait for final completion event\n");
2320 return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2323 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2344 inc_deq(xhci, ep->ring);
2346 return xhci_td_cleanup(xhci, td, ep->ring, status);
2352 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2360 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx);
2371 xhci_warn(xhci, "WARN Successful completion on short TX\n");
2372 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2379 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2393 if (xhci->quirks & XHCI_NO_SOFT_RETRY ||
2400 xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,
2412 sum_trb_lengths(xhci, ep_ring, ep_trb) +
2416 xhci_warn(xhci, "bad transfer trb length %d in event trb\n",
2421 return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2429 static int handle_tx_event(struct xhci_hcd *xhci,
2453 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
2455 xhci_err(xhci, "ERROR Invalid Transfer event\n");
2459 xdev = xhci->devs[slot_id];
2461 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2464 xhci_err(xhci,
2470 /* Some transfer events don't always point to a trb, see xhci 4.17.4 */
2477 xhci_dbg(xhci, "Stream transaction error ep %u no id\n",
2480 xhci_handle_halted_endpoint(xhci, ep, 0, NULL,
2483 xhci_handle_halted_endpoint(xhci, ep, 0, NULL,
2491 xhci_err(xhci, "ERROR Transfer event for unknown stream ring slot %u ep %u\n",
2512 if (xhci->quirks & XHCI_TRUST_TX_LENGTH ||
2516 xhci_warn_ratelimited(xhci,
2523 xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n",
2527 xhci_dbg(xhci,
2532 xhci_dbg(xhci,
2538 xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id,
2544 xhci_dbg(xhci, "Split transaction error for slot %u ep %u\n",
2549 xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n",
2554 xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n",
2560 xhci_warn(xhci,
2567 xhci_warn(xhci,
2573 xhci_warn(xhci,
2578 xhci_warn(xhci,
2588 xhci_dbg(xhci, "underrun event on endpoint\n");
2590 xhci_dbg(xhci, "Underrun Event for slot %d ep %d "
2596 xhci_dbg(xhci, "overrun event on endpoint\n");
2598 xhci_dbg(xhci, "Overrun Event for slot %d ep %d "
2611 xhci_dbg(xhci,
2617 xhci_dbg(xhci,
2624 xhci_warn(xhci,
2630 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) {
2634 xhci_warn(xhci,
2656 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n",
2662 xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n",
2666 xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
2668 xhci_handle_halted_endpoint(xhci, ep,
2679 xhci_dbg(xhci, "All tds on the ep_ring skipped. Clear skip flag for slot %u ep %u.\n",
2690 ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,
2709 skip_isoc_td(xhci, td, ep, status);
2717 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) &&
2724 * xhci 4.10.2 states isoc endpoints should continue
2728 * xhci 4.9.1 states that if there are errors in mult-TRB
2738 ep_seg = trb_in_td(xhci, td_next->start_seg, td_next->first_trb,
2742 xhci_dbg(xhci, "Missing TD completion event after mid TD error\n");
2745 inc_deq(xhci, ep_ring);
2746 xhci_td_cleanup(xhci, td, ep_ring, td->status);
2753 xhci_err(xhci,
2758 trb_in_td(xhci, ep_ring->deq_seg,
2770 xhci_dbg(xhci,
2792 xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
2794 xhci_handle_halted_endpoint(xhci, ep,
2804 process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event);
2806 process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event);
2808 process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event);
2819 inc_deq(xhci, xhci->event_ring);
2832 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
2834 xhci->event_ring->deq_seg,
2835 xhci->event_ring->dequeue),
2845 * xhci->lock between event processing (e.g. to pass up port status changes).
2849 static int xhci_handle_event(struct xhci_hcd *xhci)
2856 if (!xhci->event_ring || !xhci->event_ring->dequeue) {
2857 xhci_err(xhci, "ERROR event ring not ready\n");
2861 event = xhci->event_ring->dequeue;
2864 xhci->event_ring->cycle_state)
2867 trace_xhci_handle_event(xhci->event_ring, &event->generic);
2877 handle_cmd_completion(xhci, &event->event_cmd);
2880 handle_port_status(xhci, event);
2884 ret = handle_tx_event(xhci, &event->trans_event);
2889 handle_device_notification(xhci, event);
2894 handle_vendor_event(xhci, event);
2896 xhci_warn(xhci, "ERROR unknown event type %d\n",
2903 if (xhci->xhc_state & XHCI_STATE_DYING) {
2904 xhci_dbg(xhci, "xHCI host dying, returning from "
2911 inc_deq(xhci, xhci->event_ring);
2924 static void xhci_update_erst_dequeue(struct xhci_hcd *xhci,
2930 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2932 if (event_ring_deq != xhci->event_ring->dequeue) {
2933 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
2934 xhci->event_ring->dequeue);
2936 xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr\n");
2952 xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue);
2962 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
2970 spin_lock_irqsave(&xhci->lock, flags);
2972 status = readl(&xhci->op_regs->status);
2974 xhci_hc_died(xhci);
2983 xhci_warn(xhci, "WARNING: Host System Error\n");
2984 xhci_halt(xhci);
2995 writel(status, &xhci->op_regs->status);
2999 irq_pending = readl(&xhci->ir_set->irq_pending);
3001 writel(irq_pending, &xhci->ir_set->irq_pending);
3004 if (xhci->xhc_state & XHCI_STATE_DYING ||
3005 xhci->xhc_state & XHCI_STATE_HALTED) {
3006 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. "
3011 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
3012 xhci_write_64(xhci, temp_64 | ERST_EHB,
3013 &xhci->ir_set->erst_dequeue);
3018 event_ring_deq = xhci->event_ring->dequeue;
3022 while (xhci_handle_event(xhci) > 0) {
3025 xhci_update_erst_dequeue(xhci, event_ring_deq);
3026 event_ring_deq = xhci->event_ring->dequeue;
3031 xhci_update_erst_dequeue(xhci, event_ring_deq);
3035 spin_unlock_irqrestore(&xhci->lock, flags);
3054 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
3070 inc_enq(xhci, ring, more_trbs_coming);
3077 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
3089 xhci_warn(xhci, "WARN urb submitted to disabled ep\n");
3092 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n");
3097 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n");
3102 xhci_err(xhci, "ERROR unknown endpoint state for ep\n");
3111 if (room_on_ring(xhci, ep_ring, num_trbs))
3114 if (ep_ring == xhci->cmd_ring) {
3115 xhci_err(xhci, "Do not support expand command ring\n");
3119 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion,
3122 if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed,
3124 xhci_err(xhci, "Ring expansion failed\n");
3133 if (!xhci_link_trb_quirk(xhci) &&
3135 (xhci->quirks & XHCI_AMD_0x96_HOST)))
3155 static int prepare_transfer(struct xhci_hcd *xhci,
3168 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
3172 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n",
3177 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
3261 static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id,
3274 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id);
3277 static void check_interval(struct xhci_hcd *xhci, struct urb *urb,
3313 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3318 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index);
3319 check_interval(xhci, urb, ep_ctx);
3321 return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index);
3344 static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred,
3351 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST))
3360 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100))
3371 static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len,
3374 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
3387 xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n",
3393 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len);
3413 xhci_warn(xhci, "WARN Wrong bounce buffer write length: %zu != %d\n",
3428 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n");
3435 xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len);
3441 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3459 ring = xhci_urb_to_transfer_ring(xhci, urb);
3476 ret = prepare_transfer(xhci, xhci->devs[slot_id],
3525 if (xhci_align_td(xhci, urb, enqd_len,
3553 remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len,
3560 queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt,
3585 ret = prepare_transfer(xhci, xhci->devs[slot_id],
3591 queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field);
3596 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
3601 /* Caller must have locked xhci->lock */
3602 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3615 ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
3635 ret = prepare_transfer(xhci, xhci->devs[slot_id],
3662 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) {
3671 queue_trb(xhci, ep_ring, true,
3698 remainder = xhci_td_remainder(xhci, 0,
3707 queue_trb(xhci, ep_ring, true,
3724 queue_trb(xhci, ep_ring, false,
3731 giveback_first_trb(xhci, slot_id, ep_index, 0,
3744 static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci,
3749 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER)
3764 static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci,
3770 if (xhci->hci_version < 0x100)
3796 static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci,
3816 ist = HCS_IST(xhci->hcs_params2) & 0x7;
3817 if (HCS_IST(xhci->hcs_params2) & (1 << 3))
3833 current_frame_id = readl(&xhci->run_regs->microframe_index);
3841 xhci_dbg(xhci, "%s: index %d, reg 0x%x start_frame_id 0x%x, end_frame_id 0x%x, start_frame 0x%x\n",
3842 __func__, index, readl(&xhci->run_regs->microframe_index),
3870 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n",
3873 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n");
3881 static bool trb_block_event_intr(struct xhci_hcd *xhci, int num_tds, int i)
3883 if (xhci->hci_version < 0x100)
3892 if (i && xhci->quirks & XHCI_AVOID_BEI)
3899 static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3917 xep = &xhci->devs[slot_id]->eps[ep_index];
3918 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;
3922 xhci_dbg(xhci, "Isoc URB with zero packets?\n");
3947 burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count);
3948 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci,
3953 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index,
3965 HCC_CFC(xhci->hcc_params)) {
3966 frame_id = xhci_get_isoc_frame_id(xhci, urb, i);
3980 /* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */
4006 if (trb_block_event_intr(xhci, num_tds, i))
4015 remainder = xhci_td_remainder(xhci, running_total,
4022 /* xhci 1.1 with ETE uses TD Size field for TBC */
4029 queue_trb(xhci, ep_ring, more_trbs_coming,
4042 xhci_err(xhci, "ISOC TD length unmatch\n");
4049 if (HCC_CFC(xhci->hcc_params))
4052 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
4053 if (xhci->quirks & XHCI_AMD_PLL_FIX)
4056 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++;
4058 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
4074 td_to_noop(xhci, ep_ring, &urb_priv->td[0], true);
4088 * Update interval as xhci_queue_intr_tx does. Use xhci frame_index to
4092 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
4104 xdev = xhci->devs[slot_id];
4105 xep = &xhci->devs[slot_id]->eps[ep_index];
4107 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
4117 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
4126 check_interval(xhci, urb, ep_ctx);
4129 if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {
4136 start_frame = readl(&xhci->run_regs->microframe_index);
4142 ist = HCS_IST(xhci->hcs_params2) & 0x7;
4143 if (HCS_IST(xhci->hcs_params2) & (1 << 3))
4164 return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index);
4174 * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
4177 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
4181 int reserved_trbs = xhci->cmd_ring_reserved_trbs;
4184 if ((xhci->xhc_state & XHCI_STATE_DYING) ||
4185 (xhci->xhc_state & XHCI_STATE_HALTED)) {
4186 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n");
4193 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING,
4196 xhci_err(xhci, "ERR: No room for command on command ring\n");
4198 xhci_err(xhci, "ERR: Reserved TRB counting for "
4203 cmd->command_trb = xhci->cmd_ring->enqueue;
4206 if (list_empty(&xhci->cmd_list)) {
4207 xhci->current_cmd = cmd;
4208 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
4211 list_add_tail(&cmd->cmd_list, &xhci->cmd_list);
4213 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3,
4214 field4 | xhci->cmd_ring->cycle_state);
4219 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd,
4222 return queue_command(xhci, cmd, 0, 0, 0,
4227 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
4230 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4236 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
4239 return queue_command(xhci, cmd, field1, field2, field3, field4, false);
4243 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
4246 return queue_command(xhci, cmd, 0, 0, 0,
4252 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci,
4256 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4263 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd,
4266 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4276 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd,
4284 return queue_command(xhci, cmd, 0, 0, 0,
4289 void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci,
4303 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
4315 xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
4316 xhci_warn(xhci, "WARN deq seg = %p, deq pt = %p\n",
4320 ep = &xhci->devs[slot_id]->eps[ep_index];
4322 xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
4323 xhci_warn(xhci, "A Set TR Deq Ptr command is pending.\n");
4328 cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC);
4336 ret = queue_command(xhci, cmd,
4341 xhci_free_command(xhci, cmd);
4353 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd,
4364 return queue_command(xhci, cmd, 0, 0, 0,