Lines Matching refs:xhci
58 #include "xhci.h"
59 #include "xhci-trace.h"
61 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
142 static void next_trb(struct xhci_hcd *xhci,
158 void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring)
178 xhci_warn(xhci, "Missing link TRB at end of segment\n");
188 xhci_warn(xhci, "Ring is an endless link TRB loop\n");
214 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
224 xhci_err(xhci, "Tried to move enqueue past ring segment\n");
248 (xhci->quirks & XHCI_AMD_0x96_HOST)) &&
249 !xhci_link_trb_quirk(xhci)) {
266 xhci_warn(xhci, "%s: Ring link TRB loop\n", __func__);
280 static unsigned int xhci_num_trbs_free(struct xhci_hcd *xhci, struct xhci_ring *ring)
316 static unsigned int xhci_ring_expansion_needed(struct xhci_hcd *xhci, struct xhci_ring *ring,
342 xhci_dbg(xhci, "Ring expansion by %d segments needed\n",
344 xhci_dbg(xhci, "Adding %d trbs moves enq %d trbs into deq seg\n",
355 void xhci_ring_cmd_db(struct xhci_hcd *xhci)
357 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING))
360 xhci_dbg(xhci, "// Ding dong!\n");
364 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]);
366 readl(&xhci->dba->doorbell[0]);
369 static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay)
371 return mod_delayed_work(system_wq, &xhci->cmd_timer, delay);
374 static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci)
376 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command,
383 * This must be called with command ring stopped and xhci->lock held.
385 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci,
391 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) {
398 xhci_dbg(xhci, "Turn aborted command %p to no-op\n",
409 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
412 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) &&
413 !(xhci->xhc_state & XHCI_STATE_DYING)) {
414 xhci->current_cmd = cur_cmd;
415 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
416 xhci_ring_cmd_db(xhci);
420 /* Must be called with xhci->lock held, releases and aquires lock back */
421 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags)
423 struct xhci_segment *new_seg = xhci->cmd_ring->deq_seg;
424 union xhci_trb *new_deq = xhci->cmd_ring->dequeue;
428 xhci_dbg(xhci, "Abort command ring\n");
430 reinit_completion(&xhci->cmd_ring_stop_completion);
440 next_trb(xhci, NULL, &new_seg, &new_deq);
442 next_trb(xhci, NULL, &new_seg, &new_deq);
445 xhci_write_64(xhci, crcr | CMD_RING_ABORT, &xhci->op_regs->cmd_ring);
453 ret = xhci_handshake(&xhci->op_regs->cmd_ring,
456 xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret);
457 xhci_halt(xhci);
458 xhci_hc_died(xhci);
467 spin_unlock_irqrestore(&xhci->lock, flags);
468 ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion,
470 spin_lock_irqsave(&xhci->lock, flags);
472 xhci_dbg(xhci, "No stop event for abort, ring start fail?\n");
473 xhci_cleanup_command_queue(xhci);
475 xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci));
480 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci,
485 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id];
486 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
507 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
514 ep = &xhci->devs[slot_id]->eps[ep_index];
519 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0);
527 xhci_ring_ep_doorbell(xhci, slot_id, ep_index,
532 void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
536 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
539 static struct xhci_virt_ep *xhci_get_virt_ep(struct xhci_hcd *xhci,
544 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id);
548 xhci_warn(xhci, "Invalid endpoint index %u\n", ep_index);
551 if (!xhci->devs[slot_id]) {
552 xhci_warn(xhci, "No xhci virt device for slot_id %u\n", slot_id);
556 return &xhci->devs[slot_id]->eps[ep_index];
559 static struct xhci_ring *xhci_virt_ep_to_ring(struct xhci_hcd *xhci,
571 xhci_warn(xhci, "Invalid stream_id %u request for slot_id %u ep_index %u\n",
583 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci,
589 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
593 return xhci_virt_ep_to_ring(xhci, ep, stream_id);
603 static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev,
616 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
620 static int xhci_move_dequeue_past_td(struct xhci_hcd *xhci,
624 struct xhci_virt_device *dev = xhci->devs[slot_id];
638 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,
641 xhci_warn(xhci, "WARN can't find new dequeue, invalid stream ID %u\n",
657 xhci_dbg(xhci, "ep ring empty, Set new dequeue = enqueue");
660 xhci_warn(xhci, "Can't find new dequeue state, missing td\n");
665 hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id);
690 next_trb(xhci, ep_ring, &new_seg, &new_deq);
694 xhci_err(xhci, "Error: Failed finding new dequeue state\n");
705 xhci_warn(xhci, "Can't find dma of new dequeue ptr\n");
706 xhci_warn(xhci, "deq seg = %p, deq ptr = %p\n", new_seg, new_deq);
711 xhci_warn(xhci, "Set TR Deq already pending, don't submit for 0x%pad\n",
717 cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC);
719 xhci_warn(xhci, "Can't alloc Set TR Deq cmd 0x%pad\n", &addr);
725 ret = queue_command(xhci, cmd,
731 xhci_free_command(xhci, cmd);
737 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
746 xhci_ring_cmd_db(xhci);
754 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
770 next_trb(xhci, ep_ring, &seg, &trb);
775 * Must be called with xhci->lock held in interrupt context,
776 * releases and re-acquires xhci->lock
778 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
786 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
787 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
788 if (xhci->quirks & XHCI_AMD_PLL_FIX)
798 static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
801 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
822 xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n",
832 static int xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td,
841 xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);
849 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
867 xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
874 xhci_giveback_urb_in_irq(xhci, td, status);
890 ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb);
893 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n",
895 xhci_td_cleanup(ep->xhci, td, ring, td->status);
897 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n",
900 if (ep->xhci->xhc_state & XHCI_STATE_DYING)
905 static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id,
911 command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
917 xhci_dbg(xhci, "%s-reset ep %u, slot %u\n",
921 ret = xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type);
924 xhci_err(xhci, "ERROR queuing reset endpoint for slot %d ep_index %d, %d\n",
929 static int xhci_handle_halted_endpoint(struct xhci_hcd *xhci,
954 xhci_dbg(xhci, "Reset ep command for ep_index %d already pending\n",
959 err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type);
965 xhci_ring_cmd_db(xhci);
981 struct xhci_hcd *xhci;
990 xhci = ep->xhci;
993 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
999 ring = xhci_urb_to_transfer_ring(xhci, td->urb);
1001 xhci_warn(xhci, "WARN Cancelled URB %p has invalid stream ID %u.\n",
1011 hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index,
1016 trb_in_td(xhci, td->start_seg, td->first_trb, td->last_trb, hw_deq, false)) {
1026 xhci_dbg(xhci,
1034 td_to_noop(xhci, ring, td, false);
1043 err = xhci_move_dequeue_past_td(xhci, slot_id, ep->ep_index,
1051 xhci_dbg(xhci, "Failed to clear cancelled cached URB %p, mark clear anyway\n",
1053 td_to_noop(xhci, ring, td, false);
1070 hw_deq = xhci_get_hw_deq(ep->xhci, ep->vdev, ep->ep_index, 0);
1073 if (trb_in_td(ep->xhci, td->start_seg, td->first_trb,
1090 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
1102 if (!xhci->devs[slot_id])
1103 xhci_warn(xhci, "Stop endpoint command completion for disabled slot %u\n",
1109 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1113 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
1134 xhci_dbg(xhci, "Stop ep completion raced with stall, reset ep\n");
1144 err = xhci_handle_halted_endpoint(xhci, ep, td, reset_type);
1151 xhci_dbg(xhci, "Stop ep completion ctx error, ep is running\n");
1153 command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1158 xhci_queue_stop_endpoint(xhci, command, slot_id, ep_index, 0);
1159 xhci_ring_cmd_db(xhci);
1173 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1176 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring)
1187 xhci_unmap_td_bounce_buffer(xhci, ring, cur_td);
1191 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
1195 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci,
1203 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1217 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1220 xhci_kill_ring_urbs(xhci, ring);
1226 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1229 xhci_kill_ring_urbs(xhci, ring);
1238 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
1248 * Call with xhci->lock held.
1251 void xhci_hc_died(struct xhci_hcd *xhci)
1255 if (xhci->xhc_state & XHCI_STATE_DYING)
1258 xhci_err(xhci, "xHCI host controller not responding, assume dead\n");
1259 xhci->xhc_state |= XHCI_STATE_DYING;
1261 xhci_cleanup_command_queue(xhci);
1264 for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) {
1265 if (!xhci->devs[i])
1268 xhci_kill_endpoint_urbs(xhci, i, j);
1272 if (!(xhci->xhc_state & XHCI_STATE_REMOVING))
1273 usb_hc_died(xhci_to_hcd(xhci));
1276 static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci,
1307 xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
1320 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
1333 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1337 ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id);
1339 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n",
1345 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
1346 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx);
1356 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n");
1359 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n");
1363 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1368 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n",
1372 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n",
1392 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1399 update_ring_for_set_deq_completion(xhci, ep->vdev,
1402 xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n");
1403 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n",
1410 ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb);
1413 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n",
1415 xhci_td_cleanup(ep->xhci, td, ep_ring, td->status);
1417 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n",
1426 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1429 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
1437 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
1441 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
1447 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
1460 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1463 static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
1472 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id)
1477 virt_dev = xhci->devs[slot_id];
1481 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
1484 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
1486 xhci_free_device_endpoint_resources(xhci, virt_dev, true);
1489 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
1503 virt_dev = xhci->devs[slot_id];
1508 xhci_warn(xhci, "Could not get input context, bad type.\n");
1517 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index);
1523 static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id)
1528 vdev = xhci->devs[slot_id];
1531 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1535 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id)
1540 vdev = xhci->devs[slot_id];
1542 xhci_warn(xhci, "Reset device command completion for disabled slot %u\n",
1546 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1549 xhci_dbg(xhci, "Completed reset device command.\n");
1552 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
1555 if (!(xhci->quirks & XHCI_NEC_HOST)) {
1556 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n");
1559 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1577 void xhci_cleanup_command_queue(struct xhci_hcd *xhci)
1580 xhci->current_cmd = NULL;
1581 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list)
1587 struct xhci_hcd *xhci;
1594 xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer);
1596 spin_lock_irqsave(&xhci->lock, flags);
1602 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) {
1603 spin_unlock_irqrestore(&xhci->lock, flags);
1607 cmd_field3 = le32_to_cpu(xhci->current_cmd->command_trb->generic.field[3]);
1608 usbsts = readl(&xhci->op_regs->status);
1609 xhci_dbg(xhci, "Command timeout, USBSTS:%s\n", xhci_decode_usbsts(str, usbsts));
1611 /* Bail out and tear down xhci if a stop endpoint command failed */
1615 xhci_warn(xhci, "xHCI host not responding to stop endpoint command\n");
1617 ep = xhci_get_virt_ep(xhci, TRB_TO_SLOT_ID(cmd_field3),
1622 xhci_halt(xhci);
1623 xhci_hc_died(xhci);
1628 xhci->current_cmd->status = COMP_COMMAND_ABORTED;
1631 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
1633 xhci_hc_died(xhci);
1637 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) &&
1640 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED;
1641 xhci_dbg(xhci, "Command timeout\n");
1642 xhci_abort_cmd_ring(xhci, flags);
1647 if (xhci->xhc_state & XHCI_STATE_REMOVING) {
1648 xhci_dbg(xhci, "host removed, ring start fail?\n");
1649 xhci_cleanup_command_queue(xhci);
1655 xhci_dbg(xhci, "Command timeout on stopped ring\n");
1656 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd);
1659 spin_unlock_irqrestore(&xhci->lock, flags);
1663 static void handle_cmd_completion(struct xhci_hcd *xhci,
1675 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id);
1680 cmd_trb = xhci->cmd_ring->dequeue;
1682 trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic);
1684 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
1691 xhci_warn(xhci,
1696 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list);
1698 cancel_delayed_work(&xhci->cmd_timer);
1704 complete_all(&xhci->cmd_ring_stop_completion);
1708 if (cmd->command_trb != xhci->cmd_ring->dequeue) {
1709 xhci_err(xhci,
1721 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
1723 if (xhci->current_cmd == cmd)
1724 xhci->current_cmd = NULL;
1732 xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code);
1735 xhci_handle_cmd_disable_slot(xhci, slot_id);
1739 xhci_handle_cmd_config_ep(xhci, slot_id, cmd_comp_code);
1744 xhci_handle_cmd_addr_dev(xhci, slot_id);
1750 xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb,
1756 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code);
1766 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code);
1770 * Use the SLOT_ID from the command TRB instead (xhci 4.6.11)
1774 xhci_handle_cmd_reset_dev(xhci, slot_id);
1777 xhci_handle_cmd_nec_get_fw(xhci, event);
1781 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type);
1786 if (!list_is_singular(&xhci->cmd_list)) {
1787 xhci->current_cmd = list_first_entry(&cmd->cmd_list,
1789 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
1790 } else if (xhci->current_cmd == cmd) {
1791 xhci->current_cmd = NULL;
1797 inc_deq(xhci, xhci->cmd_ring);
1800 static void handle_vendor_event(struct xhci_hcd *xhci,
1803 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
1804 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
1805 handle_cmd_completion(xhci, &event->event_cmd);
1808 static void handle_device_notification(struct xhci_hcd *xhci,
1815 if (!xhci->devs[slot_id]) {
1816 xhci_warn(xhci, "Device Notification event for "
1821 xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n",
1823 udev = xhci->devs[slot_id]->udev;
1840 static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci)
1842 struct usb_hcd *hcd = xhci_to_hcd(xhci);
1857 static void handle_port_status(struct xhci_hcd *xhci,
1873 xhci_warn(xhci,
1877 max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1880 xhci_warn(xhci, "Port change event with invalid port ID %d\n",
1882 inc_deq(xhci, ir->event_ring);
1886 port = &xhci->hw_ports[port_id - 1];
1888 xhci_warn(xhci, "Port change event, no port for port ID %u\n",
1895 if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) {
1896 xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n");
1906 xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n",
1912 xhci_dbg(xhci, "resume root hub\n");
1918 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1);
1919 if (slot_id && xhci->devs[slot_id])
1920 xhci->devs[slot_id]->flags |= VDEV_PORT_ERROR;
1924 xhci_dbg(xhci, "port resume event for port %d\n", port_id);
1926 cmd_reg = readl(&xhci->op_regs->command);
1928 xhci_warn(xhci, "xHC is not running.\n");
1933 xhci_dbg(xhci, "remote wake SS port %d\n", port_id);
1939 xhci_test_and_clear_bit(xhci, port, PORT_PLC);
1941 xhci_set_link_state(xhci, port, XDEV_U0);
1948 xhci_dbg(xhci, "resume HS port %d\n", port_id);
1969 xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
1978 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1);
1979 if (slot_id && xhci->devs[slot_id])
1980 xhci_ring_device(xhci, slot_id);
1982 xhci_test_and_clear_bit(xhci, port, PORT_PLC);
1991 * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or
2003 xhci_test_and_clear_bit(xhci, port, PORT_PLC);
2004 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) &&
2006 xhci_cavium_reset_phy_quirk(xhci);
2011 inc_deq(xhci, ir->event_ring);
2027 xhci_dbg(xhci, "%s: starting usb%d port polling.\n",
2030 spin_unlock(&xhci->lock);
2033 spin_lock(&xhci->lock);
2042 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci,
2067 xhci_warn(xhci,
2103 static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td,
2111 (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) &&
2126 static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci,
2146 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code)
2152 xhci_dbg(xhci, "Vendor defined info completion code %u\n",
2154 xhci_dbg(xhci, "Treating code as success.\n");
2160 static int finish_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2166 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index);
2200 xhci_dbg(xhci, "Already resolving halted ep for 0x%llx\n",
2209 xhci_clear_hub_tt_buffer(xhci, td, ep);
2210 xhci_handle_halted_endpoint(xhci, ep, td, EP_HARD_RESET);
2214 * xhci internal endpoint state will go to a "halt" state for
2224 xhci_clear_hub_tt_buffer(xhci, td, ep);
2226 xhci_handle_halted_endpoint(xhci, ep, td, EP_HARD_RESET);
2236 inc_deq(xhci, ep_ring);
2238 return xhci_td_cleanup(xhci, td, ep_ring, td->status);
2242 static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring,
2249 for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) {
2259 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2269 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index);
2277 xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n",
2291 xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n");
2306 xhci_warn(xhci, "WARN: unexpected TRB Type %d\n",
2313 if (!xhci_requires_manual_halt_cleanup(xhci,
2316 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n",
2340 xhci_dbg(xhci, "Waiting for status stage event\n");
2349 return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2355 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2385 if (xhci->quirks & XHCI_TRUST_TX_LENGTH)
2438 frame->actual_length = sum_trb_lengths(xhci, ep->ring, ep_trb) +
2448 xhci_dbg(xhci, "Error mid isoc TD, wait for final completion event\n");
2453 return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2456 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2476 inc_deq(xhci, ep->ring);
2478 return xhci_td_cleanup(xhci, td, ep->ring, status);
2484 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep,
2492 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx);
2503 xhci_warn(xhci, "WARN Successful completion on short TX\n");
2504 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2511 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2525 if (xhci->quirks & XHCI_NO_SOFT_RETRY ||
2532 xhci_handle_halted_endpoint(xhci, ep, td, EP_SOFT_RESET);
2543 sum_trb_lengths(xhci, ep_ring, ep_trb) +
2547 xhci_warn(xhci, "bad transfer trb length %d in event trb\n",
2552 return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2560 static int handle_tx_event(struct xhci_hcd *xhci,
2583 ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
2585 xhci_err(xhci, "ERROR Invalid Transfer event\n");
2590 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
2593 xhci_err(xhci,
2599 /* Some transfer events don't always point to a trb, see xhci 4.17.4 */
2606 xhci_dbg(xhci, "Stream transaction error ep %u no id\n",
2609 xhci_handle_halted_endpoint(xhci, ep, NULL,
2612 xhci_handle_halted_endpoint(xhci, ep, NULL,
2620 xhci_err(xhci, "ERROR Transfer event for unknown stream ring slot %u ep %u\n",
2638 if (xhci->quirks & XHCI_TRUST_TX_LENGTH ||
2642 xhci_warn_ratelimited(xhci,
2650 xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n",
2654 xhci_dbg(xhci,
2659 xhci_dbg(xhci,
2665 xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id,
2670 xhci_dbg(xhci, "Split transaction error for slot %u ep %u\n",
2675 xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n",
2680 xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n",
2686 xhci_warn(xhci,
2693 xhci_warn(xhci,
2699 xhci_warn(xhci,
2704 xhci_warn(xhci,
2714 xhci_dbg(xhci, "underrun event on endpoint\n");
2716 xhci_dbg(xhci, "Underrun Event for slot %d ep %d "
2722 xhci_dbg(xhci, "overrun event on endpoint\n");
2724 xhci_dbg(xhci, "Overrun Event for slot %d ep %d "
2737 xhci_dbg(xhci,
2743 xhci_dbg(xhci,
2750 xhci_warn(xhci,
2756 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) {
2760 xhci_warn(xhci,
2782 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n",
2788 xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n",
2792 xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
2794 xhci_handle_halted_endpoint(xhci, ep, NULL,
2803 xhci_dbg(xhci, "All tds on the ep_ring skipped. Clear skip flag for slot %u ep %u.\n",
2814 ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,
2833 skip_isoc_td(xhci, td, ep, status);
2841 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) &&
2848 * xhci 4.10.2 states isoc endpoints should continue
2852 * xhci 4.9.1 states that if there are errors in mult-TRB
2862 ep_seg = trb_in_td(xhci, td_next->start_seg, td_next->first_trb,
2866 xhci_dbg(xhci, "Missing TD completion event after mid TD error\n");
2869 inc_deq(xhci, ep_ring);
2870 xhci_td_cleanup(xhci, td, ep_ring, td->status);
2877 xhci_err(xhci,
2882 trb_in_td(xhci, ep_ring->deq_seg,
2894 xhci_dbg(xhci,
2916 xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
2918 xhci_handle_halted_endpoint(xhci, ep, td,
2927 process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event);
2929 process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event);
2931 process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event);
2942 inc_deq(xhci, ir->event_ring);
2955 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
2968 * xhci->lock between event processing (e.g. to pass up port status changes).
2972 static int xhci_handle_event(struct xhci_hcd *xhci, struct xhci_interrupter *ir)
2981 xhci_err(xhci, "ERROR interrupter not ready\n");
3003 handle_cmd_completion(xhci, &event->event_cmd);
3006 handle_port_status(xhci, ir, event);
3010 ret = handle_tx_event(xhci, ir, &event->trans_event);
3015 handle_device_notification(xhci, event);
3019 handle_vendor_event(xhci, event, trb_type);
3021 xhci_warn(xhci, "ERROR unknown event type %d\n", trb_type);
3026 if (xhci->xhc_state & XHCI_STATE_DYING) {
3027 xhci_dbg(xhci, "xHCI host dying, returning from "
3034 inc_deq(xhci, ir->event_ring);
3047 static void xhci_update_erst_dequeue(struct xhci_hcd *xhci,
3055 temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue);
3061 xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr\n");
3078 xhci_write_64(xhci, temp_64, &ir->ir_set->erst_dequeue);
3088 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3096 spin_lock(&xhci->lock);
3098 status = readl(&xhci->op_regs->status);
3100 xhci_hc_died(xhci);
3109 xhci_warn(xhci, "WARNING: Host Controller Error\n");
3114 xhci_warn(xhci, "WARNING: Host System Error\n");
3115 xhci_halt(xhci);
3126 writel(status, &xhci->op_regs->status);
3129 ir = xhci->interrupter;
3137 if (xhci->xhc_state & XHCI_STATE_DYING ||
3138 xhci->xhc_state & XHCI_STATE_HALTED) {
3139 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. "
3144 temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue);
3145 xhci_write_64(xhci, temp_64 | ERST_EHB,
3155 while (xhci_handle_event(xhci, ir) > 0) {
3158 xhci_update_erst_dequeue(xhci, ir, event_ring_deq, false);
3162 if (xhci->isoc_bei_interval > AVOID_BEI_INTERVAL_MIN)
3163 xhci->isoc_bei_interval = xhci->isoc_bei_interval / 2;
3168 xhci_update_erst_dequeue(xhci, ir, event_ring_deq, true);
3172 spin_unlock(&xhci->lock);
3192 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
3208 inc_enq(xhci, ring, more_trbs_coming);
3215 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
3228 xhci_warn(xhci, "WARN urb submitted to disabled ep\n");
3231 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n");
3236 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n");
3242 xhci_err(xhci, "ERROR unknown endpoint state for ep\n");
3250 if (ep_ring != xhci->cmd_ring) {
3251 new_segs = xhci_ring_expansion_needed(xhci, ep_ring, num_trbs);
3252 } else if (xhci_num_trbs_free(xhci, ep_ring) <= num_trbs) {
3253 xhci_err(xhci, "Do not support expand command ring\n");
3258 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion,
3260 if (xhci_ring_expansion(xhci, ep_ring, new_segs, mem_flags)) {
3261 xhci_err(xhci, "Ring expansion failed\n");
3270 if (!xhci_link_trb_quirk(xhci) &&
3272 (xhci->quirks & XHCI_AMD_0x96_HOST)))
3291 xhci_warn(xhci, "Ring is an endless link TRB loop\n");
3297 xhci_warn(xhci, "Missing link TRB at end of ring segment\n");
3304 static int prepare_transfer(struct xhci_hcd *xhci,
3317 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
3319 ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index,
3322 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n",
3327 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
3411 static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id,
3424 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id);
3427 static void check_interval(struct xhci_hcd *xhci, struct urb *urb,
3463 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3468 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index);
3469 check_interval(xhci, urb, ep_ctx);
3471 return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index);
3494 static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred,
3501 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST))
3510 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100))
3521 static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len,
3524 struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
3537 xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n",
3543 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len);
3563 xhci_warn(xhci, "WARN Wrong bounce buffer write length: %zu != %d\n",
3578 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n");
3585 xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len);
3591 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3609 ring = xhci_urb_to_transfer_ring(xhci, urb);
3626 ret = prepare_transfer(xhci, xhci->devs[slot_id],
3675 if (xhci_align_td(xhci, urb, enqd_len,
3703 remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len,
3710 queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt,
3735 ret = prepare_transfer(xhci, xhci->devs[slot_id],
3741 queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field);
3746 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
3751 /* Caller must have locked xhci->lock */
3752 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3765 ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
3785 ret = prepare_transfer(xhci, xhci->devs[slot_id],
3812 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) {
3821 queue_trb(xhci, ep_ring, true,
3848 remainder = xhci_td_remainder(xhci, 0,
3857 queue_trb(xhci, ep_ring, true,
3874 queue_trb(xhci, ep_ring, false,
3881 giveback_first_trb(xhci, slot_id, ep_index, 0,
3894 static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci,
3899 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER)
3914 static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci,
3920 if (xhci->hci_version < 0x100)
3946 static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci,
3966 ist = HCS_IST(xhci->hcs_params2) & 0x7;
3967 if (HCS_IST(xhci->hcs_params2) & (1 << 3))
3983 current_frame_id = readl(&xhci->run_regs->microframe_index);
3991 xhci_dbg(xhci, "%s: index %d, reg 0x%x start_frame_id 0x%x, end_frame_id 0x%x, start_frame 0x%x\n",
3992 __func__, index, readl(&xhci->run_regs->microframe_index),
4020 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n",
4023 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n");
4031 static bool trb_block_event_intr(struct xhci_hcd *xhci, int num_tds, int i)
4033 if (xhci->hci_version < 0x100)
4042 if (i && xhci->quirks & XHCI_AVOID_BEI)
4043 return !!(i % xhci->isoc_bei_interval);
4049 static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
4067 xep = &xhci->devs[slot_id]->eps[ep_index];
4068 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;
4072 xhci_dbg(xhci, "Isoc URB with zero packets?\n");
4097 burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count);
4098 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci,
4103 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index,
4115 HCC_CFC(xhci->hcc_params)) {
4116 frame_id = xhci_get_isoc_frame_id(xhci, urb, i);
4130 /* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */
4156 if (trb_block_event_intr(xhci, num_tds, i))
4165 remainder = xhci_td_remainder(xhci, running_total,
4172 /* xhci 1.1 with ETE uses TD Size field for TBC */
4179 queue_trb(xhci, ep_ring, more_trbs_coming,
4192 xhci_err(xhci, "ISOC TD length unmatch\n");
4199 if (HCC_CFC(xhci->hcc_params))
4202 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
4203 if (xhci->quirks & XHCI_AMD_PLL_FIX)
4206 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++;
4208 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
4224 td_to_noop(xhci, ep_ring, &urb_priv->td[0], true);
4237 * Update interval as xhci_queue_intr_tx does. Use xhci frame_index to
4241 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
4253 xdev = xhci->devs[slot_id];
4254 xep = &xhci->devs[slot_id]->eps[ep_index];
4256 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
4266 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
4275 check_interval(xhci, urb, ep_ctx);
4278 if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {
4285 start_frame = readl(&xhci->run_regs->microframe_index);
4291 ist = HCS_IST(xhci->hcs_params2) & 0x7;
4292 if (HCS_IST(xhci->hcs_params2) & (1 << 3))
4312 return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index);
4322 * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
4325 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
4329 int reserved_trbs = xhci->cmd_ring_reserved_trbs;
4332 if ((xhci->xhc_state & XHCI_STATE_DYING) ||
4333 (xhci->xhc_state & XHCI_STATE_HALTED)) {
4334 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n");
4341 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING,
4344 xhci_err(xhci, "ERR: No room for command on command ring\n");
4346 xhci_err(xhci, "ERR: Reserved TRB counting for "
4351 cmd->command_trb = xhci->cmd_ring->enqueue;
4354 if (list_empty(&xhci->cmd_list)) {
4355 xhci->current_cmd = cmd;
4356 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
4359 list_add_tail(&cmd->cmd_list, &xhci->cmd_list);
4361 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3,
4362 field4 | xhci->cmd_ring->cycle_state);
4367 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd,
4370 return queue_command(xhci, cmd, 0, 0, 0,
4375 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
4378 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4384 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
4387 return queue_command(xhci, cmd, field1, field2, field3, field4, false);
4391 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
4394 return queue_command(xhci, cmd, 0, 0, 0,
4400 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci,
4404 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4411 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd,
4414 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
4424 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd,
4432 return queue_command(xhci, cmd, 0, 0, 0,
4436 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd,
4447 return queue_command(xhci, cmd, 0, 0, 0,