1// SPDX-License-Identifier: GPL-2.0+ 2/* 3 * Copyright (C) 2001-2004 by David Brownell 4 */ 5 6/* this file is part of ehci-hcd.c */ 7 8/*-------------------------------------------------------------------------*/ 9 10/* 11 * EHCI hardware queue manipulation ... the core. QH/QTD manipulation. 12 * 13 * Control, bulk, and interrupt traffic all use "qh" lists. They list "qtd" 14 * entries describing USB transactions, max 16-20kB/entry (with 4kB-aligned 15 * buffers needed for the larger number). We use one QH per endpoint, queue 16 * multiple urbs (all three types) per endpoint. URBs may need several qtds. 17 * 18 * ISO traffic uses "ISO TD" (itd, and sitd) records, and (along with 19 * interrupts) needs careful scheduling. Performance improvements can be 20 * an ongoing challenge. That's in "ehci-sched.c". 21 * 22 * USB 1.1 devices are handled (a) by "companion" OHCI or UHCI root hubs, 23 * or otherwise through transaction translators (TTs) in USB 2.0 hubs using 24 * (b) special fields in qh entries or (c) split iso entries. TTs will 25 * buffer low/full speed data so the host collects it at high speed. 26 */ 27 28/*-------------------------------------------------------------------------*/ 29 30/* PID Codes that are used here, from EHCI specification, Table 3-16. */ 31#define PID_CODE_IN 1 32#define PID_CODE_SETUP 2 33 34/* fill a qtd, returning how much of the buffer we were able to queue up */ 35 36static int 37qtd_fill(struct ehci_hcd *ehci, struct ehci_qtd *qtd, dma_addr_t buf, 38 size_t len, int token, int maxpacket) 39{ 40 int i, count; 41 u64 addr = buf; 42 43 /* one buffer entry per 4K ... first might be short or unaligned */ 44 qtd->hw_buf[0] = cpu_to_hc32(ehci, (u32)addr); 45 qtd->hw_buf_hi[0] = cpu_to_hc32(ehci, (u32)(addr >> 32)); 46 count = 0x1000 - (buf & 0x0fff); /* rest of that page */ 47 if (likely (len < count)) /* ... iff needed */ 48 count = len; 49 else { 50 buf += 0x1000; 51 buf &= ~0x0fff; 52 53 /* per-qtd limit: from 16K to 20K (best alignment) */ 54 for (i = 1; count < len && i < 5; i++) { 55 addr = buf; 56 qtd->hw_buf[i] = cpu_to_hc32(ehci, (u32)addr); 57 qtd->hw_buf_hi[i] = cpu_to_hc32(ehci, 58 (u32)(addr >> 32)); 59 buf += 0x1000; 60 if ((count + 0x1000) < len) 61 count += 0x1000; 62 else 63 count = len; 64 } 65 66 /* short packets may only terminate transfers */ 67 if (count != len) 68 count -= (count % maxpacket); 69 } 70 qtd->hw_token = cpu_to_hc32(ehci, (count << 16) | token); 71 qtd->length = count; 72 73 return count; 74} 75 76/*-------------------------------------------------------------------------*/ 77 78static inline void 79qh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd) 80{ 81 struct ehci_qh_hw *hw = qh->hw; 82 83 /* writes to an active overlay are unsafe */ 84 WARN_ON(qh->qh_state != QH_STATE_IDLE); 85 86 hw->hw_qtd_next = QTD_NEXT(ehci, qtd->qtd_dma); 87 hw->hw_alt_next = EHCI_LIST_END(ehci); 88 89 /* Except for control endpoints, we make hardware maintain data 90 * toggle (like OHCI) ... here (re)initialize the toggle in the QH, 91 * and set the pseudo-toggle in udev. Only usb_clear_halt() will 92 * ever clear it. 93 */ 94 if (!(hw->hw_info1 & cpu_to_hc32(ehci, QH_TOGGLE_CTL))) { 95 unsigned is_out, epnum; 96 97 is_out = qh->is_out; 98 epnum = (hc32_to_cpup(ehci, &hw->hw_info1) >> 8) & 0x0f; 99 if (unlikely(!usb_gettoggle(qh->ps.udev, epnum, is_out))) { 100 hw->hw_token &= ~cpu_to_hc32(ehci, QTD_TOGGLE); 101 usb_settoggle(qh->ps.udev, epnum, is_out, 1); 102 } 103 } 104 105 hw->hw_token &= cpu_to_hc32(ehci, QTD_TOGGLE | QTD_STS_PING); 106} 107 108/* if it weren't for a common silicon quirk (writing the dummy into the qh 109 * overlay, so qh->hw_token wrongly becomes inactive/halted), only fault 110 * recovery (including urb dequeue) would need software changes to a QH... 111 */ 112static void 113qh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh) 114{ 115 struct ehci_qtd *qtd; 116 117 qtd = list_entry(qh->qtd_list.next, struct ehci_qtd, qtd_list); 118 119 /* 120 * first qtd may already be partially processed. 121 * If we come here during unlink, the QH overlay region 122 * might have reference to the just unlinked qtd. The 123 * qtd is updated in qh_completions(). Update the QH 124 * overlay here. 125 */ 126 if (qh->hw->hw_token & ACTIVE_BIT(ehci)) { 127 qh->hw->hw_qtd_next = qtd->hw_next; 128 if (qh->should_be_inactive) 129 ehci_warn(ehci, "qh %p should be inactive!\n", qh); 130 } else { 131 qh_update(ehci, qh, qtd); 132 } 133 qh->should_be_inactive = 0; 134} 135 136/*-------------------------------------------------------------------------*/ 137 138static void qh_link_async(struct ehci_hcd *ehci, struct ehci_qh *qh); 139 140static void ehci_clear_tt_buffer_complete(struct usb_hcd *hcd, 141 struct usb_host_endpoint *ep) 142{ 143 struct ehci_hcd *ehci = hcd_to_ehci(hcd); 144 struct ehci_qh *qh = ep->hcpriv; 145 unsigned long flags; 146 147 spin_lock_irqsave(&ehci->lock, flags); 148 qh->clearing_tt = 0; 149 if (qh->qh_state == QH_STATE_IDLE && !list_empty(&qh->qtd_list) 150 && ehci->rh_state == EHCI_RH_RUNNING) 151 qh_link_async(ehci, qh); 152 spin_unlock_irqrestore(&ehci->lock, flags); 153} 154 155static void ehci_clear_tt_buffer(struct ehci_hcd *ehci, struct ehci_qh *qh, 156 struct urb *urb, u32 token) 157{ 158 159 /* If an async split transaction gets an error or is unlinked, 160 * the TT buffer may be left in an indeterminate state. We 161 * have to clear the TT buffer. 162 * 163 * Note: this routine is never called for Isochronous transfers. 164 */ 165 if (urb->dev->tt && !usb_pipeint(urb->pipe) && !qh->clearing_tt) { 166#ifdef CONFIG_DYNAMIC_DEBUG 167 struct usb_device *tt = urb->dev->tt->hub; 168 dev_dbg(&tt->dev, 169 "clear tt buffer port %d, a%d ep%d t%08x\n", 170 urb->dev->ttport, urb->dev->devnum, 171 usb_pipeendpoint(urb->pipe), token); 172#endif /* CONFIG_DYNAMIC_DEBUG */ 173 if (!ehci_is_TDI(ehci) 174 || urb->dev->tt->hub != 175 ehci_to_hcd(ehci)->self.root_hub) { 176 if (usb_hub_clear_tt_buffer(urb) == 0) 177 qh->clearing_tt = 1; 178 } else { 179 180 /* REVISIT ARC-derived cores don't clear the root 181 * hub TT buffer in this way... 182 */ 183 } 184 } 185} 186 187static int qtd_copy_status ( 188 struct ehci_hcd *ehci, 189 struct urb *urb, 190 size_t length, 191 u32 token 192) 193{ 194 int status = -EINPROGRESS; 195 196 /* count IN/OUT bytes, not SETUP (even short packets) */ 197 if (likely(QTD_PID(token) != PID_CODE_SETUP)) 198 urb->actual_length += length - QTD_LENGTH (token); 199 200 /* don't modify error codes */ 201 if (unlikely(urb->unlinked)) 202 return status; 203 204 /* force cleanup after short read; not always an error */ 205 if (unlikely (IS_SHORT_READ (token))) 206 status = -EREMOTEIO; 207 208 /* serious "can't proceed" faults reported by the hardware */ 209 if (token & QTD_STS_HALT) { 210 if (token & QTD_STS_BABBLE) { 211 /* FIXME "must" disable babbling device's port too */ 212 status = -EOVERFLOW; 213 /* 214 * When MMF is active and PID Code is IN, queue is halted. 215 * EHCI Specification, Table 4-13. 216 */ 217 } else if ((token & QTD_STS_MMF) && 218 (QTD_PID(token) == PID_CODE_IN)) { 219 status = -EPROTO; 220 /* CERR nonzero + halt --> stall */ 221 } else if (QTD_CERR(token)) { 222 status = -EPIPE; 223 224 /* In theory, more than one of the following bits can be set 225 * since they are sticky and the transaction is retried. 226 * Which to test first is rather arbitrary. 227 */ 228 } else if (token & QTD_STS_MMF) { 229 /* fs/ls interrupt xfer missed the complete-split */ 230 status = -EPROTO; 231 } else if (token & QTD_STS_DBE) { 232 status = (QTD_PID (token) == 1) /* IN ? */ 233 ? -ENOSR /* hc couldn't read data */ 234 : -ECOMM; /* hc couldn't write data */ 235 } else if (token & QTD_STS_XACT) { 236 /* timeout, bad CRC, wrong PID, etc */ 237 ehci_dbg(ehci, "devpath %s ep%d%s 3strikes\n", 238 urb->dev->devpath, 239 usb_pipeendpoint(urb->pipe), 240 usb_pipein(urb->pipe) ? "in" : "out"); 241 status = -EPROTO; 242 } else { /* unknown */ 243 status = -EPROTO; 244 } 245 } 246 247 return status; 248} 249 250static void 251ehci_urb_done(struct ehci_hcd *ehci, struct urb *urb, int status) 252{ 253 if (usb_pipetype(urb->pipe) == PIPE_INTERRUPT) { 254 /* ... update hc-wide periodic stats */ 255 ehci_to_hcd(ehci)->self.bandwidth_int_reqs--; 256 } 257 258 if (unlikely(urb->unlinked)) { 259 INCR(ehci->stats.unlink); 260 } else { 261 /* report non-error and short read status as zero */ 262 if (status == -EINPROGRESS || status == -EREMOTEIO) 263 status = 0; 264 INCR(ehci->stats.complete); 265 } 266 267#ifdef EHCI_URB_TRACE 268 ehci_dbg (ehci, 269 "%s %s urb %p ep%d%s status %d len %d/%d\n", 270 __func__, urb->dev->devpath, urb, 271 usb_pipeendpoint (urb->pipe), 272 usb_pipein (urb->pipe) ? "in" : "out", 273 status, 274 urb->actual_length, urb->transfer_buffer_length); 275#endif 276 277 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 278 usb_hcd_giveback_urb(ehci_to_hcd(ehci), urb, status); 279} 280 281static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh); 282 283/* 284 * Process and free completed qtds for a qh, returning URBs to drivers. 285 * Chases up to qh->hw_current. Returns nonzero if the caller should 286 * unlink qh. 287 */ 288static unsigned 289qh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh) 290{ 291 struct ehci_qtd *last, *end = qh->dummy; 292 struct list_head *entry, *tmp; 293 int last_status; 294 int stopped; 295 u8 state; 296 struct ehci_qh_hw *hw = qh->hw; 297 298 /* completions (or tasks on other cpus) must never clobber HALT 299 * till we've gone through and cleaned everything up, even when 300 * they add urbs to this qh's queue or mark them for unlinking. 301 * 302 * NOTE: unlinking expects to be done in queue order. 303 * 304 * It's a bug for qh->qh_state to be anything other than 305 * QH_STATE_IDLE, unless our caller is scan_async() or 306 * scan_intr(). 307 */ 308 state = qh->qh_state; 309 qh->qh_state = QH_STATE_COMPLETING; 310 stopped = (state == QH_STATE_IDLE); 311 312 rescan: 313 last = NULL; 314 last_status = -EINPROGRESS; 315 qh->dequeue_during_giveback = 0; 316 317 /* remove de-activated QTDs from front of queue. 318 * after faults (including short reads), cleanup this urb 319 * then let the queue advance. 320 * if queue is stopped, handles unlinks. 321 */ 322 list_for_each_safe (entry, tmp, &qh->qtd_list) { 323 struct ehci_qtd *qtd; 324 struct urb *urb; 325 u32 token = 0; 326 327 qtd = list_entry (entry, struct ehci_qtd, qtd_list); 328 urb = qtd->urb; 329 330 /* clean up any state from previous QTD ...*/ 331 if (last) { 332 if (likely (last->urb != urb)) { 333 ehci_urb_done(ehci, last->urb, last_status); 334 last_status = -EINPROGRESS; 335 } 336 ehci_qtd_free (ehci, last); 337 last = NULL; 338 } 339 340 /* ignore urbs submitted during completions we reported */ 341 if (qtd == end) 342 break; 343 344 /* hardware copies qtd out of qh overlay */ 345 rmb (); 346 token = hc32_to_cpu(ehci, qtd->hw_token); 347 348 /* always clean up qtds the hc de-activated */ 349 retry_xacterr: 350 if ((token & QTD_STS_ACTIVE) == 0) { 351 352 /* Report Data Buffer Error: non-fatal but useful */ 353 if (token & QTD_STS_DBE) 354 ehci_dbg(ehci, 355 "detected DataBufferErr for urb %p ep%d%s len %d, qtd %p [qh %p]\n", 356 urb, 357 usb_endpoint_num(&urb->ep->desc), 358 usb_endpoint_dir_in(&urb->ep->desc) ? "in" : "out", 359 urb->transfer_buffer_length, 360 qtd, 361 qh); 362 363 /* on STALL, error, and short reads this urb must 364 * complete and all its qtds must be recycled. 365 */ 366 if ((token & QTD_STS_HALT) != 0) { 367 368 /* retry transaction errors until we 369 * reach the software xacterr limit 370 */ 371 if ((token & QTD_STS_XACT) && 372 QTD_CERR(token) == 0 && 373 ++qh->xacterrs < QH_XACTERR_MAX && 374 !urb->unlinked) { 375 ehci_dbg(ehci, 376 "detected XactErr len %zu/%zu retry %d\n", 377 qtd->length - QTD_LENGTH(token), qtd->length, qh->xacterrs); 378 379 /* reset the token in the qtd and the 380 * qh overlay (which still contains 381 * the qtd) so that we pick up from 382 * where we left off 383 */ 384 token &= ~QTD_STS_HALT; 385 token |= QTD_STS_ACTIVE | 386 (EHCI_TUNE_CERR << 10); 387 qtd->hw_token = cpu_to_hc32(ehci, 388 token); 389 wmb(); 390 hw->hw_token = cpu_to_hc32(ehci, 391 token); 392 goto retry_xacterr; 393 } 394 stopped = 1; 395 qh->unlink_reason |= QH_UNLINK_HALTED; 396 397 /* magic dummy for some short reads; qh won't advance. 398 * that silicon quirk can kick in with this dummy too. 399 * 400 * other short reads won't stop the queue, including 401 * control transfers (status stage handles that) or 402 * most other single-qtd reads ... the queue stops if 403 * URB_SHORT_NOT_OK was set so the driver submitting 404 * the urbs could clean it up. 405 */ 406 } else if (IS_SHORT_READ (token) 407 && !(qtd->hw_alt_next 408 & EHCI_LIST_END(ehci))) { 409 stopped = 1; 410 qh->unlink_reason |= QH_UNLINK_SHORT_READ; 411 } 412 413 /* stop scanning when we reach qtds the hc is using */ 414 } else if (likely (!stopped 415 && ehci->rh_state >= EHCI_RH_RUNNING)) { 416 break; 417 418 /* scan the whole queue for unlinks whenever it stops */ 419 } else { 420 stopped = 1; 421 422 /* cancel everything if we halt, suspend, etc */ 423 if (ehci->rh_state < EHCI_RH_RUNNING) { 424 last_status = -ESHUTDOWN; 425 qh->unlink_reason |= QH_UNLINK_SHUTDOWN; 426 } 427 428 /* this qtd is active; skip it unless a previous qtd 429 * for its urb faulted, or its urb was canceled. 430 */ 431 else if (last_status == -EINPROGRESS && !urb->unlinked) 432 continue; 433 434 /* 435 * If this was the active qtd when the qh was unlinked 436 * and the overlay's token is active, then the overlay 437 * hasn't been written back to the qtd yet so use its 438 * token instead of the qtd's. After the qtd is 439 * processed and removed, the overlay won't be valid 440 * any more. 441 */ 442 if (state == QH_STATE_IDLE && 443 qh->qtd_list.next == &qtd->qtd_list && 444 (hw->hw_token & ACTIVE_BIT(ehci))) { 445 token = hc32_to_cpu(ehci, hw->hw_token); 446 hw->hw_token &= ~ACTIVE_BIT(ehci); 447 qh->should_be_inactive = 1; 448 449 /* An unlink may leave an incomplete 450 * async transaction in the TT buffer. 451 * We have to clear it. 452 */ 453 ehci_clear_tt_buffer(ehci, qh, urb, token); 454 } 455 } 456 457 /* unless we already know the urb's status, collect qtd status 458 * and update count of bytes transferred. in common short read 459 * cases with only one data qtd (including control transfers), 460 * queue processing won't halt. but with two or more qtds (for 461 * example, with a 32 KB transfer), when the first qtd gets a 462 * short read the second must be removed by hand. 463 */ 464 if (last_status == -EINPROGRESS) { 465 last_status = qtd_copy_status(ehci, urb, 466 qtd->length, token); 467 if (last_status == -EREMOTEIO 468 && (qtd->hw_alt_next 469 & EHCI_LIST_END(ehci))) 470 last_status = -EINPROGRESS; 471 472 /* As part of low/full-speed endpoint-halt processing 473 * we must clear the TT buffer (11.17.5). 474 */ 475 if (unlikely(last_status != -EINPROGRESS && 476 last_status != -EREMOTEIO)) { 477 /* The TT's in some hubs malfunction when they 478 * receive this request following a STALL (they 479 * stop sending isochronous packets). Since a 480 * STALL can't leave the TT buffer in a busy 481 * state (if you believe Figures 11-48 - 11-51 482 * in the USB 2.0 spec), we won't clear the TT 483 * buffer in this case. Strictly speaking this 484 * is a violation of the spec. 485 */ 486 if (last_status != -EPIPE) 487 ehci_clear_tt_buffer(ehci, qh, urb, 488 token); 489 } 490 } 491 492 /* if we're removing something not at the queue head, 493 * patch the hardware queue pointer. 494 */ 495 if (stopped && qtd->qtd_list.prev != &qh->qtd_list) { 496 last = list_entry (qtd->qtd_list.prev, 497 struct ehci_qtd, qtd_list); 498 last->hw_next = qtd->hw_next; 499 } 500 501 /* remove qtd; it's recycled after possible urb completion */ 502 list_del (&qtd->qtd_list); 503 last = qtd; 504 505 /* reinit the xacterr counter for the next qtd */ 506 qh->xacterrs = 0; 507 } 508 509 /* last urb's completion might still need calling */ 510 if (likely (last != NULL)) { 511 ehci_urb_done(ehci, last->urb, last_status); 512 ehci_qtd_free (ehci, last); 513 } 514 515 /* Do we need to rescan for URBs dequeued during a giveback? */ 516 if (unlikely(qh->dequeue_during_giveback)) { 517 /* If the QH is already unlinked, do the rescan now. */ 518 if (state == QH_STATE_IDLE) 519 goto rescan; 520 521 /* Otherwise the caller must unlink the QH. */ 522 } 523 524 /* restore original state; caller must unlink or relink */ 525 qh->qh_state = state; 526 527 /* be sure the hardware's done with the qh before refreshing 528 * it after fault cleanup, or recovering from silicon wrongly 529 * overlaying the dummy qtd (which reduces DMA chatter). 530 * 531 * We won't refresh a QH that's linked (after the HC 532 * stopped the queue). That avoids a race: 533 * - HC reads first part of QH; 534 * - CPU updates that first part and the token; 535 * - HC reads rest of that QH, including token 536 * Result: HC gets an inconsistent image, and then 537 * DMAs to/from the wrong memory (corrupting it). 538 * 539 * That should be rare for interrupt transfers, 540 * except maybe high bandwidth ... 541 */ 542 if (stopped != 0 || hw->hw_qtd_next == EHCI_LIST_END(ehci)) 543 qh->unlink_reason |= QH_UNLINK_DUMMY_OVERLAY; 544 545 /* Let the caller know if the QH needs to be unlinked. */ 546 return qh->unlink_reason; 547} 548 549/*-------------------------------------------------------------------------*/ 550 551/* 552 * reverse of qh_urb_transaction: free a list of TDs. 553 * used for cleanup after errors, before HC sees an URB's TDs. 554 */ 555static void qtd_list_free ( 556 struct ehci_hcd *ehci, 557 struct urb *urb, 558 struct list_head *qtd_list 559) { 560 struct list_head *entry, *temp; 561 562 list_for_each_safe (entry, temp, qtd_list) { 563 struct ehci_qtd *qtd; 564 565 qtd = list_entry (entry, struct ehci_qtd, qtd_list); 566 list_del (&qtd->qtd_list); 567 ehci_qtd_free (ehci, qtd); 568 } 569} 570 571/* 572 * create a list of filled qtds for this URB; won't link into qh. 573 */ 574static struct list_head * 575qh_urb_transaction ( 576 struct ehci_hcd *ehci, 577 struct urb *urb, 578 struct list_head *head, 579 gfp_t flags 580) { 581 struct ehci_qtd *qtd, *qtd_prev; 582 dma_addr_t buf; 583 int len, this_sg_len, maxpacket; 584 int is_input; 585 u32 token; 586 int i; 587 struct scatterlist *sg; 588 589 /* 590 * URBs map to sequences of QTDs: one logical transaction 591 */ 592 qtd = ehci_qtd_alloc (ehci, flags); 593 if (unlikely (!qtd)) 594 return NULL; 595 list_add_tail (&qtd->qtd_list, head); 596 qtd->urb = urb; 597 598 token = QTD_STS_ACTIVE; 599 token |= (EHCI_TUNE_CERR << 10); 600 /* for split transactions, SplitXState initialized to zero */ 601 602 len = urb->transfer_buffer_length; 603 is_input = usb_pipein (urb->pipe); 604 if (usb_pipecontrol (urb->pipe)) { 605 /* SETUP pid */ 606 qtd_fill(ehci, qtd, urb->setup_dma, 607 sizeof (struct usb_ctrlrequest), 608 token | (2 /* "setup" */ << 8), 8); 609 610 /* ... and always at least one more pid */ 611 token ^= QTD_TOGGLE; 612 qtd_prev = qtd; 613 qtd = ehci_qtd_alloc (ehci, flags); 614 if (unlikely (!qtd)) 615 goto cleanup; 616 qtd->urb = urb; 617 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma); 618 list_add_tail (&qtd->qtd_list, head); 619 620 /* for zero length DATA stages, STATUS is always IN */ 621 if (len == 0) 622 token |= (1 /* "in" */ << 8); 623 } 624 625 /* 626 * data transfer stage: buffer setup 627 */ 628 i = urb->num_mapped_sgs; 629 if (len > 0 && i > 0) { 630 sg = urb->sg; 631 buf = sg_dma_address(sg); 632 633 /* urb->transfer_buffer_length may be smaller than the 634 * size of the scatterlist (or vice versa) 635 */ 636 this_sg_len = min_t(int, sg_dma_len(sg), len); 637 } else { 638 sg = NULL; 639 buf = urb->transfer_dma; 640 this_sg_len = len; 641 } 642 643 if (is_input) 644 token |= (1 /* "in" */ << 8); 645 /* else it's already initted to "out" pid (0 << 8) */ 646 647 maxpacket = usb_maxpacket(urb->dev, urb->pipe, !is_input); 648 649 /* 650 * buffer gets wrapped in one or more qtds; 651 * last one may be "short" (including zero len) 652 * and may serve as a control status ack 653 */ 654 for (;;) { 655 int this_qtd_len; 656 657 this_qtd_len = qtd_fill(ehci, qtd, buf, this_sg_len, token, 658 maxpacket); 659 this_sg_len -= this_qtd_len; 660 len -= this_qtd_len; 661 buf += this_qtd_len; 662 663 /* 664 * short reads advance to a "magic" dummy instead of the next 665 * qtd ... that forces the queue to stop, for manual cleanup. 666 * (this will usually be overridden later.) 667 */ 668 if (is_input) 669 qtd->hw_alt_next = ehci->async->hw->hw_alt_next; 670 671 /* qh makes control packets use qtd toggle; maybe switch it */ 672 if ((maxpacket & (this_qtd_len + (maxpacket - 1))) == 0) 673 token ^= QTD_TOGGLE; 674 675 if (likely(this_sg_len <= 0)) { 676 if (--i <= 0 || len <= 0) 677 break; 678 sg = sg_next(sg); 679 buf = sg_dma_address(sg); 680 this_sg_len = min_t(int, sg_dma_len(sg), len); 681 } 682 683 qtd_prev = qtd; 684 qtd = ehci_qtd_alloc (ehci, flags); 685 if (unlikely (!qtd)) 686 goto cleanup; 687 qtd->urb = urb; 688 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma); 689 list_add_tail (&qtd->qtd_list, head); 690 } 691 692 /* 693 * unless the caller requires manual cleanup after short reads, 694 * have the alt_next mechanism keep the queue running after the 695 * last data qtd (the only one, for control and most other cases). 696 */ 697 if (likely ((urb->transfer_flags & URB_SHORT_NOT_OK) == 0 698 || usb_pipecontrol (urb->pipe))) 699 qtd->hw_alt_next = EHCI_LIST_END(ehci); 700 701 /* 702 * control requests may need a terminating data "status" ack; 703 * other OUT ones may need a terminating short packet 704 * (zero length). 705 */ 706 if (likely (urb->transfer_buffer_length != 0)) { 707 int one_more = 0; 708 709 if (usb_pipecontrol (urb->pipe)) { 710 one_more = 1; 711 token ^= 0x0100; /* "in" <--> "out" */ 712 token |= QTD_TOGGLE; /* force DATA1 */ 713 } else if (usb_pipeout(urb->pipe) 714 && (urb->transfer_flags & URB_ZERO_PACKET) 715 && !(urb->transfer_buffer_length % maxpacket)) { 716 one_more = 1; 717 } 718 if (one_more) { 719 qtd_prev = qtd; 720 qtd = ehci_qtd_alloc (ehci, flags); 721 if (unlikely (!qtd)) 722 goto cleanup; 723 qtd->urb = urb; 724 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma); 725 list_add_tail (&qtd->qtd_list, head); 726 727 /* never any data in such packets */ 728 qtd_fill(ehci, qtd, 0, 0, token, 0); 729 } 730 } 731 732 /* by default, enable interrupt on urb completion */ 733 if (likely (!(urb->transfer_flags & URB_NO_INTERRUPT))) 734 qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC); 735 return head; 736 737cleanup: 738 qtd_list_free (ehci, urb, head); 739 return NULL; 740} 741 742/*-------------------------------------------------------------------------*/ 743 744// Would be best to create all qh's from config descriptors, 745// when each interface/altsetting is established. Unlink 746// any previous qh and cancel its urbs first; endpoints are 747// implicitly reset then (data toggle too). 748// That'd mean updating how usbcore talks to HCDs. (2.7?) 749 750 751/* 752 * Each QH holds a qtd list; a QH is used for everything except iso. 753 * 754 * For interrupt urbs, the scheduler must set the microframe scheduling 755 * mask(s) each time the QH gets scheduled. For highspeed, that's 756 * just one microframe in the s-mask. For split interrupt transactions 757 * there are additional complications: c-mask, maybe FSTNs. 758 */ 759static struct ehci_qh * 760qh_make ( 761 struct ehci_hcd *ehci, 762 struct urb *urb, 763 gfp_t flags 764) { 765 struct ehci_qh *qh = ehci_qh_alloc (ehci, flags); 766 struct usb_host_endpoint *ep; 767 u32 info1 = 0, info2 = 0; 768 int is_input, type; 769 int maxp = 0; 770 int mult; 771 struct usb_tt *tt = urb->dev->tt; 772 struct ehci_qh_hw *hw; 773 774 if (!qh) 775 return qh; 776 777 /* 778 * init endpoint/device data for this QH 779 */ 780 info1 |= usb_pipeendpoint (urb->pipe) << 8; 781 info1 |= usb_pipedevice (urb->pipe) << 0; 782 783 is_input = usb_pipein (urb->pipe); 784 type = usb_pipetype (urb->pipe); 785 ep = usb_pipe_endpoint (urb->dev, urb->pipe); 786 maxp = usb_endpoint_maxp (&ep->desc); 787 mult = usb_endpoint_maxp_mult (&ep->desc); 788 789 /* 1024 byte maxpacket is a hardware ceiling. High bandwidth 790 * acts like up to 3KB, but is built from smaller packets. 791 */ 792 if (maxp > 1024) { 793 ehci_dbg(ehci, "bogus qh maxpacket %d\n", maxp); 794 goto done; 795 } 796 797 /* Compute interrupt scheduling parameters just once, and save. 798 * - allowing for high bandwidth, how many nsec/uframe are used? 799 * - split transactions need a second CSPLIT uframe; same question 800 * - splits also need a schedule gap (for full/low speed I/O) 801 * - qh has a polling interval 802 * 803 * For control/bulk requests, the HC or TT handles these. 804 */ 805 if (type == PIPE_INTERRUPT) { 806 unsigned tmp; 807 808 qh->ps.usecs = NS_TO_US(usb_calc_bus_time(USB_SPEED_HIGH, 809 is_input, 0, mult * maxp)); 810 qh->ps.phase = NO_FRAME; 811 812 if (urb->dev->speed == USB_SPEED_HIGH) { 813 qh->ps.c_usecs = 0; 814 qh->gap_uf = 0; 815 816 if (urb->interval > 1 && urb->interval < 8) { 817 /* NOTE interval 2 or 4 uframes could work. 818 * But interval 1 scheduling is simpler, and 819 * includes high bandwidth. 820 */ 821 urb->interval = 1; 822 } else if (urb->interval > ehci->periodic_size << 3) { 823 urb->interval = ehci->periodic_size << 3; 824 } 825 qh->ps.period = urb->interval >> 3; 826 827 /* period for bandwidth allocation */ 828 tmp = min_t(unsigned, EHCI_BANDWIDTH_SIZE, 829 1 << (urb->ep->desc.bInterval - 1)); 830 831 /* Allow urb->interval to override */ 832 qh->ps.bw_uperiod = min_t(unsigned, tmp, urb->interval); 833 qh->ps.bw_period = qh->ps.bw_uperiod >> 3; 834 } else { 835 int think_time; 836 837 /* gap is f(FS/LS transfer times) */ 838 qh->gap_uf = 1 + usb_calc_bus_time (urb->dev->speed, 839 is_input, 0, maxp) / (125 * 1000); 840 841 /* FIXME this just approximates SPLIT/CSPLIT times */ 842 if (is_input) { // SPLIT, gap, CSPLIT+DATA 843 qh->ps.c_usecs = qh->ps.usecs + HS_USECS(0); 844 qh->ps.usecs = HS_USECS(1); 845 } else { // SPLIT+DATA, gap, CSPLIT 846 qh->ps.usecs += HS_USECS(1); 847 qh->ps.c_usecs = HS_USECS(0); 848 } 849 850 think_time = tt ? tt->think_time : 0; 851 qh->ps.tt_usecs = NS_TO_US(think_time + 852 usb_calc_bus_time (urb->dev->speed, 853 is_input, 0, maxp)); 854 if (urb->interval > ehci->periodic_size) 855 urb->interval = ehci->periodic_size; 856 qh->ps.period = urb->interval; 857 858 /* period for bandwidth allocation */ 859 tmp = min_t(unsigned, EHCI_BANDWIDTH_FRAMES, 860 urb->ep->desc.bInterval); 861 tmp = rounddown_pow_of_two(tmp); 862 863 /* Allow urb->interval to override */ 864 qh->ps.bw_period = min_t(unsigned, tmp, urb->interval); 865 qh->ps.bw_uperiod = qh->ps.bw_period << 3; 866 } 867 } 868 869 /* support for tt scheduling, and access to toggles */ 870 qh->ps.udev = urb->dev; 871 qh->ps.ep = urb->ep; 872 873 /* using TT? */ 874 switch (urb->dev->speed) { 875 case USB_SPEED_LOW: 876 info1 |= QH_LOW_SPEED; 877 fallthrough; 878 879 case USB_SPEED_FULL: 880 /* EPS 0 means "full" */ 881 if (type != PIPE_INTERRUPT) 882 info1 |= (EHCI_TUNE_RL_TT << 28); 883 if (type == PIPE_CONTROL) { 884 info1 |= QH_CONTROL_EP; /* for TT */ 885 info1 |= QH_TOGGLE_CTL; /* toggle from qtd */ 886 } 887 info1 |= maxp << 16; 888 889 info2 |= (EHCI_TUNE_MULT_TT << 30); 890 891 /* Some Freescale processors have an erratum in which the 892 * port number in the queue head was 0..N-1 instead of 1..N. 893 */ 894 if (ehci_has_fsl_portno_bug(ehci)) 895 info2 |= (urb->dev->ttport-1) << 23; 896 else 897 info2 |= urb->dev->ttport << 23; 898 899 /* set the address of the TT; for TDI's integrated 900 * root hub tt, leave it zeroed. 901 */ 902 if (tt && tt->hub != ehci_to_hcd(ehci)->self.root_hub) 903 info2 |= tt->hub->devnum << 16; 904 905 /* NOTE: if (PIPE_INTERRUPT) { scheduler sets c-mask } */ 906 907 break; 908 909 case USB_SPEED_HIGH: /* no TT involved */ 910 info1 |= QH_HIGH_SPEED; 911 if (type == PIPE_CONTROL) { 912 info1 |= (EHCI_TUNE_RL_HS << 28); 913 info1 |= 64 << 16; /* usb2 fixed maxpacket */ 914 info1 |= QH_TOGGLE_CTL; /* toggle from qtd */ 915 info2 |= (EHCI_TUNE_MULT_HS << 30); 916 } else if (type == PIPE_BULK) { 917 info1 |= (EHCI_TUNE_RL_HS << 28); 918 /* The USB spec says that high speed bulk endpoints 919 * always use 512 byte maxpacket. But some device 920 * vendors decided to ignore that, and MSFT is happy 921 * to help them do so. So now people expect to use 922 * such nonconformant devices with Linux too; sigh. 923 */ 924 info1 |= maxp << 16; 925 info2 |= (EHCI_TUNE_MULT_HS << 30); 926 } else { /* PIPE_INTERRUPT */ 927 info1 |= maxp << 16; 928 info2 |= mult << 30; 929 } 930 break; 931 default: 932 ehci_dbg(ehci, "bogus dev %p speed %d\n", urb->dev, 933 urb->dev->speed); 934done: 935 qh_destroy(ehci, qh); 936 return NULL; 937 } 938 939 /* NOTE: if (PIPE_INTERRUPT) { scheduler sets s-mask } */ 940 941 /* init as live, toggle clear */ 942 qh->qh_state = QH_STATE_IDLE; 943 hw = qh->hw; 944 hw->hw_info1 = cpu_to_hc32(ehci, info1); 945 hw->hw_info2 = cpu_to_hc32(ehci, info2); 946 qh->is_out = !is_input; 947 usb_settoggle (urb->dev, usb_pipeendpoint (urb->pipe), !is_input, 1); 948 return qh; 949} 950 951/*-------------------------------------------------------------------------*/ 952 953static void enable_async(struct ehci_hcd *ehci) 954{ 955 if (ehci->async_count++) 956 return; 957 958 /* Stop waiting to turn off the async schedule */ 959 ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_DISABLE_ASYNC); 960 961 /* Don't start the schedule until ASS is 0 */ 962 ehci_poll_ASS(ehci); 963 turn_on_io_watchdog(ehci); 964} 965 966static void disable_async(struct ehci_hcd *ehci) 967{ 968 if (--ehci->async_count) 969 return; 970 971 /* The async schedule and unlink lists are supposed to be empty */ 972 WARN_ON(ehci->async->qh_next.qh || !list_empty(&ehci->async_unlink) || 973 !list_empty(&ehci->async_idle)); 974 975 /* Don't turn off the schedule until ASS is 1 */ 976 ehci_poll_ASS(ehci); 977} 978 979/* move qh (and its qtds) onto async queue; maybe enable queue. */ 980 981static void qh_link_async (struct ehci_hcd *ehci, struct ehci_qh *qh) 982{ 983 __hc32 dma = QH_NEXT(ehci, qh->qh_dma); 984 struct ehci_qh *head; 985 986 /* Don't link a QH if there's a Clear-TT-Buffer pending */ 987 if (unlikely(qh->clearing_tt)) 988 return; 989 990 WARN_ON(qh->qh_state != QH_STATE_IDLE); 991 992 /* clear halt and/or toggle; and maybe recover from silicon quirk */ 993 qh_refresh(ehci, qh); 994 995 /* splice right after start */ 996 head = ehci->async; 997 qh->qh_next = head->qh_next; 998 qh->hw->hw_next = head->hw->hw_next; 999 wmb (); 1000 1001 head->qh_next.qh = qh; 1002 head->hw->hw_next = dma; 1003 1004 qh->qh_state = QH_STATE_LINKED; 1005 qh->xacterrs = 0; 1006 qh->unlink_reason = 0; 1007 /* qtd completions reported later by interrupt */ 1008 1009 enable_async(ehci); 1010} 1011 1012/*-------------------------------------------------------------------------*/ 1013 1014/* 1015 * For control/bulk/interrupt, return QH with these TDs appended. 1016 * Allocates and initializes the QH if necessary. 1017 * Returns null if it can't allocate a QH it needs to. 1018 * If the QH has TDs (urbs) already, that's great. 1019 */ 1020static struct ehci_qh *qh_append_tds ( 1021 struct ehci_hcd *ehci, 1022 struct urb *urb, 1023 struct list_head *qtd_list, 1024 int epnum, 1025 void **ptr 1026) 1027{ 1028 struct ehci_qh *qh = NULL; 1029 __hc32 qh_addr_mask = cpu_to_hc32(ehci, 0x7f); 1030 1031 qh = (struct ehci_qh *) *ptr; 1032 if (unlikely (qh == NULL)) { 1033 /* can't sleep here, we have ehci->lock... */ 1034 qh = qh_make (ehci, urb, GFP_ATOMIC); 1035 *ptr = qh; 1036 } 1037 if (likely (qh != NULL)) { 1038 struct ehci_qtd *qtd; 1039 1040 if (unlikely (list_empty (qtd_list))) 1041 qtd = NULL; 1042 else 1043 qtd = list_entry (qtd_list->next, struct ehci_qtd, 1044 qtd_list); 1045 1046 /* control qh may need patching ... */ 1047 if (unlikely (epnum == 0)) { 1048 1049 /* usb_reset_device() briefly reverts to address 0 */ 1050 if (usb_pipedevice (urb->pipe) == 0) 1051 qh->hw->hw_info1 &= ~qh_addr_mask; 1052 } 1053 1054 /* just one way to queue requests: swap with the dummy qtd. 1055 * only hc or qh_refresh() ever modify the overlay. 1056 */ 1057 if (likely (qtd != NULL)) { 1058 struct ehci_qtd *dummy; 1059 dma_addr_t dma; 1060 __hc32 token; 1061 1062 /* to avoid racing the HC, use the dummy td instead of 1063 * the first td of our list (becomes new dummy). both 1064 * tds stay deactivated until we're done, when the 1065 * HC is allowed to fetch the old dummy (4.10.2). 1066 */ 1067 token = qtd->hw_token; 1068 qtd->hw_token = HALT_BIT(ehci); 1069 1070 dummy = qh->dummy; 1071 1072 dma = dummy->qtd_dma; 1073 *dummy = *qtd; 1074 dummy->qtd_dma = dma; 1075 1076 list_del (&qtd->qtd_list); 1077 list_add (&dummy->qtd_list, qtd_list); 1078 list_splice_tail(qtd_list, &qh->qtd_list); 1079 1080 ehci_qtd_init(ehci, qtd, qtd->qtd_dma); 1081 qh->dummy = qtd; 1082 1083 /* hc must see the new dummy at list end */ 1084 dma = qtd->qtd_dma; 1085 qtd = list_entry (qh->qtd_list.prev, 1086 struct ehci_qtd, qtd_list); 1087 qtd->hw_next = QTD_NEXT(ehci, dma); 1088 1089 /* let the hc process these next qtds */ 1090 wmb (); 1091 dummy->hw_token = token; 1092 1093 urb->hcpriv = qh; 1094 } 1095 } 1096 return qh; 1097} 1098 1099/*-------------------------------------------------------------------------*/ 1100 1101static int 1102submit_async ( 1103 struct ehci_hcd *ehci, 1104 struct urb *urb, 1105 struct list_head *qtd_list, 1106 gfp_t mem_flags 1107) { 1108 int epnum; 1109 unsigned long flags; 1110 struct ehci_qh *qh = NULL; 1111 int rc; 1112 1113 epnum = urb->ep->desc.bEndpointAddress; 1114 1115#ifdef EHCI_URB_TRACE 1116 { 1117 struct ehci_qtd *qtd; 1118 qtd = list_entry(qtd_list->next, struct ehci_qtd, qtd_list); 1119 ehci_dbg(ehci, 1120 "%s %s urb %p ep%d%s len %d, qtd %p [qh %p]\n", 1121 __func__, urb->dev->devpath, urb, 1122 epnum & 0x0f, (epnum & USB_DIR_IN) ? "in" : "out", 1123 urb->transfer_buffer_length, 1124 qtd, urb->ep->hcpriv); 1125 } 1126#endif 1127 1128 spin_lock_irqsave (&ehci->lock, flags); 1129 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 1130 rc = -ESHUTDOWN; 1131 goto done; 1132 } 1133 rc = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 1134 if (unlikely(rc)) 1135 goto done; 1136 1137 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv); 1138 if (unlikely(qh == NULL)) { 1139 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 1140 rc = -ENOMEM; 1141 goto done; 1142 } 1143 1144 /* Control/bulk operations through TTs don't need scheduling, 1145 * the HC and TT handle it when the TT has a buffer ready. 1146 */ 1147 if (likely (qh->qh_state == QH_STATE_IDLE)) 1148 qh_link_async(ehci, qh); 1149 done: 1150 spin_unlock_irqrestore (&ehci->lock, flags); 1151 if (unlikely (qh == NULL)) 1152 qtd_list_free (ehci, urb, qtd_list); 1153 return rc; 1154} 1155 1156/*-------------------------------------------------------------------------*/ 1157#ifdef CONFIG_USB_HCD_TEST_MODE 1158/* 1159 * This function creates the qtds and submits them for the 1160 * SINGLE_STEP_SET_FEATURE Test. 1161 * This is done in two parts: first SETUP req for GetDesc is sent then 1162 * 15 seconds later, the IN stage for GetDesc starts to req data from dev 1163 * 1164 * is_setup : i/p arguement decides which of the two stage needs to be 1165 * performed; TRUE - SETUP and FALSE - IN+STATUS 1166 * Returns 0 if success 1167 */ 1168static int submit_single_step_set_feature( 1169 struct usb_hcd *hcd, 1170 struct urb *urb, 1171 int is_setup 1172) { 1173 struct ehci_hcd *ehci = hcd_to_ehci(hcd); 1174 struct list_head qtd_list; 1175 struct list_head *head; 1176 1177 struct ehci_qtd *qtd, *qtd_prev; 1178 dma_addr_t buf; 1179 int len, maxpacket; 1180 u32 token; 1181 1182 INIT_LIST_HEAD(&qtd_list); 1183 head = &qtd_list; 1184 1185 /* URBs map to sequences of QTDs: one logical transaction */ 1186 qtd = ehci_qtd_alloc(ehci, GFP_KERNEL); 1187 if (unlikely(!qtd)) 1188 return -1; 1189 list_add_tail(&qtd->qtd_list, head); 1190 qtd->urb = urb; 1191 1192 token = QTD_STS_ACTIVE; 1193 token |= (EHCI_TUNE_CERR << 10); 1194 1195 len = urb->transfer_buffer_length; 1196 /* 1197 * Check if the request is to perform just the SETUP stage (getDesc) 1198 * as in SINGLE_STEP_SET_FEATURE test, DATA stage (IN) happens 1199 * 15 secs after the setup 1200 */ 1201 if (is_setup) { 1202 /* SETUP pid, and interrupt after SETUP completion */ 1203 qtd_fill(ehci, qtd, urb->setup_dma, 1204 sizeof(struct usb_ctrlrequest), 1205 QTD_IOC | token | (2 /* "setup" */ << 8), 8); 1206 1207 submit_async(ehci, urb, &qtd_list, GFP_ATOMIC); 1208 return 0; /*Return now; we shall come back after 15 seconds*/ 1209 } 1210 1211 /* 1212 * IN: data transfer stage: buffer setup : start the IN txn phase for 1213 * the get_Desc SETUP which was sent 15seconds back 1214 */ 1215 token ^= QTD_TOGGLE; /*We need to start IN with DATA-1 Pid-sequence*/ 1216 buf = urb->transfer_dma; 1217 1218 token |= (1 /* "in" */ << 8); /*This is IN stage*/ 1219 1220 maxpacket = usb_maxpacket(urb->dev, urb->pipe, 0); 1221 1222 qtd_fill(ehci, qtd, buf, len, token, maxpacket); 1223 1224 /* 1225 * Our IN phase shall always be a short read; so keep the queue running 1226 * and let it advance to the next qtd which zero length OUT status 1227 */ 1228 qtd->hw_alt_next = EHCI_LIST_END(ehci); 1229 1230 /* STATUS stage for GetDesc control request */ 1231 token ^= 0x0100; /* "in" <--> "out" */ 1232 token |= QTD_TOGGLE; /* force DATA1 */ 1233 1234 qtd_prev = qtd; 1235 qtd = ehci_qtd_alloc(ehci, GFP_ATOMIC); 1236 if (unlikely(!qtd)) 1237 goto cleanup; 1238 qtd->urb = urb; 1239 qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma); 1240 list_add_tail(&qtd->qtd_list, head); 1241 1242 /* Interrupt after STATUS completion */ 1243 qtd_fill(ehci, qtd, 0, 0, token | QTD_IOC, 0); 1244 1245 submit_async(ehci, urb, &qtd_list, GFP_KERNEL); 1246 1247 return 0; 1248 1249cleanup: 1250 qtd_list_free(ehci, urb, head); 1251 return -1; 1252} 1253#endif /* CONFIG_USB_HCD_TEST_MODE */ 1254 1255/*-------------------------------------------------------------------------*/ 1256 1257static void single_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh) 1258{ 1259 struct ehci_qh *prev; 1260 1261 /* Add to the end of the list of QHs waiting for the next IAAD */ 1262 qh->qh_state = QH_STATE_UNLINK_WAIT; 1263 list_add_tail(&qh->unlink_node, &ehci->async_unlink); 1264 1265 /* Unlink it from the schedule */ 1266 prev = ehci->async; 1267 while (prev->qh_next.qh != qh) 1268 prev = prev->qh_next.qh; 1269 1270 prev->hw->hw_next = qh->hw->hw_next; 1271 prev->qh_next = qh->qh_next; 1272 if (ehci->qh_scan_next == qh) 1273 ehci->qh_scan_next = qh->qh_next.qh; 1274} 1275 1276static void start_iaa_cycle(struct ehci_hcd *ehci) 1277{ 1278 /* If the controller isn't running, we don't have to wait for it */ 1279 if (unlikely(ehci->rh_state < EHCI_RH_RUNNING)) { 1280 end_unlink_async(ehci); 1281 1282 /* Otherwise start a new IAA cycle if one isn't already running */ 1283 } else if (ehci->rh_state == EHCI_RH_RUNNING && 1284 !ehci->iaa_in_progress) { 1285 1286 /* Make sure the unlinks are all visible to the hardware */ 1287 wmb(); 1288 1289 ehci_writel(ehci, ehci->command | CMD_IAAD, 1290 &ehci->regs->command); 1291 ehci_readl(ehci, &ehci->regs->command); 1292 ehci->iaa_in_progress = true; 1293 ehci_enable_event(ehci, EHCI_HRTIMER_IAA_WATCHDOG, true); 1294 } 1295} 1296 1297static void end_iaa_cycle(struct ehci_hcd *ehci) 1298{ 1299 if (ehci->has_synopsys_hc_bug) 1300 ehci_writel(ehci, (u32) ehci->async->qh_dma, 1301 &ehci->regs->async_next); 1302 1303 /* The current IAA cycle has ended */ 1304 ehci->iaa_in_progress = false; 1305 1306 end_unlink_async(ehci); 1307} 1308 1309/* See if the async qh for the qtds being unlinked are now gone from the HC */ 1310 1311static void end_unlink_async(struct ehci_hcd *ehci) 1312{ 1313 struct ehci_qh *qh; 1314 bool early_exit; 1315 1316 if (list_empty(&ehci->async_unlink)) 1317 return; 1318 qh = list_first_entry(&ehci->async_unlink, struct ehci_qh, 1319 unlink_node); /* QH whose IAA cycle just ended */ 1320 1321 /* 1322 * If async_unlinking is set then this routine is already running, 1323 * either on the stack or on another CPU. 1324 */ 1325 early_exit = ehci->async_unlinking; 1326 1327 /* If the controller isn't running, process all the waiting QHs */ 1328 if (ehci->rh_state < EHCI_RH_RUNNING) 1329 list_splice_tail_init(&ehci->async_unlink, &ehci->async_idle); 1330 1331 /* 1332 * Intel (?) bug: The HC can write back the overlay region even 1333 * after the IAA interrupt occurs. In self-defense, always go 1334 * through two IAA cycles for each QH. 1335 */ 1336 else if (qh->qh_state == QH_STATE_UNLINK) { 1337 /* 1338 * Second IAA cycle has finished. Process only the first 1339 * waiting QH (NVIDIA (?) bug). 1340 */ 1341 list_move_tail(&qh->unlink_node, &ehci->async_idle); 1342 } 1343 1344 /* 1345 * AMD/ATI (?) bug: The HC can continue to use an active QH long 1346 * after the IAA interrupt occurs. To prevent problems, QHs that 1347 * may still be active will wait until 2 ms have passed with no 1348 * change to the hw_current and hw_token fields (this delay occurs 1349 * between the two IAA cycles). 1350 * 1351 * The EHCI spec (4.8.2) says that active QHs must not be removed 1352 * from the async schedule and recommends waiting until the QH 1353 * goes inactive. This is ridiculous because the QH will _never_ 1354 * become inactive if the endpoint NAKs indefinitely. 1355 */ 1356 1357 /* Some reasons for unlinking guarantee the QH can't be active */ 1358 else if (qh->unlink_reason & (QH_UNLINK_HALTED | 1359 QH_UNLINK_SHORT_READ | QH_UNLINK_DUMMY_OVERLAY)) 1360 goto DelayDone; 1361 1362 /* The QH can't be active if the queue was and still is empty... */ 1363 else if ((qh->unlink_reason & QH_UNLINK_QUEUE_EMPTY) && 1364 list_empty(&qh->qtd_list)) 1365 goto DelayDone; 1366 1367 /* ... or if the QH has halted */ 1368 else if (qh->hw->hw_token & cpu_to_hc32(ehci, QTD_STS_HALT)) 1369 goto DelayDone; 1370 1371 /* Otherwise we have to wait until the QH stops changing */ 1372 else { 1373 __hc32 qh_current, qh_token; 1374 1375 qh_current = qh->hw->hw_current; 1376 qh_token = qh->hw->hw_token; 1377 if (qh_current != ehci->old_current || 1378 qh_token != ehci->old_token) { 1379 ehci->old_current = qh_current; 1380 ehci->old_token = qh_token; 1381 ehci_enable_event(ehci, 1382 EHCI_HRTIMER_ACTIVE_UNLINK, true); 1383 return; 1384 } 1385 DelayDone: 1386 qh->qh_state = QH_STATE_UNLINK; 1387 early_exit = true; 1388 } 1389 ehci->old_current = ~0; /* Prepare for next QH */ 1390 1391 /* Start a new IAA cycle if any QHs are waiting for it */ 1392 if (!list_empty(&ehci->async_unlink)) 1393 start_iaa_cycle(ehci); 1394 1395 /* 1396 * Don't allow nesting or concurrent calls, 1397 * or wait for the second IAA cycle for the next QH. 1398 */ 1399 if (early_exit) 1400 return; 1401 1402 /* Process the idle QHs */ 1403 ehci->async_unlinking = true; 1404 while (!list_empty(&ehci->async_idle)) { 1405 qh = list_first_entry(&ehci->async_idle, struct ehci_qh, 1406 unlink_node); 1407 list_del(&qh->unlink_node); 1408 1409 qh->qh_state = QH_STATE_IDLE; 1410 qh->qh_next.qh = NULL; 1411 1412 if (!list_empty(&qh->qtd_list)) 1413 qh_completions(ehci, qh); 1414 if (!list_empty(&qh->qtd_list) && 1415 ehci->rh_state == EHCI_RH_RUNNING) 1416 qh_link_async(ehci, qh); 1417 disable_async(ehci); 1418 } 1419 ehci->async_unlinking = false; 1420} 1421 1422static void start_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh); 1423 1424static void unlink_empty_async(struct ehci_hcd *ehci) 1425{ 1426 struct ehci_qh *qh; 1427 struct ehci_qh *qh_to_unlink = NULL; 1428 int count = 0; 1429 1430 /* Find the last async QH which has been empty for a timer cycle */ 1431 for (qh = ehci->async->qh_next.qh; qh; qh = qh->qh_next.qh) { 1432 if (list_empty(&qh->qtd_list) && 1433 qh->qh_state == QH_STATE_LINKED) { 1434 ++count; 1435 if (qh->unlink_cycle != ehci->async_unlink_cycle) 1436 qh_to_unlink = qh; 1437 } 1438 } 1439 1440 /* If nothing else is being unlinked, unlink the last empty QH */ 1441 if (list_empty(&ehci->async_unlink) && qh_to_unlink) { 1442 qh_to_unlink->unlink_reason |= QH_UNLINK_QUEUE_EMPTY; 1443 start_unlink_async(ehci, qh_to_unlink); 1444 --count; 1445 } 1446 1447 /* Other QHs will be handled later */ 1448 if (count > 0) { 1449 ehci_enable_event(ehci, EHCI_HRTIMER_ASYNC_UNLINKS, true); 1450 ++ehci->async_unlink_cycle; 1451 } 1452} 1453 1454#ifdef CONFIG_PM 1455 1456/* The root hub is suspended; unlink all the async QHs */ 1457static void unlink_empty_async_suspended(struct ehci_hcd *ehci) 1458{ 1459 struct ehci_qh *qh; 1460 1461 while (ehci->async->qh_next.qh) { 1462 qh = ehci->async->qh_next.qh; 1463 WARN_ON(!list_empty(&qh->qtd_list)); 1464 single_unlink_async(ehci, qh); 1465 } 1466} 1467 1468#endif 1469 1470/* makes sure the async qh will become idle */ 1471/* caller must own ehci->lock */ 1472 1473static void start_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh) 1474{ 1475 /* If the QH isn't linked then there's nothing we can do. */ 1476 if (qh->qh_state != QH_STATE_LINKED) 1477 return; 1478 1479 single_unlink_async(ehci, qh); 1480 start_iaa_cycle(ehci); 1481} 1482 1483/*-------------------------------------------------------------------------*/ 1484 1485static void scan_async (struct ehci_hcd *ehci) 1486{ 1487 struct ehci_qh *qh; 1488 bool check_unlinks_later = false; 1489 1490 ehci->qh_scan_next = ehci->async->qh_next.qh; 1491 while (ehci->qh_scan_next) { 1492 qh = ehci->qh_scan_next; 1493 ehci->qh_scan_next = qh->qh_next.qh; 1494 1495 /* clean any finished work for this qh */ 1496 if (!list_empty(&qh->qtd_list)) { 1497 int temp; 1498 1499 /* 1500 * Unlinks could happen here; completion reporting 1501 * drops the lock. That's why ehci->qh_scan_next 1502 * always holds the next qh to scan; if the next qh 1503 * gets unlinked then ehci->qh_scan_next is adjusted 1504 * in single_unlink_async(). 1505 */ 1506 temp = qh_completions(ehci, qh); 1507 if (unlikely(temp)) { 1508 start_unlink_async(ehci, qh); 1509 } else if (list_empty(&qh->qtd_list) 1510 && qh->qh_state == QH_STATE_LINKED) { 1511 qh->unlink_cycle = ehci->async_unlink_cycle; 1512 check_unlinks_later = true; 1513 } 1514 } 1515 } 1516 1517 /* 1518 * Unlink empty entries, reducing DMA usage as well 1519 * as HCD schedule-scanning costs. Delay for any qh 1520 * we just scanned, there's a not-unusual case that it 1521 * doesn't stay idle for long. 1522 */ 1523 if (check_unlinks_later && ehci->rh_state == EHCI_RH_RUNNING && 1524 !(ehci->enabled_hrtimer_events & 1525 BIT(EHCI_HRTIMER_ASYNC_UNLINKS))) { 1526 ehci_enable_event(ehci, EHCI_HRTIMER_ASYNC_UNLINKS, true); 1527 ++ehci->async_unlink_cycle; 1528 } 1529} 1530