1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. 4 * 5 */ 6 7#include <linux/delay.h> 8#include <linux/device.h> 9#include <linux/dma-direction.h> 10#include <linux/dma-mapping.h> 11#include <linux/interrupt.h> 12#include <linux/list.h> 13#include <linux/mhi.h> 14#include <linux/module.h> 15#include <linux/skbuff.h> 16#include <linux/slab.h> 17#include "internal.h" 18 19int __must_check mhi_read_reg(struct mhi_controller *mhi_cntrl, 20 void __iomem *base, u32 offset, u32 *out) 21{ 22 return mhi_cntrl->read_reg(mhi_cntrl, base + offset, out); 23} 24 25int __must_check mhi_read_reg_field(struct mhi_controller *mhi_cntrl, 26 void __iomem *base, u32 offset, 27 u32 mask, u32 *out) 28{ 29 u32 tmp; 30 int ret; 31 32 ret = mhi_read_reg(mhi_cntrl, base, offset, &tmp); 33 if (ret) 34 return ret; 35 36 *out = (tmp & mask) >> __ffs(mask); 37 38 return 0; 39} 40 41int __must_check mhi_poll_reg_field(struct mhi_controller *mhi_cntrl, 42 void __iomem *base, u32 offset, 43 u32 mask, u32 val, u32 delayus) 44{ 45 int ret; 46 u32 out, retry = (mhi_cntrl->timeout_ms * 1000) / delayus; 47 48 while (retry--) { 49 ret = mhi_read_reg_field(mhi_cntrl, base, offset, mask, &out); 50 if (ret) 51 return ret; 52 53 if (out == val) 54 return 0; 55 56 fsleep(delayus); 57 } 58 59 return -ETIMEDOUT; 60} 61 62void mhi_write_reg(struct mhi_controller *mhi_cntrl, void __iomem *base, 63 u32 offset, u32 val) 64{ 65 mhi_cntrl->write_reg(mhi_cntrl, base + offset, val); 66} 67 68int __must_check mhi_write_reg_field(struct mhi_controller *mhi_cntrl, 69 void __iomem *base, u32 offset, u32 mask, 70 u32 val) 71{ 72 int ret; 73 u32 tmp; 74 75 ret = mhi_read_reg(mhi_cntrl, base, offset, &tmp); 76 if (ret) 77 return ret; 78 79 tmp &= ~mask; 80 tmp |= (val << __ffs(mask)); 81 mhi_write_reg(mhi_cntrl, base, offset, tmp); 82 83 return 0; 84} 85 86void mhi_write_db(struct mhi_controller *mhi_cntrl, void __iomem *db_addr, 87 dma_addr_t db_val) 88{ 89 mhi_write_reg(mhi_cntrl, db_addr, 4, upper_32_bits(db_val)); 90 mhi_write_reg(mhi_cntrl, db_addr, 0, lower_32_bits(db_val)); 91} 92 93void mhi_db_brstmode(struct mhi_controller *mhi_cntrl, 94 struct db_cfg *db_cfg, 95 void __iomem *db_addr, 96 dma_addr_t db_val) 97{ 98 if (db_cfg->db_mode) { 99 db_cfg->db_val = db_val; 100 mhi_write_db(mhi_cntrl, db_addr, db_val); 101 db_cfg->db_mode = 0; 102 } 103} 104 105void mhi_db_brstmode_disable(struct mhi_controller *mhi_cntrl, 106 struct db_cfg *db_cfg, 107 void __iomem *db_addr, 108 dma_addr_t db_val) 109{ 110 db_cfg->db_val = db_val; 111 mhi_write_db(mhi_cntrl, db_addr, db_val); 112} 113 114void mhi_ring_er_db(struct mhi_event *mhi_event) 115{ 116 struct mhi_ring *ring = &mhi_event->ring; 117 118 mhi_event->db_cfg.process_db(mhi_event->mhi_cntrl, &mhi_event->db_cfg, 119 ring->db_addr, le64_to_cpu(*ring->ctxt_wp)); 120} 121 122void mhi_ring_cmd_db(struct mhi_controller *mhi_cntrl, struct mhi_cmd *mhi_cmd) 123{ 124 dma_addr_t db; 125 struct mhi_ring *ring = &mhi_cmd->ring; 126 127 db = ring->iommu_base + (ring->wp - ring->base); 128 *ring->ctxt_wp = cpu_to_le64(db); 129 mhi_write_db(mhi_cntrl, ring->db_addr, db); 130} 131 132void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl, 133 struct mhi_chan *mhi_chan) 134{ 135 struct mhi_ring *ring = &mhi_chan->tre_ring; 136 dma_addr_t db; 137 138 db = ring->iommu_base + (ring->wp - ring->base); 139 140 /* 141 * Writes to the new ring element must be visible to the hardware 142 * before letting h/w know there is new element to fetch. 143 */ 144 dma_wmb(); 145 *ring->ctxt_wp = cpu_to_le64(db); 146 147 mhi_chan->db_cfg.process_db(mhi_cntrl, &mhi_chan->db_cfg, 148 ring->db_addr, db); 149} 150 151enum mhi_ee_type mhi_get_exec_env(struct mhi_controller *mhi_cntrl) 152{ 153 u32 exec; 154 int ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_EXECENV, &exec); 155 156 return (ret) ? MHI_EE_MAX : exec; 157} 158EXPORT_SYMBOL_GPL(mhi_get_exec_env); 159 160enum mhi_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl) 161{ 162 u32 state; 163 int ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, MHISTATUS, 164 MHISTATUS_MHISTATE_MASK, &state); 165 return ret ? MHI_STATE_MAX : state; 166} 167EXPORT_SYMBOL_GPL(mhi_get_mhi_state); 168 169void mhi_soc_reset(struct mhi_controller *mhi_cntrl) 170{ 171 if (mhi_cntrl->reset) { 172 mhi_cntrl->reset(mhi_cntrl); 173 return; 174 } 175 176 /* Generic MHI SoC reset */ 177 mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, 178 MHI_SOC_RESET_REQ); 179} 180EXPORT_SYMBOL_GPL(mhi_soc_reset); 181 182int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl, 183 struct mhi_buf_info *buf_info) 184{ 185 buf_info->p_addr = dma_map_single(mhi_cntrl->cntrl_dev, 186 buf_info->v_addr, buf_info->len, 187 buf_info->dir); 188 if (dma_mapping_error(mhi_cntrl->cntrl_dev, buf_info->p_addr)) 189 return -ENOMEM; 190 191 return 0; 192} 193 194int mhi_map_single_use_bb(struct mhi_controller *mhi_cntrl, 195 struct mhi_buf_info *buf_info) 196{ 197 void *buf = dma_alloc_coherent(mhi_cntrl->cntrl_dev, buf_info->len, 198 &buf_info->p_addr, GFP_ATOMIC); 199 200 if (!buf) 201 return -ENOMEM; 202 203 if (buf_info->dir == DMA_TO_DEVICE) 204 memcpy(buf, buf_info->v_addr, buf_info->len); 205 206 buf_info->bb_addr = buf; 207 208 return 0; 209} 210 211void mhi_unmap_single_no_bb(struct mhi_controller *mhi_cntrl, 212 struct mhi_buf_info *buf_info) 213{ 214 dma_unmap_single(mhi_cntrl->cntrl_dev, buf_info->p_addr, buf_info->len, 215 buf_info->dir); 216} 217 218void mhi_unmap_single_use_bb(struct mhi_controller *mhi_cntrl, 219 struct mhi_buf_info *buf_info) 220{ 221 if (buf_info->dir == DMA_FROM_DEVICE) 222 memcpy(buf_info->v_addr, buf_info->bb_addr, buf_info->len); 223 224 dma_free_coherent(mhi_cntrl->cntrl_dev, buf_info->len, 225 buf_info->bb_addr, buf_info->p_addr); 226} 227 228static int get_nr_avail_ring_elements(struct mhi_controller *mhi_cntrl, 229 struct mhi_ring *ring) 230{ 231 int nr_el; 232 233 if (ring->wp < ring->rp) { 234 nr_el = ((ring->rp - ring->wp) / ring->el_size) - 1; 235 } else { 236 nr_el = (ring->rp - ring->base) / ring->el_size; 237 nr_el += ((ring->base + ring->len - ring->wp) / 238 ring->el_size) - 1; 239 } 240 241 return nr_el; 242} 243 244static void *mhi_to_virtual(struct mhi_ring *ring, dma_addr_t addr) 245{ 246 return (addr - ring->iommu_base) + ring->base; 247} 248 249static void mhi_add_ring_element(struct mhi_controller *mhi_cntrl, 250 struct mhi_ring *ring) 251{ 252 ring->wp += ring->el_size; 253 if (ring->wp >= (ring->base + ring->len)) 254 ring->wp = ring->base; 255 /* smp update */ 256 smp_wmb(); 257} 258 259static void mhi_del_ring_element(struct mhi_controller *mhi_cntrl, 260 struct mhi_ring *ring) 261{ 262 ring->rp += ring->el_size; 263 if (ring->rp >= (ring->base + ring->len)) 264 ring->rp = ring->base; 265 /* smp update */ 266 smp_wmb(); 267} 268 269static bool is_valid_ring_ptr(struct mhi_ring *ring, dma_addr_t addr) 270{ 271 return addr >= ring->iommu_base && addr < ring->iommu_base + ring->len && 272 !(addr & (sizeof(struct mhi_ring_element) - 1)); 273} 274 275int mhi_destroy_device(struct device *dev, void *data) 276{ 277 struct mhi_chan *ul_chan, *dl_chan; 278 struct mhi_device *mhi_dev; 279 struct mhi_controller *mhi_cntrl; 280 enum mhi_ee_type ee = MHI_EE_MAX; 281 282 if (dev->bus != &mhi_bus_type) 283 return 0; 284 285 mhi_dev = to_mhi_device(dev); 286 mhi_cntrl = mhi_dev->mhi_cntrl; 287 288 /* Only destroy virtual devices thats attached to bus */ 289 if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER) 290 return 0; 291 292 ul_chan = mhi_dev->ul_chan; 293 dl_chan = mhi_dev->dl_chan; 294 295 /* 296 * If execution environment is specified, remove only those devices that 297 * started in them based on ee_mask for the channels as we move on to a 298 * different execution environment 299 */ 300 if (data) 301 ee = *(enum mhi_ee_type *)data; 302 303 /* 304 * For the suspend and resume case, this function will get called 305 * without mhi_unregister_controller(). Hence, we need to drop the 306 * references to mhi_dev created for ul and dl channels. We can 307 * be sure that there will be no instances of mhi_dev left after 308 * this. 309 */ 310 if (ul_chan) { 311 if (ee != MHI_EE_MAX && !(ul_chan->ee_mask & BIT(ee))) 312 return 0; 313 314 put_device(&ul_chan->mhi_dev->dev); 315 } 316 317 if (dl_chan) { 318 if (ee != MHI_EE_MAX && !(dl_chan->ee_mask & BIT(ee))) 319 return 0; 320 321 put_device(&dl_chan->mhi_dev->dev); 322 } 323 324 dev_dbg(&mhi_cntrl->mhi_dev->dev, "destroy device for chan:%s\n", 325 mhi_dev->name); 326 327 /* Notify the client and remove the device from MHI bus */ 328 device_del(dev); 329 put_device(dev); 330 331 return 0; 332} 333 334int mhi_get_free_desc_count(struct mhi_device *mhi_dev, 335 enum dma_data_direction dir) 336{ 337 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; 338 struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? 339 mhi_dev->ul_chan : mhi_dev->dl_chan; 340 struct mhi_ring *tre_ring = &mhi_chan->tre_ring; 341 342 return get_nr_avail_ring_elements(mhi_cntrl, tre_ring); 343} 344EXPORT_SYMBOL_GPL(mhi_get_free_desc_count); 345 346void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason) 347{ 348 struct mhi_driver *mhi_drv; 349 350 if (!mhi_dev->dev.driver) 351 return; 352 353 mhi_drv = to_mhi_driver(mhi_dev->dev.driver); 354 355 if (mhi_drv->status_cb) 356 mhi_drv->status_cb(mhi_dev, cb_reason); 357} 358EXPORT_SYMBOL_GPL(mhi_notify); 359 360/* Bind MHI channels to MHI devices */ 361void mhi_create_devices(struct mhi_controller *mhi_cntrl) 362{ 363 struct mhi_chan *mhi_chan; 364 struct mhi_device *mhi_dev; 365 struct device *dev = &mhi_cntrl->mhi_dev->dev; 366 int i, ret; 367 368 mhi_chan = mhi_cntrl->mhi_chan; 369 for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) { 370 if (!mhi_chan->configured || mhi_chan->mhi_dev || 371 !(mhi_chan->ee_mask & BIT(mhi_cntrl->ee))) 372 continue; 373 mhi_dev = mhi_alloc_device(mhi_cntrl); 374 if (IS_ERR(mhi_dev)) 375 return; 376 377 mhi_dev->dev_type = MHI_DEVICE_XFER; 378 switch (mhi_chan->dir) { 379 case DMA_TO_DEVICE: 380 mhi_dev->ul_chan = mhi_chan; 381 mhi_dev->ul_chan_id = mhi_chan->chan; 382 break; 383 case DMA_FROM_DEVICE: 384 /* We use dl_chan as offload channels */ 385 mhi_dev->dl_chan = mhi_chan; 386 mhi_dev->dl_chan_id = mhi_chan->chan; 387 break; 388 default: 389 dev_err(dev, "Direction not supported\n"); 390 put_device(&mhi_dev->dev); 391 return; 392 } 393 394 get_device(&mhi_dev->dev); 395 mhi_chan->mhi_dev = mhi_dev; 396 397 /* Check next channel if it matches */ 398 if ((i + 1) < mhi_cntrl->max_chan && mhi_chan[1].configured) { 399 if (!strcmp(mhi_chan[1].name, mhi_chan->name)) { 400 i++; 401 mhi_chan++; 402 if (mhi_chan->dir == DMA_TO_DEVICE) { 403 mhi_dev->ul_chan = mhi_chan; 404 mhi_dev->ul_chan_id = mhi_chan->chan; 405 } else { 406 mhi_dev->dl_chan = mhi_chan; 407 mhi_dev->dl_chan_id = mhi_chan->chan; 408 } 409 get_device(&mhi_dev->dev); 410 mhi_chan->mhi_dev = mhi_dev; 411 } 412 } 413 414 /* Channel name is same for both UL and DL */ 415 mhi_dev->name = mhi_chan->name; 416 dev_set_name(&mhi_dev->dev, "%s_%s", 417 dev_name(&mhi_cntrl->mhi_dev->dev), 418 mhi_dev->name); 419 420 /* Init wakeup source if available */ 421 if (mhi_dev->dl_chan && mhi_dev->dl_chan->wake_capable) 422 device_init_wakeup(&mhi_dev->dev, true); 423 424 ret = device_add(&mhi_dev->dev); 425 if (ret) 426 put_device(&mhi_dev->dev); 427 } 428} 429 430irqreturn_t mhi_irq_handler(int irq_number, void *dev) 431{ 432 struct mhi_event *mhi_event = dev; 433 struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl; 434 struct mhi_event_ctxt *er_ctxt; 435 struct mhi_ring *ev_ring = &mhi_event->ring; 436 dma_addr_t ptr; 437 void *dev_rp; 438 439 /* 440 * If CONFIG_DEBUG_SHIRQ is set, the IRQ handler will get invoked during __free_irq() 441 * and by that time mhi_ctxt() would've freed. So check for the existence of mhi_ctxt 442 * before handling the IRQs. 443 */ 444 if (!mhi_cntrl->mhi_ctxt) { 445 dev_dbg(&mhi_cntrl->mhi_dev->dev, 446 "mhi_ctxt has been freed\n"); 447 return IRQ_HANDLED; 448 } 449 450 er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; 451 ptr = le64_to_cpu(er_ctxt->rp); 452 453 if (!is_valid_ring_ptr(ev_ring, ptr)) { 454 dev_err(&mhi_cntrl->mhi_dev->dev, 455 "Event ring rp points outside of the event ring\n"); 456 return IRQ_HANDLED; 457 } 458 459 dev_rp = mhi_to_virtual(ev_ring, ptr); 460 461 /* Only proceed if event ring has pending events */ 462 if (ev_ring->rp == dev_rp) 463 return IRQ_HANDLED; 464 465 /* For client managed event ring, notify pending data */ 466 if (mhi_event->cl_manage) { 467 struct mhi_chan *mhi_chan = mhi_event->mhi_chan; 468 struct mhi_device *mhi_dev = mhi_chan->mhi_dev; 469 470 if (mhi_dev) 471 mhi_notify(mhi_dev, MHI_CB_PENDING_DATA); 472 } else { 473 tasklet_schedule(&mhi_event->task); 474 } 475 476 return IRQ_HANDLED; 477} 478 479irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *priv) 480{ 481 struct mhi_controller *mhi_cntrl = priv; 482 struct device *dev = &mhi_cntrl->mhi_dev->dev; 483 enum mhi_state state; 484 enum mhi_pm_state pm_state = 0; 485 enum mhi_ee_type ee; 486 487 write_lock_irq(&mhi_cntrl->pm_lock); 488 if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) { 489 write_unlock_irq(&mhi_cntrl->pm_lock); 490 goto exit_intvec; 491 } 492 493 state = mhi_get_mhi_state(mhi_cntrl); 494 ee = mhi_get_exec_env(mhi_cntrl); 495 dev_dbg(dev, "local ee: %s state: %s device ee: %s state: %s\n", 496 TO_MHI_EXEC_STR(mhi_cntrl->ee), 497 mhi_state_str(mhi_cntrl->dev_state), 498 TO_MHI_EXEC_STR(ee), mhi_state_str(state)); 499 500 if (state == MHI_STATE_SYS_ERR) { 501 dev_dbg(dev, "System error detected\n"); 502 pm_state = mhi_tryset_pm_state(mhi_cntrl, 503 MHI_PM_SYS_ERR_DETECT); 504 } 505 write_unlock_irq(&mhi_cntrl->pm_lock); 506 507 if (pm_state != MHI_PM_SYS_ERR_DETECT) 508 goto exit_intvec; 509 510 switch (ee) { 511 case MHI_EE_RDDM: 512 /* proceed if power down is not already in progress */ 513 if (mhi_cntrl->rddm_image && mhi_is_active(mhi_cntrl)) { 514 mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM); 515 mhi_cntrl->ee = ee; 516 wake_up_all(&mhi_cntrl->state_event); 517 } 518 break; 519 case MHI_EE_PBL: 520 case MHI_EE_EDL: 521 case MHI_EE_PTHRU: 522 mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_FATAL_ERROR); 523 mhi_cntrl->ee = ee; 524 wake_up_all(&mhi_cntrl->state_event); 525 mhi_pm_sys_err_handler(mhi_cntrl); 526 break; 527 default: 528 wake_up_all(&mhi_cntrl->state_event); 529 mhi_pm_sys_err_handler(mhi_cntrl); 530 break; 531 } 532 533exit_intvec: 534 535 return IRQ_HANDLED; 536} 537 538irqreturn_t mhi_intvec_handler(int irq_number, void *dev) 539{ 540 struct mhi_controller *mhi_cntrl = dev; 541 542 /* Wake up events waiting for state change */ 543 wake_up_all(&mhi_cntrl->state_event); 544 545 return IRQ_WAKE_THREAD; 546} 547 548static void mhi_recycle_ev_ring_element(struct mhi_controller *mhi_cntrl, 549 struct mhi_ring *ring) 550{ 551 /* Update the WP */ 552 ring->wp += ring->el_size; 553 554 if (ring->wp >= (ring->base + ring->len)) 555 ring->wp = ring->base; 556 557 *ring->ctxt_wp = cpu_to_le64(ring->iommu_base + (ring->wp - ring->base)); 558 559 /* Update the RP */ 560 ring->rp += ring->el_size; 561 if (ring->rp >= (ring->base + ring->len)) 562 ring->rp = ring->base; 563 564 /* Update to all cores */ 565 smp_wmb(); 566} 567 568static int parse_xfer_event(struct mhi_controller *mhi_cntrl, 569 struct mhi_ring_element *event, 570 struct mhi_chan *mhi_chan) 571{ 572 struct mhi_ring *buf_ring, *tre_ring; 573 struct device *dev = &mhi_cntrl->mhi_dev->dev; 574 struct mhi_result result; 575 unsigned long flags = 0; 576 u32 ev_code; 577 578 ev_code = MHI_TRE_GET_EV_CODE(event); 579 buf_ring = &mhi_chan->buf_ring; 580 tre_ring = &mhi_chan->tre_ring; 581 582 result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ? 583 -EOVERFLOW : 0; 584 585 /* 586 * If it's a DB Event then we need to grab the lock 587 * with preemption disabled and as a write because we 588 * have to update db register and there are chances that 589 * another thread could be doing the same. 590 */ 591 if (ev_code >= MHI_EV_CC_OOB) 592 write_lock_irqsave(&mhi_chan->lock, flags); 593 else 594 read_lock_bh(&mhi_chan->lock); 595 596 if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED) 597 goto end_process_tx_event; 598 599 switch (ev_code) { 600 case MHI_EV_CC_OVERFLOW: 601 case MHI_EV_CC_EOB: 602 case MHI_EV_CC_EOT: 603 { 604 dma_addr_t ptr = MHI_TRE_GET_EV_PTR(event); 605 struct mhi_ring_element *local_rp, *ev_tre; 606 void *dev_rp; 607 struct mhi_buf_info *buf_info; 608 u16 xfer_len; 609 610 if (!is_valid_ring_ptr(tre_ring, ptr)) { 611 dev_err(&mhi_cntrl->mhi_dev->dev, 612 "Event element points outside of the tre ring\n"); 613 break; 614 } 615 /* Get the TRB this event points to */ 616 ev_tre = mhi_to_virtual(tre_ring, ptr); 617 618 dev_rp = ev_tre + 1; 619 if (dev_rp >= (tre_ring->base + tre_ring->len)) 620 dev_rp = tre_ring->base; 621 622 result.dir = mhi_chan->dir; 623 624 local_rp = tre_ring->rp; 625 while (local_rp != dev_rp) { 626 buf_info = buf_ring->rp; 627 /* If it's the last TRE, get length from the event */ 628 if (local_rp == ev_tre) 629 xfer_len = MHI_TRE_GET_EV_LEN(event); 630 else 631 xfer_len = buf_info->len; 632 633 /* Unmap if it's not pre-mapped by client */ 634 if (likely(!buf_info->pre_mapped)) 635 mhi_cntrl->unmap_single(mhi_cntrl, buf_info); 636 637 result.buf_addr = buf_info->cb_buf; 638 639 /* truncate to buf len if xfer_len is larger */ 640 result.bytes_xferd = 641 min_t(u16, xfer_len, buf_info->len); 642 mhi_del_ring_element(mhi_cntrl, buf_ring); 643 mhi_del_ring_element(mhi_cntrl, tre_ring); 644 local_rp = tre_ring->rp; 645 646 read_unlock_bh(&mhi_chan->lock); 647 648 /* notify client */ 649 mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result); 650 651 if (mhi_chan->dir == DMA_TO_DEVICE) { 652 atomic_dec(&mhi_cntrl->pending_pkts); 653 /* Release the reference got from mhi_queue() */ 654 mhi_cntrl->runtime_put(mhi_cntrl); 655 } 656 657 /* 658 * Recycle the buffer if buffer is pre-allocated, 659 * if there is an error, not much we can do apart 660 * from dropping the packet 661 */ 662 if (mhi_chan->pre_alloc) { 663 if (mhi_queue_buf(mhi_chan->mhi_dev, 664 mhi_chan->dir, 665 buf_info->cb_buf, 666 buf_info->len, MHI_EOT)) { 667 dev_err(dev, 668 "Error recycling buffer for chan:%d\n", 669 mhi_chan->chan); 670 kfree(buf_info->cb_buf); 671 } 672 } 673 674 read_lock_bh(&mhi_chan->lock); 675 } 676 break; 677 } /* CC_EOT */ 678 case MHI_EV_CC_OOB: 679 case MHI_EV_CC_DB_MODE: 680 { 681 unsigned long pm_lock_flags; 682 683 mhi_chan->db_cfg.db_mode = 1; 684 read_lock_irqsave(&mhi_cntrl->pm_lock, pm_lock_flags); 685 if (tre_ring->wp != tre_ring->rp && 686 MHI_DB_ACCESS_VALID(mhi_cntrl)) { 687 mhi_ring_chan_db(mhi_cntrl, mhi_chan); 688 } 689 read_unlock_irqrestore(&mhi_cntrl->pm_lock, pm_lock_flags); 690 break; 691 } 692 case MHI_EV_CC_BAD_TRE: 693 default: 694 dev_err(dev, "Unknown event 0x%x\n", ev_code); 695 break; 696 } /* switch(MHI_EV_READ_CODE(EV_TRB_CODE,event)) */ 697 698end_process_tx_event: 699 if (ev_code >= MHI_EV_CC_OOB) 700 write_unlock_irqrestore(&mhi_chan->lock, flags); 701 else 702 read_unlock_bh(&mhi_chan->lock); 703 704 return 0; 705} 706 707static int parse_rsc_event(struct mhi_controller *mhi_cntrl, 708 struct mhi_ring_element *event, 709 struct mhi_chan *mhi_chan) 710{ 711 struct mhi_ring *buf_ring, *tre_ring; 712 struct mhi_buf_info *buf_info; 713 struct mhi_result result; 714 int ev_code; 715 u32 cookie; /* offset to local descriptor */ 716 u16 xfer_len; 717 718 buf_ring = &mhi_chan->buf_ring; 719 tre_ring = &mhi_chan->tre_ring; 720 721 ev_code = MHI_TRE_GET_EV_CODE(event); 722 cookie = MHI_TRE_GET_EV_COOKIE(event); 723 xfer_len = MHI_TRE_GET_EV_LEN(event); 724 725 /* Received out of bound cookie */ 726 WARN_ON(cookie >= buf_ring->len); 727 728 buf_info = buf_ring->base + cookie; 729 730 result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ? 731 -EOVERFLOW : 0; 732 733 /* truncate to buf len if xfer_len is larger */ 734 result.bytes_xferd = min_t(u16, xfer_len, buf_info->len); 735 result.buf_addr = buf_info->cb_buf; 736 result.dir = mhi_chan->dir; 737 738 read_lock_bh(&mhi_chan->lock); 739 740 if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED) 741 goto end_process_rsc_event; 742 743 WARN_ON(!buf_info->used); 744 745 /* notify the client */ 746 mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result); 747 748 /* 749 * Note: We're arbitrarily incrementing RP even though, completion 750 * packet we processed might not be the same one, reason we can do this 751 * is because device guaranteed to cache descriptors in order it 752 * receive, so even though completion event is different we can re-use 753 * all descriptors in between. 754 * Example: 755 * Transfer Ring has descriptors: A, B, C, D 756 * Last descriptor host queue is D (WP) and first descriptor 757 * host queue is A (RP). 758 * The completion event we just serviced is descriptor C. 759 * Then we can safely queue descriptors to replace A, B, and C 760 * even though host did not receive any completions. 761 */ 762 mhi_del_ring_element(mhi_cntrl, tre_ring); 763 buf_info->used = false; 764 765end_process_rsc_event: 766 read_unlock_bh(&mhi_chan->lock); 767 768 return 0; 769} 770 771static void mhi_process_cmd_completion(struct mhi_controller *mhi_cntrl, 772 struct mhi_ring_element *tre) 773{ 774 dma_addr_t ptr = MHI_TRE_GET_EV_PTR(tre); 775 struct mhi_cmd *cmd_ring = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING]; 776 struct mhi_ring *mhi_ring = &cmd_ring->ring; 777 struct mhi_ring_element *cmd_pkt; 778 struct mhi_chan *mhi_chan; 779 u32 chan; 780 781 if (!is_valid_ring_ptr(mhi_ring, ptr)) { 782 dev_err(&mhi_cntrl->mhi_dev->dev, 783 "Event element points outside of the cmd ring\n"); 784 return; 785 } 786 787 cmd_pkt = mhi_to_virtual(mhi_ring, ptr); 788 789 chan = MHI_TRE_GET_CMD_CHID(cmd_pkt); 790 791 if (chan < mhi_cntrl->max_chan && 792 mhi_cntrl->mhi_chan[chan].configured) { 793 mhi_chan = &mhi_cntrl->mhi_chan[chan]; 794 write_lock_bh(&mhi_chan->lock); 795 mhi_chan->ccs = MHI_TRE_GET_EV_CODE(tre); 796 complete(&mhi_chan->completion); 797 write_unlock_bh(&mhi_chan->lock); 798 } else { 799 dev_err(&mhi_cntrl->mhi_dev->dev, 800 "Completion packet for invalid channel ID: %d\n", chan); 801 } 802 803 mhi_del_ring_element(mhi_cntrl, mhi_ring); 804} 805 806int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl, 807 struct mhi_event *mhi_event, 808 u32 event_quota) 809{ 810 struct mhi_ring_element *dev_rp, *local_rp; 811 struct mhi_ring *ev_ring = &mhi_event->ring; 812 struct mhi_event_ctxt *er_ctxt = 813 &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; 814 struct mhi_chan *mhi_chan; 815 struct device *dev = &mhi_cntrl->mhi_dev->dev; 816 u32 chan; 817 int count = 0; 818 dma_addr_t ptr = le64_to_cpu(er_ctxt->rp); 819 820 /* 821 * This is a quick check to avoid unnecessary event processing 822 * in case MHI is already in error state, but it's still possible 823 * to transition to error state while processing events 824 */ 825 if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state))) 826 return -EIO; 827 828 if (!is_valid_ring_ptr(ev_ring, ptr)) { 829 dev_err(&mhi_cntrl->mhi_dev->dev, 830 "Event ring rp points outside of the event ring\n"); 831 return -EIO; 832 } 833 834 dev_rp = mhi_to_virtual(ev_ring, ptr); 835 local_rp = ev_ring->rp; 836 837 while (dev_rp != local_rp) { 838 enum mhi_pkt_type type = MHI_TRE_GET_EV_TYPE(local_rp); 839 840 switch (type) { 841 case MHI_PKT_TYPE_BW_REQ_EVENT: 842 { 843 struct mhi_link_info *link_info; 844 845 link_info = &mhi_cntrl->mhi_link_info; 846 write_lock_irq(&mhi_cntrl->pm_lock); 847 link_info->target_link_speed = 848 MHI_TRE_GET_EV_LINKSPEED(local_rp); 849 link_info->target_link_width = 850 MHI_TRE_GET_EV_LINKWIDTH(local_rp); 851 write_unlock_irq(&mhi_cntrl->pm_lock); 852 dev_dbg(dev, "Received BW_REQ event\n"); 853 mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_BW_REQ); 854 break; 855 } 856 case MHI_PKT_TYPE_STATE_CHANGE_EVENT: 857 { 858 enum mhi_state new_state; 859 860 new_state = MHI_TRE_GET_EV_STATE(local_rp); 861 862 dev_dbg(dev, "State change event to state: %s\n", 863 mhi_state_str(new_state)); 864 865 switch (new_state) { 866 case MHI_STATE_M0: 867 mhi_pm_m0_transition(mhi_cntrl); 868 break; 869 case MHI_STATE_M1: 870 mhi_pm_m1_transition(mhi_cntrl); 871 break; 872 case MHI_STATE_M3: 873 mhi_pm_m3_transition(mhi_cntrl); 874 break; 875 case MHI_STATE_SYS_ERR: 876 { 877 enum mhi_pm_state pm_state; 878 879 dev_dbg(dev, "System error detected\n"); 880 write_lock_irq(&mhi_cntrl->pm_lock); 881 pm_state = mhi_tryset_pm_state(mhi_cntrl, 882 MHI_PM_SYS_ERR_DETECT); 883 write_unlock_irq(&mhi_cntrl->pm_lock); 884 if (pm_state == MHI_PM_SYS_ERR_DETECT) 885 mhi_pm_sys_err_handler(mhi_cntrl); 886 break; 887 } 888 default: 889 dev_err(dev, "Invalid state: %s\n", 890 mhi_state_str(new_state)); 891 } 892 893 break; 894 } 895 case MHI_PKT_TYPE_CMD_COMPLETION_EVENT: 896 mhi_process_cmd_completion(mhi_cntrl, local_rp); 897 break; 898 case MHI_PKT_TYPE_EE_EVENT: 899 { 900 enum dev_st_transition st = DEV_ST_TRANSITION_MAX; 901 enum mhi_ee_type event = MHI_TRE_GET_EV_EXECENV(local_rp); 902 903 dev_dbg(dev, "Received EE event: %s\n", 904 TO_MHI_EXEC_STR(event)); 905 switch (event) { 906 case MHI_EE_SBL: 907 st = DEV_ST_TRANSITION_SBL; 908 break; 909 case MHI_EE_WFW: 910 case MHI_EE_AMSS: 911 st = DEV_ST_TRANSITION_MISSION_MODE; 912 break; 913 case MHI_EE_FP: 914 st = DEV_ST_TRANSITION_FP; 915 break; 916 case MHI_EE_RDDM: 917 mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM); 918 write_lock_irq(&mhi_cntrl->pm_lock); 919 mhi_cntrl->ee = event; 920 write_unlock_irq(&mhi_cntrl->pm_lock); 921 wake_up_all(&mhi_cntrl->state_event); 922 break; 923 default: 924 dev_err(dev, 925 "Unhandled EE event: 0x%x\n", type); 926 } 927 if (st != DEV_ST_TRANSITION_MAX) 928 mhi_queue_state_transition(mhi_cntrl, st); 929 930 break; 931 } 932 case MHI_PKT_TYPE_TX_EVENT: 933 chan = MHI_TRE_GET_EV_CHID(local_rp); 934 935 WARN_ON(chan >= mhi_cntrl->max_chan); 936 937 /* 938 * Only process the event ring elements whose channel 939 * ID is within the maximum supported range. 940 */ 941 if (chan < mhi_cntrl->max_chan) { 942 mhi_chan = &mhi_cntrl->mhi_chan[chan]; 943 if (!mhi_chan->configured) 944 break; 945 parse_xfer_event(mhi_cntrl, local_rp, mhi_chan); 946 } 947 break; 948 default: 949 dev_err(dev, "Unhandled event type: %d\n", type); 950 break; 951 } 952 953 mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring); 954 local_rp = ev_ring->rp; 955 956 ptr = le64_to_cpu(er_ctxt->rp); 957 if (!is_valid_ring_ptr(ev_ring, ptr)) { 958 dev_err(&mhi_cntrl->mhi_dev->dev, 959 "Event ring rp points outside of the event ring\n"); 960 return -EIO; 961 } 962 963 dev_rp = mhi_to_virtual(ev_ring, ptr); 964 count++; 965 } 966 967 read_lock_bh(&mhi_cntrl->pm_lock); 968 969 /* Ring EV DB only if there is any pending element to process */ 970 if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)) && count) 971 mhi_ring_er_db(mhi_event); 972 read_unlock_bh(&mhi_cntrl->pm_lock); 973 974 return count; 975} 976 977int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl, 978 struct mhi_event *mhi_event, 979 u32 event_quota) 980{ 981 struct mhi_ring_element *dev_rp, *local_rp; 982 struct mhi_ring *ev_ring = &mhi_event->ring; 983 struct mhi_event_ctxt *er_ctxt = 984 &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; 985 int count = 0; 986 u32 chan; 987 struct mhi_chan *mhi_chan; 988 dma_addr_t ptr = le64_to_cpu(er_ctxt->rp); 989 990 if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state))) 991 return -EIO; 992 993 if (!is_valid_ring_ptr(ev_ring, ptr)) { 994 dev_err(&mhi_cntrl->mhi_dev->dev, 995 "Event ring rp points outside of the event ring\n"); 996 return -EIO; 997 } 998 999 dev_rp = mhi_to_virtual(ev_ring, ptr); 1000 local_rp = ev_ring->rp; 1001 1002 while (dev_rp != local_rp && event_quota > 0) { 1003 enum mhi_pkt_type type = MHI_TRE_GET_EV_TYPE(local_rp); 1004 1005 chan = MHI_TRE_GET_EV_CHID(local_rp); 1006 1007 WARN_ON(chan >= mhi_cntrl->max_chan); 1008 1009 /* 1010 * Only process the event ring elements whose channel 1011 * ID is within the maximum supported range. 1012 */ 1013 if (chan < mhi_cntrl->max_chan && 1014 mhi_cntrl->mhi_chan[chan].configured) { 1015 mhi_chan = &mhi_cntrl->mhi_chan[chan]; 1016 1017 if (likely(type == MHI_PKT_TYPE_TX_EVENT)) { 1018 parse_xfer_event(mhi_cntrl, local_rp, mhi_chan); 1019 event_quota--; 1020 } else if (type == MHI_PKT_TYPE_RSC_TX_EVENT) { 1021 parse_rsc_event(mhi_cntrl, local_rp, mhi_chan); 1022 event_quota--; 1023 } 1024 } 1025 1026 mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring); 1027 local_rp = ev_ring->rp; 1028 1029 ptr = le64_to_cpu(er_ctxt->rp); 1030 if (!is_valid_ring_ptr(ev_ring, ptr)) { 1031 dev_err(&mhi_cntrl->mhi_dev->dev, 1032 "Event ring rp points outside of the event ring\n"); 1033 return -EIO; 1034 } 1035 1036 dev_rp = mhi_to_virtual(ev_ring, ptr); 1037 count++; 1038 } 1039 read_lock_bh(&mhi_cntrl->pm_lock); 1040 1041 /* Ring EV DB only if there is any pending element to process */ 1042 if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)) && count) 1043 mhi_ring_er_db(mhi_event); 1044 read_unlock_bh(&mhi_cntrl->pm_lock); 1045 1046 return count; 1047} 1048 1049void mhi_ev_task(unsigned long data) 1050{ 1051 struct mhi_event *mhi_event = (struct mhi_event *)data; 1052 struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl; 1053 1054 /* process all pending events */ 1055 spin_lock_bh(&mhi_event->lock); 1056 mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX); 1057 spin_unlock_bh(&mhi_event->lock); 1058} 1059 1060void mhi_ctrl_ev_task(unsigned long data) 1061{ 1062 struct mhi_event *mhi_event = (struct mhi_event *)data; 1063 struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl; 1064 struct device *dev = &mhi_cntrl->mhi_dev->dev; 1065 enum mhi_state state; 1066 enum mhi_pm_state pm_state = 0; 1067 int ret; 1068 1069 /* 1070 * We can check PM state w/o a lock here because there is no way 1071 * PM state can change from reg access valid to no access while this 1072 * thread being executed. 1073 */ 1074 if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) { 1075 /* 1076 * We may have a pending event but not allowed to 1077 * process it since we are probably in a suspended state, 1078 * so trigger a resume. 1079 */ 1080 mhi_trigger_resume(mhi_cntrl); 1081 1082 return; 1083 } 1084 1085 /* Process ctrl events */ 1086 ret = mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX); 1087 1088 /* 1089 * We received an IRQ but no events to process, maybe device went to 1090 * SYS_ERR state? Check the state to confirm. 1091 */ 1092 if (!ret) { 1093 write_lock_irq(&mhi_cntrl->pm_lock); 1094 state = mhi_get_mhi_state(mhi_cntrl); 1095 if (state == MHI_STATE_SYS_ERR) { 1096 dev_dbg(dev, "System error detected\n"); 1097 pm_state = mhi_tryset_pm_state(mhi_cntrl, 1098 MHI_PM_SYS_ERR_DETECT); 1099 } 1100 write_unlock_irq(&mhi_cntrl->pm_lock); 1101 if (pm_state == MHI_PM_SYS_ERR_DETECT) 1102 mhi_pm_sys_err_handler(mhi_cntrl); 1103 } 1104} 1105 1106static bool mhi_is_ring_full(struct mhi_controller *mhi_cntrl, 1107 struct mhi_ring *ring) 1108{ 1109 void *tmp = ring->wp + ring->el_size; 1110 1111 if (tmp >= (ring->base + ring->len)) 1112 tmp = ring->base; 1113 1114 return (tmp == ring->rp); 1115} 1116 1117static int mhi_queue(struct mhi_device *mhi_dev, struct mhi_buf_info *buf_info, 1118 enum dma_data_direction dir, enum mhi_flags mflags) 1119{ 1120 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; 1121 struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : 1122 mhi_dev->dl_chan; 1123 struct mhi_ring *tre_ring = &mhi_chan->tre_ring; 1124 unsigned long flags; 1125 int ret; 1126 1127 if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) 1128 return -EIO; 1129 1130 ret = mhi_is_ring_full(mhi_cntrl, tre_ring); 1131 if (unlikely(ret)) 1132 return -EAGAIN; 1133 1134 ret = mhi_gen_tre(mhi_cntrl, mhi_chan, buf_info, mflags); 1135 if (unlikely(ret)) 1136 return ret; 1137 1138 read_lock_irqsave(&mhi_cntrl->pm_lock, flags); 1139 1140 /* Packet is queued, take a usage ref to exit M3 if necessary 1141 * for host->device buffer, balanced put is done on buffer completion 1142 * for device->host buffer, balanced put is after ringing the DB 1143 */ 1144 mhi_cntrl->runtime_get(mhi_cntrl); 1145 1146 /* Assert dev_wake (to exit/prevent M1/M2)*/ 1147 mhi_cntrl->wake_toggle(mhi_cntrl); 1148 1149 if (mhi_chan->dir == DMA_TO_DEVICE) 1150 atomic_inc(&mhi_cntrl->pending_pkts); 1151 1152 if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) 1153 mhi_ring_chan_db(mhi_cntrl, mhi_chan); 1154 1155 if (dir == DMA_FROM_DEVICE) 1156 mhi_cntrl->runtime_put(mhi_cntrl); 1157 1158 read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); 1159 1160 return ret; 1161} 1162 1163int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir, 1164 struct sk_buff *skb, size_t len, enum mhi_flags mflags) 1165{ 1166 struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : 1167 mhi_dev->dl_chan; 1168 struct mhi_buf_info buf_info = { }; 1169 1170 buf_info.v_addr = skb->data; 1171 buf_info.cb_buf = skb; 1172 buf_info.len = len; 1173 1174 if (unlikely(mhi_chan->pre_alloc)) 1175 return -EINVAL; 1176 1177 return mhi_queue(mhi_dev, &buf_info, dir, mflags); 1178} 1179EXPORT_SYMBOL_GPL(mhi_queue_skb); 1180 1181int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir, 1182 struct mhi_buf *mhi_buf, size_t len, enum mhi_flags mflags) 1183{ 1184 struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : 1185 mhi_dev->dl_chan; 1186 struct mhi_buf_info buf_info = { }; 1187 1188 buf_info.p_addr = mhi_buf->dma_addr; 1189 buf_info.cb_buf = mhi_buf; 1190 buf_info.pre_mapped = true; 1191 buf_info.len = len; 1192 1193 if (unlikely(mhi_chan->pre_alloc)) 1194 return -EINVAL; 1195 1196 return mhi_queue(mhi_dev, &buf_info, dir, mflags); 1197} 1198EXPORT_SYMBOL_GPL(mhi_queue_dma); 1199 1200int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan, 1201 struct mhi_buf_info *info, enum mhi_flags flags) 1202{ 1203 struct mhi_ring *buf_ring, *tre_ring; 1204 struct mhi_ring_element *mhi_tre; 1205 struct mhi_buf_info *buf_info; 1206 int eot, eob, chain, bei; 1207 int ret; 1208 1209 /* Protect accesses for reading and incrementing WP */ 1210 write_lock_bh(&mhi_chan->lock); 1211 1212 buf_ring = &mhi_chan->buf_ring; 1213 tre_ring = &mhi_chan->tre_ring; 1214 1215 buf_info = buf_ring->wp; 1216 WARN_ON(buf_info->used); 1217 buf_info->pre_mapped = info->pre_mapped; 1218 if (info->pre_mapped) 1219 buf_info->p_addr = info->p_addr; 1220 else 1221 buf_info->v_addr = info->v_addr; 1222 buf_info->cb_buf = info->cb_buf; 1223 buf_info->wp = tre_ring->wp; 1224 buf_info->dir = mhi_chan->dir; 1225 buf_info->len = info->len; 1226 1227 if (!info->pre_mapped) { 1228 ret = mhi_cntrl->map_single(mhi_cntrl, buf_info); 1229 if (ret) { 1230 write_unlock_bh(&mhi_chan->lock); 1231 return ret; 1232 } 1233 } 1234 1235 eob = !!(flags & MHI_EOB); 1236 eot = !!(flags & MHI_EOT); 1237 chain = !!(flags & MHI_CHAIN); 1238 bei = !!(mhi_chan->intmod); 1239 1240 mhi_tre = tre_ring->wp; 1241 mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr); 1242 mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(info->len); 1243 mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(bei, eot, eob, chain); 1244 1245 /* increment WP */ 1246 mhi_add_ring_element(mhi_cntrl, tre_ring); 1247 mhi_add_ring_element(mhi_cntrl, buf_ring); 1248 1249 write_unlock_bh(&mhi_chan->lock); 1250 1251 return 0; 1252} 1253 1254int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir, 1255 void *buf, size_t len, enum mhi_flags mflags) 1256{ 1257 struct mhi_buf_info buf_info = { }; 1258 1259 buf_info.v_addr = buf; 1260 buf_info.cb_buf = buf; 1261 buf_info.len = len; 1262 1263 return mhi_queue(mhi_dev, &buf_info, dir, mflags); 1264} 1265EXPORT_SYMBOL_GPL(mhi_queue_buf); 1266 1267bool mhi_queue_is_full(struct mhi_device *mhi_dev, enum dma_data_direction dir) 1268{ 1269 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; 1270 struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? 1271 mhi_dev->ul_chan : mhi_dev->dl_chan; 1272 struct mhi_ring *tre_ring = &mhi_chan->tre_ring; 1273 1274 return mhi_is_ring_full(mhi_cntrl, tre_ring); 1275} 1276EXPORT_SYMBOL_GPL(mhi_queue_is_full); 1277 1278int mhi_send_cmd(struct mhi_controller *mhi_cntrl, 1279 struct mhi_chan *mhi_chan, 1280 enum mhi_cmd_type cmd) 1281{ 1282 struct mhi_ring_element *cmd_tre = NULL; 1283 struct mhi_cmd *mhi_cmd = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING]; 1284 struct mhi_ring *ring = &mhi_cmd->ring; 1285 struct device *dev = &mhi_cntrl->mhi_dev->dev; 1286 int chan = 0; 1287 1288 if (mhi_chan) 1289 chan = mhi_chan->chan; 1290 1291 spin_lock_bh(&mhi_cmd->lock); 1292 if (!get_nr_avail_ring_elements(mhi_cntrl, ring)) { 1293 spin_unlock_bh(&mhi_cmd->lock); 1294 return -ENOMEM; 1295 } 1296 1297 /* prepare the cmd tre */ 1298 cmd_tre = ring->wp; 1299 switch (cmd) { 1300 case MHI_CMD_RESET_CHAN: 1301 cmd_tre->ptr = MHI_TRE_CMD_RESET_PTR; 1302 cmd_tre->dword[0] = MHI_TRE_CMD_RESET_DWORD0; 1303 cmd_tre->dword[1] = MHI_TRE_CMD_RESET_DWORD1(chan); 1304 break; 1305 case MHI_CMD_STOP_CHAN: 1306 cmd_tre->ptr = MHI_TRE_CMD_STOP_PTR; 1307 cmd_tre->dword[0] = MHI_TRE_CMD_STOP_DWORD0; 1308 cmd_tre->dword[1] = MHI_TRE_CMD_STOP_DWORD1(chan); 1309 break; 1310 case MHI_CMD_START_CHAN: 1311 cmd_tre->ptr = MHI_TRE_CMD_START_PTR; 1312 cmd_tre->dword[0] = MHI_TRE_CMD_START_DWORD0; 1313 cmd_tre->dword[1] = MHI_TRE_CMD_START_DWORD1(chan); 1314 break; 1315 default: 1316 dev_err(dev, "Command not supported\n"); 1317 break; 1318 } 1319 1320 /* queue to hardware */ 1321 mhi_add_ring_element(mhi_cntrl, ring); 1322 read_lock_bh(&mhi_cntrl->pm_lock); 1323 if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) 1324 mhi_ring_cmd_db(mhi_cntrl, mhi_cmd); 1325 read_unlock_bh(&mhi_cntrl->pm_lock); 1326 spin_unlock_bh(&mhi_cmd->lock); 1327 1328 return 0; 1329} 1330 1331static int mhi_update_channel_state(struct mhi_controller *mhi_cntrl, 1332 struct mhi_chan *mhi_chan, 1333 enum mhi_ch_state_type to_state) 1334{ 1335 struct device *dev = &mhi_chan->mhi_dev->dev; 1336 enum mhi_cmd_type cmd = MHI_CMD_NOP; 1337 int ret; 1338 1339 dev_dbg(dev, "%d: Updating channel state to: %s\n", mhi_chan->chan, 1340 TO_CH_STATE_TYPE_STR(to_state)); 1341 1342 switch (to_state) { 1343 case MHI_CH_STATE_TYPE_RESET: 1344 write_lock_irq(&mhi_chan->lock); 1345 if (mhi_chan->ch_state != MHI_CH_STATE_STOP && 1346 mhi_chan->ch_state != MHI_CH_STATE_ENABLED && 1347 mhi_chan->ch_state != MHI_CH_STATE_SUSPENDED) { 1348 write_unlock_irq(&mhi_chan->lock); 1349 return -EINVAL; 1350 } 1351 mhi_chan->ch_state = MHI_CH_STATE_DISABLED; 1352 write_unlock_irq(&mhi_chan->lock); 1353 1354 cmd = MHI_CMD_RESET_CHAN; 1355 break; 1356 case MHI_CH_STATE_TYPE_STOP: 1357 if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED) 1358 return -EINVAL; 1359 1360 cmd = MHI_CMD_STOP_CHAN; 1361 break; 1362 case MHI_CH_STATE_TYPE_START: 1363 if (mhi_chan->ch_state != MHI_CH_STATE_STOP && 1364 mhi_chan->ch_state != MHI_CH_STATE_DISABLED) 1365 return -EINVAL; 1366 1367 cmd = MHI_CMD_START_CHAN; 1368 break; 1369 default: 1370 dev_err(dev, "%d: Channel state update to %s not allowed\n", 1371 mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state)); 1372 return -EINVAL; 1373 } 1374 1375 /* bring host and device out of suspended states */ 1376 ret = mhi_device_get_sync(mhi_cntrl->mhi_dev); 1377 if (ret) 1378 return ret; 1379 mhi_cntrl->runtime_get(mhi_cntrl); 1380 1381 reinit_completion(&mhi_chan->completion); 1382 ret = mhi_send_cmd(mhi_cntrl, mhi_chan, cmd); 1383 if (ret) { 1384 dev_err(dev, "%d: Failed to send %s channel command\n", 1385 mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state)); 1386 goto exit_channel_update; 1387 } 1388 1389 ret = wait_for_completion_timeout(&mhi_chan->completion, 1390 msecs_to_jiffies(mhi_cntrl->timeout_ms)); 1391 if (!ret || mhi_chan->ccs != MHI_EV_CC_SUCCESS) { 1392 dev_err(dev, 1393 "%d: Failed to receive %s channel command completion\n", 1394 mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state)); 1395 ret = -EIO; 1396 goto exit_channel_update; 1397 } 1398 1399 ret = 0; 1400 1401 if (to_state != MHI_CH_STATE_TYPE_RESET) { 1402 write_lock_irq(&mhi_chan->lock); 1403 mhi_chan->ch_state = (to_state == MHI_CH_STATE_TYPE_START) ? 1404 MHI_CH_STATE_ENABLED : MHI_CH_STATE_STOP; 1405 write_unlock_irq(&mhi_chan->lock); 1406 } 1407 1408 dev_dbg(dev, "%d: Channel state change to %s successful\n", 1409 mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state)); 1410 1411exit_channel_update: 1412 mhi_cntrl->runtime_put(mhi_cntrl); 1413 mhi_device_put(mhi_cntrl->mhi_dev); 1414 1415 return ret; 1416} 1417 1418static void mhi_unprepare_channel(struct mhi_controller *mhi_cntrl, 1419 struct mhi_chan *mhi_chan) 1420{ 1421 int ret; 1422 struct device *dev = &mhi_chan->mhi_dev->dev; 1423 1424 mutex_lock(&mhi_chan->mutex); 1425 1426 if (!(BIT(mhi_cntrl->ee) & mhi_chan->ee_mask)) { 1427 dev_dbg(dev, "Current EE: %s Required EE Mask: 0x%x\n", 1428 TO_MHI_EXEC_STR(mhi_cntrl->ee), mhi_chan->ee_mask); 1429 goto exit_unprepare_channel; 1430 } 1431 1432 /* no more processing events for this channel */ 1433 ret = mhi_update_channel_state(mhi_cntrl, mhi_chan, 1434 MHI_CH_STATE_TYPE_RESET); 1435 if (ret) 1436 dev_err(dev, "%d: Failed to reset channel, still resetting\n", 1437 mhi_chan->chan); 1438 1439exit_unprepare_channel: 1440 write_lock_irq(&mhi_chan->lock); 1441 mhi_chan->ch_state = MHI_CH_STATE_DISABLED; 1442 write_unlock_irq(&mhi_chan->lock); 1443 1444 if (!mhi_chan->offload_ch) { 1445 mhi_reset_chan(mhi_cntrl, mhi_chan); 1446 mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan); 1447 } 1448 dev_dbg(dev, "%d: successfully reset\n", mhi_chan->chan); 1449 1450 mutex_unlock(&mhi_chan->mutex); 1451} 1452 1453int mhi_prepare_channel(struct mhi_controller *mhi_cntrl, 1454 struct mhi_chan *mhi_chan, unsigned int flags) 1455{ 1456 int ret = 0; 1457 struct device *dev = &mhi_chan->mhi_dev->dev; 1458 1459 if (!(BIT(mhi_cntrl->ee) & mhi_chan->ee_mask)) { 1460 dev_err(dev, "Current EE: %s Required EE Mask: 0x%x\n", 1461 TO_MHI_EXEC_STR(mhi_cntrl->ee), mhi_chan->ee_mask); 1462 return -ENOTCONN; 1463 } 1464 1465 mutex_lock(&mhi_chan->mutex); 1466 1467 /* Check of client manages channel context for offload channels */ 1468 if (!mhi_chan->offload_ch) { 1469 ret = mhi_init_chan_ctxt(mhi_cntrl, mhi_chan); 1470 if (ret) 1471 goto error_init_chan; 1472 } 1473 1474 ret = mhi_update_channel_state(mhi_cntrl, mhi_chan, 1475 MHI_CH_STATE_TYPE_START); 1476 if (ret) 1477 goto error_pm_state; 1478 1479 if (mhi_chan->dir == DMA_FROM_DEVICE) 1480 mhi_chan->pre_alloc = !!(flags & MHI_CH_INBOUND_ALLOC_BUFS); 1481 1482 /* Pre-allocate buffer for xfer ring */ 1483 if (mhi_chan->pre_alloc) { 1484 int nr_el = get_nr_avail_ring_elements(mhi_cntrl, 1485 &mhi_chan->tre_ring); 1486 size_t len = mhi_cntrl->buffer_len; 1487 1488 while (nr_el--) { 1489 void *buf; 1490 struct mhi_buf_info info = { }; 1491 1492 buf = kmalloc(len, GFP_KERNEL); 1493 if (!buf) { 1494 ret = -ENOMEM; 1495 goto error_pre_alloc; 1496 } 1497 1498 /* Prepare transfer descriptors */ 1499 info.v_addr = buf; 1500 info.cb_buf = buf; 1501 info.len = len; 1502 ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &info, MHI_EOT); 1503 if (ret) { 1504 kfree(buf); 1505 goto error_pre_alloc; 1506 } 1507 } 1508 1509 read_lock_bh(&mhi_cntrl->pm_lock); 1510 if (MHI_DB_ACCESS_VALID(mhi_cntrl)) { 1511 read_lock_irq(&mhi_chan->lock); 1512 mhi_ring_chan_db(mhi_cntrl, mhi_chan); 1513 read_unlock_irq(&mhi_chan->lock); 1514 } 1515 read_unlock_bh(&mhi_cntrl->pm_lock); 1516 } 1517 1518 mutex_unlock(&mhi_chan->mutex); 1519 1520 return 0; 1521 1522error_pm_state: 1523 if (!mhi_chan->offload_ch) 1524 mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan); 1525 1526error_init_chan: 1527 mutex_unlock(&mhi_chan->mutex); 1528 1529 return ret; 1530 1531error_pre_alloc: 1532 mutex_unlock(&mhi_chan->mutex); 1533 mhi_unprepare_channel(mhi_cntrl, mhi_chan); 1534 1535 return ret; 1536} 1537 1538static void mhi_mark_stale_events(struct mhi_controller *mhi_cntrl, 1539 struct mhi_event *mhi_event, 1540 struct mhi_event_ctxt *er_ctxt, 1541 int chan) 1542 1543{ 1544 struct mhi_ring_element *dev_rp, *local_rp; 1545 struct mhi_ring *ev_ring; 1546 struct device *dev = &mhi_cntrl->mhi_dev->dev; 1547 unsigned long flags; 1548 dma_addr_t ptr; 1549 1550 dev_dbg(dev, "Marking all events for chan: %d as stale\n", chan); 1551 1552 ev_ring = &mhi_event->ring; 1553 1554 /* mark all stale events related to channel as STALE event */ 1555 spin_lock_irqsave(&mhi_event->lock, flags); 1556 1557 ptr = le64_to_cpu(er_ctxt->rp); 1558 if (!is_valid_ring_ptr(ev_ring, ptr)) { 1559 dev_err(&mhi_cntrl->mhi_dev->dev, 1560 "Event ring rp points outside of the event ring\n"); 1561 dev_rp = ev_ring->rp; 1562 } else { 1563 dev_rp = mhi_to_virtual(ev_ring, ptr); 1564 } 1565 1566 local_rp = ev_ring->rp; 1567 while (dev_rp != local_rp) { 1568 if (MHI_TRE_GET_EV_TYPE(local_rp) == MHI_PKT_TYPE_TX_EVENT && 1569 chan == MHI_TRE_GET_EV_CHID(local_rp)) 1570 local_rp->dword[1] = MHI_TRE_EV_DWORD1(chan, 1571 MHI_PKT_TYPE_STALE_EVENT); 1572 local_rp++; 1573 if (local_rp == (ev_ring->base + ev_ring->len)) 1574 local_rp = ev_ring->base; 1575 } 1576 1577 dev_dbg(dev, "Finished marking events as stale events\n"); 1578 spin_unlock_irqrestore(&mhi_event->lock, flags); 1579} 1580 1581static void mhi_reset_data_chan(struct mhi_controller *mhi_cntrl, 1582 struct mhi_chan *mhi_chan) 1583{ 1584 struct mhi_ring *buf_ring, *tre_ring; 1585 struct mhi_result result; 1586 1587 /* Reset any pending buffers */ 1588 buf_ring = &mhi_chan->buf_ring; 1589 tre_ring = &mhi_chan->tre_ring; 1590 result.transaction_status = -ENOTCONN; 1591 result.bytes_xferd = 0; 1592 while (tre_ring->rp != tre_ring->wp) { 1593 struct mhi_buf_info *buf_info = buf_ring->rp; 1594 1595 if (mhi_chan->dir == DMA_TO_DEVICE) { 1596 atomic_dec(&mhi_cntrl->pending_pkts); 1597 /* Release the reference got from mhi_queue() */ 1598 mhi_cntrl->runtime_put(mhi_cntrl); 1599 } 1600 1601 if (!buf_info->pre_mapped) 1602 mhi_cntrl->unmap_single(mhi_cntrl, buf_info); 1603 1604 mhi_del_ring_element(mhi_cntrl, buf_ring); 1605 mhi_del_ring_element(mhi_cntrl, tre_ring); 1606 1607 if (mhi_chan->pre_alloc) { 1608 kfree(buf_info->cb_buf); 1609 } else { 1610 result.buf_addr = buf_info->cb_buf; 1611 mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result); 1612 } 1613 } 1614} 1615 1616void mhi_reset_chan(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan) 1617{ 1618 struct mhi_event *mhi_event; 1619 struct mhi_event_ctxt *er_ctxt; 1620 int chan = mhi_chan->chan; 1621 1622 /* Nothing to reset, client doesn't queue buffers */ 1623 if (mhi_chan->offload_ch) 1624 return; 1625 1626 read_lock_bh(&mhi_cntrl->pm_lock); 1627 mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index]; 1628 er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_chan->er_index]; 1629 1630 mhi_mark_stale_events(mhi_cntrl, mhi_event, er_ctxt, chan); 1631 1632 mhi_reset_data_chan(mhi_cntrl, mhi_chan); 1633 1634 read_unlock_bh(&mhi_cntrl->pm_lock); 1635} 1636 1637static int __mhi_prepare_for_transfer(struct mhi_device *mhi_dev, unsigned int flags) 1638{ 1639 int ret, dir; 1640 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; 1641 struct mhi_chan *mhi_chan; 1642 1643 for (dir = 0; dir < 2; dir++) { 1644 mhi_chan = dir ? mhi_dev->dl_chan : mhi_dev->ul_chan; 1645 if (!mhi_chan) 1646 continue; 1647 1648 ret = mhi_prepare_channel(mhi_cntrl, mhi_chan, flags); 1649 if (ret) 1650 goto error_open_chan; 1651 } 1652 1653 return 0; 1654 1655error_open_chan: 1656 for (--dir; dir >= 0; dir--) { 1657 mhi_chan = dir ? mhi_dev->dl_chan : mhi_dev->ul_chan; 1658 if (!mhi_chan) 1659 continue; 1660 1661 mhi_unprepare_channel(mhi_cntrl, mhi_chan); 1662 } 1663 1664 return ret; 1665} 1666 1667int mhi_prepare_for_transfer(struct mhi_device *mhi_dev) 1668{ 1669 return __mhi_prepare_for_transfer(mhi_dev, 0); 1670} 1671EXPORT_SYMBOL_GPL(mhi_prepare_for_transfer); 1672 1673int mhi_prepare_for_transfer_autoqueue(struct mhi_device *mhi_dev) 1674{ 1675 return __mhi_prepare_for_transfer(mhi_dev, MHI_CH_INBOUND_ALLOC_BUFS); 1676} 1677EXPORT_SYMBOL_GPL(mhi_prepare_for_transfer_autoqueue); 1678 1679void mhi_unprepare_from_transfer(struct mhi_device *mhi_dev) 1680{ 1681 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; 1682 struct mhi_chan *mhi_chan; 1683 int dir; 1684 1685 for (dir = 0; dir < 2; dir++) { 1686 mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan; 1687 if (!mhi_chan) 1688 continue; 1689 1690 mhi_unprepare_channel(mhi_cntrl, mhi_chan); 1691 } 1692} 1693EXPORT_SYMBOL_GPL(mhi_unprepare_from_transfer); 1694