1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * ISHTP client logic 4 * 5 * Copyright (c) 2003-2016, Intel Corporation. 6 */ 7 8#include <linux/slab.h> 9#include <linux/sched.h> 10#include <linux/wait.h> 11#include <linux/delay.h> 12#include <linux/dma-mapping.h> 13#include <asm/cacheflush.h> 14#include "hbm.h" 15#include "client.h" 16 17int ishtp_cl_get_tx_free_buffer_size(struct ishtp_cl *cl) 18{ 19 unsigned long tx_free_flags; 20 int size; 21 22 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags); 23 size = cl->tx_ring_free_size * cl->device->fw_client->props.max_msg_length; 24 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags); 25 26 return size; 27} 28EXPORT_SYMBOL(ishtp_cl_get_tx_free_buffer_size); 29 30int ishtp_cl_get_tx_free_rings(struct ishtp_cl *cl) 31{ 32 return cl->tx_ring_free_size; 33} 34EXPORT_SYMBOL(ishtp_cl_get_tx_free_rings); 35 36/** 37 * ishtp_read_list_flush() - Flush read queue 38 * @cl: ishtp client instance 39 * 40 * Used to remove all entries from read queue for a client 41 */ 42static void ishtp_read_list_flush(struct ishtp_cl *cl) 43{ 44 struct ishtp_cl_rb *rb; 45 struct ishtp_cl_rb *next; 46 unsigned long flags; 47 48 spin_lock_irqsave(&cl->dev->read_list_spinlock, flags); 49 list_for_each_entry_safe(rb, next, &cl->dev->read_list.list, list) 50 if (rb->cl && ishtp_cl_cmp_id(cl, rb->cl)) { 51 list_del(&rb->list); 52 ishtp_io_rb_free(rb); 53 } 54 spin_unlock_irqrestore(&cl->dev->read_list_spinlock, flags); 55} 56 57/** 58 * ishtp_cl_flush_queues() - Flush all queues for a client 59 * @cl: ishtp client instance 60 * 61 * Used to remove all queues for a client. This is called when a client device 62 * needs reset due to error, S3 resume or during module removal 63 * 64 * Return: 0 on success else -EINVAL if device is NULL 65 */ 66int ishtp_cl_flush_queues(struct ishtp_cl *cl) 67{ 68 if (WARN_ON(!cl || !cl->dev)) 69 return -EINVAL; 70 71 ishtp_read_list_flush(cl); 72 73 return 0; 74} 75EXPORT_SYMBOL(ishtp_cl_flush_queues); 76 77/** 78 * ishtp_cl_init() - Initialize all fields of a client device 79 * @cl: ishtp client instance 80 * @dev: ishtp device 81 * 82 * Initializes a client device fields: Init spinlocks, init queues etc. 83 * This function is called during new client creation 84 */ 85static void ishtp_cl_init(struct ishtp_cl *cl, struct ishtp_device *dev) 86{ 87 memset(cl, 0, sizeof(struct ishtp_cl)); 88 init_waitqueue_head(&cl->wait_ctrl_res); 89 spin_lock_init(&cl->free_list_spinlock); 90 spin_lock_init(&cl->in_process_spinlock); 91 spin_lock_init(&cl->tx_list_spinlock); 92 spin_lock_init(&cl->tx_free_list_spinlock); 93 spin_lock_init(&cl->fc_spinlock); 94 INIT_LIST_HEAD(&cl->link); 95 cl->dev = dev; 96 97 INIT_LIST_HEAD(&cl->free_rb_list.list); 98 INIT_LIST_HEAD(&cl->tx_list.list); 99 INIT_LIST_HEAD(&cl->tx_free_list.list); 100 INIT_LIST_HEAD(&cl->in_process_list.list); 101 102 cl->rx_ring_size = CL_DEF_RX_RING_SIZE; 103 cl->tx_ring_size = CL_DEF_TX_RING_SIZE; 104 cl->tx_ring_free_size = cl->tx_ring_size; 105 106 /* dma */ 107 cl->last_tx_path = CL_TX_PATH_IPC; 108 cl->last_dma_acked = 1; 109 cl->last_dma_addr = NULL; 110 cl->last_ipc_acked = 1; 111} 112 113/** 114 * ishtp_cl_allocate() - allocates client structure and sets it up. 115 * @cl_device: ishtp client device 116 * 117 * Allocate memory for new client device and call to initialize each field. 118 * 119 * Return: The allocated client instance or NULL on failure 120 */ 121struct ishtp_cl *ishtp_cl_allocate(struct ishtp_cl_device *cl_device) 122{ 123 struct ishtp_cl *cl; 124 125 cl = kmalloc(sizeof(struct ishtp_cl), GFP_KERNEL); 126 if (!cl) 127 return NULL; 128 129 ishtp_cl_init(cl, cl_device->ishtp_dev); 130 return cl; 131} 132EXPORT_SYMBOL(ishtp_cl_allocate); 133 134/** 135 * ishtp_cl_free() - Frees a client device 136 * @cl: client device instance 137 * 138 * Frees a client device 139 */ 140void ishtp_cl_free(struct ishtp_cl *cl) 141{ 142 struct ishtp_device *dev; 143 unsigned long flags; 144 145 if (!cl) 146 return; 147 148 dev = cl->dev; 149 if (!dev) 150 return; 151 152 spin_lock_irqsave(&dev->cl_list_lock, flags); 153 ishtp_cl_free_rx_ring(cl); 154 ishtp_cl_free_tx_ring(cl); 155 kfree(cl); 156 spin_unlock_irqrestore(&dev->cl_list_lock, flags); 157} 158EXPORT_SYMBOL(ishtp_cl_free); 159 160/** 161 * ishtp_cl_link() - Reserve a host id and link the client instance 162 * @cl: client device instance 163 * 164 * This allocates a single bit in the hostmap. This function will make sure 165 * that not many client sessions are opened at the same time. Once allocated 166 * the client device instance is added to the ishtp device in the current 167 * client list 168 * 169 * Return: 0 or error code on failure 170 */ 171int ishtp_cl_link(struct ishtp_cl *cl) 172{ 173 struct ishtp_device *dev; 174 unsigned long flags, flags_cl; 175 int id, ret = 0; 176 177 if (WARN_ON(!cl || !cl->dev)) 178 return -EINVAL; 179 180 dev = cl->dev; 181 182 spin_lock_irqsave(&dev->device_lock, flags); 183 184 if (dev->open_handle_count >= ISHTP_MAX_OPEN_HANDLE_COUNT) { 185 ret = -EMFILE; 186 goto unlock_dev; 187 } 188 189 id = find_first_zero_bit(dev->host_clients_map, ISHTP_CLIENTS_MAX); 190 191 if (id >= ISHTP_CLIENTS_MAX) { 192 spin_unlock_irqrestore(&dev->device_lock, flags); 193 dev_err(&cl->device->dev, "id exceeded %d", ISHTP_CLIENTS_MAX); 194 return -ENOENT; 195 } 196 197 dev->open_handle_count++; 198 cl->host_client_id = id; 199 spin_lock_irqsave(&dev->cl_list_lock, flags_cl); 200 if (dev->dev_state != ISHTP_DEV_ENABLED) { 201 ret = -ENODEV; 202 goto unlock_cl; 203 } 204 list_add_tail(&cl->link, &dev->cl_list); 205 set_bit(id, dev->host_clients_map); 206 cl->state = ISHTP_CL_INITIALIZING; 207 208unlock_cl: 209 spin_unlock_irqrestore(&dev->cl_list_lock, flags_cl); 210unlock_dev: 211 spin_unlock_irqrestore(&dev->device_lock, flags); 212 return ret; 213} 214EXPORT_SYMBOL(ishtp_cl_link); 215 216/** 217 * ishtp_cl_unlink() - remove fw_cl from the client device list 218 * @cl: client device instance 219 * 220 * Remove a previously linked device to a ishtp device 221 */ 222void ishtp_cl_unlink(struct ishtp_cl *cl) 223{ 224 struct ishtp_device *dev; 225 struct ishtp_cl *pos; 226 unsigned long flags; 227 228 /* don't shout on error exit path */ 229 if (!cl || !cl->dev) 230 return; 231 232 dev = cl->dev; 233 234 spin_lock_irqsave(&dev->device_lock, flags); 235 if (dev->open_handle_count > 0) { 236 clear_bit(cl->host_client_id, dev->host_clients_map); 237 dev->open_handle_count--; 238 } 239 spin_unlock_irqrestore(&dev->device_lock, flags); 240 241 /* 242 * This checks that 'cl' is actually linked into device's structure, 243 * before attempting 'list_del' 244 */ 245 spin_lock_irqsave(&dev->cl_list_lock, flags); 246 list_for_each_entry(pos, &dev->cl_list, link) 247 if (cl->host_client_id == pos->host_client_id) { 248 list_del_init(&pos->link); 249 break; 250 } 251 spin_unlock_irqrestore(&dev->cl_list_lock, flags); 252} 253EXPORT_SYMBOL(ishtp_cl_unlink); 254 255/** 256 * ishtp_cl_disconnect() - Send disconnect request to firmware 257 * @cl: client device instance 258 * 259 * Send a disconnect request for a client to firmware. 260 * 261 * Return: 0 if successful disconnect response from the firmware or error 262 * code on failure 263 */ 264int ishtp_cl_disconnect(struct ishtp_cl *cl) 265{ 266 struct ishtp_device *dev; 267 268 if (WARN_ON(!cl || !cl->dev)) 269 return -ENODEV; 270 271 dev = cl->dev; 272 273 dev->print_log(dev, "%s() state %d\n", __func__, cl->state); 274 275 if (cl->state != ISHTP_CL_DISCONNECTING) { 276 dev->print_log(dev, "%s() Disconnect in progress\n", __func__); 277 return 0; 278 } 279 280 if (ishtp_hbm_cl_disconnect_req(dev, cl)) { 281 dev->print_log(dev, "%s() Failed to disconnect\n", __func__); 282 dev_err(&cl->device->dev, "failed to disconnect.\n"); 283 return -ENODEV; 284 } 285 286 wait_event_interruptible_timeout(cl->wait_ctrl_res, 287 (dev->dev_state != ISHTP_DEV_ENABLED || 288 cl->state == ISHTP_CL_DISCONNECTED), 289 ishtp_secs_to_jiffies(ISHTP_CL_CONNECT_TIMEOUT)); 290 291 /* 292 * If FW reset arrived, this will happen. Don't check cl->, 293 * as 'cl' may be freed already 294 */ 295 if (dev->dev_state != ISHTP_DEV_ENABLED) { 296 dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n", 297 __func__); 298 return -ENODEV; 299 } 300 301 if (cl->state == ISHTP_CL_DISCONNECTED) { 302 dev->print_log(dev, "%s() successful\n", __func__); 303 return 0; 304 } 305 306 return -ENODEV; 307} 308EXPORT_SYMBOL(ishtp_cl_disconnect); 309 310/** 311 * ishtp_cl_is_other_connecting() - Check other client is connecting 312 * @cl: client device instance 313 * 314 * Checks if other client with the same fw client id is connecting 315 * 316 * Return: true if other client is connected else false 317 */ 318static bool ishtp_cl_is_other_connecting(struct ishtp_cl *cl) 319{ 320 struct ishtp_device *dev; 321 struct ishtp_cl *pos; 322 unsigned long flags; 323 324 if (WARN_ON(!cl || !cl->dev)) 325 return false; 326 327 dev = cl->dev; 328 spin_lock_irqsave(&dev->cl_list_lock, flags); 329 list_for_each_entry(pos, &dev->cl_list, link) { 330 if ((pos->state == ISHTP_CL_CONNECTING) && (pos != cl) && 331 cl->fw_client_id == pos->fw_client_id) { 332 spin_unlock_irqrestore(&dev->cl_list_lock, flags); 333 return true; 334 } 335 } 336 spin_unlock_irqrestore(&dev->cl_list_lock, flags); 337 338 return false; 339} 340 341/** 342 * ishtp_cl_connect() - Send connect request to firmware 343 * @cl: client device instance 344 * 345 * Send a connect request for a client to firmware. If successful it will 346 * RX and TX ring buffers 347 * 348 * Return: 0 if successful connect response from the firmware and able 349 * to bind and allocate ring buffers or error code on failure 350 */ 351int ishtp_cl_connect(struct ishtp_cl *cl) 352{ 353 struct ishtp_device *dev; 354 int rets; 355 356 if (WARN_ON(!cl || !cl->dev)) 357 return -ENODEV; 358 359 dev = cl->dev; 360 361 dev->print_log(dev, "%s() current_state = %d\n", __func__, cl->state); 362 363 if (ishtp_cl_is_other_connecting(cl)) { 364 dev->print_log(dev, "%s() Busy\n", __func__); 365 return -EBUSY; 366 } 367 368 if (ishtp_hbm_cl_connect_req(dev, cl)) { 369 dev->print_log(dev, "%s() HBM connect req fail\n", __func__); 370 return -ENODEV; 371 } 372 373 rets = wait_event_interruptible_timeout(cl->wait_ctrl_res, 374 (dev->dev_state == ISHTP_DEV_ENABLED && 375 (cl->state == ISHTP_CL_CONNECTED || 376 cl->state == ISHTP_CL_DISCONNECTED)), 377 ishtp_secs_to_jiffies( 378 ISHTP_CL_CONNECT_TIMEOUT)); 379 /* 380 * If FW reset arrived, this will happen. Don't check cl->, 381 * as 'cl' may be freed already 382 */ 383 if (dev->dev_state != ISHTP_DEV_ENABLED) { 384 dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n", 385 __func__); 386 return -EFAULT; 387 } 388 389 if (cl->state != ISHTP_CL_CONNECTED) { 390 dev->print_log(dev, "%s() state != ISHTP_CL_CONNECTED\n", 391 __func__); 392 return -EFAULT; 393 } 394 395 rets = cl->status; 396 if (rets) { 397 dev->print_log(dev, "%s() Invalid status\n", __func__); 398 return rets; 399 } 400 401 rets = ishtp_cl_device_bind(cl); 402 if (rets) { 403 dev->print_log(dev, "%s() Bind error\n", __func__); 404 ishtp_cl_disconnect(cl); 405 return rets; 406 } 407 408 rets = ishtp_cl_alloc_rx_ring(cl); 409 if (rets) { 410 dev->print_log(dev, "%s() Alloc RX ring failed\n", __func__); 411 /* if failed allocation, disconnect */ 412 ishtp_cl_disconnect(cl); 413 return rets; 414 } 415 416 rets = ishtp_cl_alloc_tx_ring(cl); 417 if (rets) { 418 dev->print_log(dev, "%s() Alloc TX ring failed\n", __func__); 419 /* if failed allocation, disconnect */ 420 ishtp_cl_free_rx_ring(cl); 421 ishtp_cl_disconnect(cl); 422 return rets; 423 } 424 425 /* Upon successful connection and allocation, emit flow-control */ 426 rets = ishtp_cl_read_start(cl); 427 428 dev->print_log(dev, "%s() successful\n", __func__); 429 430 return rets; 431} 432EXPORT_SYMBOL(ishtp_cl_connect); 433 434/** 435 * ishtp_cl_read_start() - Prepare to read client message 436 * @cl: client device instance 437 * 438 * Get a free buffer from pool of free read buffers and add to read buffer 439 * pool to add contents. Send a flow control request to firmware to be able 440 * send next message. 441 * 442 * Return: 0 if successful or error code on failure 443 */ 444int ishtp_cl_read_start(struct ishtp_cl *cl) 445{ 446 struct ishtp_device *dev; 447 struct ishtp_cl_rb *rb; 448 int rets; 449 int i; 450 unsigned long flags; 451 unsigned long dev_flags; 452 453 if (WARN_ON(!cl || !cl->dev)) 454 return -ENODEV; 455 456 dev = cl->dev; 457 458 if (cl->state != ISHTP_CL_CONNECTED) 459 return -ENODEV; 460 461 if (dev->dev_state != ISHTP_DEV_ENABLED) 462 return -ENODEV; 463 464 i = ishtp_fw_cl_by_id(dev, cl->fw_client_id); 465 if (i < 0) { 466 dev_err(&cl->device->dev, "no such fw client %d\n", 467 cl->fw_client_id); 468 return -ENODEV; 469 } 470 471 /* The current rb is the head of the free rb list */ 472 spin_lock_irqsave(&cl->free_list_spinlock, flags); 473 if (list_empty(&cl->free_rb_list.list)) { 474 dev_warn(&cl->device->dev, 475 "[ishtp-ish] Rx buffers pool is empty\n"); 476 rets = -ENOMEM; 477 rb = NULL; 478 spin_unlock_irqrestore(&cl->free_list_spinlock, flags); 479 goto out; 480 } 481 rb = list_entry(cl->free_rb_list.list.next, struct ishtp_cl_rb, list); 482 list_del_init(&rb->list); 483 spin_unlock_irqrestore(&cl->free_list_spinlock, flags); 484 485 rb->cl = cl; 486 rb->buf_idx = 0; 487 488 INIT_LIST_HEAD(&rb->list); 489 rets = 0; 490 491 /* 492 * This must be BEFORE sending flow control - 493 * response in ISR may come too fast... 494 */ 495 spin_lock_irqsave(&dev->read_list_spinlock, dev_flags); 496 list_add_tail(&rb->list, &dev->read_list.list); 497 spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags); 498 if (ishtp_hbm_cl_flow_control_req(dev, cl)) { 499 rets = -ENODEV; 500 goto out; 501 } 502out: 503 /* if ishtp_hbm_cl_flow_control_req failed, return rb to free list */ 504 if (rets && rb) { 505 spin_lock_irqsave(&dev->read_list_spinlock, dev_flags); 506 list_del(&rb->list); 507 spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags); 508 509 spin_lock_irqsave(&cl->free_list_spinlock, flags); 510 list_add_tail(&rb->list, &cl->free_rb_list.list); 511 spin_unlock_irqrestore(&cl->free_list_spinlock, flags); 512 } 513 return rets; 514} 515 516/** 517 * ishtp_cl_send() - Send a message to firmware 518 * @cl: client device instance 519 * @buf: message buffer 520 * @length: length of message 521 * 522 * If the client is correct state to send message, this function gets a buffer 523 * from tx ring buffers, copy the message data and call to send the message 524 * using ishtp_cl_send_msg() 525 * 526 * Return: 0 if successful or error code on failure 527 */ 528int ishtp_cl_send(struct ishtp_cl *cl, uint8_t *buf, size_t length) 529{ 530 struct ishtp_device *dev; 531 int id; 532 struct ishtp_cl_tx_ring *cl_msg; 533 int have_msg_to_send = 0; 534 unsigned long tx_flags, tx_free_flags; 535 536 if (WARN_ON(!cl || !cl->dev)) 537 return -ENODEV; 538 539 dev = cl->dev; 540 541 if (cl->state != ISHTP_CL_CONNECTED) { 542 ++cl->err_send_msg; 543 return -EPIPE; 544 } 545 546 if (dev->dev_state != ISHTP_DEV_ENABLED) { 547 ++cl->err_send_msg; 548 return -ENODEV; 549 } 550 551 /* Check if we have fw client device */ 552 id = ishtp_fw_cl_by_id(dev, cl->fw_client_id); 553 if (id < 0) { 554 ++cl->err_send_msg; 555 return -ENOENT; 556 } 557 558 if (length > dev->fw_clients[id].props.max_msg_length) { 559 ++cl->err_send_msg; 560 return -EMSGSIZE; 561 } 562 563 /* No free bufs */ 564 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags); 565 if (list_empty(&cl->tx_free_list.list)) { 566 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, 567 tx_free_flags); 568 ++cl->err_send_msg; 569 return -ENOMEM; 570 } 571 572 cl_msg = list_first_entry(&cl->tx_free_list.list, 573 struct ishtp_cl_tx_ring, list); 574 if (!cl_msg->send_buf.data) { 575 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, 576 tx_free_flags); 577 return -EIO; 578 /* Should not happen, as free list is pre-allocated */ 579 } 580 /* 581 * This is safe, as 'length' is already checked for not exceeding 582 * max ISHTP message size per client 583 */ 584 list_del_init(&cl_msg->list); 585 --cl->tx_ring_free_size; 586 587 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags); 588 memcpy(cl_msg->send_buf.data, buf, length); 589 cl_msg->send_buf.size = length; 590 spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags); 591 have_msg_to_send = !list_empty(&cl->tx_list.list); 592 list_add_tail(&cl_msg->list, &cl->tx_list.list); 593 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 594 595 if (!have_msg_to_send && cl->ishtp_flow_ctrl_creds > 0) 596 ishtp_cl_send_msg(dev, cl); 597 598 return 0; 599} 600EXPORT_SYMBOL(ishtp_cl_send); 601 602/** 603 * ishtp_cl_read_complete() - read complete 604 * @rb: Pointer to client request block 605 * 606 * If the message is completely received call ishtp_cl_bus_rx_event() 607 * to process message 608 */ 609static void ishtp_cl_read_complete(struct ishtp_cl_rb *rb) 610{ 611 unsigned long flags; 612 int schedule_work_flag = 0; 613 struct ishtp_cl *cl = rb->cl; 614 615 spin_lock_irqsave(&cl->in_process_spinlock, flags); 616 /* 617 * if in-process list is empty, then need to schedule 618 * the processing thread 619 */ 620 schedule_work_flag = list_empty(&cl->in_process_list.list); 621 list_add_tail(&rb->list, &cl->in_process_list.list); 622 spin_unlock_irqrestore(&cl->in_process_spinlock, flags); 623 624 if (schedule_work_flag) 625 ishtp_cl_bus_rx_event(cl->device); 626} 627 628/** 629 * ipc_tx_send() - IPC tx send function 630 * @prm: Pointer to client device instance 631 * 632 * Send message over IPC. Message will be split into fragments 633 * if message size is bigger than IPC FIFO size, and all 634 * fragments will be sent one by one. 635 */ 636static void ipc_tx_send(void *prm) 637{ 638 struct ishtp_cl *cl = prm; 639 struct ishtp_cl_tx_ring *cl_msg; 640 size_t rem; 641 struct ishtp_device *dev = (cl ? cl->dev : NULL); 642 struct ishtp_msg_hdr ishtp_hdr; 643 unsigned long tx_flags, tx_free_flags; 644 unsigned char *pmsg; 645 646 if (!dev) 647 return; 648 649 /* 650 * Other conditions if some critical error has 651 * occurred before this callback is called 652 */ 653 if (dev->dev_state != ISHTP_DEV_ENABLED) 654 return; 655 656 if (cl->state != ISHTP_CL_CONNECTED) 657 return; 658 659 spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags); 660 if (list_empty(&cl->tx_list.list)) { 661 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 662 return; 663 } 664 665 if (cl->ishtp_flow_ctrl_creds != 1 && !cl->sending) { 666 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 667 return; 668 } 669 670 if (!cl->sending) { 671 --cl->ishtp_flow_ctrl_creds; 672 cl->last_ipc_acked = 0; 673 cl->last_tx_path = CL_TX_PATH_IPC; 674 cl->sending = 1; 675 } 676 677 cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring, 678 list); 679 rem = cl_msg->send_buf.size - cl->tx_offs; 680 681 while (rem > 0) { 682 ishtp_hdr.host_addr = cl->host_client_id; 683 ishtp_hdr.fw_addr = cl->fw_client_id; 684 ishtp_hdr.reserved = 0; 685 pmsg = cl_msg->send_buf.data + cl->tx_offs; 686 687 if (rem <= dev->mtu) { 688 /* Last fragment or only one packet */ 689 ishtp_hdr.length = rem; 690 ishtp_hdr.msg_complete = 1; 691 /* Submit to IPC queue with no callback */ 692 ishtp_write_message(dev, &ishtp_hdr, pmsg); 693 cl->tx_offs = 0; 694 cl->sending = 0; 695 696 break; 697 } else { 698 /* Send ipc fragment */ 699 ishtp_hdr.length = dev->mtu; 700 ishtp_hdr.msg_complete = 0; 701 /* All fregments submitted to IPC queue with no callback */ 702 ishtp_write_message(dev, &ishtp_hdr, pmsg); 703 cl->tx_offs += dev->mtu; 704 rem = cl_msg->send_buf.size - cl->tx_offs; 705 } 706 } 707 708 list_del_init(&cl_msg->list); 709 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 710 711 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags); 712 list_add_tail(&cl_msg->list, &cl->tx_free_list.list); 713 ++cl->tx_ring_free_size; 714 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, 715 tx_free_flags); 716} 717 718/** 719 * ishtp_cl_send_msg_ipc() -Send message using IPC 720 * @dev: ISHTP device instance 721 * @cl: Pointer to client device instance 722 * 723 * Send message over IPC not using DMA 724 */ 725static void ishtp_cl_send_msg_ipc(struct ishtp_device *dev, 726 struct ishtp_cl *cl) 727{ 728 /* If last DMA message wasn't acked yet, leave this one in Tx queue */ 729 if (cl->last_tx_path == CL_TX_PATH_DMA && cl->last_dma_acked == 0) 730 return; 731 732 cl->tx_offs = 0; 733 ipc_tx_send(cl); 734 ++cl->send_msg_cnt_ipc; 735} 736 737/** 738 * ishtp_cl_send_msg_dma() -Send message using DMA 739 * @dev: ISHTP device instance 740 * @cl: Pointer to client device instance 741 * 742 * Send message using DMA 743 */ 744static void ishtp_cl_send_msg_dma(struct ishtp_device *dev, 745 struct ishtp_cl *cl) 746{ 747 struct ishtp_msg_hdr hdr; 748 struct dma_xfer_hbm dma_xfer; 749 unsigned char *msg_addr; 750 int off; 751 struct ishtp_cl_tx_ring *cl_msg; 752 unsigned long tx_flags, tx_free_flags; 753 754 /* If last IPC message wasn't acked yet, leave this one in Tx queue */ 755 if (cl->last_tx_path == CL_TX_PATH_IPC && cl->last_ipc_acked == 0) 756 return; 757 758 spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags); 759 if (list_empty(&cl->tx_list.list)) { 760 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 761 return; 762 } 763 764 cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring, 765 list); 766 767 msg_addr = ishtp_cl_get_dma_send_buf(dev, cl_msg->send_buf.size); 768 if (!msg_addr) { 769 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 770 if (dev->transfer_path == CL_TX_PATH_DEFAULT) 771 ishtp_cl_send_msg_ipc(dev, cl); 772 return; 773 } 774 775 list_del_init(&cl_msg->list); /* Must be before write */ 776 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); 777 778 --cl->ishtp_flow_ctrl_creds; 779 cl->last_dma_acked = 0; 780 cl->last_dma_addr = msg_addr; 781 cl->last_tx_path = CL_TX_PATH_DMA; 782 783 /* write msg to dma buf */ 784 memcpy(msg_addr, cl_msg->send_buf.data, cl_msg->send_buf.size); 785 786 /* 787 * if current fw don't support cache snooping, driver have to 788 * flush the cache manually. 789 */ 790 if (dev->ops->dma_no_cache_snooping && 791 dev->ops->dma_no_cache_snooping(dev)) 792 clflush_cache_range(msg_addr, cl_msg->send_buf.size); 793 794 /* send dma_xfer hbm msg */ 795 off = msg_addr - (unsigned char *)dev->ishtp_host_dma_tx_buf; 796 ishtp_hbm_hdr(&hdr, sizeof(struct dma_xfer_hbm)); 797 dma_xfer.hbm = DMA_XFER; 798 dma_xfer.fw_client_id = cl->fw_client_id; 799 dma_xfer.host_client_id = cl->host_client_id; 800 dma_xfer.reserved = 0; 801 dma_xfer.msg_addr = dev->ishtp_host_dma_tx_buf_phys + off; 802 dma_xfer.msg_length = cl_msg->send_buf.size; 803 dma_xfer.reserved2 = 0; 804 ishtp_write_message(dev, &hdr, (unsigned char *)&dma_xfer); 805 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags); 806 list_add_tail(&cl_msg->list, &cl->tx_free_list.list); 807 ++cl->tx_ring_free_size; 808 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags); 809 ++cl->send_msg_cnt_dma; 810} 811 812/** 813 * ishtp_cl_send_msg() -Send message using DMA or IPC 814 * @dev: ISHTP device instance 815 * @cl: Pointer to client device instance 816 * 817 * Send message using DMA or IPC based on transfer_path 818 */ 819void ishtp_cl_send_msg(struct ishtp_device *dev, struct ishtp_cl *cl) 820{ 821 if (dev->transfer_path == CL_TX_PATH_DMA) 822 ishtp_cl_send_msg_dma(dev, cl); 823 else 824 ishtp_cl_send_msg_ipc(dev, cl); 825} 826 827/** 828 * recv_ishtp_cl_msg() -Receive client message 829 * @dev: ISHTP device instance 830 * @ishtp_hdr: Pointer to message header 831 * 832 * Receive and dispatch ISHTP client messages. This function executes in ISR 833 * or work queue context 834 */ 835void recv_ishtp_cl_msg(struct ishtp_device *dev, 836 struct ishtp_msg_hdr *ishtp_hdr) 837{ 838 struct ishtp_cl *cl; 839 struct ishtp_cl_rb *rb; 840 struct ishtp_cl_rb *new_rb; 841 unsigned char *buffer = NULL; 842 struct ishtp_cl_rb *complete_rb = NULL; 843 unsigned long flags; 844 845 if (ishtp_hdr->reserved) { 846 dev_err(dev->devc, "corrupted message header.\n"); 847 goto eoi; 848 } 849 850 if (ishtp_hdr->length > IPC_PAYLOAD_SIZE) { 851 dev_err(dev->devc, 852 "ISHTP message length in hdr exceeds IPC MTU\n"); 853 goto eoi; 854 } 855 856 spin_lock_irqsave(&dev->read_list_spinlock, flags); 857 list_for_each_entry(rb, &dev->read_list.list, list) { 858 cl = rb->cl; 859 if (!cl || !(cl->host_client_id == ishtp_hdr->host_addr && 860 cl->fw_client_id == ishtp_hdr->fw_addr) || 861 !(cl->state == ISHTP_CL_CONNECTED)) 862 continue; 863 864 /* If no Rx buffer is allocated, disband the rb */ 865 if (rb->buffer.size == 0 || rb->buffer.data == NULL) { 866 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 867 dev_err(&cl->device->dev, 868 "Rx buffer is not allocated.\n"); 869 list_del(&rb->list); 870 ishtp_io_rb_free(rb); 871 cl->status = -ENOMEM; 872 goto eoi; 873 } 874 875 /* 876 * If message buffer overflown (exceeds max. client msg 877 * size, drop message and return to free buffer. 878 * Do we need to disconnect such a client? (We don't send 879 * back FC, so communication will be stuck anyway) 880 */ 881 if (rb->buffer.size < ishtp_hdr->length + rb->buf_idx) { 882 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 883 dev_err(&cl->device->dev, 884 "message overflow. size %d len %d idx %ld\n", 885 rb->buffer.size, ishtp_hdr->length, 886 rb->buf_idx); 887 list_del(&rb->list); 888 ishtp_cl_io_rb_recycle(rb); 889 cl->status = -EIO; 890 goto eoi; 891 } 892 893 buffer = rb->buffer.data + rb->buf_idx; 894 dev->ops->ishtp_read(dev, buffer, ishtp_hdr->length); 895 896 rb->buf_idx += ishtp_hdr->length; 897 if (ishtp_hdr->msg_complete) { 898 /* Last fragment in message - it's complete */ 899 cl->status = 0; 900 list_del(&rb->list); 901 complete_rb = rb; 902 903 --cl->out_flow_ctrl_creds; 904 /* 905 * the whole msg arrived, send a new FC, and add a new 906 * rb buffer for the next coming msg 907 */ 908 spin_lock(&cl->free_list_spinlock); 909 910 if (!list_empty(&cl->free_rb_list.list)) { 911 new_rb = list_entry(cl->free_rb_list.list.next, 912 struct ishtp_cl_rb, list); 913 list_del_init(&new_rb->list); 914 spin_unlock(&cl->free_list_spinlock); 915 new_rb->cl = cl; 916 new_rb->buf_idx = 0; 917 INIT_LIST_HEAD(&new_rb->list); 918 list_add_tail(&new_rb->list, 919 &dev->read_list.list); 920 921 ishtp_hbm_cl_flow_control_req(dev, cl); 922 } else { 923 spin_unlock(&cl->free_list_spinlock); 924 } 925 } 926 /* One more fragment in message (even if this was last) */ 927 ++cl->recv_msg_num_frags; 928 929 /* 930 * We can safely break here (and in BH too), 931 * a single input message can go only to a single request! 932 */ 933 break; 934 } 935 936 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 937 /* If it's nobody's message, just read and discard it */ 938 if (!buffer) { 939 uint8_t rd_msg_buf[ISHTP_RD_MSG_BUF_SIZE]; 940 941 dev_err(dev->devc, "Dropped Rx msg - no request\n"); 942 dev->ops->ishtp_read(dev, rd_msg_buf, ishtp_hdr->length); 943 goto eoi; 944 } 945 946 if (complete_rb) { 947 cl = complete_rb->cl; 948 cl->ts_rx = ktime_get(); 949 ++cl->recv_msg_cnt_ipc; 950 ishtp_cl_read_complete(complete_rb); 951 } 952eoi: 953 return; 954} 955 956/** 957 * recv_ishtp_cl_msg_dma() -Receive client message 958 * @dev: ISHTP device instance 959 * @msg: message pointer 960 * @hbm: hbm buffer 961 * 962 * Receive and dispatch ISHTP client messages using DMA. This function executes 963 * in ISR or work queue context 964 */ 965void recv_ishtp_cl_msg_dma(struct ishtp_device *dev, void *msg, 966 struct dma_xfer_hbm *hbm) 967{ 968 struct ishtp_cl *cl; 969 struct ishtp_cl_rb *rb; 970 struct ishtp_cl_rb *new_rb; 971 unsigned char *buffer = NULL; 972 struct ishtp_cl_rb *complete_rb = NULL; 973 unsigned long flags; 974 975 spin_lock_irqsave(&dev->read_list_spinlock, flags); 976 977 list_for_each_entry(rb, &dev->read_list.list, list) { 978 cl = rb->cl; 979 if (!cl || !(cl->host_client_id == hbm->host_client_id && 980 cl->fw_client_id == hbm->fw_client_id) || 981 !(cl->state == ISHTP_CL_CONNECTED)) 982 continue; 983 984 /* 985 * If no Rx buffer is allocated, disband the rb 986 */ 987 if (rb->buffer.size == 0 || rb->buffer.data == NULL) { 988 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 989 dev_err(&cl->device->dev, 990 "response buffer is not allocated.\n"); 991 list_del(&rb->list); 992 ishtp_io_rb_free(rb); 993 cl->status = -ENOMEM; 994 goto eoi; 995 } 996 997 /* 998 * If message buffer overflown (exceeds max. client msg 999 * size, drop message and return to free buffer. 1000 * Do we need to disconnect such a client? (We don't send 1001 * back FC, so communication will be stuck anyway) 1002 */ 1003 if (rb->buffer.size < hbm->msg_length) { 1004 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 1005 dev_err(&cl->device->dev, 1006 "message overflow. size %d len %d idx %ld\n", 1007 rb->buffer.size, hbm->msg_length, rb->buf_idx); 1008 list_del(&rb->list); 1009 ishtp_cl_io_rb_recycle(rb); 1010 cl->status = -EIO; 1011 goto eoi; 1012 } 1013 1014 buffer = rb->buffer.data; 1015 1016 /* 1017 * if current fw don't support cache snooping, driver have to 1018 * flush the cache manually. 1019 */ 1020 if (dev->ops->dma_no_cache_snooping && 1021 dev->ops->dma_no_cache_snooping(dev)) 1022 clflush_cache_range(msg, hbm->msg_length); 1023 1024 memcpy(buffer, msg, hbm->msg_length); 1025 rb->buf_idx = hbm->msg_length; 1026 1027 /* Last fragment in message - it's complete */ 1028 cl->status = 0; 1029 list_del(&rb->list); 1030 complete_rb = rb; 1031 1032 --cl->out_flow_ctrl_creds; 1033 /* 1034 * the whole msg arrived, send a new FC, and add a new 1035 * rb buffer for the next coming msg 1036 */ 1037 spin_lock(&cl->free_list_spinlock); 1038 1039 if (!list_empty(&cl->free_rb_list.list)) { 1040 new_rb = list_entry(cl->free_rb_list.list.next, 1041 struct ishtp_cl_rb, list); 1042 list_del_init(&new_rb->list); 1043 spin_unlock(&cl->free_list_spinlock); 1044 new_rb->cl = cl; 1045 new_rb->buf_idx = 0; 1046 INIT_LIST_HEAD(&new_rb->list); 1047 list_add_tail(&new_rb->list, 1048 &dev->read_list.list); 1049 1050 ishtp_hbm_cl_flow_control_req(dev, cl); 1051 } else { 1052 spin_unlock(&cl->free_list_spinlock); 1053 } 1054 1055 /* One more fragment in message (this is always last) */ 1056 ++cl->recv_msg_num_frags; 1057 1058 /* 1059 * We can safely break here (and in BH too), 1060 * a single input message can go only to a single request! 1061 */ 1062 break; 1063 } 1064 1065 spin_unlock_irqrestore(&dev->read_list_spinlock, flags); 1066 /* If it's nobody's message, just read and discard it */ 1067 if (!buffer) { 1068 dev_err(dev->devc, "Dropped Rx (DMA) msg - no request\n"); 1069 goto eoi; 1070 } 1071 1072 if (complete_rb) { 1073 cl = complete_rb->cl; 1074 cl->ts_rx = ktime_get(); 1075 ++cl->recv_msg_cnt_dma; 1076 ishtp_cl_read_complete(complete_rb); 1077 } 1078eoi: 1079 return; 1080} 1081 1082void *ishtp_get_client_data(struct ishtp_cl *cl) 1083{ 1084 return cl->client_data; 1085} 1086EXPORT_SYMBOL(ishtp_get_client_data); 1087 1088void ishtp_set_client_data(struct ishtp_cl *cl, void *data) 1089{ 1090 cl->client_data = data; 1091} 1092EXPORT_SYMBOL(ishtp_set_client_data); 1093 1094struct ishtp_device *ishtp_get_ishtp_device(struct ishtp_cl *cl) 1095{ 1096 return cl->dev; 1097} 1098EXPORT_SYMBOL(ishtp_get_ishtp_device); 1099 1100void ishtp_set_tx_ring_size(struct ishtp_cl *cl, int size) 1101{ 1102 cl->tx_ring_size = size; 1103} 1104EXPORT_SYMBOL(ishtp_set_tx_ring_size); 1105 1106void ishtp_set_rx_ring_size(struct ishtp_cl *cl, int size) 1107{ 1108 cl->rx_ring_size = size; 1109} 1110EXPORT_SYMBOL(ishtp_set_rx_ring_size); 1111 1112void ishtp_set_connection_state(struct ishtp_cl *cl, int state) 1113{ 1114 cl->state = state; 1115} 1116EXPORT_SYMBOL(ishtp_set_connection_state); 1117 1118void ishtp_cl_set_fw_client_id(struct ishtp_cl *cl, int fw_client_id) 1119{ 1120 cl->fw_client_id = fw_client_id; 1121} 1122EXPORT_SYMBOL(ishtp_cl_set_fw_client_id); 1123