1/* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * BSD LICENSE 15 * 16 * Copyright(c) 2012 Intel Corporation. All rights reserved. 17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 23 * * Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * * Redistributions in binary form must reproduce the above copy 26 * notice, this list of conditions and the following disclaimer in 27 * the documentation and/or other materials provided with the 28 * distribution. 29 * * Neither the name of Intel Corporation nor the names of its 30 * contributors may be used to endorse or promote products derived 31 * from this software without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 44 * 45 * PCIe NTB Transport Linux driver 46 * 47 * Contact Information: 48 * Jon Mason <jon.mason@intel.com> 49 */ 50#include <linux/debugfs.h> 51#include <linux/delay.h> 52#include <linux/dmaengine.h> 53#include <linux/dma-mapping.h> 54#include <linux/errno.h> 55#include <linux/export.h> 56#include <linux/interrupt.h> 57#include <linux/module.h> 58#include <linux/pci.h> 59#include <linux/slab.h> 60#include <linux/types.h> 61#include <linux/uaccess.h> 62#include "linux/ntb.h" 63#include "linux/ntb_transport.h" 64 65#define NTB_TRANSPORT_VERSION 4 66#define NTB_TRANSPORT_VER "4" 67#define NTB_TRANSPORT_NAME "ntb_transport" 68#define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB" 69#define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2) 70 71MODULE_DESCRIPTION(NTB_TRANSPORT_DESC); 72MODULE_VERSION(NTB_TRANSPORT_VER); 73MODULE_LICENSE("Dual BSD/GPL"); 74MODULE_AUTHOR("Intel Corporation"); 75 76static unsigned long max_mw_size; 77module_param(max_mw_size, ulong, 0644); 78MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); 79 80static unsigned int transport_mtu = 0x10000; 81module_param(transport_mtu, uint, 0644); 82MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); 83 84static unsigned char max_num_clients; 85module_param(max_num_clients, byte, 0644); 86MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients"); 87 88static unsigned int copy_bytes = 1024; 89module_param(copy_bytes, uint, 0644); 90MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); 91 92static bool use_dma; 93module_param(use_dma, bool, 0644); 94MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy"); 95 96static bool use_msi; 97#ifdef CONFIG_NTB_MSI 98module_param(use_msi, bool, 0644); 99MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells"); 100#endif 101 102static struct dentry *nt_debugfs_dir; 103 104/* Only two-ports NTB devices are supported */ 105#define PIDX NTB_DEF_PEER_IDX 106 107struct ntb_queue_entry { 108 /* ntb_queue list reference */ 109 struct list_head entry; 110 /* pointers to data to be transferred */ 111 void *cb_data; 112 void *buf; 113 unsigned int len; 114 unsigned int flags; 115 int retries; 116 int errors; 117 unsigned int tx_index; 118 unsigned int rx_index; 119 120 struct ntb_transport_qp *qp; 121 union { 122 struct ntb_payload_header __iomem *tx_hdr; 123 struct ntb_payload_header *rx_hdr; 124 }; 125}; 126 127struct ntb_rx_info { 128 unsigned int entry; 129}; 130 131struct ntb_transport_qp { 132 struct ntb_transport_ctx *transport; 133 struct ntb_dev *ndev; 134 void *cb_data; 135 struct dma_chan *tx_dma_chan; 136 struct dma_chan *rx_dma_chan; 137 138 bool client_ready; 139 bool link_is_up; 140 bool active; 141 142 u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ 143 u64 qp_bit; 144 145 struct ntb_rx_info __iomem *rx_info; 146 struct ntb_rx_info *remote_rx_info; 147 148 void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, 149 void *data, int len); 150 struct list_head tx_free_q; 151 spinlock_t ntb_tx_free_q_lock; 152 void __iomem *tx_mw; 153 phys_addr_t tx_mw_phys; 154 size_t tx_mw_size; 155 dma_addr_t tx_mw_dma_addr; 156 unsigned int tx_index; 157 unsigned int tx_max_entry; 158 unsigned int tx_max_frame; 159 160 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 161 void *data, int len); 162 struct list_head rx_post_q; 163 struct list_head rx_pend_q; 164 struct list_head rx_free_q; 165 /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */ 166 spinlock_t ntb_rx_q_lock; 167 void *rx_buff; 168 unsigned int rx_index; 169 unsigned int rx_max_entry; 170 unsigned int rx_max_frame; 171 unsigned int rx_alloc_entry; 172 dma_cookie_t last_cookie; 173 struct tasklet_struct rxc_db_work; 174 175 void (*event_handler)(void *data, int status); 176 struct delayed_work link_work; 177 struct work_struct link_cleanup; 178 179 struct dentry *debugfs_dir; 180 struct dentry *debugfs_stats; 181 182 /* Stats */ 183 u64 rx_bytes; 184 u64 rx_pkts; 185 u64 rx_ring_empty; 186 u64 rx_err_no_buf; 187 u64 rx_err_oflow; 188 u64 rx_err_ver; 189 u64 rx_memcpy; 190 u64 rx_async; 191 u64 tx_bytes; 192 u64 tx_pkts; 193 u64 tx_ring_full; 194 u64 tx_err_no_buf; 195 u64 tx_memcpy; 196 u64 tx_async; 197 198 bool use_msi; 199 int msi_irq; 200 struct ntb_msi_desc msi_desc; 201 struct ntb_msi_desc peer_msi_desc; 202}; 203 204struct ntb_transport_mw { 205 phys_addr_t phys_addr; 206 resource_size_t phys_size; 207 void __iomem *vbase; 208 size_t xlat_size; 209 size_t buff_size; 210 size_t alloc_size; 211 void *alloc_addr; 212 void *virt_addr; 213 dma_addr_t dma_addr; 214}; 215 216struct ntb_transport_client_dev { 217 struct list_head entry; 218 struct ntb_transport_ctx *nt; 219 struct device dev; 220}; 221 222struct ntb_transport_ctx { 223 struct list_head entry; 224 struct list_head client_devs; 225 226 struct ntb_dev *ndev; 227 228 struct ntb_transport_mw *mw_vec; 229 struct ntb_transport_qp *qp_vec; 230 unsigned int mw_count; 231 unsigned int qp_count; 232 u64 qp_bitmap; 233 u64 qp_bitmap_free; 234 235 bool use_msi; 236 unsigned int msi_spad_offset; 237 u64 msi_db_mask; 238 239 bool link_is_up; 240 struct delayed_work link_work; 241 struct work_struct link_cleanup; 242 243 struct dentry *debugfs_node_dir; 244}; 245 246enum { 247 DESC_DONE_FLAG = BIT(0), 248 LINK_DOWN_FLAG = BIT(1), 249}; 250 251struct ntb_payload_header { 252 unsigned int ver; 253 unsigned int len; 254 unsigned int flags; 255}; 256 257enum { 258 VERSION = 0, 259 QP_LINKS, 260 NUM_QPS, 261 NUM_MWS, 262 MW0_SZ_HIGH, 263 MW0_SZ_LOW, 264}; 265 266#define dev_client_dev(__dev) \ 267 container_of((__dev), struct ntb_transport_client_dev, dev) 268 269#define drv_client(__drv) \ 270 container_of((__drv), struct ntb_transport_client, driver) 271 272#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) 273#define NTB_QP_DEF_NUM_ENTRIES 100 274#define NTB_LINK_DOWN_TIMEOUT 10 275 276static void ntb_transport_rxc_db(unsigned long data); 277static const struct ntb_ctx_ops ntb_transport_ops; 278static struct ntb_client ntb_transport_client; 279static int ntb_async_tx_submit(struct ntb_transport_qp *qp, 280 struct ntb_queue_entry *entry); 281static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset); 282static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset); 283static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset); 284 285 286static int ntb_transport_bus_match(struct device *dev, 287 struct device_driver *drv) 288{ 289 return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); 290} 291 292static int ntb_transport_bus_probe(struct device *dev) 293{ 294 const struct ntb_transport_client *client; 295 int rc; 296 297 get_device(dev); 298 299 client = drv_client(dev->driver); 300 rc = client->probe(dev); 301 if (rc) 302 put_device(dev); 303 304 return rc; 305} 306 307static int ntb_transport_bus_remove(struct device *dev) 308{ 309 const struct ntb_transport_client *client; 310 311 client = drv_client(dev->driver); 312 client->remove(dev); 313 314 put_device(dev); 315 316 return 0; 317} 318 319static struct bus_type ntb_transport_bus = { 320 .name = "ntb_transport", 321 .match = ntb_transport_bus_match, 322 .probe = ntb_transport_bus_probe, 323 .remove = ntb_transport_bus_remove, 324}; 325 326static LIST_HEAD(ntb_transport_list); 327 328static int ntb_bus_init(struct ntb_transport_ctx *nt) 329{ 330 list_add_tail(&nt->entry, &ntb_transport_list); 331 return 0; 332} 333 334static void ntb_bus_remove(struct ntb_transport_ctx *nt) 335{ 336 struct ntb_transport_client_dev *client_dev, *cd; 337 338 list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) { 339 dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n", 340 dev_name(&client_dev->dev)); 341 list_del(&client_dev->entry); 342 device_unregister(&client_dev->dev); 343 } 344 345 list_del(&nt->entry); 346} 347 348static void ntb_transport_client_release(struct device *dev) 349{ 350 struct ntb_transport_client_dev *client_dev; 351 352 client_dev = dev_client_dev(dev); 353 kfree(client_dev); 354} 355 356/** 357 * ntb_transport_unregister_client_dev - Unregister NTB client device 358 * @device_name: Name of NTB client device 359 * 360 * Unregister an NTB client device with the NTB transport layer 361 */ 362void ntb_transport_unregister_client_dev(char *device_name) 363{ 364 struct ntb_transport_client_dev *client, *cd; 365 struct ntb_transport_ctx *nt; 366 367 list_for_each_entry(nt, &ntb_transport_list, entry) 368 list_for_each_entry_safe(client, cd, &nt->client_devs, entry) 369 if (!strncmp(dev_name(&client->dev), device_name, 370 strlen(device_name))) { 371 list_del(&client->entry); 372 device_unregister(&client->dev); 373 } 374} 375EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev); 376 377/** 378 * ntb_transport_register_client_dev - Register NTB client device 379 * @device_name: Name of NTB client device 380 * 381 * Register an NTB client device with the NTB transport layer 382 */ 383int ntb_transport_register_client_dev(char *device_name) 384{ 385 struct ntb_transport_client_dev *client_dev; 386 struct ntb_transport_ctx *nt; 387 int node; 388 int rc, i = 0; 389 390 if (list_empty(&ntb_transport_list)) 391 return -ENODEV; 392 393 list_for_each_entry(nt, &ntb_transport_list, entry) { 394 struct device *dev; 395 396 node = dev_to_node(&nt->ndev->dev); 397 398 client_dev = kzalloc_node(sizeof(*client_dev), 399 GFP_KERNEL, node); 400 if (!client_dev) { 401 rc = -ENOMEM; 402 goto err; 403 } 404 405 dev = &client_dev->dev; 406 407 /* setup and register client devices */ 408 dev_set_name(dev, "%s%d", device_name, i); 409 dev->bus = &ntb_transport_bus; 410 dev->release = ntb_transport_client_release; 411 dev->parent = &nt->ndev->dev; 412 413 rc = device_register(dev); 414 if (rc) { 415 put_device(dev); 416 goto err; 417 } 418 419 list_add_tail(&client_dev->entry, &nt->client_devs); 420 i++; 421 } 422 423 return 0; 424 425err: 426 ntb_transport_unregister_client_dev(device_name); 427 428 return rc; 429} 430EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev); 431 432/** 433 * ntb_transport_register_client - Register NTB client driver 434 * @drv: NTB client driver to be registered 435 * 436 * Register an NTB client driver with the NTB transport layer 437 * 438 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 439 */ 440int ntb_transport_register_client(struct ntb_transport_client *drv) 441{ 442 drv->driver.bus = &ntb_transport_bus; 443 444 if (list_empty(&ntb_transport_list)) 445 return -ENODEV; 446 447 return driver_register(&drv->driver); 448} 449EXPORT_SYMBOL_GPL(ntb_transport_register_client); 450 451/** 452 * ntb_transport_unregister_client - Unregister NTB client driver 453 * @drv: NTB client driver to be unregistered 454 * 455 * Unregister an NTB client driver with the NTB transport layer 456 * 457 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 458 */ 459void ntb_transport_unregister_client(struct ntb_transport_client *drv) 460{ 461 driver_unregister(&drv->driver); 462} 463EXPORT_SYMBOL_GPL(ntb_transport_unregister_client); 464 465static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, 466 loff_t *offp) 467{ 468 struct ntb_transport_qp *qp; 469 char *buf; 470 ssize_t ret, out_offset, out_count; 471 472 qp = filp->private_data; 473 474 if (!qp || !qp->link_is_up) 475 return 0; 476 477 out_count = 1000; 478 479 buf = kmalloc(out_count, GFP_KERNEL); 480 if (!buf) 481 return -ENOMEM; 482 483 out_offset = 0; 484 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 485 "\nNTB QP stats:\n\n"); 486 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 487 "rx_bytes - \t%llu\n", qp->rx_bytes); 488 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 489 "rx_pkts - \t%llu\n", qp->rx_pkts); 490 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 491 "rx_memcpy - \t%llu\n", qp->rx_memcpy); 492 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 493 "rx_async - \t%llu\n", qp->rx_async); 494 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 495 "rx_ring_empty - %llu\n", qp->rx_ring_empty); 496 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 497 "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); 498 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 499 "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); 500 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 501 "rx_err_ver - \t%llu\n", qp->rx_err_ver); 502 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 503 "rx_buff - \t0x%p\n", qp->rx_buff); 504 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 505 "rx_index - \t%u\n", qp->rx_index); 506 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 507 "rx_max_entry - \t%u\n", qp->rx_max_entry); 508 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 509 "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry); 510 511 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 512 "tx_bytes - \t%llu\n", qp->tx_bytes); 513 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 514 "tx_pkts - \t%llu\n", qp->tx_pkts); 515 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 516 "tx_memcpy - \t%llu\n", qp->tx_memcpy); 517 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 518 "tx_async - \t%llu\n", qp->tx_async); 519 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 520 "tx_ring_full - \t%llu\n", qp->tx_ring_full); 521 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 522 "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); 523 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 524 "tx_mw - \t0x%p\n", qp->tx_mw); 525 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 526 "tx_index (H) - \t%u\n", qp->tx_index); 527 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 528 "RRI (T) - \t%u\n", 529 qp->remote_rx_info->entry); 530 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 531 "tx_max_entry - \t%u\n", qp->tx_max_entry); 532 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 533 "free tx - \t%u\n", 534 ntb_transport_tx_free_entry(qp)); 535 536 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 537 "\n"); 538 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 539 "Using TX DMA - \t%s\n", 540 qp->tx_dma_chan ? "Yes" : "No"); 541 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 542 "Using RX DMA - \t%s\n", 543 qp->rx_dma_chan ? "Yes" : "No"); 544 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 545 "QP Link - \t%s\n", 546 qp->link_is_up ? "Up" : "Down"); 547 out_offset += scnprintf(buf + out_offset, out_count - out_offset, 548 "\n"); 549 550 if (out_offset > out_count) 551 out_offset = out_count; 552 553 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); 554 kfree(buf); 555 return ret; 556} 557 558static const struct file_operations ntb_qp_debugfs_stats = { 559 .owner = THIS_MODULE, 560 .open = simple_open, 561 .read = debugfs_read, 562}; 563 564static void ntb_list_add(spinlock_t *lock, struct list_head *entry, 565 struct list_head *list) 566{ 567 unsigned long flags; 568 569 spin_lock_irqsave(lock, flags); 570 list_add_tail(entry, list); 571 spin_unlock_irqrestore(lock, flags); 572} 573 574static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, 575 struct list_head *list) 576{ 577 struct ntb_queue_entry *entry; 578 unsigned long flags; 579 580 spin_lock_irqsave(lock, flags); 581 if (list_empty(list)) { 582 entry = NULL; 583 goto out; 584 } 585 entry = list_first_entry(list, struct ntb_queue_entry, entry); 586 list_del(&entry->entry); 587 588out: 589 spin_unlock_irqrestore(lock, flags); 590 591 return entry; 592} 593 594static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock, 595 struct list_head *list, 596 struct list_head *to_list) 597{ 598 struct ntb_queue_entry *entry; 599 unsigned long flags; 600 601 spin_lock_irqsave(lock, flags); 602 603 if (list_empty(list)) { 604 entry = NULL; 605 } else { 606 entry = list_first_entry(list, struct ntb_queue_entry, entry); 607 list_move_tail(&entry->entry, to_list); 608 } 609 610 spin_unlock_irqrestore(lock, flags); 611 612 return entry; 613} 614 615static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, 616 unsigned int qp_num) 617{ 618 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 619 struct ntb_transport_mw *mw; 620 struct ntb_dev *ndev = nt->ndev; 621 struct ntb_queue_entry *entry; 622 unsigned int rx_size, num_qps_mw; 623 unsigned int mw_num, mw_count, qp_count; 624 unsigned int i; 625 int node; 626 627 mw_count = nt->mw_count; 628 qp_count = nt->qp_count; 629 630 mw_num = QP_TO_MW(nt, qp_num); 631 mw = &nt->mw_vec[mw_num]; 632 633 if (!mw->virt_addr) 634 return -ENOMEM; 635 636 if (mw_num < qp_count % mw_count) 637 num_qps_mw = qp_count / mw_count + 1; 638 else 639 num_qps_mw = qp_count / mw_count; 640 641 rx_size = (unsigned int)mw->xlat_size / num_qps_mw; 642 qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); 643 rx_size -= sizeof(struct ntb_rx_info); 644 645 qp->remote_rx_info = qp->rx_buff + rx_size; 646 647 /* Due to housekeeping, there must be atleast 2 buffs */ 648 qp->rx_max_frame = min(transport_mtu, rx_size / 2); 649 qp->rx_max_entry = rx_size / qp->rx_max_frame; 650 qp->rx_index = 0; 651 652 /* 653 * Checking to see if we have more entries than the default. 654 * We should add additional entries if that is the case so we 655 * can be in sync with the transport frames. 656 */ 657 node = dev_to_node(&ndev->dev); 658 for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) { 659 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 660 if (!entry) 661 return -ENOMEM; 662 663 entry->qp = qp; 664 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, 665 &qp->rx_free_q); 666 qp->rx_alloc_entry++; 667 } 668 669 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 670 671 /* setup the hdr offsets with 0's */ 672 for (i = 0; i < qp->rx_max_entry; i++) { 673 void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) - 674 sizeof(struct ntb_payload_header)); 675 memset(offset, 0, sizeof(struct ntb_payload_header)); 676 } 677 678 qp->rx_pkts = 0; 679 qp->tx_pkts = 0; 680 qp->tx_index = 0; 681 682 return 0; 683} 684 685static irqreturn_t ntb_transport_isr(int irq, void *dev) 686{ 687 struct ntb_transport_qp *qp = dev; 688 689 tasklet_schedule(&qp->rxc_db_work); 690 691 return IRQ_HANDLED; 692} 693 694static void ntb_transport_setup_qp_peer_msi(struct ntb_transport_ctx *nt, 695 unsigned int qp_num) 696{ 697 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 698 int spad = qp_num * 2 + nt->msi_spad_offset; 699 700 if (!nt->use_msi) 701 return; 702 703 if (spad >= ntb_spad_count(nt->ndev)) 704 return; 705 706 qp->peer_msi_desc.addr_offset = 707 ntb_peer_spad_read(qp->ndev, PIDX, spad); 708 qp->peer_msi_desc.data = 709 ntb_peer_spad_read(qp->ndev, PIDX, spad + 1); 710 711 dev_dbg(&qp->ndev->pdev->dev, "QP%d Peer MSI addr=%x data=%x\n", 712 qp_num, qp->peer_msi_desc.addr_offset, qp->peer_msi_desc.data); 713 714 if (qp->peer_msi_desc.addr_offset) { 715 qp->use_msi = true; 716 dev_info(&qp->ndev->pdev->dev, 717 "Using MSI interrupts for QP%d\n", qp_num); 718 } 719} 720 721static void ntb_transport_setup_qp_msi(struct ntb_transport_ctx *nt, 722 unsigned int qp_num) 723{ 724 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 725 int spad = qp_num * 2 + nt->msi_spad_offset; 726 int rc; 727 728 if (!nt->use_msi) 729 return; 730 731 if (spad >= ntb_spad_count(nt->ndev)) { 732 dev_warn_once(&qp->ndev->pdev->dev, 733 "Not enough SPADS to use MSI interrupts\n"); 734 return; 735 } 736 737 ntb_spad_write(qp->ndev, spad, 0); 738 ntb_spad_write(qp->ndev, spad + 1, 0); 739 740 if (!qp->msi_irq) { 741 qp->msi_irq = ntbm_msi_request_irq(qp->ndev, ntb_transport_isr, 742 KBUILD_MODNAME, qp, 743 &qp->msi_desc); 744 if (qp->msi_irq < 0) { 745 dev_warn(&qp->ndev->pdev->dev, 746 "Unable to allocate MSI interrupt for qp%d\n", 747 qp_num); 748 return; 749 } 750 } 751 752 rc = ntb_spad_write(qp->ndev, spad, qp->msi_desc.addr_offset); 753 if (rc) 754 goto err_free_interrupt; 755 756 rc = ntb_spad_write(qp->ndev, spad + 1, qp->msi_desc.data); 757 if (rc) 758 goto err_free_interrupt; 759 760 dev_dbg(&qp->ndev->pdev->dev, "QP%d MSI %d addr=%x data=%x\n", 761 qp_num, qp->msi_irq, qp->msi_desc.addr_offset, 762 qp->msi_desc.data); 763 764 return; 765 766err_free_interrupt: 767 devm_free_irq(&nt->ndev->dev, qp->msi_irq, qp); 768} 769 770static void ntb_transport_msi_peer_desc_changed(struct ntb_transport_ctx *nt) 771{ 772 int i; 773 774 dev_dbg(&nt->ndev->pdev->dev, "Peer MSI descriptors changed"); 775 776 for (i = 0; i < nt->qp_count; i++) 777 ntb_transport_setup_qp_peer_msi(nt, i); 778} 779 780static void ntb_transport_msi_desc_changed(void *data) 781{ 782 struct ntb_transport_ctx *nt = data; 783 int i; 784 785 dev_dbg(&nt->ndev->pdev->dev, "MSI descriptors changed"); 786 787 for (i = 0; i < nt->qp_count; i++) 788 ntb_transport_setup_qp_msi(nt, i); 789 790 ntb_peer_db_set(nt->ndev, nt->msi_db_mask); 791} 792 793static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) 794{ 795 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 796 struct pci_dev *pdev = nt->ndev->pdev; 797 798 if (!mw->virt_addr) 799 return; 800 801 ntb_mw_clear_trans(nt->ndev, PIDX, num_mw); 802 dma_free_coherent(&pdev->dev, mw->alloc_size, 803 mw->alloc_addr, mw->dma_addr); 804 mw->xlat_size = 0; 805 mw->buff_size = 0; 806 mw->alloc_size = 0; 807 mw->alloc_addr = NULL; 808 mw->virt_addr = NULL; 809} 810 811static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw, 812 struct device *dma_dev, size_t align) 813{ 814 dma_addr_t dma_addr; 815 void *alloc_addr, *virt_addr; 816 int rc; 817 818 alloc_addr = dma_alloc_coherent(dma_dev, mw->alloc_size, 819 &dma_addr, GFP_KERNEL); 820 if (!alloc_addr) { 821 dev_err(dma_dev, "Unable to alloc MW buff of size %zu\n", 822 mw->alloc_size); 823 return -ENOMEM; 824 } 825 virt_addr = alloc_addr; 826 827 /* 828 * we must ensure that the memory address allocated is BAR size 829 * aligned in order for the XLAT register to take the value. This 830 * is a requirement of the hardware. It is recommended to setup CMA 831 * for BAR sizes equal or greater than 4MB. 832 */ 833 if (!IS_ALIGNED(dma_addr, align)) { 834 if (mw->alloc_size > mw->buff_size) { 835 virt_addr = PTR_ALIGN(alloc_addr, align); 836 dma_addr = ALIGN(dma_addr, align); 837 } else { 838 rc = -ENOMEM; 839 goto err; 840 } 841 } 842 843 mw->alloc_addr = alloc_addr; 844 mw->virt_addr = virt_addr; 845 mw->dma_addr = dma_addr; 846 847 return 0; 848 849err: 850 dma_free_coherent(dma_dev, mw->alloc_size, alloc_addr, dma_addr); 851 852 return rc; 853} 854 855static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, 856 resource_size_t size) 857{ 858 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 859 struct pci_dev *pdev = nt->ndev->pdev; 860 size_t xlat_size, buff_size; 861 resource_size_t xlat_align; 862 resource_size_t xlat_align_size; 863 int rc; 864 865 if (!size) 866 return -EINVAL; 867 868 rc = ntb_mw_get_align(nt->ndev, PIDX, num_mw, &xlat_align, 869 &xlat_align_size, NULL); 870 if (rc) 871 return rc; 872 873 xlat_size = round_up(size, xlat_align_size); 874 buff_size = round_up(size, xlat_align); 875 876 /* No need to re-setup */ 877 if (mw->xlat_size == xlat_size) 878 return 0; 879 880 if (mw->buff_size) 881 ntb_free_mw(nt, num_mw); 882 883 /* Alloc memory for receiving data. Must be aligned */ 884 mw->xlat_size = xlat_size; 885 mw->buff_size = buff_size; 886 mw->alloc_size = buff_size; 887 888 rc = ntb_alloc_mw_buffer(mw, &pdev->dev, xlat_align); 889 if (rc) { 890 mw->alloc_size *= 2; 891 rc = ntb_alloc_mw_buffer(mw, &pdev->dev, xlat_align); 892 if (rc) { 893 dev_err(&pdev->dev, 894 "Unable to alloc aligned MW buff\n"); 895 mw->xlat_size = 0; 896 mw->buff_size = 0; 897 mw->alloc_size = 0; 898 return rc; 899 } 900 } 901 902 /* Notify HW the memory location of the receive buffer */ 903 rc = ntb_mw_set_trans(nt->ndev, PIDX, num_mw, mw->dma_addr, 904 mw->xlat_size); 905 if (rc) { 906 dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw); 907 ntb_free_mw(nt, num_mw); 908 return -EIO; 909 } 910 911 return 0; 912} 913 914static void ntb_qp_link_context_reset(struct ntb_transport_qp *qp) 915{ 916 qp->link_is_up = false; 917 qp->active = false; 918 919 qp->tx_index = 0; 920 qp->rx_index = 0; 921 qp->rx_bytes = 0; 922 qp->rx_pkts = 0; 923 qp->rx_ring_empty = 0; 924 qp->rx_err_no_buf = 0; 925 qp->rx_err_oflow = 0; 926 qp->rx_err_ver = 0; 927 qp->rx_memcpy = 0; 928 qp->rx_async = 0; 929 qp->tx_bytes = 0; 930 qp->tx_pkts = 0; 931 qp->tx_ring_full = 0; 932 qp->tx_err_no_buf = 0; 933 qp->tx_memcpy = 0; 934 qp->tx_async = 0; 935} 936 937static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp) 938{ 939 ntb_qp_link_context_reset(qp); 940 if (qp->remote_rx_info) 941 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 942} 943 944static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 945{ 946 struct ntb_transport_ctx *nt = qp->transport; 947 struct pci_dev *pdev = nt->ndev->pdev; 948 949 dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num); 950 951 cancel_delayed_work_sync(&qp->link_work); 952 ntb_qp_link_down_reset(qp); 953 954 if (qp->event_handler) 955 qp->event_handler(qp->cb_data, qp->link_is_up); 956} 957 958static void ntb_qp_link_cleanup_work(struct work_struct *work) 959{ 960 struct ntb_transport_qp *qp = container_of(work, 961 struct ntb_transport_qp, 962 link_cleanup); 963 struct ntb_transport_ctx *nt = qp->transport; 964 965 ntb_qp_link_cleanup(qp); 966 967 if (nt->link_is_up) 968 schedule_delayed_work(&qp->link_work, 969 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 970} 971 972static void ntb_qp_link_down(struct ntb_transport_qp *qp) 973{ 974 schedule_work(&qp->link_cleanup); 975} 976 977static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) 978{ 979 struct ntb_transport_qp *qp; 980 u64 qp_bitmap_alloc; 981 unsigned int i, count; 982 983 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 984 985 /* Pass along the info to any clients */ 986 for (i = 0; i < nt->qp_count; i++) 987 if (qp_bitmap_alloc & BIT_ULL(i)) { 988 qp = &nt->qp_vec[i]; 989 ntb_qp_link_cleanup(qp); 990 cancel_work_sync(&qp->link_cleanup); 991 cancel_delayed_work_sync(&qp->link_work); 992 } 993 994 if (!nt->link_is_up) 995 cancel_delayed_work_sync(&nt->link_work); 996 997 for (i = 0; i < nt->mw_count; i++) 998 ntb_free_mw(nt, i); 999 1000 /* The scratchpad registers keep the values if the remote side 1001 * goes down, blast them now to give them a sane value the next 1002 * time they are accessed 1003 */ 1004 count = ntb_spad_count(nt->ndev); 1005 for (i = 0; i < count; i++) 1006 ntb_spad_write(nt->ndev, i, 0); 1007} 1008 1009static void ntb_transport_link_cleanup_work(struct work_struct *work) 1010{ 1011 struct ntb_transport_ctx *nt = 1012 container_of(work, struct ntb_transport_ctx, link_cleanup); 1013 1014 ntb_transport_link_cleanup(nt); 1015} 1016 1017static void ntb_transport_event_callback(void *data) 1018{ 1019 struct ntb_transport_ctx *nt = data; 1020 1021 if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1) 1022 schedule_delayed_work(&nt->link_work, 0); 1023 else 1024 schedule_work(&nt->link_cleanup); 1025} 1026 1027static void ntb_transport_link_work(struct work_struct *work) 1028{ 1029 struct ntb_transport_ctx *nt = 1030 container_of(work, struct ntb_transport_ctx, link_work.work); 1031 struct ntb_dev *ndev = nt->ndev; 1032 struct pci_dev *pdev = ndev->pdev; 1033 resource_size_t size; 1034 u32 val; 1035 int rc = 0, i, spad; 1036 1037 /* send the local info, in the opposite order of the way we read it */ 1038 1039 if (nt->use_msi) { 1040 rc = ntb_msi_setup_mws(ndev); 1041 if (rc) { 1042 dev_warn(&pdev->dev, 1043 "Failed to register MSI memory window: %d\n", 1044 rc); 1045 nt->use_msi = false; 1046 } 1047 } 1048 1049 for (i = 0; i < nt->qp_count; i++) 1050 ntb_transport_setup_qp_msi(nt, i); 1051 1052 for (i = 0; i < nt->mw_count; i++) { 1053 size = nt->mw_vec[i].phys_size; 1054 1055 if (max_mw_size && size > max_mw_size) 1056 size = max_mw_size; 1057 1058 spad = MW0_SZ_HIGH + (i * 2); 1059 ntb_peer_spad_write(ndev, PIDX, spad, upper_32_bits(size)); 1060 1061 spad = MW0_SZ_LOW + (i * 2); 1062 ntb_peer_spad_write(ndev, PIDX, spad, lower_32_bits(size)); 1063 } 1064 1065 ntb_peer_spad_write(ndev, PIDX, NUM_MWS, nt->mw_count); 1066 1067 ntb_peer_spad_write(ndev, PIDX, NUM_QPS, nt->qp_count); 1068 1069 ntb_peer_spad_write(ndev, PIDX, VERSION, NTB_TRANSPORT_VERSION); 1070 1071 /* Query the remote side for its info */ 1072 val = ntb_spad_read(ndev, VERSION); 1073 dev_dbg(&pdev->dev, "Remote version = %d\n", val); 1074 if (val != NTB_TRANSPORT_VERSION) 1075 goto out; 1076 1077 val = ntb_spad_read(ndev, NUM_QPS); 1078 dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); 1079 if (val != nt->qp_count) 1080 goto out; 1081 1082 val = ntb_spad_read(ndev, NUM_MWS); 1083 dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); 1084 if (val != nt->mw_count) 1085 goto out; 1086 1087 for (i = 0; i < nt->mw_count; i++) { 1088 u64 val64; 1089 1090 val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2)); 1091 val64 = (u64)val << 32; 1092 1093 val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2)); 1094 val64 |= val; 1095 1096 dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64); 1097 1098 rc = ntb_set_mw(nt, i, val64); 1099 if (rc) 1100 goto out1; 1101 } 1102 1103 nt->link_is_up = true; 1104 1105 for (i = 0; i < nt->qp_count; i++) { 1106 struct ntb_transport_qp *qp = &nt->qp_vec[i]; 1107 1108 ntb_transport_setup_qp_mw(nt, i); 1109 ntb_transport_setup_qp_peer_msi(nt, i); 1110 1111 if (qp->client_ready) 1112 schedule_delayed_work(&qp->link_work, 0); 1113 } 1114 1115 return; 1116 1117out1: 1118 for (i = 0; i < nt->mw_count; i++) 1119 ntb_free_mw(nt, i); 1120 1121 /* if there's an actual failure, we should just bail */ 1122 if (rc < 0) 1123 return; 1124 1125out: 1126 if (ntb_link_is_up(ndev, NULL, NULL) == 1) 1127 schedule_delayed_work(&nt->link_work, 1128 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 1129} 1130 1131static void ntb_qp_link_work(struct work_struct *work) 1132{ 1133 struct ntb_transport_qp *qp = container_of(work, 1134 struct ntb_transport_qp, 1135 link_work.work); 1136 struct pci_dev *pdev = qp->ndev->pdev; 1137 struct ntb_transport_ctx *nt = qp->transport; 1138 int val; 1139 1140 WARN_ON(!nt->link_is_up); 1141 1142 val = ntb_spad_read(nt->ndev, QP_LINKS); 1143 1144 ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num)); 1145 1146 /* query remote spad for qp ready bits */ 1147 dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val); 1148 1149 /* See if the remote side is up */ 1150 if (val & BIT(qp->qp_num)) { 1151 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); 1152 qp->link_is_up = true; 1153 qp->active = true; 1154 1155 if (qp->event_handler) 1156 qp->event_handler(qp->cb_data, qp->link_is_up); 1157 1158 if (qp->active) 1159 tasklet_schedule(&qp->rxc_db_work); 1160 } else if (nt->link_is_up) 1161 schedule_delayed_work(&qp->link_work, 1162 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 1163} 1164 1165static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, 1166 unsigned int qp_num) 1167{ 1168 struct ntb_transport_qp *qp; 1169 phys_addr_t mw_base; 1170 resource_size_t mw_size; 1171 unsigned int num_qps_mw, tx_size; 1172 unsigned int mw_num, mw_count, qp_count; 1173 u64 qp_offset; 1174 1175 mw_count = nt->mw_count; 1176 qp_count = nt->qp_count; 1177 1178 mw_num = QP_TO_MW(nt, qp_num); 1179 1180 qp = &nt->qp_vec[qp_num]; 1181 qp->qp_num = qp_num; 1182 qp->transport = nt; 1183 qp->ndev = nt->ndev; 1184 qp->client_ready = false; 1185 qp->event_handler = NULL; 1186 ntb_qp_link_context_reset(qp); 1187 1188 if (mw_num < qp_count % mw_count) 1189 num_qps_mw = qp_count / mw_count + 1; 1190 else 1191 num_qps_mw = qp_count / mw_count; 1192 1193 mw_base = nt->mw_vec[mw_num].phys_addr; 1194 mw_size = nt->mw_vec[mw_num].phys_size; 1195 1196 if (max_mw_size && mw_size > max_mw_size) 1197 mw_size = max_mw_size; 1198 1199 tx_size = (unsigned int)mw_size / num_qps_mw; 1200 qp_offset = tx_size * (qp_num / mw_count); 1201 1202 qp->tx_mw_size = tx_size; 1203 qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; 1204 if (!qp->tx_mw) 1205 return -EINVAL; 1206 1207 qp->tx_mw_phys = mw_base + qp_offset; 1208 if (!qp->tx_mw_phys) 1209 return -EINVAL; 1210 1211 tx_size -= sizeof(struct ntb_rx_info); 1212 qp->rx_info = qp->tx_mw + tx_size; 1213 1214 /* Due to housekeeping, there must be atleast 2 buffs */ 1215 qp->tx_max_frame = min(transport_mtu, tx_size / 2); 1216 qp->tx_max_entry = tx_size / qp->tx_max_frame; 1217 1218 if (nt->debugfs_node_dir) { 1219 char debugfs_name[4]; 1220 1221 snprintf(debugfs_name, 4, "qp%d", qp_num); 1222 qp->debugfs_dir = debugfs_create_dir(debugfs_name, 1223 nt->debugfs_node_dir); 1224 1225 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, 1226 qp->debugfs_dir, qp, 1227 &ntb_qp_debugfs_stats); 1228 } else { 1229 qp->debugfs_dir = NULL; 1230 qp->debugfs_stats = NULL; 1231 } 1232 1233 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); 1234 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); 1235 1236 spin_lock_init(&qp->ntb_rx_q_lock); 1237 spin_lock_init(&qp->ntb_tx_free_q_lock); 1238 1239 INIT_LIST_HEAD(&qp->rx_post_q); 1240 INIT_LIST_HEAD(&qp->rx_pend_q); 1241 INIT_LIST_HEAD(&qp->rx_free_q); 1242 INIT_LIST_HEAD(&qp->tx_free_q); 1243 1244 tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db, 1245 (unsigned long)qp); 1246 1247 return 0; 1248} 1249 1250static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) 1251{ 1252 struct ntb_transport_ctx *nt; 1253 struct ntb_transport_mw *mw; 1254 unsigned int mw_count, qp_count, spad_count, max_mw_count_for_spads; 1255 u64 qp_bitmap; 1256 int node; 1257 int rc, i; 1258 1259 mw_count = ntb_peer_mw_count(ndev); 1260 1261 if (!ndev->ops->mw_set_trans) { 1262 dev_err(&ndev->dev, "Inbound MW based NTB API is required\n"); 1263 return -EINVAL; 1264 } 1265 1266 if (ntb_db_is_unsafe(ndev)) 1267 dev_dbg(&ndev->dev, 1268 "doorbell is unsafe, proceed anyway...\n"); 1269 if (ntb_spad_is_unsafe(ndev)) 1270 dev_dbg(&ndev->dev, 1271 "scratchpad is unsafe, proceed anyway...\n"); 1272 1273 if (ntb_peer_port_count(ndev) != NTB_DEF_PEER_CNT) 1274 dev_warn(&ndev->dev, "Multi-port NTB devices unsupported\n"); 1275 1276 node = dev_to_node(&ndev->dev); 1277 1278 nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node); 1279 if (!nt) 1280 return -ENOMEM; 1281 1282 nt->ndev = ndev; 1283 1284 /* 1285 * If we are using MSI, and have at least one extra memory window, 1286 * we will reserve the last MW for the MSI window. 1287 */ 1288 if (use_msi && mw_count > 1) { 1289 rc = ntb_msi_init(ndev, ntb_transport_msi_desc_changed); 1290 if (!rc) { 1291 mw_count -= 1; 1292 nt->use_msi = true; 1293 } 1294 } 1295 1296 spad_count = ntb_spad_count(ndev); 1297 1298 /* Limit the MW's based on the availability of scratchpads */ 1299 1300 if (spad_count < NTB_TRANSPORT_MIN_SPADS) { 1301 nt->mw_count = 0; 1302 rc = -EINVAL; 1303 goto err; 1304 } 1305 1306 max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2; 1307 nt->mw_count = min(mw_count, max_mw_count_for_spads); 1308 1309 nt->msi_spad_offset = nt->mw_count * 2 + MW0_SZ_HIGH; 1310 1311 nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec), 1312 GFP_KERNEL, node); 1313 if (!nt->mw_vec) { 1314 rc = -ENOMEM; 1315 goto err; 1316 } 1317 1318 for (i = 0; i < mw_count; i++) { 1319 mw = &nt->mw_vec[i]; 1320 1321 rc = ntb_peer_mw_get_addr(ndev, i, &mw->phys_addr, 1322 &mw->phys_size); 1323 if (rc) 1324 goto err1; 1325 1326 mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size); 1327 if (!mw->vbase) { 1328 rc = -ENOMEM; 1329 goto err1; 1330 } 1331 1332 mw->buff_size = 0; 1333 mw->xlat_size = 0; 1334 mw->virt_addr = NULL; 1335 mw->dma_addr = 0; 1336 } 1337 1338 qp_bitmap = ntb_db_valid_mask(ndev); 1339 1340 qp_count = ilog2(qp_bitmap); 1341 if (nt->use_msi) { 1342 qp_count -= 1; 1343 nt->msi_db_mask = 1 << qp_count; 1344 ntb_db_clear_mask(ndev, nt->msi_db_mask); 1345 } 1346 1347 if (max_num_clients && max_num_clients < qp_count) 1348 qp_count = max_num_clients; 1349 else if (nt->mw_count < qp_count) 1350 qp_count = nt->mw_count; 1351 1352 qp_bitmap &= BIT_ULL(qp_count) - 1; 1353 1354 nt->qp_count = qp_count; 1355 nt->qp_bitmap = qp_bitmap; 1356 nt->qp_bitmap_free = qp_bitmap; 1357 1358 nt->qp_vec = kcalloc_node(qp_count, sizeof(*nt->qp_vec), 1359 GFP_KERNEL, node); 1360 if (!nt->qp_vec) { 1361 rc = -ENOMEM; 1362 goto err1; 1363 } 1364 1365 if (nt_debugfs_dir) { 1366 nt->debugfs_node_dir = 1367 debugfs_create_dir(pci_name(ndev->pdev), 1368 nt_debugfs_dir); 1369 } 1370 1371 for (i = 0; i < qp_count; i++) { 1372 rc = ntb_transport_init_queue(nt, i); 1373 if (rc) 1374 goto err2; 1375 } 1376 1377 INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); 1378 INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); 1379 1380 rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops); 1381 if (rc) 1382 goto err2; 1383 1384 INIT_LIST_HEAD(&nt->client_devs); 1385 rc = ntb_bus_init(nt); 1386 if (rc) 1387 goto err3; 1388 1389 nt->link_is_up = false; 1390 ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1391 ntb_link_event(ndev); 1392 1393 return 0; 1394 1395err3: 1396 ntb_clear_ctx(ndev); 1397err2: 1398 kfree(nt->qp_vec); 1399err1: 1400 while (i--) { 1401 mw = &nt->mw_vec[i]; 1402 iounmap(mw->vbase); 1403 } 1404 kfree(nt->mw_vec); 1405err: 1406 kfree(nt); 1407 return rc; 1408} 1409 1410static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev) 1411{ 1412 struct ntb_transport_ctx *nt = ndev->ctx; 1413 struct ntb_transport_qp *qp; 1414 u64 qp_bitmap_alloc; 1415 int i; 1416 1417 ntb_transport_link_cleanup(nt); 1418 cancel_work_sync(&nt->link_cleanup); 1419 cancel_delayed_work_sync(&nt->link_work); 1420 1421 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 1422 1423 /* verify that all the qp's are freed */ 1424 for (i = 0; i < nt->qp_count; i++) { 1425 qp = &nt->qp_vec[i]; 1426 if (qp_bitmap_alloc & BIT_ULL(i)) 1427 ntb_transport_free_queue(qp); 1428 debugfs_remove_recursive(qp->debugfs_dir); 1429 } 1430 1431 ntb_link_disable(ndev); 1432 ntb_clear_ctx(ndev); 1433 1434 ntb_bus_remove(nt); 1435 1436 for (i = nt->mw_count; i--; ) { 1437 ntb_free_mw(nt, i); 1438 iounmap(nt->mw_vec[i].vbase); 1439 } 1440 1441 kfree(nt->qp_vec); 1442 kfree(nt->mw_vec); 1443 kfree(nt); 1444} 1445 1446static void ntb_complete_rxc(struct ntb_transport_qp *qp) 1447{ 1448 struct ntb_queue_entry *entry; 1449 void *cb_data; 1450 unsigned int len; 1451 unsigned long irqflags; 1452 1453 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); 1454 1455 while (!list_empty(&qp->rx_post_q)) { 1456 entry = list_first_entry(&qp->rx_post_q, 1457 struct ntb_queue_entry, entry); 1458 if (!(entry->flags & DESC_DONE_FLAG)) 1459 break; 1460 1461 entry->rx_hdr->flags = 0; 1462 iowrite32(entry->rx_index, &qp->rx_info->entry); 1463 1464 cb_data = entry->cb_data; 1465 len = entry->len; 1466 1467 list_move_tail(&entry->entry, &qp->rx_free_q); 1468 1469 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); 1470 1471 if (qp->rx_handler && qp->client_ready) 1472 qp->rx_handler(qp, qp->cb_data, cb_data, len); 1473 1474 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); 1475 } 1476 1477 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); 1478} 1479 1480static void ntb_rx_copy_callback(void *data, 1481 const struct dmaengine_result *res) 1482{ 1483 struct ntb_queue_entry *entry = data; 1484 1485 /* we need to check DMA results if we are using DMA */ 1486 if (res) { 1487 enum dmaengine_tx_result dma_err = res->result; 1488 1489 switch (dma_err) { 1490 case DMA_TRANS_READ_FAILED: 1491 case DMA_TRANS_WRITE_FAILED: 1492 entry->errors++; 1493 fallthrough; 1494 case DMA_TRANS_ABORTED: 1495 { 1496 struct ntb_transport_qp *qp = entry->qp; 1497 void *offset = qp->rx_buff + qp->rx_max_frame * 1498 qp->rx_index; 1499 1500 ntb_memcpy_rx(entry, offset); 1501 qp->rx_memcpy++; 1502 return; 1503 } 1504 1505 case DMA_TRANS_NOERROR: 1506 default: 1507 break; 1508 } 1509 } 1510 1511 entry->flags |= DESC_DONE_FLAG; 1512 1513 ntb_complete_rxc(entry->qp); 1514} 1515 1516static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) 1517{ 1518 void *buf = entry->buf; 1519 size_t len = entry->len; 1520 1521 memcpy(buf, offset, len); 1522 1523 /* Ensure that the data is fully copied out before clearing the flag */ 1524 wmb(); 1525 1526 ntb_rx_copy_callback(entry, NULL); 1527} 1528 1529static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset) 1530{ 1531 struct dma_async_tx_descriptor *txd; 1532 struct ntb_transport_qp *qp = entry->qp; 1533 struct dma_chan *chan = qp->rx_dma_chan; 1534 struct dma_device *device; 1535 size_t pay_off, buff_off, len; 1536 struct dmaengine_unmap_data *unmap; 1537 dma_cookie_t cookie; 1538 void *buf = entry->buf; 1539 1540 len = entry->len; 1541 device = chan->device; 1542 pay_off = (size_t)offset & ~PAGE_MASK; 1543 buff_off = (size_t)buf & ~PAGE_MASK; 1544 1545 if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) 1546 goto err; 1547 1548 unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); 1549 if (!unmap) 1550 goto err; 1551 1552 unmap->len = len; 1553 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset), 1554 pay_off, len, DMA_TO_DEVICE); 1555 if (dma_mapping_error(device->dev, unmap->addr[0])) 1556 goto err_get_unmap; 1557 1558 unmap->to_cnt = 1; 1559 1560 unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf), 1561 buff_off, len, DMA_FROM_DEVICE); 1562 if (dma_mapping_error(device->dev, unmap->addr[1])) 1563 goto err_get_unmap; 1564 1565 unmap->from_cnt = 1; 1566 1567 txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], 1568 unmap->addr[0], len, 1569 DMA_PREP_INTERRUPT); 1570 if (!txd) 1571 goto err_get_unmap; 1572 1573 txd->callback_result = ntb_rx_copy_callback; 1574 txd->callback_param = entry; 1575 dma_set_unmap(txd, unmap); 1576 1577 cookie = dmaengine_submit(txd); 1578 if (dma_submit_error(cookie)) 1579 goto err_set_unmap; 1580 1581 dmaengine_unmap_put(unmap); 1582 1583 qp->last_cookie = cookie; 1584 1585 qp->rx_async++; 1586 1587 return 0; 1588 1589err_set_unmap: 1590 dmaengine_unmap_put(unmap); 1591err_get_unmap: 1592 dmaengine_unmap_put(unmap); 1593err: 1594 return -ENXIO; 1595} 1596 1597static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) 1598{ 1599 struct ntb_transport_qp *qp = entry->qp; 1600 struct dma_chan *chan = qp->rx_dma_chan; 1601 int res; 1602 1603 if (!chan) 1604 goto err; 1605 1606 if (entry->len < copy_bytes) 1607 goto err; 1608 1609 res = ntb_async_rx_submit(entry, offset); 1610 if (res < 0) 1611 goto err; 1612 1613 if (!entry->retries) 1614 qp->rx_async++; 1615 1616 return; 1617 1618err: 1619 ntb_memcpy_rx(entry, offset); 1620 qp->rx_memcpy++; 1621} 1622 1623static int ntb_process_rxc(struct ntb_transport_qp *qp) 1624{ 1625 struct ntb_payload_header *hdr; 1626 struct ntb_queue_entry *entry; 1627 void *offset; 1628 1629 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; 1630 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); 1631 1632 dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n", 1633 qp->qp_num, hdr->ver, hdr->len, hdr->flags); 1634 1635 if (!(hdr->flags & DESC_DONE_FLAG)) { 1636 dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n"); 1637 qp->rx_ring_empty++; 1638 return -EAGAIN; 1639 } 1640 1641 if (hdr->flags & LINK_DOWN_FLAG) { 1642 dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n"); 1643 ntb_qp_link_down(qp); 1644 hdr->flags = 0; 1645 return -EAGAIN; 1646 } 1647 1648 if (hdr->ver != (u32)qp->rx_pkts) { 1649 dev_dbg(&qp->ndev->pdev->dev, 1650 "version mismatch, expected %llu - got %u\n", 1651 qp->rx_pkts, hdr->ver); 1652 qp->rx_err_ver++; 1653 return -EIO; 1654 } 1655 1656 entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q); 1657 if (!entry) { 1658 dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n"); 1659 qp->rx_err_no_buf++; 1660 return -EAGAIN; 1661 } 1662 1663 entry->rx_hdr = hdr; 1664 entry->rx_index = qp->rx_index; 1665 1666 if (hdr->len > entry->len) { 1667 dev_dbg(&qp->ndev->pdev->dev, 1668 "receive buffer overflow! Wanted %d got %d\n", 1669 hdr->len, entry->len); 1670 qp->rx_err_oflow++; 1671 1672 entry->len = -EIO; 1673 entry->flags |= DESC_DONE_FLAG; 1674 1675 ntb_complete_rxc(qp); 1676 } else { 1677 dev_dbg(&qp->ndev->pdev->dev, 1678 "RX OK index %u ver %u size %d into buf size %d\n", 1679 qp->rx_index, hdr->ver, hdr->len, entry->len); 1680 1681 qp->rx_bytes += hdr->len; 1682 qp->rx_pkts++; 1683 1684 entry->len = hdr->len; 1685 1686 ntb_async_rx(entry, offset); 1687 } 1688 1689 qp->rx_index++; 1690 qp->rx_index %= qp->rx_max_entry; 1691 1692 return 0; 1693} 1694 1695static void ntb_transport_rxc_db(unsigned long data) 1696{ 1697 struct ntb_transport_qp *qp = (void *)data; 1698 int rc, i; 1699 1700 dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n", 1701 __func__, qp->qp_num); 1702 1703 /* Limit the number of packets processed in a single interrupt to 1704 * provide fairness to others 1705 */ 1706 for (i = 0; i < qp->rx_max_entry; i++) { 1707 rc = ntb_process_rxc(qp); 1708 if (rc) 1709 break; 1710 } 1711 1712 if (i && qp->rx_dma_chan) 1713 dma_async_issue_pending(qp->rx_dma_chan); 1714 1715 if (i == qp->rx_max_entry) { 1716 /* there is more work to do */ 1717 if (qp->active) 1718 tasklet_schedule(&qp->rxc_db_work); 1719 } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) { 1720 /* the doorbell bit is set: clear it */ 1721 ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num)); 1722 /* ntb_db_read ensures ntb_db_clear write is committed */ 1723 ntb_db_read(qp->ndev); 1724 1725 /* an interrupt may have arrived between finishing 1726 * ntb_process_rxc and clearing the doorbell bit: 1727 * there might be some more work to do. 1728 */ 1729 if (qp->active) 1730 tasklet_schedule(&qp->rxc_db_work); 1731 } 1732} 1733 1734static void ntb_tx_copy_callback(void *data, 1735 const struct dmaengine_result *res) 1736{ 1737 struct ntb_queue_entry *entry = data; 1738 struct ntb_transport_qp *qp = entry->qp; 1739 struct ntb_payload_header __iomem *hdr = entry->tx_hdr; 1740 1741 /* we need to check DMA results if we are using DMA */ 1742 if (res) { 1743 enum dmaengine_tx_result dma_err = res->result; 1744 1745 switch (dma_err) { 1746 case DMA_TRANS_READ_FAILED: 1747 case DMA_TRANS_WRITE_FAILED: 1748 entry->errors++; 1749 fallthrough; 1750 case DMA_TRANS_ABORTED: 1751 { 1752 void __iomem *offset = 1753 qp->tx_mw + qp->tx_max_frame * 1754 entry->tx_index; 1755 1756 /* resubmit via CPU */ 1757 ntb_memcpy_tx(entry, offset); 1758 qp->tx_memcpy++; 1759 return; 1760 } 1761 1762 case DMA_TRANS_NOERROR: 1763 default: 1764 break; 1765 } 1766 } 1767 1768 iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); 1769 1770 if (qp->use_msi) 1771 ntb_msi_peer_trigger(qp->ndev, PIDX, &qp->peer_msi_desc); 1772 else 1773 ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); 1774 1775 /* The entry length can only be zero if the packet is intended to be a 1776 * "link down" or similar. Since no payload is being sent in these 1777 * cases, there is nothing to add to the completion queue. 1778 */ 1779 if (entry->len > 0) { 1780 qp->tx_bytes += entry->len; 1781 1782 if (qp->tx_handler) 1783 qp->tx_handler(qp, qp->cb_data, entry->cb_data, 1784 entry->len); 1785 } 1786 1787 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); 1788} 1789 1790static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) 1791{ 1792#ifdef ARCH_HAS_NOCACHE_UACCESS 1793 /* 1794 * Using non-temporal mov to improve performance on non-cached 1795 * writes, even though we aren't actually copying from user space. 1796 */ 1797 __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); 1798#else 1799 memcpy_toio(offset, entry->buf, entry->len); 1800#endif 1801 1802 /* Ensure that the data is fully copied out before setting the flags */ 1803 wmb(); 1804 1805 ntb_tx_copy_callback(entry, NULL); 1806} 1807 1808static int ntb_async_tx_submit(struct ntb_transport_qp *qp, 1809 struct ntb_queue_entry *entry) 1810{ 1811 struct dma_async_tx_descriptor *txd; 1812 struct dma_chan *chan = qp->tx_dma_chan; 1813 struct dma_device *device; 1814 size_t len = entry->len; 1815 void *buf = entry->buf; 1816 size_t dest_off, buff_off; 1817 struct dmaengine_unmap_data *unmap; 1818 dma_addr_t dest; 1819 dma_cookie_t cookie; 1820 1821 device = chan->device; 1822 dest = qp->tx_mw_dma_addr + qp->tx_max_frame * entry->tx_index; 1823 buff_off = (size_t)buf & ~PAGE_MASK; 1824 dest_off = (size_t)dest & ~PAGE_MASK; 1825 1826 if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) 1827 goto err; 1828 1829 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); 1830 if (!unmap) 1831 goto err; 1832 1833 unmap->len = len; 1834 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf), 1835 buff_off, len, DMA_TO_DEVICE); 1836 if (dma_mapping_error(device->dev, unmap->addr[0])) 1837 goto err_get_unmap; 1838 1839 unmap->to_cnt = 1; 1840 1841 txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len, 1842 DMA_PREP_INTERRUPT); 1843 if (!txd) 1844 goto err_get_unmap; 1845 1846 txd->callback_result = ntb_tx_copy_callback; 1847 txd->callback_param = entry; 1848 dma_set_unmap(txd, unmap); 1849 1850 cookie = dmaengine_submit(txd); 1851 if (dma_submit_error(cookie)) 1852 goto err_set_unmap; 1853 1854 dmaengine_unmap_put(unmap); 1855 1856 dma_async_issue_pending(chan); 1857 1858 return 0; 1859err_set_unmap: 1860 dmaengine_unmap_put(unmap); 1861err_get_unmap: 1862 dmaengine_unmap_put(unmap); 1863err: 1864 return -ENXIO; 1865} 1866 1867static void ntb_async_tx(struct ntb_transport_qp *qp, 1868 struct ntb_queue_entry *entry) 1869{ 1870 struct ntb_payload_header __iomem *hdr; 1871 struct dma_chan *chan = qp->tx_dma_chan; 1872 void __iomem *offset; 1873 int res; 1874 1875 entry->tx_index = qp->tx_index; 1876 offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index; 1877 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); 1878 entry->tx_hdr = hdr; 1879 1880 iowrite32(entry->len, &hdr->len); 1881 iowrite32((u32)qp->tx_pkts, &hdr->ver); 1882 1883 if (!chan) 1884 goto err; 1885 1886 if (entry->len < copy_bytes) 1887 goto err; 1888 1889 res = ntb_async_tx_submit(qp, entry); 1890 if (res < 0) 1891 goto err; 1892 1893 if (!entry->retries) 1894 qp->tx_async++; 1895 1896 return; 1897 1898err: 1899 ntb_memcpy_tx(entry, offset); 1900 qp->tx_memcpy++; 1901} 1902 1903static int ntb_process_tx(struct ntb_transport_qp *qp, 1904 struct ntb_queue_entry *entry) 1905{ 1906 if (qp->tx_index == qp->remote_rx_info->entry) { 1907 qp->tx_ring_full++; 1908 return -EAGAIN; 1909 } 1910 1911 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 1912 if (qp->tx_handler) 1913 qp->tx_handler(qp, qp->cb_data, NULL, -EIO); 1914 1915 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1916 &qp->tx_free_q); 1917 return 0; 1918 } 1919 1920 ntb_async_tx(qp, entry); 1921 1922 qp->tx_index++; 1923 qp->tx_index %= qp->tx_max_entry; 1924 1925 qp->tx_pkts++; 1926 1927 return 0; 1928} 1929 1930static void ntb_send_link_down(struct ntb_transport_qp *qp) 1931{ 1932 struct pci_dev *pdev = qp->ndev->pdev; 1933 struct ntb_queue_entry *entry; 1934 int i, rc; 1935 1936 if (!qp->link_is_up) 1937 return; 1938 1939 dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num); 1940 1941 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1942 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1943 if (entry) 1944 break; 1945 msleep(100); 1946 } 1947 1948 if (!entry) 1949 return; 1950 1951 entry->cb_data = NULL; 1952 entry->buf = NULL; 1953 entry->len = 0; 1954 entry->flags = LINK_DOWN_FLAG; 1955 1956 rc = ntb_process_tx(qp, entry); 1957 if (rc) 1958 dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", 1959 qp->qp_num); 1960 1961 ntb_qp_link_down_reset(qp); 1962} 1963 1964static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node) 1965{ 1966 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; 1967} 1968 1969/** 1970 * ntb_transport_create_queue - Create a new NTB transport layer queue 1971 * @rx_handler: receive callback function 1972 * @tx_handler: transmit callback function 1973 * @event_handler: event callback function 1974 * 1975 * Create a new NTB transport layer queue and provide the queue with a callback 1976 * routine for both transmit and receive. The receive callback routine will be 1977 * used to pass up data when the transport has received it on the queue. The 1978 * transmit callback routine will be called when the transport has completed the 1979 * transmission of the data on the queue and the data is ready to be freed. 1980 * 1981 * RETURNS: pointer to newly created ntb_queue, NULL on error. 1982 */ 1983struct ntb_transport_qp * 1984ntb_transport_create_queue(void *data, struct device *client_dev, 1985 const struct ntb_queue_handlers *handlers) 1986{ 1987 struct ntb_dev *ndev; 1988 struct pci_dev *pdev; 1989 struct ntb_transport_ctx *nt; 1990 struct ntb_queue_entry *entry; 1991 struct ntb_transport_qp *qp; 1992 u64 qp_bit; 1993 unsigned int free_queue; 1994 dma_cap_mask_t dma_mask; 1995 int node; 1996 int i; 1997 1998 ndev = dev_ntb(client_dev->parent); 1999 pdev = ndev->pdev; 2000 nt = ndev->ctx; 2001 2002 node = dev_to_node(&ndev->dev); 2003 2004 free_queue = ffs(nt->qp_bitmap_free); 2005 if (!free_queue) 2006 goto err; 2007 2008 /* decrement free_queue to make it zero based */ 2009 free_queue--; 2010 2011 qp = &nt->qp_vec[free_queue]; 2012 qp_bit = BIT_ULL(qp->qp_num); 2013 2014 nt->qp_bitmap_free &= ~qp_bit; 2015 2016 qp->cb_data = data; 2017 qp->rx_handler = handlers->rx_handler; 2018 qp->tx_handler = handlers->tx_handler; 2019 qp->event_handler = handlers->event_handler; 2020 2021 dma_cap_zero(dma_mask); 2022 dma_cap_set(DMA_MEMCPY, dma_mask); 2023 2024 if (use_dma) { 2025 qp->tx_dma_chan = 2026 dma_request_channel(dma_mask, ntb_dma_filter_fn, 2027 (void *)(unsigned long)node); 2028 if (!qp->tx_dma_chan) 2029 dev_info(&pdev->dev, "Unable to allocate TX DMA channel\n"); 2030 2031 qp->rx_dma_chan = 2032 dma_request_channel(dma_mask, ntb_dma_filter_fn, 2033 (void *)(unsigned long)node); 2034 if (!qp->rx_dma_chan) 2035 dev_info(&pdev->dev, "Unable to allocate RX DMA channel\n"); 2036 } else { 2037 qp->tx_dma_chan = NULL; 2038 qp->rx_dma_chan = NULL; 2039 } 2040 2041 qp->tx_mw_dma_addr = 0; 2042 if (qp->tx_dma_chan) { 2043 qp->tx_mw_dma_addr = 2044 dma_map_resource(qp->tx_dma_chan->device->dev, 2045 qp->tx_mw_phys, qp->tx_mw_size, 2046 DMA_FROM_DEVICE, 0); 2047 if (dma_mapping_error(qp->tx_dma_chan->device->dev, 2048 qp->tx_mw_dma_addr)) { 2049 qp->tx_mw_dma_addr = 0; 2050 goto err1; 2051 } 2052 } 2053 2054 dev_dbg(&pdev->dev, "Using %s memcpy for TX\n", 2055 qp->tx_dma_chan ? "DMA" : "CPU"); 2056 2057 dev_dbg(&pdev->dev, "Using %s memcpy for RX\n", 2058 qp->rx_dma_chan ? "DMA" : "CPU"); 2059 2060 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 2061 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 2062 if (!entry) 2063 goto err1; 2064 2065 entry->qp = qp; 2066 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, 2067 &qp->rx_free_q); 2068 } 2069 qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES; 2070 2071 for (i = 0; i < qp->tx_max_entry; i++) { 2072 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 2073 if (!entry) 2074 goto err2; 2075 2076 entry->qp = qp; 2077 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 2078 &qp->tx_free_q); 2079 } 2080 2081 ntb_db_clear(qp->ndev, qp_bit); 2082 ntb_db_clear_mask(qp->ndev, qp_bit); 2083 2084 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); 2085 2086 return qp; 2087 2088err2: 2089 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 2090 kfree(entry); 2091err1: 2092 qp->rx_alloc_entry = 0; 2093 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 2094 kfree(entry); 2095 if (qp->tx_mw_dma_addr) 2096 dma_unmap_resource(qp->tx_dma_chan->device->dev, 2097 qp->tx_mw_dma_addr, qp->tx_mw_size, 2098 DMA_FROM_DEVICE, 0); 2099 if (qp->tx_dma_chan) 2100 dma_release_channel(qp->tx_dma_chan); 2101 if (qp->rx_dma_chan) 2102 dma_release_channel(qp->rx_dma_chan); 2103 nt->qp_bitmap_free |= qp_bit; 2104err: 2105 return NULL; 2106} 2107EXPORT_SYMBOL_GPL(ntb_transport_create_queue); 2108 2109/** 2110 * ntb_transport_free_queue - Frees NTB transport queue 2111 * @qp: NTB queue to be freed 2112 * 2113 * Frees NTB transport queue 2114 */ 2115void ntb_transport_free_queue(struct ntb_transport_qp *qp) 2116{ 2117 struct pci_dev *pdev; 2118 struct ntb_queue_entry *entry; 2119 u64 qp_bit; 2120 2121 if (!qp) 2122 return; 2123 2124 pdev = qp->ndev->pdev; 2125 2126 qp->active = false; 2127 2128 if (qp->tx_dma_chan) { 2129 struct dma_chan *chan = qp->tx_dma_chan; 2130 /* Putting the dma_chan to NULL will force any new traffic to be 2131 * processed by the CPU instead of the DAM engine 2132 */ 2133 qp->tx_dma_chan = NULL; 2134 2135 /* Try to be nice and wait for any queued DMA engine 2136 * transactions to process before smashing it with a rock 2137 */ 2138 dma_sync_wait(chan, qp->last_cookie); 2139 dmaengine_terminate_all(chan); 2140 2141 dma_unmap_resource(chan->device->dev, 2142 qp->tx_mw_dma_addr, qp->tx_mw_size, 2143 DMA_FROM_DEVICE, 0); 2144 2145 dma_release_channel(chan); 2146 } 2147 2148 if (qp->rx_dma_chan) { 2149 struct dma_chan *chan = qp->rx_dma_chan; 2150 /* Putting the dma_chan to NULL will force any new traffic to be 2151 * processed by the CPU instead of the DAM engine 2152 */ 2153 qp->rx_dma_chan = NULL; 2154 2155 /* Try to be nice and wait for any queued DMA engine 2156 * transactions to process before smashing it with a rock 2157 */ 2158 dma_sync_wait(chan, qp->last_cookie); 2159 dmaengine_terminate_all(chan); 2160 dma_release_channel(chan); 2161 } 2162 2163 qp_bit = BIT_ULL(qp->qp_num); 2164 2165 ntb_db_set_mask(qp->ndev, qp_bit); 2166 tasklet_kill(&qp->rxc_db_work); 2167 2168 cancel_delayed_work_sync(&qp->link_work); 2169 2170 qp->cb_data = NULL; 2171 qp->rx_handler = NULL; 2172 qp->tx_handler = NULL; 2173 qp->event_handler = NULL; 2174 2175 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 2176 kfree(entry); 2177 2178 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) { 2179 dev_warn(&pdev->dev, "Freeing item from non-empty rx_pend_q\n"); 2180 kfree(entry); 2181 } 2182 2183 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) { 2184 dev_warn(&pdev->dev, "Freeing item from non-empty rx_post_q\n"); 2185 kfree(entry); 2186 } 2187 2188 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 2189 kfree(entry); 2190 2191 qp->transport->qp_bitmap_free |= qp_bit; 2192 2193 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); 2194} 2195EXPORT_SYMBOL_GPL(ntb_transport_free_queue); 2196 2197/** 2198 * ntb_transport_rx_remove - Dequeues enqueued rx packet 2199 * @qp: NTB queue to be freed 2200 * @len: pointer to variable to write enqueued buffers length 2201 * 2202 * Dequeues unused buffers from receive queue. Should only be used during 2203 * shutdown of qp. 2204 * 2205 * RETURNS: NULL error value on error, or void* for success. 2206 */ 2207void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) 2208{ 2209 struct ntb_queue_entry *entry; 2210 void *buf; 2211 2212 if (!qp || qp->client_ready) 2213 return NULL; 2214 2215 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q); 2216 if (!entry) 2217 return NULL; 2218 2219 buf = entry->cb_data; 2220 *len = entry->len; 2221 2222 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q); 2223 2224 return buf; 2225} 2226EXPORT_SYMBOL_GPL(ntb_transport_rx_remove); 2227 2228/** 2229 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry 2230 * @qp: NTB transport layer queue the entry is to be enqueued on 2231 * @cb: per buffer pointer for callback function to use 2232 * @data: pointer to data buffer that incoming packets will be copied into 2233 * @len: length of the data buffer 2234 * 2235 * Enqueue a new receive buffer onto the transport queue into which a NTB 2236 * payload can be received into. 2237 * 2238 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 2239 */ 2240int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 2241 unsigned int len) 2242{ 2243 struct ntb_queue_entry *entry; 2244 2245 if (!qp) 2246 return -EINVAL; 2247 2248 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q); 2249 if (!entry) 2250 return -ENOMEM; 2251 2252 entry->cb_data = cb; 2253 entry->buf = data; 2254 entry->len = len; 2255 entry->flags = 0; 2256 entry->retries = 0; 2257 entry->errors = 0; 2258 entry->rx_index = 0; 2259 2260 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q); 2261 2262 if (qp->active) 2263 tasklet_schedule(&qp->rxc_db_work); 2264 2265 return 0; 2266} 2267EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue); 2268 2269/** 2270 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 2271 * @qp: NTB transport layer queue the entry is to be enqueued on 2272 * @cb: per buffer pointer for callback function to use 2273 * @data: pointer to data buffer that will be sent 2274 * @len: length of the data buffer 2275 * 2276 * Enqueue a new transmit buffer onto the transport queue from which a NTB 2277 * payload will be transmitted. This assumes that a lock is being held to 2278 * serialize access to the qp. 2279 * 2280 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 2281 */ 2282int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 2283 unsigned int len) 2284{ 2285 struct ntb_queue_entry *entry; 2286 int rc; 2287 2288 if (!qp || !len) 2289 return -EINVAL; 2290 2291 /* If the qp link is down already, just ignore. */ 2292 if (!qp->link_is_up) 2293 return 0; 2294 2295 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 2296 if (!entry) { 2297 qp->tx_err_no_buf++; 2298 return -EBUSY; 2299 } 2300 2301 entry->cb_data = cb; 2302 entry->buf = data; 2303 entry->len = len; 2304 entry->flags = 0; 2305 entry->errors = 0; 2306 entry->retries = 0; 2307 entry->tx_index = 0; 2308 2309 rc = ntb_process_tx(qp, entry); 2310 if (rc) 2311 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 2312 &qp->tx_free_q); 2313 2314 return rc; 2315} 2316EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue); 2317 2318/** 2319 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 2320 * @qp: NTB transport layer queue to be enabled 2321 * 2322 * Notify NTB transport layer of client readiness to use queue 2323 */ 2324void ntb_transport_link_up(struct ntb_transport_qp *qp) 2325{ 2326 if (!qp) 2327 return; 2328 2329 qp->client_ready = true; 2330 2331 if (qp->transport->link_is_up) 2332 schedule_delayed_work(&qp->link_work, 0); 2333} 2334EXPORT_SYMBOL_GPL(ntb_transport_link_up); 2335 2336/** 2337 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 2338 * @qp: NTB transport layer queue to be disabled 2339 * 2340 * Notify NTB transport layer of client's desire to no longer receive data on 2341 * transport queue specified. It is the client's responsibility to ensure all 2342 * entries on queue are purged or otherwise handled appropriately. 2343 */ 2344void ntb_transport_link_down(struct ntb_transport_qp *qp) 2345{ 2346 int val; 2347 2348 if (!qp) 2349 return; 2350 2351 qp->client_ready = false; 2352 2353 val = ntb_spad_read(qp->ndev, QP_LINKS); 2354 2355 ntb_peer_spad_write(qp->ndev, PIDX, QP_LINKS, val & ~BIT(qp->qp_num)); 2356 2357 if (qp->link_is_up) 2358 ntb_send_link_down(qp); 2359 else 2360 cancel_delayed_work_sync(&qp->link_work); 2361} 2362EXPORT_SYMBOL_GPL(ntb_transport_link_down); 2363 2364/** 2365 * ntb_transport_link_query - Query transport link state 2366 * @qp: NTB transport layer queue to be queried 2367 * 2368 * Query connectivity to the remote system of the NTB transport queue 2369 * 2370 * RETURNS: true for link up or false for link down 2371 */ 2372bool ntb_transport_link_query(struct ntb_transport_qp *qp) 2373{ 2374 if (!qp) 2375 return false; 2376 2377 return qp->link_is_up; 2378} 2379EXPORT_SYMBOL_GPL(ntb_transport_link_query); 2380 2381/** 2382 * ntb_transport_qp_num - Query the qp number 2383 * @qp: NTB transport layer queue to be queried 2384 * 2385 * Query qp number of the NTB transport queue 2386 * 2387 * RETURNS: a zero based number specifying the qp number 2388 */ 2389unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) 2390{ 2391 if (!qp) 2392 return 0; 2393 2394 return qp->qp_num; 2395} 2396EXPORT_SYMBOL_GPL(ntb_transport_qp_num); 2397 2398/** 2399 * ntb_transport_max_size - Query the max payload size of a qp 2400 * @qp: NTB transport layer queue to be queried 2401 * 2402 * Query the maximum payload size permissible on the given qp 2403 * 2404 * RETURNS: the max payload size of a qp 2405 */ 2406unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) 2407{ 2408 unsigned int max_size; 2409 unsigned int copy_align; 2410 struct dma_chan *rx_chan, *tx_chan; 2411 2412 if (!qp) 2413 return 0; 2414 2415 rx_chan = qp->rx_dma_chan; 2416 tx_chan = qp->tx_dma_chan; 2417 2418 copy_align = max(rx_chan ? rx_chan->device->copy_align : 0, 2419 tx_chan ? tx_chan->device->copy_align : 0); 2420 2421 /* If DMA engine usage is possible, try to find the max size for that */ 2422 max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header); 2423 max_size = round_down(max_size, 1 << copy_align); 2424 2425 return max_size; 2426} 2427EXPORT_SYMBOL_GPL(ntb_transport_max_size); 2428 2429unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp) 2430{ 2431 unsigned int head = qp->tx_index; 2432 unsigned int tail = qp->remote_rx_info->entry; 2433 2434 return tail >= head ? tail - head : qp->tx_max_entry + tail - head; 2435} 2436EXPORT_SYMBOL_GPL(ntb_transport_tx_free_entry); 2437 2438static void ntb_transport_doorbell_callback(void *data, int vector) 2439{ 2440 struct ntb_transport_ctx *nt = data; 2441 struct ntb_transport_qp *qp; 2442 u64 db_bits; 2443 unsigned int qp_num; 2444 2445 if (ntb_db_read(nt->ndev) & nt->msi_db_mask) { 2446 ntb_transport_msi_peer_desc_changed(nt); 2447 ntb_db_clear(nt->ndev, nt->msi_db_mask); 2448 } 2449 2450 db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free & 2451 ntb_db_vector_mask(nt->ndev, vector)); 2452 2453 while (db_bits) { 2454 qp_num = __ffs(db_bits); 2455 qp = &nt->qp_vec[qp_num]; 2456 2457 if (qp->active) 2458 tasklet_schedule(&qp->rxc_db_work); 2459 2460 db_bits &= ~BIT_ULL(qp_num); 2461 } 2462} 2463 2464static const struct ntb_ctx_ops ntb_transport_ops = { 2465 .link_event = ntb_transport_event_callback, 2466 .db_event = ntb_transport_doorbell_callback, 2467}; 2468 2469static struct ntb_client ntb_transport_client = { 2470 .ops = { 2471 .probe = ntb_transport_probe, 2472 .remove = ntb_transport_free, 2473 }, 2474}; 2475 2476static int __init ntb_transport_init(void) 2477{ 2478 int rc; 2479 2480 pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER); 2481 2482 if (debugfs_initialized()) 2483 nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 2484 2485 rc = bus_register(&ntb_transport_bus); 2486 if (rc) 2487 goto err_bus; 2488 2489 rc = ntb_register_client(&ntb_transport_client); 2490 if (rc) 2491 goto err_client; 2492 2493 return 0; 2494 2495err_client: 2496 bus_unregister(&ntb_transport_bus); 2497err_bus: 2498 debugfs_remove_recursive(nt_debugfs_dir); 2499 return rc; 2500} 2501module_init(ntb_transport_init); 2502 2503static void __exit ntb_transport_exit(void) 2504{ 2505 ntb_unregister_client(&ntb_transport_client); 2506 bus_unregister(&ntb_transport_bus); 2507 debugfs_remove_recursive(nt_debugfs_dir); 2508} 2509module_exit(ntb_transport_exit); 2510