1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Microsemi Switchtec(tm) PCIe Management Driver 4 * Copyright (c) 2017, Microsemi Corporation 5 */ 6 7#include <linux/interrupt.h> 8#include <linux/io-64-nonatomic-lo-hi.h> 9#include <linux/delay.h> 10#include <linux/kthread.h> 11#include <linux/module.h> 12#include <linux/ntb.h> 13#include <linux/pci.h> 14#include <linux/switchtec.h> 15 16MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver"); 17MODULE_VERSION("0.1"); 18MODULE_LICENSE("GPL"); 19MODULE_AUTHOR("Microsemi Corporation"); 20 21static ulong max_mw_size = SZ_2M; 22module_param(max_mw_size, ulong, 0644); 23MODULE_PARM_DESC(max_mw_size, 24 "Max memory window size reported to the upper layer"); 25 26static bool use_lut_mws; 27module_param(use_lut_mws, bool, 0644); 28MODULE_PARM_DESC(use_lut_mws, 29 "Enable the use of the LUT based memory windows"); 30 31#define SWITCHTEC_NTB_MAGIC 0x45CC0001 32#define MAX_MWS 128 33 34struct shared_mw { 35 u32 magic; 36 u32 link_sta; 37 u32 partition_id; 38 u64 mw_sizes[MAX_MWS]; 39 u32 spad[128]; 40}; 41 42#define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry) 43#define LUT_SIZE SZ_64K 44 45struct switchtec_ntb { 46 struct ntb_dev ntb; 47 struct switchtec_dev *stdev; 48 49 int self_partition; 50 int peer_partition; 51 52 int doorbell_irq; 53 int message_irq; 54 55 struct ntb_info_regs __iomem *mmio_ntb; 56 struct ntb_ctrl_regs __iomem *mmio_ctrl; 57 struct ntb_dbmsg_regs __iomem *mmio_dbmsg; 58 struct ntb_ctrl_regs __iomem *mmio_self_ctrl; 59 struct ntb_ctrl_regs __iomem *mmio_peer_ctrl; 60 struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg; 61 struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg; 62 63 void __iomem *mmio_xlink_win; 64 65 struct shared_mw *self_shared; 66 struct shared_mw __iomem *peer_shared; 67 dma_addr_t self_shared_dma; 68 69 u64 db_mask; 70 u64 db_valid_mask; 71 int db_shift; 72 int db_peer_shift; 73 74 /* synchronize rmw access of db_mask and hw reg */ 75 spinlock_t db_mask_lock; 76 77 int nr_direct_mw; 78 int nr_lut_mw; 79 int nr_rsvd_luts; 80 int direct_mw_to_bar[MAX_DIRECT_MW]; 81 82 int peer_nr_direct_mw; 83 int peer_nr_lut_mw; 84 int peer_direct_mw_to_bar[MAX_DIRECT_MW]; 85 86 bool link_is_up; 87 enum ntb_speed link_speed; 88 enum ntb_width link_width; 89 struct work_struct check_link_status_work; 90 bool link_force_down; 91}; 92 93static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb) 94{ 95 return container_of(ntb, struct switchtec_ntb, ntb); 96} 97 98static int switchtec_ntb_part_op(struct switchtec_ntb *sndev, 99 struct ntb_ctrl_regs __iomem *ctl, 100 u32 op, int wait_status) 101{ 102 static const char * const op_text[] = { 103 [NTB_CTRL_PART_OP_LOCK] = "lock", 104 [NTB_CTRL_PART_OP_CFG] = "configure", 105 [NTB_CTRL_PART_OP_RESET] = "reset", 106 }; 107 108 int i; 109 u32 ps; 110 int status; 111 112 switch (op) { 113 case NTB_CTRL_PART_OP_LOCK: 114 status = NTB_CTRL_PART_STATUS_LOCKING; 115 break; 116 case NTB_CTRL_PART_OP_CFG: 117 status = NTB_CTRL_PART_STATUS_CONFIGURING; 118 break; 119 case NTB_CTRL_PART_OP_RESET: 120 status = NTB_CTRL_PART_STATUS_RESETTING; 121 break; 122 default: 123 return -EINVAL; 124 } 125 126 iowrite32(op, &ctl->partition_op); 127 128 for (i = 0; i < 1000; i++) { 129 if (msleep_interruptible(50) != 0) { 130 iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op); 131 return -EINTR; 132 } 133 134 ps = ioread32(&ctl->partition_status) & 0xFFFF; 135 136 if (ps != status) 137 break; 138 } 139 140 if (ps == wait_status) 141 return 0; 142 143 if (ps == status) { 144 dev_err(&sndev->stdev->dev, 145 "Timed out while performing %s (%d). (%08x)\n", 146 op_text[op], op, 147 ioread32(&ctl->partition_status)); 148 149 return -ETIMEDOUT; 150 } 151 152 return -EIO; 153} 154 155static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx, 156 u32 val) 157{ 158 if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg)) 159 return -EINVAL; 160 161 iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg); 162 163 return 0; 164} 165 166static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx) 167{ 168 struct switchtec_ntb *sndev = ntb_sndev(ntb); 169 int nr_direct_mw = sndev->peer_nr_direct_mw; 170 int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts; 171 172 if (pidx != NTB_DEF_PEER_IDX) 173 return -EINVAL; 174 175 if (!use_lut_mws) 176 nr_lut_mw = 0; 177 178 return nr_direct_mw + nr_lut_mw; 179} 180 181static int lut_index(struct switchtec_ntb *sndev, int mw_idx) 182{ 183 return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts; 184} 185 186static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx) 187{ 188 return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts; 189} 190 191static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, 192 int widx, resource_size_t *addr_align, 193 resource_size_t *size_align, 194 resource_size_t *size_max) 195{ 196 struct switchtec_ntb *sndev = ntb_sndev(ntb); 197 int lut; 198 resource_size_t size; 199 200 if (pidx != NTB_DEF_PEER_IDX) 201 return -EINVAL; 202 203 lut = widx >= sndev->peer_nr_direct_mw; 204 size = ioread64(&sndev->peer_shared->mw_sizes[widx]); 205 206 if (size == 0) 207 return -EINVAL; 208 209 if (addr_align) 210 *addr_align = lut ? size : SZ_4K; 211 212 if (size_align) 213 *size_align = lut ? size : SZ_4K; 214 215 if (size_max) 216 *size_max = size; 217 218 return 0; 219} 220 221static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx) 222{ 223 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 224 int bar = sndev->peer_direct_mw_to_bar[idx]; 225 u32 ctl_val; 226 227 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 228 ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN; 229 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 230 iowrite32(0, &ctl->bar_entry[bar].win_size); 231 iowrite32(0, &ctl->bar_ext_entry[bar].win_size); 232 iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr); 233} 234 235static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx) 236{ 237 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 238 239 iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]); 240} 241 242static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx, 243 dma_addr_t addr, resource_size_t size) 244{ 245 int xlate_pos = ilog2(size); 246 int bar = sndev->peer_direct_mw_to_bar[idx]; 247 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 248 u32 ctl_val; 249 250 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 251 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; 252 253 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 254 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000), 255 &ctl->bar_entry[bar].win_size); 256 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size); 257 iowrite64(sndev->self_partition | addr, 258 &ctl->bar_entry[bar].xlate_addr); 259} 260 261static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx, 262 dma_addr_t addr, resource_size_t size) 263{ 264 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 265 266 iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr), 267 &ctl->lut_entry[peer_lut_index(sndev, idx)]); 268} 269 270static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, 271 dma_addr_t addr, resource_size_t size) 272{ 273 struct switchtec_ntb *sndev = ntb_sndev(ntb); 274 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 275 int xlate_pos = ilog2(size); 276 int nr_direct_mw = sndev->peer_nr_direct_mw; 277 int rc; 278 279 if (pidx != NTB_DEF_PEER_IDX) 280 return -EINVAL; 281 282 dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n", 283 widx, pidx, &addr, &size); 284 285 if (widx >= switchtec_ntb_mw_count(ntb, pidx)) 286 return -EINVAL; 287 288 if (size != 0 && xlate_pos < 12) 289 return -EINVAL; 290 291 if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) { 292 /* 293 * In certain circumstances we can get a buffer that is 294 * not aligned to its size. (Most of the time 295 * dma_alloc_coherent ensures this). This can happen when 296 * using large buffers allocated by the CMA 297 * (see CMA_CONFIG_ALIGNMENT) 298 */ 299 dev_err(&sndev->stdev->dev, 300 "ERROR: Memory window address is not aligned to it's size!\n"); 301 return -EINVAL; 302 } 303 304 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 305 NTB_CTRL_PART_STATUS_LOCKED); 306 if (rc) 307 return rc; 308 309 if (size == 0) { 310 if (widx < nr_direct_mw) 311 switchtec_ntb_mw_clr_direct(sndev, widx); 312 else 313 switchtec_ntb_mw_clr_lut(sndev, widx); 314 } else { 315 if (widx < nr_direct_mw) 316 switchtec_ntb_mw_set_direct(sndev, widx, addr, size); 317 else 318 switchtec_ntb_mw_set_lut(sndev, widx, addr, size); 319 } 320 321 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 322 NTB_CTRL_PART_STATUS_NORMAL); 323 324 if (rc == -EIO) { 325 dev_err(&sndev->stdev->dev, 326 "Hardware reported an error configuring mw %d: %08x\n", 327 widx, ioread32(&ctl->bar_error)); 328 329 if (widx < nr_direct_mw) 330 switchtec_ntb_mw_clr_direct(sndev, widx); 331 else 332 switchtec_ntb_mw_clr_lut(sndev, widx); 333 334 switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 335 NTB_CTRL_PART_STATUS_NORMAL); 336 } 337 338 return rc; 339} 340 341static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb) 342{ 343 struct switchtec_ntb *sndev = ntb_sndev(ntb); 344 int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts; 345 346 return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0); 347} 348 349static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev, 350 int idx, phys_addr_t *base, 351 resource_size_t *size) 352{ 353 int bar = sndev->direct_mw_to_bar[idx]; 354 size_t offset = 0; 355 356 if (bar < 0) 357 return -EINVAL; 358 359 if (idx == 0) { 360 /* 361 * This is the direct BAR shared with the LUTs 362 * which means the actual window will be offset 363 * by the size of all the LUT entries. 364 */ 365 366 offset = LUT_SIZE * sndev->nr_lut_mw; 367 } 368 369 if (base) 370 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; 371 372 if (size) { 373 *size = pci_resource_len(sndev->ntb.pdev, bar) - offset; 374 if (offset && *size > offset) 375 *size = offset; 376 377 if (*size > max_mw_size) 378 *size = max_mw_size; 379 } 380 381 return 0; 382} 383 384static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev, 385 int idx, phys_addr_t *base, 386 resource_size_t *size) 387{ 388 int bar = sndev->direct_mw_to_bar[0]; 389 int offset; 390 391 offset = LUT_SIZE * lut_index(sndev, idx); 392 393 if (base) 394 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; 395 396 if (size) 397 *size = LUT_SIZE; 398 399 return 0; 400} 401 402static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 403 phys_addr_t *base, 404 resource_size_t *size) 405{ 406 struct switchtec_ntb *sndev = ntb_sndev(ntb); 407 408 if (idx < sndev->nr_direct_mw) 409 return switchtec_ntb_direct_get_addr(sndev, idx, base, size); 410 else if (idx < switchtec_ntb_peer_mw_count(ntb)) 411 return switchtec_ntb_lut_get_addr(sndev, idx, base, size); 412 else 413 return -EINVAL; 414} 415 416static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev, 417 int partition, 418 enum ntb_speed *speed, 419 enum ntb_width *width) 420{ 421 struct switchtec_dev *stdev = sndev->stdev; 422 423 u32 pff = ioread32(&stdev->mmio_part_cfg[partition].vep_pff_inst_id); 424 u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]); 425 426 if (speed) 427 *speed = (linksta >> 16) & 0xF; 428 429 if (width) 430 *width = (linksta >> 20) & 0x3F; 431} 432 433static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev) 434{ 435 enum ntb_speed self_speed, peer_speed; 436 enum ntb_width self_width, peer_width; 437 438 if (!sndev->link_is_up) { 439 sndev->link_speed = NTB_SPEED_NONE; 440 sndev->link_width = NTB_WIDTH_NONE; 441 return; 442 } 443 444 switchtec_ntb_part_link_speed(sndev, sndev->self_partition, 445 &self_speed, &self_width); 446 switchtec_ntb_part_link_speed(sndev, sndev->peer_partition, 447 &peer_speed, &peer_width); 448 449 sndev->link_speed = min(self_speed, peer_speed); 450 sndev->link_width = min(self_width, peer_width); 451} 452 453static int crosslink_is_enabled(struct switchtec_ntb *sndev) 454{ 455 struct ntb_info_regs __iomem *inf = sndev->mmio_ntb; 456 457 return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled); 458} 459 460static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev) 461{ 462 int i; 463 u32 msg_map = 0; 464 465 if (!crosslink_is_enabled(sndev)) 466 return; 467 468 for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) { 469 int m = i | sndev->self_partition << 2; 470 471 msg_map |= m << i * 8; 472 } 473 474 iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map); 475 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 476 &sndev->mmio_peer_dbmsg->odb_mask); 477} 478 479enum switchtec_msg { 480 LINK_MESSAGE = 0, 481 MSG_LINK_UP = 1, 482 MSG_LINK_DOWN = 2, 483 MSG_CHECK_LINK = 3, 484 MSG_LINK_FORCE_DOWN = 4, 485}; 486 487static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev); 488 489static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev) 490{ 491 int link_sta; 492 int old = sndev->link_is_up; 493 494 link_sta = sndev->self_shared->link_sta; 495 if (link_sta) { 496 u64 peer = ioread64(&sndev->peer_shared->magic); 497 498 if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC) 499 link_sta = peer >> 32; 500 else 501 link_sta = 0; 502 } 503 504 sndev->link_is_up = link_sta; 505 switchtec_ntb_set_link_speed(sndev); 506 507 if (link_sta != old) { 508 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK); 509 ntb_link_event(&sndev->ntb); 510 dev_info(&sndev->stdev->dev, "ntb link %s\n", 511 link_sta ? "up" : "down"); 512 513 if (link_sta) 514 crosslink_init_dbmsgs(sndev); 515 } 516} 517 518static void check_link_status_work(struct work_struct *work) 519{ 520 struct switchtec_ntb *sndev; 521 522 sndev = container_of(work, struct switchtec_ntb, 523 check_link_status_work); 524 525 if (sndev->link_force_down) { 526 sndev->link_force_down = false; 527 switchtec_ntb_reinit_peer(sndev); 528 529 if (sndev->link_is_up) { 530 sndev->link_is_up = 0; 531 ntb_link_event(&sndev->ntb); 532 dev_info(&sndev->stdev->dev, "ntb link forced down\n"); 533 } 534 535 return; 536 } 537 538 switchtec_ntb_link_status_update(sndev); 539} 540 541static void switchtec_ntb_check_link(struct switchtec_ntb *sndev, 542 enum switchtec_msg msg) 543{ 544 if (msg == MSG_LINK_FORCE_DOWN) 545 sndev->link_force_down = true; 546 547 schedule_work(&sndev->check_link_status_work); 548} 549 550static void switchtec_ntb_link_notification(struct switchtec_dev *stdev) 551{ 552 struct switchtec_ntb *sndev = stdev->sndev; 553 554 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 555} 556 557static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb, 558 enum ntb_speed *speed, 559 enum ntb_width *width) 560{ 561 struct switchtec_ntb *sndev = ntb_sndev(ntb); 562 563 if (speed) 564 *speed = sndev->link_speed; 565 if (width) 566 *width = sndev->link_width; 567 568 return sndev->link_is_up; 569} 570 571static int switchtec_ntb_link_enable(struct ntb_dev *ntb, 572 enum ntb_speed max_speed, 573 enum ntb_width max_width) 574{ 575 struct switchtec_ntb *sndev = ntb_sndev(ntb); 576 577 dev_dbg(&sndev->stdev->dev, "enabling link\n"); 578 579 sndev->self_shared->link_sta = 1; 580 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); 581 582 switchtec_ntb_link_status_update(sndev); 583 584 return 0; 585} 586 587static int switchtec_ntb_link_disable(struct ntb_dev *ntb) 588{ 589 struct switchtec_ntb *sndev = ntb_sndev(ntb); 590 591 dev_dbg(&sndev->stdev->dev, "disabling link\n"); 592 593 sndev->self_shared->link_sta = 0; 594 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN); 595 596 switchtec_ntb_link_status_update(sndev); 597 598 return 0; 599} 600 601static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb) 602{ 603 struct switchtec_ntb *sndev = ntb_sndev(ntb); 604 605 return sndev->db_valid_mask; 606} 607 608static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb) 609{ 610 return 1; 611} 612 613static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 614{ 615 struct switchtec_ntb *sndev = ntb_sndev(ntb); 616 617 if (db_vector < 0 || db_vector > 1) 618 return 0; 619 620 return sndev->db_valid_mask; 621} 622 623static u64 switchtec_ntb_db_read(struct ntb_dev *ntb) 624{ 625 u64 ret; 626 struct switchtec_ntb *sndev = ntb_sndev(ntb); 627 628 ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift; 629 630 return ret & sndev->db_valid_mask; 631} 632 633static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 634{ 635 struct switchtec_ntb *sndev = ntb_sndev(ntb); 636 637 iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb); 638 639 return 0; 640} 641 642static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 643{ 644 unsigned long irqflags; 645 struct switchtec_ntb *sndev = ntb_sndev(ntb); 646 647 if (db_bits & ~sndev->db_valid_mask) 648 return -EINVAL; 649 650 spin_lock_irqsave(&sndev->db_mask_lock, irqflags); 651 652 sndev->db_mask |= db_bits << sndev->db_shift; 653 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 654 655 spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); 656 657 return 0; 658} 659 660static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 661{ 662 unsigned long irqflags; 663 struct switchtec_ntb *sndev = ntb_sndev(ntb); 664 665 if (db_bits & ~sndev->db_valid_mask) 666 return -EINVAL; 667 668 spin_lock_irqsave(&sndev->db_mask_lock, irqflags); 669 670 sndev->db_mask &= ~(db_bits << sndev->db_shift); 671 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 672 673 spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); 674 675 return 0; 676} 677 678static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb) 679{ 680 struct switchtec_ntb *sndev = ntb_sndev(ntb); 681 682 return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask; 683} 684 685static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb, 686 phys_addr_t *db_addr, 687 resource_size_t *db_size, 688 u64 *db_data, 689 int db_bit) 690{ 691 struct switchtec_ntb *sndev = ntb_sndev(ntb); 692 unsigned long offset; 693 694 if (unlikely(db_bit >= BITS_PER_LONG_LONG)) 695 return -EINVAL; 696 697 offset = (unsigned long)sndev->mmio_peer_dbmsg->odb - 698 (unsigned long)sndev->stdev->mmio; 699 700 offset += sndev->db_shift / 8; 701 702 if (db_addr) 703 *db_addr = pci_resource_start(ntb->pdev, 0) + offset; 704 if (db_size) 705 *db_size = sizeof(u32); 706 if (db_data) 707 *db_data = BIT_ULL(db_bit) << sndev->db_peer_shift; 708 709 return 0; 710} 711 712static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 713{ 714 struct switchtec_ntb *sndev = ntb_sndev(ntb); 715 716 iowrite64(db_bits << sndev->db_peer_shift, 717 &sndev->mmio_peer_dbmsg->odb); 718 719 return 0; 720} 721 722static int switchtec_ntb_spad_count(struct ntb_dev *ntb) 723{ 724 struct switchtec_ntb *sndev = ntb_sndev(ntb); 725 726 return ARRAY_SIZE(sndev->self_shared->spad); 727} 728 729static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx) 730{ 731 struct switchtec_ntb *sndev = ntb_sndev(ntb); 732 733 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) 734 return 0; 735 736 if (!sndev->self_shared) 737 return 0; 738 739 return sndev->self_shared->spad[idx]; 740} 741 742static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val) 743{ 744 struct switchtec_ntb *sndev = ntb_sndev(ntb); 745 746 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) 747 return -EINVAL; 748 749 if (!sndev->self_shared) 750 return -EIO; 751 752 sndev->self_shared->spad[idx] = val; 753 754 return 0; 755} 756 757static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, 758 int sidx) 759{ 760 struct switchtec_ntb *sndev = ntb_sndev(ntb); 761 762 if (pidx != NTB_DEF_PEER_IDX) 763 return -EINVAL; 764 765 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) 766 return 0; 767 768 if (!sndev->peer_shared) 769 return 0; 770 771 return ioread32(&sndev->peer_shared->spad[sidx]); 772} 773 774static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, 775 int sidx, u32 val) 776{ 777 struct switchtec_ntb *sndev = ntb_sndev(ntb); 778 779 if (pidx != NTB_DEF_PEER_IDX) 780 return -EINVAL; 781 782 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) 783 return -EINVAL; 784 785 if (!sndev->peer_shared) 786 return -EIO; 787 788 iowrite32(val, &sndev->peer_shared->spad[sidx]); 789 790 return 0; 791} 792 793static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, 794 int sidx, phys_addr_t *spad_addr) 795{ 796 struct switchtec_ntb *sndev = ntb_sndev(ntb); 797 unsigned long offset; 798 799 if (pidx != NTB_DEF_PEER_IDX) 800 return -EINVAL; 801 802 offset = (unsigned long)&sndev->peer_shared->spad[sidx] - 803 (unsigned long)sndev->stdev->mmio; 804 805 if (spad_addr) 806 *spad_addr = pci_resource_start(ntb->pdev, 0) + offset; 807 808 return 0; 809} 810 811static const struct ntb_dev_ops switchtec_ntb_ops = { 812 .mw_count = switchtec_ntb_mw_count, 813 .mw_get_align = switchtec_ntb_mw_get_align, 814 .mw_set_trans = switchtec_ntb_mw_set_trans, 815 .peer_mw_count = switchtec_ntb_peer_mw_count, 816 .peer_mw_get_addr = switchtec_ntb_peer_mw_get_addr, 817 .link_is_up = switchtec_ntb_link_is_up, 818 .link_enable = switchtec_ntb_link_enable, 819 .link_disable = switchtec_ntb_link_disable, 820 .db_valid_mask = switchtec_ntb_db_valid_mask, 821 .db_vector_count = switchtec_ntb_db_vector_count, 822 .db_vector_mask = switchtec_ntb_db_vector_mask, 823 .db_read = switchtec_ntb_db_read, 824 .db_clear = switchtec_ntb_db_clear, 825 .db_set_mask = switchtec_ntb_db_set_mask, 826 .db_clear_mask = switchtec_ntb_db_clear_mask, 827 .db_read_mask = switchtec_ntb_db_read_mask, 828 .peer_db_addr = switchtec_ntb_peer_db_addr, 829 .peer_db_set = switchtec_ntb_peer_db_set, 830 .spad_count = switchtec_ntb_spad_count, 831 .spad_read = switchtec_ntb_spad_read, 832 .spad_write = switchtec_ntb_spad_write, 833 .peer_spad_read = switchtec_ntb_peer_spad_read, 834 .peer_spad_write = switchtec_ntb_peer_spad_write, 835 .peer_spad_addr = switchtec_ntb_peer_spad_addr, 836}; 837 838static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev) 839{ 840 u64 tpart_vec; 841 int self; 842 u64 part_map; 843 int bit; 844 845 sndev->ntb.pdev = sndev->stdev->pdev; 846 sndev->ntb.topo = NTB_TOPO_SWITCH; 847 sndev->ntb.ops = &switchtec_ntb_ops; 848 849 INIT_WORK(&sndev->check_link_status_work, check_link_status_work); 850 sndev->link_force_down = false; 851 852 sndev->self_partition = sndev->stdev->partition; 853 854 sndev->mmio_ntb = sndev->stdev->mmio_ntb; 855 856 self = sndev->self_partition; 857 tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high); 858 tpart_vec <<= 32; 859 tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low); 860 861 part_map = ioread64(&sndev->mmio_ntb->ep_map); 862 part_map &= ~(1 << sndev->self_partition); 863 864 if (!ffs(tpart_vec)) { 865 if (sndev->stdev->partition_count != 2) { 866 dev_err(&sndev->stdev->dev, 867 "ntb target partition not defined\n"); 868 return -ENODEV; 869 } 870 871 bit = ffs(part_map); 872 if (!bit) { 873 dev_err(&sndev->stdev->dev, 874 "peer partition is not NT partition\n"); 875 return -ENODEV; 876 } 877 878 sndev->peer_partition = bit - 1; 879 } else { 880 if (ffs(tpart_vec) != fls(tpart_vec)) { 881 dev_err(&sndev->stdev->dev, 882 "ntb driver only supports 1 pair of 1-1 ntb mapping\n"); 883 return -ENODEV; 884 } 885 886 sndev->peer_partition = ffs(tpart_vec) - 1; 887 if (!(part_map & (1ULL << sndev->peer_partition))) { 888 dev_err(&sndev->stdev->dev, 889 "ntb target partition is not NT partition\n"); 890 return -ENODEV; 891 } 892 } 893 894 dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n", 895 sndev->self_partition, sndev->stdev->partition_count); 896 897 sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb + 898 SWITCHTEC_NTB_REG_CTRL_OFFSET; 899 sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb + 900 SWITCHTEC_NTB_REG_DBMSG_OFFSET; 901 902 sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition]; 903 sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition]; 904 sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition]; 905 sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg; 906 907 return 0; 908} 909 910static int config_rsvd_lut_win(struct switchtec_ntb *sndev, 911 struct ntb_ctrl_regs __iomem *ctl, 912 int lut_idx, int partition, u64 addr) 913{ 914 int peer_bar = sndev->peer_direct_mw_to_bar[0]; 915 u32 ctl_val; 916 int rc; 917 918 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 919 NTB_CTRL_PART_STATUS_LOCKED); 920 if (rc) 921 return rc; 922 923 ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl); 924 ctl_val &= 0xFF; 925 ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN; 926 ctl_val |= ilog2(LUT_SIZE) << 8; 927 ctl_val |= (sndev->nr_lut_mw - 1) << 14; 928 iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl); 929 930 iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr), 931 &ctl->lut_entry[lut_idx]); 932 933 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 934 NTB_CTRL_PART_STATUS_NORMAL); 935 if (rc) { 936 u32 bar_error, lut_error; 937 938 bar_error = ioread32(&ctl->bar_error); 939 lut_error = ioread32(&ctl->lut_error); 940 dev_err(&sndev->stdev->dev, 941 "Error setting up reserved lut window: %08x / %08x\n", 942 bar_error, lut_error); 943 return rc; 944 } 945 946 return 0; 947} 948 949static int config_req_id_table(struct switchtec_ntb *sndev, 950 struct ntb_ctrl_regs __iomem *mmio_ctrl, 951 int *req_ids, int count) 952{ 953 int i, rc = 0; 954 u32 error; 955 u32 proxy_id; 956 957 if (ioread32(&mmio_ctrl->req_id_table_size) < count) { 958 dev_err(&sndev->stdev->dev, 959 "Not enough requester IDs available.\n"); 960 return -EFAULT; 961 } 962 963 rc = switchtec_ntb_part_op(sndev, mmio_ctrl, 964 NTB_CTRL_PART_OP_LOCK, 965 NTB_CTRL_PART_STATUS_LOCKED); 966 if (rc) 967 return rc; 968 969 iowrite32(NTB_PART_CTRL_ID_PROT_DIS, 970 &mmio_ctrl->partition_ctrl); 971 972 for (i = 0; i < count; i++) { 973 iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN, 974 &mmio_ctrl->req_id_table[i]); 975 976 proxy_id = ioread32(&mmio_ctrl->req_id_table[i]); 977 dev_dbg(&sndev->stdev->dev, 978 "Requester ID %02X:%02X.%X -> BB:%02X.%X\n", 979 req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F, 980 req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F, 981 (proxy_id >> 1) & 0x7); 982 } 983 984 rc = switchtec_ntb_part_op(sndev, mmio_ctrl, 985 NTB_CTRL_PART_OP_CFG, 986 NTB_CTRL_PART_STATUS_NORMAL); 987 988 if (rc == -EIO) { 989 error = ioread32(&mmio_ctrl->req_id_error); 990 dev_err(&sndev->stdev->dev, 991 "Error setting up the requester ID table: %08x\n", 992 error); 993 } 994 995 return 0; 996} 997 998static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx, 999 u64 *mw_addrs, int mw_count) 1000{ 1001 int rc, i; 1002 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl; 1003 u64 addr; 1004 size_t size, offset; 1005 int bar; 1006 int xlate_pos; 1007 u32 ctl_val; 1008 1009 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 1010 NTB_CTRL_PART_STATUS_LOCKED); 1011 if (rc) 1012 return rc; 1013 1014 for (i = 0; i < sndev->nr_lut_mw; i++) { 1015 if (i == ntb_lut_idx) 1016 continue; 1017 1018 addr = mw_addrs[0] + LUT_SIZE * i; 1019 1020 iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) | 1021 addr), 1022 &ctl->lut_entry[i]); 1023 } 1024 1025 sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count); 1026 1027 for (i = 0; i < sndev->nr_direct_mw; i++) { 1028 bar = sndev->direct_mw_to_bar[i]; 1029 offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0; 1030 addr = mw_addrs[i] + offset; 1031 size = pci_resource_len(sndev->ntb.pdev, bar) - offset; 1032 xlate_pos = ilog2(size); 1033 1034 if (offset && size > offset) 1035 size = offset; 1036 1037 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 1038 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; 1039 1040 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 1041 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000), 1042 &ctl->bar_entry[bar].win_size); 1043 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size); 1044 iowrite64(sndev->peer_partition | addr, 1045 &ctl->bar_entry[bar].xlate_addr); 1046 } 1047 1048 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 1049 NTB_CTRL_PART_STATUS_NORMAL); 1050 if (rc) { 1051 u32 bar_error, lut_error; 1052 1053 bar_error = ioread32(&ctl->bar_error); 1054 lut_error = ioread32(&ctl->lut_error); 1055 dev_err(&sndev->stdev->dev, 1056 "Error setting up cross link windows: %08x / %08x\n", 1057 bar_error, lut_error); 1058 return rc; 1059 } 1060 1061 return 0; 1062} 1063 1064static int crosslink_setup_req_ids(struct switchtec_ntb *sndev, 1065 struct ntb_ctrl_regs __iomem *mmio_ctrl) 1066{ 1067 int req_ids[16]; 1068 int i; 1069 u32 proxy_id; 1070 1071 for (i = 0; i < ARRAY_SIZE(req_ids); i++) { 1072 proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]); 1073 1074 if (!(proxy_id & NTB_CTRL_REQ_ID_EN)) 1075 break; 1076 1077 req_ids[i] = ((proxy_id >> 1) & 0xFF); 1078 } 1079 1080 return config_req_id_table(sndev, mmio_ctrl, req_ids, i); 1081} 1082 1083/* 1084 * In crosslink configuration there is a virtual partition in the 1085 * middle of the two switches. The BARs in this partition have to be 1086 * enumerated and assigned addresses. 1087 */ 1088static int crosslink_enum_partition(struct switchtec_ntb *sndev, 1089 u64 *bar_addrs) 1090{ 1091 struct part_cfg_regs __iomem *part_cfg = 1092 &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition]; 1093 u32 pff = ioread32(&part_cfg->vep_pff_inst_id); 1094 struct pff_csr_regs __iomem *mmio_pff = 1095 &sndev->stdev->mmio_pff_csr[pff]; 1096 const u64 bar_space = 0x1000000000LL; 1097 u64 bar_addr; 1098 int bar_cnt = 0; 1099 int i; 1100 1101 iowrite16(0x6, &mmio_pff->pcicmd); 1102 1103 for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) { 1104 iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]); 1105 bar_addr = ioread64(&mmio_pff->pci_bar64[i]); 1106 bar_addr &= ~0xf; 1107 1108 dev_dbg(&sndev->stdev->dev, 1109 "Crosslink BAR%d addr: %llx\n", 1110 i*2, bar_addr); 1111 1112 if (bar_addr != bar_space * i) 1113 continue; 1114 1115 bar_addrs[bar_cnt++] = bar_addr; 1116 } 1117 1118 return bar_cnt; 1119} 1120 1121static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev) 1122{ 1123 int rc; 1124 int bar = sndev->direct_mw_to_bar[0]; 1125 const int ntb_lut_idx = 1; 1126 u64 bar_addrs[6]; 1127 u64 addr; 1128 int offset; 1129 int bar_cnt; 1130 1131 if (!crosslink_is_enabled(sndev)) 1132 return 0; 1133 1134 dev_info(&sndev->stdev->dev, "Using crosslink configuration\n"); 1135 sndev->ntb.topo = NTB_TOPO_CROSSLINK; 1136 1137 bar_cnt = crosslink_enum_partition(sndev, bar_addrs); 1138 if (bar_cnt < sndev->nr_direct_mw + 1) { 1139 dev_err(&sndev->stdev->dev, 1140 "Error enumerating crosslink partition\n"); 1141 return -EINVAL; 1142 } 1143 1144 addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET + 1145 SWITCHTEC_NTB_REG_DBMSG_OFFSET + 1146 sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition); 1147 1148 offset = addr & (LUT_SIZE - 1); 1149 addr -= offset; 1150 1151 rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx, 1152 sndev->peer_partition, addr); 1153 if (rc) 1154 return rc; 1155 1156 rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1], 1157 bar_cnt - 1); 1158 if (rc) 1159 return rc; 1160 1161 rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl); 1162 if (rc) 1163 return rc; 1164 1165 sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar, 1166 LUT_SIZE, LUT_SIZE); 1167 if (!sndev->mmio_xlink_win) { 1168 rc = -ENOMEM; 1169 return rc; 1170 } 1171 1172 sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset; 1173 sndev->nr_rsvd_luts++; 1174 1175 crosslink_init_dbmsgs(sndev); 1176 1177 return 0; 1178} 1179 1180static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev) 1181{ 1182 if (sndev->mmio_xlink_win) 1183 pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win); 1184} 1185 1186static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl) 1187{ 1188 int i; 1189 int cnt = 0; 1190 1191 for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) { 1192 u32 r = ioread32(&ctrl->bar_entry[i].ctl); 1193 1194 if (r & NTB_CTRL_BAR_VALID) 1195 map[cnt++] = i; 1196 } 1197 1198 return cnt; 1199} 1200 1201static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) 1202{ 1203 sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar, 1204 sndev->mmio_self_ctrl); 1205 1206 sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); 1207 sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); 1208 1209 dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n", 1210 sndev->nr_direct_mw, sndev->nr_lut_mw); 1211 1212 sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar, 1213 sndev->mmio_peer_ctrl); 1214 1215 sndev->peer_nr_lut_mw = 1216 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); 1217 sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); 1218 1219 dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n", 1220 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); 1221 1222} 1223 1224/* 1225 * There are 64 doorbells in the switch hardware but this is 1226 * shared among all partitions. So we must split them in half 1227 * (32 for each partition). However, the message interrupts are 1228 * also shared with the top 4 doorbells so we just limit this to 1229 * 28 doorbells per partition. 1230 * 1231 * In crosslink mode, each side has it's own dbmsg register so 1232 * they can each use all 60 of the available doorbells. 1233 */ 1234static void switchtec_ntb_init_db(struct switchtec_ntb *sndev) 1235{ 1236 sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL; 1237 1238 if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) { 1239 sndev->db_shift = 0; 1240 sndev->db_peer_shift = 0; 1241 sndev->db_valid_mask = sndev->db_mask; 1242 } else if (sndev->self_partition < sndev->peer_partition) { 1243 sndev->db_shift = 0; 1244 sndev->db_peer_shift = 32; 1245 sndev->db_valid_mask = 0x0FFFFFFF; 1246 } else { 1247 sndev->db_shift = 32; 1248 sndev->db_peer_shift = 0; 1249 sndev->db_valid_mask = 0x0FFFFFFF; 1250 } 1251 1252 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 1253 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 1254 &sndev->mmio_peer_dbmsg->odb_mask); 1255 1256 dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n", 1257 sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask); 1258} 1259 1260static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev) 1261{ 1262 int i; 1263 u32 msg_map = 0; 1264 1265 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 1266 int m = i | sndev->peer_partition << 2; 1267 1268 msg_map |= m << i * 8; 1269 } 1270 1271 iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map); 1272 1273 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) 1274 iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK, 1275 &sndev->mmio_self_dbmsg->imsg[i]); 1276} 1277 1278static int 1279switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev) 1280{ 1281 int req_ids[2]; 1282 1283 /* 1284 * Root Complex Requester ID (which is 0:00.0) 1285 */ 1286 req_ids[0] = 0; 1287 1288 /* 1289 * Host Bridge Requester ID (as read from the mmap address) 1290 */ 1291 req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id); 1292 1293 return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids, 1294 ARRAY_SIZE(req_ids)); 1295} 1296 1297static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) 1298{ 1299 int i; 1300 1301 memset(sndev->self_shared, 0, LUT_SIZE); 1302 sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC; 1303 sndev->self_shared->partition_id = sndev->stdev->partition; 1304 1305 for (i = 0; i < sndev->nr_direct_mw; i++) { 1306 int bar = sndev->direct_mw_to_bar[i]; 1307 resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar); 1308 1309 if (i == 0) 1310 sz = min_t(resource_size_t, sz, 1311 LUT_SIZE * sndev->nr_lut_mw); 1312 1313 sndev->self_shared->mw_sizes[i] = sz; 1314 } 1315 1316 for (i = 0; i < sndev->nr_lut_mw; i++) { 1317 int idx = sndev->nr_direct_mw + i; 1318 1319 sndev->self_shared->mw_sizes[idx] = LUT_SIZE; 1320 } 1321} 1322 1323static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev) 1324{ 1325 int self_bar = sndev->direct_mw_to_bar[0]; 1326 int rc; 1327 1328 sndev->nr_rsvd_luts++; 1329 sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev, 1330 LUT_SIZE, 1331 &sndev->self_shared_dma, 1332 GFP_KERNEL); 1333 if (!sndev->self_shared) { 1334 dev_err(&sndev->stdev->dev, 1335 "unable to allocate memory for shared mw\n"); 1336 return -ENOMEM; 1337 } 1338 1339 switchtec_ntb_init_shared(sndev); 1340 1341 rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0, 1342 sndev->self_partition, 1343 sndev->self_shared_dma); 1344 if (rc) 1345 goto unalloc_and_exit; 1346 1347 sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE); 1348 if (!sndev->peer_shared) { 1349 rc = -ENOMEM; 1350 goto unalloc_and_exit; 1351 } 1352 1353 dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n"); 1354 return 0; 1355 1356unalloc_and_exit: 1357 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1358 sndev->self_shared, sndev->self_shared_dma); 1359 1360 return rc; 1361} 1362 1363static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev) 1364{ 1365 if (sndev->peer_shared) 1366 pci_iounmap(sndev->stdev->pdev, sndev->peer_shared); 1367 1368 if (sndev->self_shared) 1369 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1370 sndev->self_shared, 1371 sndev->self_shared_dma); 1372 sndev->nr_rsvd_luts--; 1373} 1374 1375static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev) 1376{ 1377 struct switchtec_ntb *sndev = dev; 1378 1379 dev_dbg(&sndev->stdev->dev, "doorbell\n"); 1380 1381 ntb_db_event(&sndev->ntb, 0); 1382 1383 return IRQ_HANDLED; 1384} 1385 1386static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev) 1387{ 1388 int i; 1389 struct switchtec_ntb *sndev = dev; 1390 1391 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 1392 u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]); 1393 1394 if (msg & NTB_DBMSG_IMSG_STATUS) { 1395 dev_dbg(&sndev->stdev->dev, "message: %d %08x\n", 1396 i, (u32)msg); 1397 iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status); 1398 1399 if (i == LINK_MESSAGE) 1400 switchtec_ntb_check_link(sndev, msg); 1401 } 1402 } 1403 1404 return IRQ_HANDLED; 1405} 1406 1407static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev) 1408{ 1409 int i; 1410 int rc; 1411 int doorbell_irq = 0; 1412 int message_irq = 0; 1413 int event_irq; 1414 int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map); 1415 1416 event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number); 1417 1418 while (doorbell_irq == event_irq) 1419 doorbell_irq++; 1420 while (message_irq == doorbell_irq || 1421 message_irq == event_irq) 1422 message_irq++; 1423 1424 dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n", 1425 event_irq, doorbell_irq, message_irq); 1426 1427 for (i = 0; i < idb_vecs - 4; i++) 1428 iowrite8(doorbell_irq, 1429 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 1430 1431 for (; i < idb_vecs; i++) 1432 iowrite8(message_irq, 1433 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 1434 1435 sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq); 1436 sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq); 1437 1438 rc = request_irq(sndev->doorbell_irq, 1439 switchtec_ntb_doorbell_isr, 0, 1440 "switchtec_ntb_doorbell", sndev); 1441 if (rc) 1442 return rc; 1443 1444 rc = request_irq(sndev->message_irq, 1445 switchtec_ntb_message_isr, 0, 1446 "switchtec_ntb_message", sndev); 1447 if (rc) { 1448 free_irq(sndev->doorbell_irq, sndev); 1449 return rc; 1450 } 1451 1452 return 0; 1453} 1454 1455static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev) 1456{ 1457 free_irq(sndev->doorbell_irq, sndev); 1458 free_irq(sndev->message_irq, sndev); 1459} 1460 1461static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev) 1462{ 1463 int rc; 1464 1465 if (crosslink_is_enabled(sndev)) 1466 return 0; 1467 1468 dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n"); 1469 rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0, 1470 sndev->self_partition, 1471 sndev->self_shared_dma); 1472 return rc; 1473} 1474 1475static int switchtec_ntb_add(struct device *dev, 1476 struct class_interface *class_intf) 1477{ 1478 struct switchtec_dev *stdev = to_stdev(dev); 1479 struct switchtec_ntb *sndev; 1480 int rc; 1481 1482 stdev->sndev = NULL; 1483 1484 if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8)) 1485 return -ENODEV; 1486 1487 sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev)); 1488 if (!sndev) 1489 return -ENOMEM; 1490 1491 sndev->stdev = stdev; 1492 rc = switchtec_ntb_init_sndev(sndev); 1493 if (rc) 1494 goto free_and_exit; 1495 1496 switchtec_ntb_init_mw(sndev); 1497 1498 rc = switchtec_ntb_init_req_id_table(sndev); 1499 if (rc) 1500 goto free_and_exit; 1501 1502 rc = switchtec_ntb_init_crosslink(sndev); 1503 if (rc) 1504 goto free_and_exit; 1505 1506 switchtec_ntb_init_db(sndev); 1507 switchtec_ntb_init_msgs(sndev); 1508 1509 rc = switchtec_ntb_init_shared_mw(sndev); 1510 if (rc) 1511 goto deinit_crosslink; 1512 1513 rc = switchtec_ntb_init_db_msg_irq(sndev); 1514 if (rc) 1515 goto deinit_shared_and_exit; 1516 1517 /* 1518 * If this host crashed, the other host may think the link is 1519 * still up. Tell them to force it down (it will go back up 1520 * once we register the ntb device). 1521 */ 1522 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN); 1523 1524 rc = ntb_register_device(&sndev->ntb); 1525 if (rc) 1526 goto deinit_and_exit; 1527 1528 stdev->sndev = sndev; 1529 stdev->link_notifier = switchtec_ntb_link_notification; 1530 dev_info(dev, "NTB device registered\n"); 1531 1532 return 0; 1533 1534deinit_and_exit: 1535 switchtec_ntb_deinit_db_msg_irq(sndev); 1536deinit_shared_and_exit: 1537 switchtec_ntb_deinit_shared_mw(sndev); 1538deinit_crosslink: 1539 switchtec_ntb_deinit_crosslink(sndev); 1540free_and_exit: 1541 kfree(sndev); 1542 dev_err(dev, "failed to register ntb device: %d\n", rc); 1543 return rc; 1544} 1545 1546static void switchtec_ntb_remove(struct device *dev, 1547 struct class_interface *class_intf) 1548{ 1549 struct switchtec_dev *stdev = to_stdev(dev); 1550 struct switchtec_ntb *sndev = stdev->sndev; 1551 1552 if (!sndev) 1553 return; 1554 1555 stdev->link_notifier = NULL; 1556 stdev->sndev = NULL; 1557 ntb_unregister_device(&sndev->ntb); 1558 switchtec_ntb_deinit_db_msg_irq(sndev); 1559 switchtec_ntb_deinit_shared_mw(sndev); 1560 switchtec_ntb_deinit_crosslink(sndev); 1561 kfree(sndev); 1562 dev_info(dev, "ntb device unregistered\n"); 1563} 1564 1565static struct class_interface switchtec_interface = { 1566 .add_dev = switchtec_ntb_add, 1567 .remove_dev = switchtec_ntb_remove, 1568}; 1569 1570static int __init switchtec_ntb_init(void) 1571{ 1572 switchtec_interface.class = switchtec_class; 1573 return class_interface_register(&switchtec_interface); 1574} 1575module_init(switchtec_ntb_init); 1576 1577static void __exit switchtec_ntb_exit(void) 1578{ 1579 class_interface_unregister(&switchtec_interface); 1580} 1581module_exit(switchtec_ntb_exit); 1582