1/* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 8 * Copyright (C) 2016 T-Platforms. All Rights Reserved. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * BSD LICENSE 15 * 16 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 17 * Copyright (C) 2016 T-Platforms. All Rights Reserved. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 23 * * Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * * Redistributions in binary form must reproduce the above copy 26 * notice, this list of conditions and the following disclaimer in 27 * the documentation and/or other materials provided with the 28 * distribution. 29 * * Neither the name of AMD Corporation nor the names of its 30 * contributors may be used to endorse or promote products derived 31 * from this software without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 44 * 45 * AMD PCIe NTB Linux driver 46 * 47 * Contact Information: 48 * Xiangliang Yu <Xiangliang.Yu@amd.com> 49 */ 50 51#include <linux/debugfs.h> 52#include <linux/delay.h> 53#include <linux/init.h> 54#include <linux/interrupt.h> 55#include <linux/module.h> 56#include <linux/acpi.h> 57#include <linux/pci.h> 58#include <linux/random.h> 59#include <linux/slab.h> 60#include <linux/ntb.h> 61 62#include "ntb_hw_amd.h" 63 64#define NTB_NAME "ntb_hw_amd" 65#define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver" 66#define NTB_VER "1.0" 67 68MODULE_DESCRIPTION(NTB_DESC); 69MODULE_VERSION(NTB_VER); 70MODULE_LICENSE("Dual BSD/GPL"); 71MODULE_AUTHOR("AMD Inc."); 72 73static const struct file_operations amd_ntb_debugfs_info; 74static struct dentry *debugfs_dir; 75 76static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx) 77{ 78 if (idx < 0 || idx > ndev->mw_count) 79 return -EINVAL; 80 81 return ndev->dev_data->mw_idx << idx; 82} 83 84static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx) 85{ 86 if (pidx != NTB_DEF_PEER_IDX) 87 return -EINVAL; 88 89 return ntb_ndev(ntb)->mw_count; 90} 91 92static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, 93 resource_size_t *addr_align, 94 resource_size_t *size_align, 95 resource_size_t *size_max) 96{ 97 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 98 int bar; 99 100 if (pidx != NTB_DEF_PEER_IDX) 101 return -EINVAL; 102 103 bar = ndev_mw_to_bar(ndev, idx); 104 if (bar < 0) 105 return bar; 106 107 if (addr_align) 108 *addr_align = SZ_4K; 109 110 if (size_align) 111 *size_align = 1; 112 113 if (size_max) 114 *size_max = pci_resource_len(ndev->ntb.pdev, bar); 115 116 return 0; 117} 118 119static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, 120 dma_addr_t addr, resource_size_t size) 121{ 122 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 123 unsigned long xlat_reg, limit_reg = 0; 124 resource_size_t mw_size; 125 void __iomem *mmio, *peer_mmio; 126 u64 base_addr, limit, reg_val; 127 int bar; 128 129 if (pidx != NTB_DEF_PEER_IDX) 130 return -EINVAL; 131 132 bar = ndev_mw_to_bar(ndev, idx); 133 if (bar < 0) 134 return bar; 135 136 mw_size = pci_resource_len(ntb->pdev, bar); 137 138 /* make sure the range fits in the usable mw size */ 139 if (size > mw_size) 140 return -EINVAL; 141 142 mmio = ndev->self_mmio; 143 peer_mmio = ndev->peer_mmio; 144 145 base_addr = pci_resource_start(ntb->pdev, bar); 146 147 if (bar != 1) { 148 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2); 149 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2); 150 151 /* Set the limit if supported */ 152 limit = size; 153 154 /* set and verify setting the translation address */ 155 write64(addr, peer_mmio + xlat_reg); 156 reg_val = read64(peer_mmio + xlat_reg); 157 if (reg_val != addr) { 158 write64(0, peer_mmio + xlat_reg); 159 return -EIO; 160 } 161 162 /* set and verify setting the limit */ 163 write64(limit, peer_mmio + limit_reg); 164 reg_val = read64(peer_mmio + limit_reg); 165 if (reg_val != limit) { 166 write64(base_addr, mmio + limit_reg); 167 write64(0, peer_mmio + xlat_reg); 168 return -EIO; 169 } 170 } else { 171 xlat_reg = AMD_BAR1XLAT_OFFSET; 172 limit_reg = AMD_BAR1LMT_OFFSET; 173 174 /* Set the limit if supported */ 175 limit = size; 176 177 /* set and verify setting the translation address */ 178 write64(addr, peer_mmio + xlat_reg); 179 reg_val = read64(peer_mmio + xlat_reg); 180 if (reg_val != addr) { 181 write64(0, peer_mmio + xlat_reg); 182 return -EIO; 183 } 184 185 /* set and verify setting the limit */ 186 writel(limit, peer_mmio + limit_reg); 187 reg_val = readl(peer_mmio + limit_reg); 188 if (reg_val != limit) { 189 writel(base_addr, mmio + limit_reg); 190 writel(0, peer_mmio + xlat_reg); 191 return -EIO; 192 } 193 } 194 195 return 0; 196} 197 198static int amd_ntb_get_link_status(struct amd_ntb_dev *ndev) 199{ 200 struct pci_dev *pdev = NULL; 201 struct pci_dev *pci_swds = NULL; 202 struct pci_dev *pci_swus = NULL; 203 u32 stat; 204 int rc; 205 206 if (ndev->ntb.topo == NTB_TOPO_SEC) { 207 /* Locate the pointer to Downstream Switch for this device */ 208 pci_swds = pci_upstream_bridge(ndev->ntb.pdev); 209 if (pci_swds) { 210 /* 211 * Locate the pointer to Upstream Switch for 212 * the Downstream Switch. 213 */ 214 pci_swus = pci_upstream_bridge(pci_swds); 215 if (pci_swus) { 216 rc = pcie_capability_read_dword(pci_swus, 217 PCI_EXP_LNKCTL, 218 &stat); 219 if (rc) 220 return 0; 221 } else { 222 return 0; 223 } 224 } else { 225 return 0; 226 } 227 } else if (ndev->ntb.topo == NTB_TOPO_PRI) { 228 /* 229 * For NTB primary, we simply read the Link Status and control 230 * register of the NTB device itself. 231 */ 232 pdev = ndev->ntb.pdev; 233 rc = pcie_capability_read_dword(pdev, PCI_EXP_LNKCTL, &stat); 234 if (rc) 235 return 0; 236 } else { 237 /* Catch all for everything else */ 238 return 0; 239 } 240 241 ndev->lnk_sta = stat; 242 243 return 1; 244} 245 246static int amd_link_is_up(struct amd_ntb_dev *ndev) 247{ 248 int ret; 249 250 /* 251 * We consider the link to be up under two conditions: 252 * 253 * - When a link-up event is received. This is indicated by 254 * AMD_LINK_UP_EVENT set in peer_sta. 255 * - When driver on both sides of the link have been loaded. 256 * This is indicated by bit 1 being set in the peer 257 * SIDEINFO register. 258 * 259 * This function should return 1 when the latter of the above 260 * two conditions is true. 261 * 262 * Now consider the sequence of events - Link-Up event occurs, 263 * then the peer side driver loads. In this case, we would have 264 * received LINK_UP event and bit 1 of peer SIDEINFO is also 265 * set. What happens now if the link goes down? Bit 1 of 266 * peer SIDEINFO remains set, but LINK_DOWN bit is set in 267 * peer_sta. So we should return 0 from this function. Not only 268 * that, we clear bit 1 of peer SIDEINFO to 0, since the peer 269 * side driver did not even get a chance to clear it before 270 * the link went down. This can be the case of surprise link 271 * removal. 272 * 273 * LINK_UP event will always occur before the peer side driver 274 * gets loaded the very first time. So there can be a case when 275 * the LINK_UP event has occurred, but the peer side driver hasn't 276 * yet loaded. We return 0 in that case. 277 * 278 * There is also a special case when the primary side driver is 279 * unloaded and then loaded again. Since there is no change in 280 * the status of NTB secondary in this case, there is no Link-Up 281 * or Link-Down notification received. We recognize this condition 282 * with peer_sta being set to 0. 283 * 284 * If bit 1 of peer SIDEINFO register is not set, then we 285 * simply return 0 irrespective of the link up or down status 286 * set in peer_sta. 287 */ 288 ret = amd_poll_link(ndev); 289 if (ret) { 290 /* 291 * We need to check the below only for NTB primary. For NTB 292 * secondary, simply checking the result of PSIDE_INFO 293 * register will suffice. 294 */ 295 if (ndev->ntb.topo == NTB_TOPO_PRI) { 296 if ((ndev->peer_sta & AMD_LINK_UP_EVENT) || 297 (ndev->peer_sta == 0)) 298 return ret; 299 else if (ndev->peer_sta & AMD_LINK_DOWN_EVENT) { 300 /* Clear peer sideinfo register */ 301 amd_clear_side_info_reg(ndev, true); 302 303 return 0; 304 } 305 } else { /* NTB_TOPO_SEC */ 306 return ret; 307 } 308 } 309 310 return 0; 311} 312 313static u64 amd_ntb_link_is_up(struct ntb_dev *ntb, 314 enum ntb_speed *speed, 315 enum ntb_width *width) 316{ 317 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 318 int ret = 0; 319 320 if (amd_link_is_up(ndev)) { 321 if (speed) 322 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); 323 if (width) 324 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); 325 326 dev_dbg(&ntb->pdev->dev, "link is up.\n"); 327 328 ret = 1; 329 } else { 330 if (speed) 331 *speed = NTB_SPEED_NONE; 332 if (width) 333 *width = NTB_WIDTH_NONE; 334 335 dev_dbg(&ntb->pdev->dev, "link is down.\n"); 336 } 337 338 return ret; 339} 340 341static int amd_ntb_link_enable(struct ntb_dev *ntb, 342 enum ntb_speed max_speed, 343 enum ntb_width max_width) 344{ 345 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 346 void __iomem *mmio = ndev->self_mmio; 347 348 /* Enable event interrupt */ 349 ndev->int_mask &= ~AMD_EVENT_INTMASK; 350 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 351 352 if (ndev->ntb.topo == NTB_TOPO_SEC) 353 return -EINVAL; 354 dev_dbg(&ntb->pdev->dev, "Enabling Link.\n"); 355 356 return 0; 357} 358 359static int amd_ntb_link_disable(struct ntb_dev *ntb) 360{ 361 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 362 void __iomem *mmio = ndev->self_mmio; 363 364 /* Disable event interrupt */ 365 ndev->int_mask |= AMD_EVENT_INTMASK; 366 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 367 368 if (ndev->ntb.topo == NTB_TOPO_SEC) 369 return -EINVAL; 370 dev_dbg(&ntb->pdev->dev, "Enabling Link.\n"); 371 372 return 0; 373} 374 375static int amd_ntb_peer_mw_count(struct ntb_dev *ntb) 376{ 377 /* The same as for inbound MWs */ 378 return ntb_ndev(ntb)->mw_count; 379} 380 381static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 382 phys_addr_t *base, resource_size_t *size) 383{ 384 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 385 int bar; 386 387 bar = ndev_mw_to_bar(ndev, idx); 388 if (bar < 0) 389 return bar; 390 391 if (base) 392 *base = pci_resource_start(ndev->ntb.pdev, bar); 393 394 if (size) 395 *size = pci_resource_len(ndev->ntb.pdev, bar); 396 397 return 0; 398} 399 400static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb) 401{ 402 return ntb_ndev(ntb)->db_valid_mask; 403} 404 405static int amd_ntb_db_vector_count(struct ntb_dev *ntb) 406{ 407 return ntb_ndev(ntb)->db_count; 408} 409 410static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 411{ 412 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 413 414 if (db_vector < 0 || db_vector > ndev->db_count) 415 return 0; 416 417 return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector); 418} 419 420static u64 amd_ntb_db_read(struct ntb_dev *ntb) 421{ 422 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 423 void __iomem *mmio = ndev->self_mmio; 424 425 return (u64)readw(mmio + AMD_DBSTAT_OFFSET); 426} 427 428static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 429{ 430 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 431 void __iomem *mmio = ndev->self_mmio; 432 433 writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET); 434 435 return 0; 436} 437 438static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 439{ 440 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 441 void __iomem *mmio = ndev->self_mmio; 442 unsigned long flags; 443 444 if (db_bits & ~ndev->db_valid_mask) 445 return -EINVAL; 446 447 spin_lock_irqsave(&ndev->db_mask_lock, flags); 448 ndev->db_mask |= db_bits; 449 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 450 spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 451 452 return 0; 453} 454 455static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 456{ 457 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 458 void __iomem *mmio = ndev->self_mmio; 459 unsigned long flags; 460 461 if (db_bits & ~ndev->db_valid_mask) 462 return -EINVAL; 463 464 spin_lock_irqsave(&ndev->db_mask_lock, flags); 465 ndev->db_mask &= ~db_bits; 466 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 467 spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 468 469 return 0; 470} 471 472static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 473{ 474 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 475 void __iomem *mmio = ndev->self_mmio; 476 477 writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET); 478 479 return 0; 480} 481 482static int amd_ntb_spad_count(struct ntb_dev *ntb) 483{ 484 return ntb_ndev(ntb)->spad_count; 485} 486 487static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx) 488{ 489 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 490 void __iomem *mmio = ndev->self_mmio; 491 u32 offset; 492 493 if (idx < 0 || idx >= ndev->spad_count) 494 return 0; 495 496 offset = ndev->self_spad + (idx << 2); 497 return readl(mmio + AMD_SPAD_OFFSET + offset); 498} 499 500static int amd_ntb_spad_write(struct ntb_dev *ntb, 501 int idx, u32 val) 502{ 503 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 504 void __iomem *mmio = ndev->self_mmio; 505 u32 offset; 506 507 if (idx < 0 || idx >= ndev->spad_count) 508 return -EINVAL; 509 510 offset = ndev->self_spad + (idx << 2); 511 writel(val, mmio + AMD_SPAD_OFFSET + offset); 512 513 return 0; 514} 515 516static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) 517{ 518 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 519 void __iomem *mmio = ndev->self_mmio; 520 u32 offset; 521 522 if (sidx < 0 || sidx >= ndev->spad_count) 523 return -EINVAL; 524 525 offset = ndev->peer_spad + (sidx << 2); 526 return readl(mmio + AMD_SPAD_OFFSET + offset); 527} 528 529static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, 530 int sidx, u32 val) 531{ 532 struct amd_ntb_dev *ndev = ntb_ndev(ntb); 533 void __iomem *mmio = ndev->self_mmio; 534 u32 offset; 535 536 if (sidx < 0 || sidx >= ndev->spad_count) 537 return -EINVAL; 538 539 offset = ndev->peer_spad + (sidx << 2); 540 writel(val, mmio + AMD_SPAD_OFFSET + offset); 541 542 return 0; 543} 544 545static const struct ntb_dev_ops amd_ntb_ops = { 546 .mw_count = amd_ntb_mw_count, 547 .mw_get_align = amd_ntb_mw_get_align, 548 .mw_set_trans = amd_ntb_mw_set_trans, 549 .peer_mw_count = amd_ntb_peer_mw_count, 550 .peer_mw_get_addr = amd_ntb_peer_mw_get_addr, 551 .link_is_up = amd_ntb_link_is_up, 552 .link_enable = amd_ntb_link_enable, 553 .link_disable = amd_ntb_link_disable, 554 .db_valid_mask = amd_ntb_db_valid_mask, 555 .db_vector_count = amd_ntb_db_vector_count, 556 .db_vector_mask = amd_ntb_db_vector_mask, 557 .db_read = amd_ntb_db_read, 558 .db_clear = amd_ntb_db_clear, 559 .db_set_mask = amd_ntb_db_set_mask, 560 .db_clear_mask = amd_ntb_db_clear_mask, 561 .peer_db_set = amd_ntb_peer_db_set, 562 .spad_count = amd_ntb_spad_count, 563 .spad_read = amd_ntb_spad_read, 564 .spad_write = amd_ntb_spad_write, 565 .peer_spad_read = amd_ntb_peer_spad_read, 566 .peer_spad_write = amd_ntb_peer_spad_write, 567}; 568 569static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit) 570{ 571 void __iomem *mmio = ndev->self_mmio; 572 int reg; 573 574 reg = readl(mmio + AMD_SMUACK_OFFSET); 575 reg |= bit; 576 writel(reg, mmio + AMD_SMUACK_OFFSET); 577} 578 579static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) 580{ 581 void __iomem *mmio = ndev->self_mmio; 582 struct device *dev = &ndev->ntb.pdev->dev; 583 u32 status; 584 585 status = readl(mmio + AMD_INTSTAT_OFFSET); 586 if (!(status & AMD_EVENT_INTMASK)) 587 return; 588 589 dev_dbg(dev, "status = 0x%x and vec = %d\n", status, vec); 590 591 status &= AMD_EVENT_INTMASK; 592 switch (status) { 593 case AMD_PEER_FLUSH_EVENT: 594 ndev->peer_sta |= AMD_PEER_FLUSH_EVENT; 595 dev_info(dev, "Flush is done.\n"); 596 break; 597 case AMD_PEER_RESET_EVENT: 598 case AMD_LINK_DOWN_EVENT: 599 ndev->peer_sta |= status; 600 if (status == AMD_LINK_DOWN_EVENT) 601 ndev->peer_sta &= ~AMD_LINK_UP_EVENT; 602 603 amd_ack_smu(ndev, status); 604 605 /* link down first */ 606 ntb_link_event(&ndev->ntb); 607 /* polling peer status */ 608 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 609 610 break; 611 case AMD_PEER_D3_EVENT: 612 case AMD_PEER_PMETO_EVENT: 613 case AMD_LINK_UP_EVENT: 614 ndev->peer_sta |= status; 615 if (status == AMD_LINK_UP_EVENT) 616 ndev->peer_sta &= ~AMD_LINK_DOWN_EVENT; 617 else if (status == AMD_PEER_D3_EVENT) 618 ndev->peer_sta &= ~AMD_PEER_D0_EVENT; 619 620 amd_ack_smu(ndev, status); 621 622 /* link down */ 623 ntb_link_event(&ndev->ntb); 624 625 break; 626 case AMD_PEER_D0_EVENT: 627 mmio = ndev->peer_mmio; 628 status = readl(mmio + AMD_PMESTAT_OFFSET); 629 /* check if this is WAKEUP event */ 630 if (status & 0x1) 631 dev_info(dev, "Wakeup is done.\n"); 632 633 ndev->peer_sta |= AMD_PEER_D0_EVENT; 634 ndev->peer_sta &= ~AMD_PEER_D3_EVENT; 635 amd_ack_smu(ndev, AMD_PEER_D0_EVENT); 636 637 /* start a timer to poll link status */ 638 schedule_delayed_work(&ndev->hb_timer, 639 AMD_LINK_HB_TIMEOUT); 640 break; 641 default: 642 dev_info(dev, "event status = 0x%x.\n", status); 643 break; 644 } 645 646 /* Clear the interrupt status */ 647 writel(status, mmio + AMD_INTSTAT_OFFSET); 648} 649 650static void amd_handle_db_event(struct amd_ntb_dev *ndev, int vec) 651{ 652 struct device *dev = &ndev->ntb.pdev->dev; 653 u64 status; 654 655 status = amd_ntb_db_read(&ndev->ntb); 656 657 dev_dbg(dev, "status = 0x%llx and vec = %d\n", status, vec); 658 659 /* 660 * Since we had reserved highest order bit of DB for signaling peer of 661 * a special event, this is the only status bit we should be concerned 662 * here now. 663 */ 664 if (status & BIT(ndev->db_last_bit)) { 665 ntb_db_clear(&ndev->ntb, BIT(ndev->db_last_bit)); 666 /* send link down event notification */ 667 ntb_link_event(&ndev->ntb); 668 669 /* 670 * If we are here, that means the peer has signalled a special 671 * event which notifies that the peer driver has been 672 * un-loaded for some reason. Since there is a chance that the 673 * peer will load its driver again sometime, we schedule link 674 * polling routine. 675 */ 676 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 677 } 678} 679 680static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec) 681{ 682 dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec); 683 684 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1)) 685 amd_handle_event(ndev, vec); 686 687 if (vec < AMD_DB_CNT) { 688 amd_handle_db_event(ndev, vec); 689 ntb_db_event(&ndev->ntb, vec); 690 } 691 692 return IRQ_HANDLED; 693} 694 695static irqreturn_t ndev_vec_isr(int irq, void *dev) 696{ 697 struct amd_ntb_vec *nvec = dev; 698 699 return ndev_interrupt(nvec->ndev, nvec->num); 700} 701 702static irqreturn_t ndev_irq_isr(int irq, void *dev) 703{ 704 struct amd_ntb_dev *ndev = dev; 705 706 return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq); 707} 708 709static int ndev_init_isr(struct amd_ntb_dev *ndev, 710 int msix_min, int msix_max) 711{ 712 struct pci_dev *pdev; 713 int rc, i, msix_count, node; 714 715 pdev = ndev->ntb.pdev; 716 717 node = dev_to_node(&pdev->dev); 718 719 ndev->db_mask = ndev->db_valid_mask; 720 721 /* Try to set up msix irq */ 722 ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec), 723 GFP_KERNEL, node); 724 if (!ndev->vec) 725 goto err_msix_vec_alloc; 726 727 ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix), 728 GFP_KERNEL, node); 729 if (!ndev->msix) 730 goto err_msix_alloc; 731 732 for (i = 0; i < msix_max; ++i) 733 ndev->msix[i].entry = i; 734 735 msix_count = pci_enable_msix_range(pdev, ndev->msix, 736 msix_min, msix_max); 737 if (msix_count < 0) 738 goto err_msix_enable; 739 740 /* NOTE: Disable MSIX if msix count is less than 16 because of 741 * hardware limitation. 742 */ 743 if (msix_count < msix_min) { 744 pci_disable_msix(pdev); 745 goto err_msix_enable; 746 } 747 748 for (i = 0; i < msix_count; ++i) { 749 ndev->vec[i].ndev = ndev; 750 ndev->vec[i].num = i; 751 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, 752 "ndev_vec_isr", &ndev->vec[i]); 753 if (rc) 754 goto err_msix_request; 755 } 756 757 dev_dbg(&pdev->dev, "Using msix interrupts\n"); 758 ndev->db_count = msix_min; 759 ndev->msix_vec_count = msix_max; 760 return 0; 761 762err_msix_request: 763 while (i-- > 0) 764 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 765 pci_disable_msix(pdev); 766err_msix_enable: 767 kfree(ndev->msix); 768err_msix_alloc: 769 kfree(ndev->vec); 770err_msix_vec_alloc: 771 ndev->msix = NULL; 772 ndev->vec = NULL; 773 774 /* Try to set up msi irq */ 775 rc = pci_enable_msi(pdev); 776 if (rc) 777 goto err_msi_enable; 778 779 rc = request_irq(pdev->irq, ndev_irq_isr, 0, 780 "ndev_irq_isr", ndev); 781 if (rc) 782 goto err_msi_request; 783 784 dev_dbg(&pdev->dev, "Using msi interrupts\n"); 785 ndev->db_count = 1; 786 ndev->msix_vec_count = 1; 787 return 0; 788 789err_msi_request: 790 pci_disable_msi(pdev); 791err_msi_enable: 792 793 /* Try to set up intx irq */ 794 pci_intx(pdev, 1); 795 796 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, 797 "ndev_irq_isr", ndev); 798 if (rc) 799 goto err_intx_request; 800 801 dev_dbg(&pdev->dev, "Using intx interrupts\n"); 802 ndev->db_count = 1; 803 ndev->msix_vec_count = 1; 804 return 0; 805 806err_intx_request: 807 return rc; 808} 809 810static void ndev_deinit_isr(struct amd_ntb_dev *ndev) 811{ 812 struct pci_dev *pdev; 813 void __iomem *mmio = ndev->self_mmio; 814 int i; 815 816 pdev = ndev->ntb.pdev; 817 818 /* Mask all doorbell interrupts */ 819 ndev->db_mask = ndev->db_valid_mask; 820 writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 821 822 if (ndev->msix) { 823 i = ndev->msix_vec_count; 824 while (i--) 825 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 826 pci_disable_msix(pdev); 827 kfree(ndev->msix); 828 kfree(ndev->vec); 829 } else { 830 free_irq(pdev->irq, ndev); 831 if (pci_dev_msi_enabled(pdev)) 832 pci_disable_msi(pdev); 833 else 834 pci_intx(pdev, 0); 835 } 836} 837 838static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, 839 size_t count, loff_t *offp) 840{ 841 struct amd_ntb_dev *ndev; 842 void __iomem *mmio; 843 char *buf; 844 size_t buf_size; 845 ssize_t ret, off; 846 union { u64 v64; u32 v32; u16 v16; } u; 847 848 ndev = filp->private_data; 849 mmio = ndev->self_mmio; 850 851 buf_size = min(count, 0x800ul); 852 853 buf = kmalloc(buf_size, GFP_KERNEL); 854 if (!buf) 855 return -ENOMEM; 856 857 off = 0; 858 859 off += scnprintf(buf + off, buf_size - off, 860 "NTB Device Information:\n"); 861 862 off += scnprintf(buf + off, buf_size - off, 863 "Connection Topology -\t%s\n", 864 ntb_topo_string(ndev->ntb.topo)); 865 866 off += scnprintf(buf + off, buf_size - off, 867 "LNK STA -\t\t%#06x\n", ndev->lnk_sta); 868 869 if (!amd_link_is_up(ndev)) { 870 off += scnprintf(buf + off, buf_size - off, 871 "Link Status -\t\tDown\n"); 872 } else { 873 off += scnprintf(buf + off, buf_size - off, 874 "Link Status -\t\tUp\n"); 875 off += scnprintf(buf + off, buf_size - off, 876 "Link Speed -\t\tPCI-E Gen %u\n", 877 NTB_LNK_STA_SPEED(ndev->lnk_sta)); 878 off += scnprintf(buf + off, buf_size - off, 879 "Link Width -\t\tx%u\n", 880 NTB_LNK_STA_WIDTH(ndev->lnk_sta)); 881 } 882 883 off += scnprintf(buf + off, buf_size - off, 884 "Memory Window Count -\t%u\n", ndev->mw_count); 885 off += scnprintf(buf + off, buf_size - off, 886 "Scratchpad Count -\t%u\n", ndev->spad_count); 887 off += scnprintf(buf + off, buf_size - off, 888 "Doorbell Count -\t%u\n", ndev->db_count); 889 off += scnprintf(buf + off, buf_size - off, 890 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count); 891 892 off += scnprintf(buf + off, buf_size - off, 893 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); 894 895 u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET); 896 off += scnprintf(buf + off, buf_size - off, 897 "Doorbell Mask -\t\t\t%#06x\n", u.v32); 898 899 u.v32 = readl(mmio + AMD_DBSTAT_OFFSET); 900 off += scnprintf(buf + off, buf_size - off, 901 "Doorbell Bell -\t\t\t%#06x\n", u.v32); 902 903 off += scnprintf(buf + off, buf_size - off, 904 "\nNTB Incoming XLAT:\n"); 905 906 u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET); 907 off += scnprintf(buf + off, buf_size - off, 908 "XLAT1 -\t\t%#018llx\n", u.v64); 909 910 u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET); 911 off += scnprintf(buf + off, buf_size - off, 912 "XLAT23 -\t\t%#018llx\n", u.v64); 913 914 u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET); 915 off += scnprintf(buf + off, buf_size - off, 916 "XLAT45 -\t\t%#018llx\n", u.v64); 917 918 u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET); 919 off += scnprintf(buf + off, buf_size - off, 920 "LMT1 -\t\t\t%#06x\n", u.v32); 921 922 u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET); 923 off += scnprintf(buf + off, buf_size - off, 924 "LMT23 -\t\t\t%#018llx\n", u.v64); 925 926 u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET); 927 off += scnprintf(buf + off, buf_size - off, 928 "LMT45 -\t\t\t%#018llx\n", u.v64); 929 930 ret = simple_read_from_buffer(ubuf, count, offp, buf, off); 931 kfree(buf); 932 return ret; 933} 934 935static void ndev_init_debugfs(struct amd_ntb_dev *ndev) 936{ 937 if (!debugfs_dir) { 938 ndev->debugfs_dir = NULL; 939 ndev->debugfs_info = NULL; 940 } else { 941 ndev->debugfs_dir = 942 debugfs_create_dir(pci_name(ndev->ntb.pdev), 943 debugfs_dir); 944 ndev->debugfs_info = 945 debugfs_create_file("info", S_IRUSR, 946 ndev->debugfs_dir, ndev, 947 &amd_ntb_debugfs_info); 948 } 949} 950 951static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev) 952{ 953 debugfs_remove_recursive(ndev->debugfs_dir); 954} 955 956static inline void ndev_init_struct(struct amd_ntb_dev *ndev, 957 struct pci_dev *pdev) 958{ 959 ndev->ntb.pdev = pdev; 960 ndev->ntb.topo = NTB_TOPO_NONE; 961 ndev->ntb.ops = &amd_ntb_ops; 962 ndev->int_mask = AMD_EVENT_INTMASK; 963 spin_lock_init(&ndev->db_mask_lock); 964} 965 966static int amd_poll_link(struct amd_ntb_dev *ndev) 967{ 968 void __iomem *mmio = ndev->peer_mmio; 969 u32 reg; 970 971 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 972 reg &= AMD_SIDE_READY; 973 974 dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg); 975 976 ndev->cntl_sta = reg; 977 978 amd_ntb_get_link_status(ndev); 979 980 return ndev->cntl_sta; 981} 982 983static void amd_link_hb(struct work_struct *work) 984{ 985 struct amd_ntb_dev *ndev = hb_ndev(work); 986 987 if (amd_poll_link(ndev)) 988 ntb_link_event(&ndev->ntb); 989 990 if (!amd_link_is_up(ndev)) 991 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 992} 993 994static int amd_init_isr(struct amd_ntb_dev *ndev) 995{ 996 return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT); 997} 998 999static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer) 1000{ 1001 void __iomem *mmio = NULL; 1002 unsigned int reg; 1003 1004 if (peer) 1005 mmio = ndev->peer_mmio; 1006 else 1007 mmio = ndev->self_mmio; 1008 1009 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 1010 if (!(reg & AMD_SIDE_READY)) { 1011 reg |= AMD_SIDE_READY; 1012 writel(reg, mmio + AMD_SIDEINFO_OFFSET); 1013 } 1014} 1015 1016static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer) 1017{ 1018 void __iomem *mmio = NULL; 1019 unsigned int reg; 1020 1021 if (peer) 1022 mmio = ndev->peer_mmio; 1023 else 1024 mmio = ndev->self_mmio; 1025 1026 reg = readl(mmio + AMD_SIDEINFO_OFFSET); 1027 if (reg & AMD_SIDE_READY) { 1028 reg &= ~AMD_SIDE_READY; 1029 writel(reg, mmio + AMD_SIDEINFO_OFFSET); 1030 readl(mmio + AMD_SIDEINFO_OFFSET); 1031 } 1032} 1033 1034static void amd_init_side_info(struct amd_ntb_dev *ndev) 1035{ 1036 void __iomem *mmio = ndev->self_mmio; 1037 u32 ntb_ctl; 1038 1039 amd_set_side_info_reg(ndev, false); 1040 1041 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 1042 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); 1043 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 1044} 1045 1046static void amd_deinit_side_info(struct amd_ntb_dev *ndev) 1047{ 1048 void __iomem *mmio = ndev->self_mmio; 1049 u32 ntb_ctl; 1050 1051 amd_clear_side_info_reg(ndev, false); 1052 1053 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 1054 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); 1055 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 1056} 1057 1058static int amd_init_ntb(struct amd_ntb_dev *ndev) 1059{ 1060 void __iomem *mmio = ndev->self_mmio; 1061 1062 ndev->mw_count = ndev->dev_data->mw_count; 1063 ndev->spad_count = AMD_SPADS_CNT; 1064 ndev->db_count = AMD_DB_CNT; 1065 1066 switch (ndev->ntb.topo) { 1067 case NTB_TOPO_PRI: 1068 case NTB_TOPO_SEC: 1069 ndev->spad_count >>= 1; 1070 if (ndev->ntb.topo == NTB_TOPO_PRI) { 1071 ndev->self_spad = 0; 1072 ndev->peer_spad = 0x20; 1073 } else { 1074 ndev->self_spad = 0x20; 1075 ndev->peer_spad = 0; 1076 } 1077 1078 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb); 1079 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 1080 1081 break; 1082 default: 1083 dev_err(&ndev->ntb.pdev->dev, 1084 "AMD NTB does not support B2B mode.\n"); 1085 return -EINVAL; 1086 } 1087 1088 /* Mask event interrupts */ 1089 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 1090 1091 return 0; 1092} 1093 1094static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev) 1095{ 1096 void __iomem *mmio = ndev->self_mmio; 1097 u32 info; 1098 1099 info = readl(mmio + AMD_SIDEINFO_OFFSET); 1100 if (info & AMD_SIDE_MASK) 1101 return NTB_TOPO_SEC; 1102 else 1103 return NTB_TOPO_PRI; 1104} 1105 1106static int amd_init_dev(struct amd_ntb_dev *ndev) 1107{ 1108 void __iomem *mmio = ndev->self_mmio; 1109 struct pci_dev *pdev; 1110 int rc = 0; 1111 1112 pdev = ndev->ntb.pdev; 1113 1114 ndev->ntb.topo = amd_get_topo(ndev); 1115 dev_dbg(&pdev->dev, "AMD NTB topo is %s\n", 1116 ntb_topo_string(ndev->ntb.topo)); 1117 1118 rc = amd_init_ntb(ndev); 1119 if (rc) 1120 return rc; 1121 1122 rc = amd_init_isr(ndev); 1123 if (rc) { 1124 dev_err(&pdev->dev, "fail to init isr.\n"); 1125 return rc; 1126 } 1127 1128 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 1129 /* 1130 * We reserve the highest order bit of the DB register which will 1131 * be used to notify peer when the driver on this side is being 1132 * un-loaded. 1133 */ 1134 ndev->db_last_bit = 1135 find_last_bit((unsigned long *)&ndev->db_valid_mask, 1136 hweight64(ndev->db_valid_mask)); 1137 writew((u16)~BIT(ndev->db_last_bit), mmio + AMD_DBMASK_OFFSET); 1138 /* 1139 * Since now there is one less bit to account for, the DB count 1140 * and DB mask should be adjusted accordingly. 1141 */ 1142 ndev->db_count -= 1; 1143 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 1144 1145 /* Enable Link-Up and Link-Down event interrupts */ 1146 ndev->int_mask &= ~(AMD_LINK_UP_EVENT | AMD_LINK_DOWN_EVENT); 1147 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 1148 1149 return 0; 1150} 1151 1152static void amd_deinit_dev(struct amd_ntb_dev *ndev) 1153{ 1154 cancel_delayed_work_sync(&ndev->hb_timer); 1155 1156 ndev_deinit_isr(ndev); 1157} 1158 1159static int amd_ntb_init_pci(struct amd_ntb_dev *ndev, 1160 struct pci_dev *pdev) 1161{ 1162 int rc; 1163 1164 pci_set_drvdata(pdev, ndev); 1165 1166 rc = pci_enable_device(pdev); 1167 if (rc) 1168 goto err_pci_enable; 1169 1170 rc = pci_request_regions(pdev, NTB_NAME); 1171 if (rc) 1172 goto err_pci_regions; 1173 1174 pci_set_master(pdev); 1175 1176 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1177 if (rc) { 1178 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1179 if (rc) 1180 goto err_dma_mask; 1181 dev_warn(&pdev->dev, "Cannot DMA highmem\n"); 1182 } 1183 1184 ndev->self_mmio = pci_iomap(pdev, 0, 0); 1185 if (!ndev->self_mmio) { 1186 rc = -EIO; 1187 goto err_dma_mask; 1188 } 1189 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET; 1190 1191 return 0; 1192 1193err_dma_mask: 1194 pci_release_regions(pdev); 1195err_pci_regions: 1196 pci_disable_device(pdev); 1197err_pci_enable: 1198 pci_set_drvdata(pdev, NULL); 1199 return rc; 1200} 1201 1202static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev) 1203{ 1204 struct pci_dev *pdev = ndev->ntb.pdev; 1205 1206 pci_iounmap(pdev, ndev->self_mmio); 1207 1208 pci_release_regions(pdev); 1209 pci_disable_device(pdev); 1210 pci_set_drvdata(pdev, NULL); 1211} 1212 1213static int amd_ntb_pci_probe(struct pci_dev *pdev, 1214 const struct pci_device_id *id) 1215{ 1216 struct amd_ntb_dev *ndev; 1217 int rc, node; 1218 1219 node = dev_to_node(&pdev->dev); 1220 1221 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1222 if (!ndev) { 1223 rc = -ENOMEM; 1224 goto err_ndev; 1225 } 1226 1227 ndev->dev_data = (struct ntb_dev_data *)id->driver_data; 1228 1229 ndev_init_struct(ndev, pdev); 1230 1231 rc = amd_ntb_init_pci(ndev, pdev); 1232 if (rc) 1233 goto err_init_pci; 1234 1235 rc = amd_init_dev(ndev); 1236 if (rc) 1237 goto err_init_dev; 1238 1239 /* write side info */ 1240 amd_init_side_info(ndev); 1241 1242 amd_poll_link(ndev); 1243 1244 ndev_init_debugfs(ndev); 1245 1246 rc = ntb_register_device(&ndev->ntb); 1247 if (rc) 1248 goto err_register; 1249 1250 dev_info(&pdev->dev, "NTB device registered.\n"); 1251 1252 return 0; 1253 1254err_register: 1255 ndev_deinit_debugfs(ndev); 1256 amd_deinit_dev(ndev); 1257err_init_dev: 1258 amd_ntb_deinit_pci(ndev); 1259err_init_pci: 1260 kfree(ndev); 1261err_ndev: 1262 return rc; 1263} 1264 1265static void amd_ntb_pci_remove(struct pci_dev *pdev) 1266{ 1267 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); 1268 1269 /* 1270 * Clear the READY bit in SIDEINFO register before sending DB event 1271 * to the peer. This will make sure that when the peer handles the 1272 * DB event, it correctly reads this bit as being 0. 1273 */ 1274 amd_deinit_side_info(ndev); 1275 ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit)); 1276 ntb_unregister_device(&ndev->ntb); 1277 ndev_deinit_debugfs(ndev); 1278 amd_deinit_dev(ndev); 1279 amd_ntb_deinit_pci(ndev); 1280 kfree(ndev); 1281} 1282 1283static void amd_ntb_pci_shutdown(struct pci_dev *pdev) 1284{ 1285 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); 1286 1287 /* Send link down notification */ 1288 ntb_link_event(&ndev->ntb); 1289 1290 amd_deinit_side_info(ndev); 1291 ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit)); 1292 ntb_unregister_device(&ndev->ntb); 1293 ndev_deinit_debugfs(ndev); 1294 amd_deinit_dev(ndev); 1295 amd_ntb_deinit_pci(ndev); 1296 kfree(ndev); 1297} 1298 1299static const struct file_operations amd_ntb_debugfs_info = { 1300 .owner = THIS_MODULE, 1301 .open = simple_open, 1302 .read = ndev_debugfs_read, 1303}; 1304 1305static const struct ntb_dev_data dev_data[] = { 1306 { /* for device 145b */ 1307 .mw_count = 3, 1308 .mw_idx = 1, 1309 }, 1310 { /* for device 148b */ 1311 .mw_count = 2, 1312 .mw_idx = 2, 1313 }, 1314}; 1315 1316static const struct pci_device_id amd_ntb_pci_tbl[] = { 1317 { PCI_VDEVICE(AMD, 0x145b), (kernel_ulong_t)&dev_data[0] }, 1318 { PCI_VDEVICE(AMD, 0x148b), (kernel_ulong_t)&dev_data[1] }, 1319 { PCI_VDEVICE(AMD, 0x14c0), (kernel_ulong_t)&dev_data[1] }, 1320 { PCI_VDEVICE(AMD, 0x14c3), (kernel_ulong_t)&dev_data[1] }, 1321 { PCI_VDEVICE(HYGON, 0x145b), (kernel_ulong_t)&dev_data[0] }, 1322 { 0, } 1323}; 1324MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl); 1325 1326static struct pci_driver amd_ntb_pci_driver = { 1327 .name = KBUILD_MODNAME, 1328 .id_table = amd_ntb_pci_tbl, 1329 .probe = amd_ntb_pci_probe, 1330 .remove = amd_ntb_pci_remove, 1331 .shutdown = amd_ntb_pci_shutdown, 1332}; 1333 1334static int __init amd_ntb_pci_driver_init(void) 1335{ 1336 int ret; 1337 pr_info("%s %s\n", NTB_DESC, NTB_VER); 1338 1339 if (debugfs_initialized()) 1340 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 1341 1342 ret = pci_register_driver(&amd_ntb_pci_driver); 1343 if (ret) 1344 debugfs_remove_recursive(debugfs_dir); 1345 1346 return ret; 1347} 1348module_init(amd_ntb_pci_driver_init); 1349 1350static void __exit amd_ntb_pci_driver_exit(void) 1351{ 1352 pci_unregister_driver(&amd_ntb_pci_driver); 1353 debugfs_remove_recursive(debugfs_dir); 1354} 1355module_exit(amd_ntb_pci_driver_exit); 1356