1// SPDX-License-Identifier: GPL-2.0 2// Copyright (c) 2017 Cadence 3// Cadence PCIe host controller driver. 4// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com> 5 6#include <linux/delay.h> 7#include <linux/kernel.h> 8#include <linux/list_sort.h> 9#include <linux/of_address.h> 10#include <linux/of_pci.h> 11#include <linux/platform_device.h> 12 13#include "pcie-cadence.h" 14 15#define LINK_RETRAIN_TIMEOUT HZ 16 17static u64 bar_max_size[] = { 18 [RP_BAR0] = _ULL(128 * SZ_2G), 19 [RP_BAR1] = SZ_2G, 20 [RP_NO_BAR] = _BITULL(63), 21}; 22 23static u8 bar_aperture_mask[] = { 24 [RP_BAR0] = 0x1F, 25 [RP_BAR1] = 0xF, 26}; 27 28void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn, 29 int where) 30{ 31 struct pci_host_bridge *bridge = pci_find_host_bridge(bus); 32 struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge); 33 struct cdns_pcie *pcie = &rc->pcie; 34 unsigned int busn = bus->number; 35 u32 addr0, desc0; 36 37 if (pci_is_root_bus(bus)) { 38 /* 39 * Only the root port (devfn == 0) is connected to this bus. 40 * All other PCI devices are behind some bridge hence on another 41 * bus. 42 */ 43 if (devfn) 44 return NULL; 45 46 return pcie->reg_base + (where & 0xfff); 47 } 48 /* Check that the link is up */ 49 if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1)) 50 return NULL; 51 /* Clear AXI link-down status */ 52 cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0); 53 54 /* Update Output registers for AXI region 0. */ 55 addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) | 56 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) | 57 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn); 58 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0); 59 60 /* Configuration Type 0 or Type 1 access. */ 61 desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID | 62 CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0); 63 /* 64 * The bus number was already set once for all in desc1 by 65 * cdns_pcie_host_init_address_translation(). 66 */ 67 if (busn == bridge->busnr + 1) 68 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0; 69 else 70 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1; 71 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0); 72 73 return rc->cfg_base + (where & 0xfff); 74} 75 76static struct pci_ops cdns_pcie_host_ops = { 77 .map_bus = cdns_pci_map_bus, 78 .read = pci_generic_config_read, 79 .write = pci_generic_config_write, 80}; 81 82static int cdns_pcie_host_training_complete(struct cdns_pcie *pcie) 83{ 84 u32 pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET; 85 unsigned long end_jiffies; 86 u16 lnk_stat; 87 88 /* Wait for link training to complete. Exit after timeout. */ 89 end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT; 90 do { 91 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA); 92 if (!(lnk_stat & PCI_EXP_LNKSTA_LT)) 93 break; 94 usleep_range(0, 1000); 95 } while (time_before(jiffies, end_jiffies)); 96 97 if (!(lnk_stat & PCI_EXP_LNKSTA_LT)) 98 return 0; 99 100 return -ETIMEDOUT; 101} 102 103static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie) 104{ 105 struct device *dev = pcie->dev; 106 int retries; 107 108 /* Check if the link is up or not */ 109 for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) { 110 if (cdns_pcie_link_up(pcie)) { 111 dev_info(dev, "Link up\n"); 112 return 0; 113 } 114 usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX); 115 } 116 117 return -ETIMEDOUT; 118} 119 120static int cdns_pcie_retrain(struct cdns_pcie *pcie) 121{ 122 u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET; 123 u16 lnk_stat, lnk_ctl; 124 int ret = 0; 125 126 /* 127 * Set retrain bit if current speed is 2.5 GB/s, 128 * but the PCIe root port support is > 2.5 GB/s. 129 */ 130 131 lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off + 132 PCI_EXP_LNKCAP)); 133 if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB) 134 return ret; 135 136 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA); 137 if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) { 138 lnk_ctl = cdns_pcie_rp_readw(pcie, 139 pcie_cap_off + PCI_EXP_LNKCTL); 140 lnk_ctl |= PCI_EXP_LNKCTL_RL; 141 cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL, 142 lnk_ctl); 143 144 ret = cdns_pcie_host_training_complete(pcie); 145 if (ret) 146 return ret; 147 148 ret = cdns_pcie_host_wait_for_link(pcie); 149 } 150 return ret; 151} 152 153static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc) 154{ 155 struct cdns_pcie *pcie = &rc->pcie; 156 int ret; 157 158 ret = cdns_pcie_host_wait_for_link(pcie); 159 160 /* 161 * Retrain link for Gen2 training defect 162 * if quirk flag is set. 163 */ 164 if (!ret && rc->quirk_retrain_flag) 165 ret = cdns_pcie_retrain(pcie); 166 167 return ret; 168} 169 170static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc) 171{ 172 struct cdns_pcie *pcie = &rc->pcie; 173 u32 value, ctrl; 174 u32 id; 175 176 /* 177 * Set the root complex BAR configuration register: 178 * - disable both BAR0 and BAR1. 179 * - enable Prefetchable Memory Base and Limit registers in type 1 180 * config space (64 bits). 181 * - enable IO Base and Limit registers in type 1 config 182 * space (32 bits). 183 */ 184 ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED; 185 value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) | 186 CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) | 187 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE | 188 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS | 189 CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE | 190 CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS; 191 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value); 192 193 /* Set root port configuration space */ 194 if (rc->vendor_id != 0xffff) { 195 id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) | 196 CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id); 197 cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id); 198 } 199 200 if (rc->device_id != 0xffff) 201 cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id); 202 203 cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0); 204 cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0); 205 cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); 206 207 return 0; 208} 209 210static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc, 211 enum cdns_pcie_rp_bar bar, 212 u64 cpu_addr, u64 size, 213 unsigned long flags) 214{ 215 struct cdns_pcie *pcie = &rc->pcie; 216 u32 addr0, addr1, aperture, value; 217 218 if (!rc->avail_ib_bar[bar]) 219 return -EBUSY; 220 221 rc->avail_ib_bar[bar] = false; 222 223 aperture = ilog2(size); 224 addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) | 225 (lower_32_bits(cpu_addr) & GENMASK(31, 8)); 226 addr1 = upper_32_bits(cpu_addr); 227 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0); 228 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1); 229 230 if (bar == RP_NO_BAR) 231 return 0; 232 233 value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG); 234 value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) | 235 LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) | 236 LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) | 237 LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) | 238 LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2)); 239 if (size + cpu_addr >= SZ_4G) { 240 if (!(flags & IORESOURCE_PREFETCH)) 241 value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar); 242 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar); 243 } else { 244 if (!(flags & IORESOURCE_PREFETCH)) 245 value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar); 246 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar); 247 } 248 249 value |= LM_RC_BAR_CFG_APERTURE(bar, aperture); 250 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value); 251 252 return 0; 253} 254 255static enum cdns_pcie_rp_bar 256cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size) 257{ 258 enum cdns_pcie_rp_bar bar, sel_bar; 259 260 sel_bar = RP_BAR_UNDEFINED; 261 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) { 262 if (!rc->avail_ib_bar[bar]) 263 continue; 264 265 if (size <= bar_max_size[bar]) { 266 if (sel_bar == RP_BAR_UNDEFINED) { 267 sel_bar = bar; 268 continue; 269 } 270 271 if (bar_max_size[bar] < bar_max_size[sel_bar]) 272 sel_bar = bar; 273 } 274 } 275 276 return sel_bar; 277} 278 279static enum cdns_pcie_rp_bar 280cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size) 281{ 282 enum cdns_pcie_rp_bar bar, sel_bar; 283 284 sel_bar = RP_BAR_UNDEFINED; 285 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) { 286 if (!rc->avail_ib_bar[bar]) 287 continue; 288 289 if (size >= bar_max_size[bar]) { 290 if (sel_bar == RP_BAR_UNDEFINED) { 291 sel_bar = bar; 292 continue; 293 } 294 295 if (bar_max_size[bar] > bar_max_size[sel_bar]) 296 sel_bar = bar; 297 } 298 } 299 300 return sel_bar; 301} 302 303static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc, 304 struct resource_entry *entry) 305{ 306 u64 cpu_addr, pci_addr, size, winsize; 307 struct cdns_pcie *pcie = &rc->pcie; 308 struct device *dev = pcie->dev; 309 enum cdns_pcie_rp_bar bar; 310 unsigned long flags; 311 int ret; 312 313 cpu_addr = entry->res->start; 314 pci_addr = entry->res->start - entry->offset; 315 flags = entry->res->flags; 316 size = resource_size(entry->res); 317 318 if (entry->offset) { 319 dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n", 320 pci_addr, cpu_addr); 321 return -EINVAL; 322 } 323 324 while (size > 0) { 325 /* 326 * Try to find a minimum BAR whose size is greater than 327 * or equal to the remaining resource_entry size. This will 328 * fail if the size of each of the available BARs is less than 329 * the remaining resource_entry size. 330 * If a minimum BAR is found, IB ATU will be configured and 331 * exited. 332 */ 333 bar = cdns_pcie_host_find_min_bar(rc, size); 334 if (bar != RP_BAR_UNDEFINED) { 335 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, 336 size, flags); 337 if (ret) 338 dev_err(dev, "IB BAR: %d config failed\n", bar); 339 return ret; 340 } 341 342 /* 343 * If the control reaches here, it would mean the remaining 344 * resource_entry size cannot be fitted in a single BAR. So we 345 * find a maximum BAR whose size is less than or equal to the 346 * remaining resource_entry size and split the resource entry 347 * so that part of resource entry is fitted inside the maximum 348 * BAR. The remaining size would be fitted during the next 349 * iteration of the loop. 350 * If a maximum BAR is not found, there is no way we can fit 351 * this resource_entry, so we error out. 352 */ 353 bar = cdns_pcie_host_find_max_bar(rc, size); 354 if (bar == RP_BAR_UNDEFINED) { 355 dev_err(dev, "No free BAR to map cpu_addr %llx\n", 356 cpu_addr); 357 return -EINVAL; 358 } 359 360 winsize = bar_max_size[bar]; 361 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize, 362 flags); 363 if (ret) { 364 dev_err(dev, "IB BAR: %d config failed\n", bar); 365 return ret; 366 } 367 368 size -= winsize; 369 cpu_addr += winsize; 370 } 371 372 return 0; 373} 374 375static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a, 376 const struct list_head *b) 377{ 378 struct resource_entry *entry1, *entry2; 379 380 entry1 = container_of(a, struct resource_entry, node); 381 entry2 = container_of(b, struct resource_entry, node); 382 383 return resource_size(entry2->res) - resource_size(entry1->res); 384} 385 386static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc) 387{ 388 struct cdns_pcie *pcie = &rc->pcie; 389 struct device *dev = pcie->dev; 390 struct device_node *np = dev->of_node; 391 struct pci_host_bridge *bridge; 392 struct resource_entry *entry; 393 u32 no_bar_nbits = 32; 394 int err; 395 396 bridge = pci_host_bridge_from_priv(rc); 397 if (!bridge) 398 return -ENOMEM; 399 400 if (list_empty(&bridge->dma_ranges)) { 401 of_property_read_u32(np, "cdns,no-bar-match-nbits", 402 &no_bar_nbits); 403 err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0, 404 (u64)1 << no_bar_nbits, 0); 405 if (err) 406 dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR); 407 return err; 408 } 409 410 list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp); 411 412 resource_list_for_each_entry(entry, &bridge->dma_ranges) { 413 err = cdns_pcie_host_bar_config(rc, entry); 414 if (err) { 415 dev_err(dev, "Fail to configure IB using dma-ranges\n"); 416 return err; 417 } 418 } 419 420 return 0; 421} 422 423static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc) 424{ 425 struct cdns_pcie *pcie = &rc->pcie; 426 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc); 427 struct resource *cfg_res = rc->cfg_res; 428 struct resource_entry *entry; 429 u64 cpu_addr = cfg_res->start; 430 u32 addr0, addr1, desc1; 431 int r, busnr = 0; 432 433 entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); 434 if (entry) 435 busnr = entry->res->start; 436 437 /* 438 * Reserve region 0 for PCI configure space accesses: 439 * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by 440 * cdns_pci_map_bus(), other region registers are set here once for all. 441 */ 442 addr1 = 0; /* Should be programmed to zero. */ 443 desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr); 444 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1); 445 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1); 446 447 if (pcie->ops->cpu_addr_fixup) 448 cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr); 449 450 addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) | 451 (lower_32_bits(cpu_addr) & GENMASK(31, 8)); 452 addr1 = upper_32_bits(cpu_addr); 453 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0); 454 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1); 455 456 r = 1; 457 resource_list_for_each_entry(entry, &bridge->windows) { 458 struct resource *res = entry->res; 459 u64 pci_addr = res->start - entry->offset; 460 461 if (resource_type(res) == IORESOURCE_IO) 462 cdns_pcie_set_outbound_region(pcie, busnr, 0, r, 463 true, 464 pci_pio_to_address(res->start), 465 pci_addr, 466 resource_size(res)); 467 else 468 cdns_pcie_set_outbound_region(pcie, busnr, 0, r, 469 false, 470 res->start, 471 pci_addr, 472 resource_size(res)); 473 474 r++; 475 } 476 477 return cdns_pcie_host_map_dma_ranges(rc); 478} 479 480static int cdns_pcie_host_init(struct device *dev, 481 struct cdns_pcie_rc *rc) 482{ 483 int err; 484 485 err = cdns_pcie_host_init_root_port(rc); 486 if (err) 487 return err; 488 489 return cdns_pcie_host_init_address_translation(rc); 490} 491 492int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) 493{ 494 struct device *dev = rc->pcie.dev; 495 struct platform_device *pdev = to_platform_device(dev); 496 struct device_node *np = dev->of_node; 497 struct pci_host_bridge *bridge; 498 enum cdns_pcie_rp_bar bar; 499 struct cdns_pcie *pcie; 500 struct resource *res; 501 int ret; 502 503 bridge = pci_host_bridge_from_priv(rc); 504 if (!bridge) 505 return -ENOMEM; 506 507 pcie = &rc->pcie; 508 pcie->is_rc = true; 509 510 rc->vendor_id = 0xffff; 511 of_property_read_u32(np, "vendor-id", &rc->vendor_id); 512 513 rc->device_id = 0xffff; 514 of_property_read_u32(np, "device-id", &rc->device_id); 515 516 pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg"); 517 if (IS_ERR(pcie->reg_base)) { 518 dev_err(dev, "missing \"reg\"\n"); 519 return PTR_ERR(pcie->reg_base); 520 } 521 522 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); 523 rc->cfg_base = devm_pci_remap_cfg_resource(dev, res); 524 if (IS_ERR(rc->cfg_base)) 525 return PTR_ERR(rc->cfg_base); 526 rc->cfg_res = res; 527 528 if (rc->quirk_detect_quiet_flag) 529 cdns_pcie_detect_quiet_min_delay_set(&rc->pcie); 530 531 ret = cdns_pcie_start_link(pcie); 532 if (ret) { 533 dev_err(dev, "Failed to start link\n"); 534 return ret; 535 } 536 537 ret = cdns_pcie_host_start_link(rc); 538 if (ret) 539 dev_dbg(dev, "PCIe link never came up\n"); 540 541 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) 542 rc->avail_ib_bar[bar] = true; 543 544 ret = cdns_pcie_host_init(dev, rc); 545 if (ret) 546 return ret; 547 548 if (!bridge->ops) 549 bridge->ops = &cdns_pcie_host_ops; 550 551 ret = pci_host_probe(bridge); 552 if (ret < 0) 553 goto err_init; 554 555 return 0; 556 557 err_init: 558 pm_runtime_put_sync(dev); 559 560 return ret; 561} 562