1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2014 Hauke Mehrtens <hauke@hauke-m.de> 4 * Copyright (C) 2015 Broadcom Corporation 5 */ 6 7#include <linux/kernel.h> 8#include <linux/pci.h> 9#include <linux/msi.h> 10#include <linux/clk.h> 11#include <linux/module.h> 12#include <linux/mbus.h> 13#include <linux/slab.h> 14#include <linux/delay.h> 15#include <linux/interrupt.h> 16#include <linux/irqchip/arm-gic-v3.h> 17#include <linux/platform_device.h> 18#include <linux/of_address.h> 19#include <linux/of_pci.h> 20#include <linux/of_irq.h> 21#include <linux/of_platform.h> 22#include <linux/phy/phy.h> 23 24#include "pcie-iproc.h" 25 26#define EP_PERST_SOURCE_SELECT_SHIFT 2 27#define EP_PERST_SOURCE_SELECT BIT(EP_PERST_SOURCE_SELECT_SHIFT) 28#define EP_MODE_SURVIVE_PERST_SHIFT 1 29#define EP_MODE_SURVIVE_PERST BIT(EP_MODE_SURVIVE_PERST_SHIFT) 30#define RC_PCIE_RST_OUTPUT_SHIFT 0 31#define RC_PCIE_RST_OUTPUT BIT(RC_PCIE_RST_OUTPUT_SHIFT) 32#define PAXC_RESET_MASK 0x7f 33 34#define GIC_V3_CFG_SHIFT 0 35#define GIC_V3_CFG BIT(GIC_V3_CFG_SHIFT) 36 37#define MSI_ENABLE_CFG_SHIFT 0 38#define MSI_ENABLE_CFG BIT(MSI_ENABLE_CFG_SHIFT) 39 40#define CFG_IND_ADDR_MASK 0x00001ffc 41 42#define CFG_ADDR_BUS_NUM_SHIFT 20 43#define CFG_ADDR_BUS_NUM_MASK 0x0ff00000 44#define CFG_ADDR_DEV_NUM_SHIFT 15 45#define CFG_ADDR_DEV_NUM_MASK 0x000f8000 46#define CFG_ADDR_FUNC_NUM_SHIFT 12 47#define CFG_ADDR_FUNC_NUM_MASK 0x00007000 48#define CFG_ADDR_REG_NUM_SHIFT 2 49#define CFG_ADDR_REG_NUM_MASK 0x00000ffc 50#define CFG_ADDR_CFG_TYPE_SHIFT 0 51#define CFG_ADDR_CFG_TYPE_MASK 0x00000003 52 53#define SYS_RC_INTX_MASK 0xf 54 55#define PCIE_PHYLINKUP_SHIFT 3 56#define PCIE_PHYLINKUP BIT(PCIE_PHYLINKUP_SHIFT) 57#define PCIE_DL_ACTIVE_SHIFT 2 58#define PCIE_DL_ACTIVE BIT(PCIE_DL_ACTIVE_SHIFT) 59 60#define APB_ERR_EN_SHIFT 0 61#define APB_ERR_EN BIT(APB_ERR_EN_SHIFT) 62 63#define CFG_RD_SUCCESS 0 64#define CFG_RD_UR 1 65#define CFG_RD_CRS 2 66#define CFG_RD_CA 3 67#define CFG_RETRY_STATUS 0xffff0001 68#define CFG_RETRY_STATUS_TIMEOUT_US 500000 /* 500 milliseconds */ 69 70/* derive the enum index of the outbound/inbound mapping registers */ 71#define MAP_REG(base_reg, index) ((base_reg) + (index) * 2) 72 73/* 74 * Maximum number of outbound mapping window sizes that can be supported by any 75 * OARR/OMAP mapping pair 76 */ 77#define MAX_NUM_OB_WINDOW_SIZES 4 78 79#define OARR_VALID_SHIFT 0 80#define OARR_VALID BIT(OARR_VALID_SHIFT) 81#define OARR_SIZE_CFG_SHIFT 1 82 83/* 84 * Maximum number of inbound mapping region sizes that can be supported by an 85 * IARR 86 */ 87#define MAX_NUM_IB_REGION_SIZES 9 88 89#define IMAP_VALID_SHIFT 0 90#define IMAP_VALID BIT(IMAP_VALID_SHIFT) 91 92#define IPROC_PCI_PM_CAP 0x48 93#define IPROC_PCI_PM_CAP_MASK 0xffff 94#define IPROC_PCI_EXP_CAP 0xac 95 96#define IPROC_PCIE_REG_INVALID 0xffff 97 98/** 99 * iProc PCIe outbound mapping controller specific parameters 100 * 101 * @window_sizes: list of supported outbound mapping window sizes in MB 102 * @nr_sizes: number of supported outbound mapping window sizes 103 */ 104struct iproc_pcie_ob_map { 105 resource_size_t window_sizes[MAX_NUM_OB_WINDOW_SIZES]; 106 unsigned int nr_sizes; 107}; 108 109static const struct iproc_pcie_ob_map paxb_ob_map[] = { 110 { 111 /* OARR0/OMAP0 */ 112 .window_sizes = { 128, 256 }, 113 .nr_sizes = 2, 114 }, 115 { 116 /* OARR1/OMAP1 */ 117 .window_sizes = { 128, 256 }, 118 .nr_sizes = 2, 119 }, 120}; 121 122static const struct iproc_pcie_ob_map paxb_v2_ob_map[] = { 123 { 124 /* OARR0/OMAP0 */ 125 .window_sizes = { 128, 256 }, 126 .nr_sizes = 2, 127 }, 128 { 129 /* OARR1/OMAP1 */ 130 .window_sizes = { 128, 256 }, 131 .nr_sizes = 2, 132 }, 133 { 134 /* OARR2/OMAP2 */ 135 .window_sizes = { 128, 256, 512, 1024 }, 136 .nr_sizes = 4, 137 }, 138 { 139 /* OARR3/OMAP3 */ 140 .window_sizes = { 128, 256, 512, 1024 }, 141 .nr_sizes = 4, 142 }, 143}; 144 145/** 146 * iProc PCIe inbound mapping type 147 */ 148enum iproc_pcie_ib_map_type { 149 /* for DDR memory */ 150 IPROC_PCIE_IB_MAP_MEM = 0, 151 152 /* for device I/O memory */ 153 IPROC_PCIE_IB_MAP_IO, 154 155 /* invalid or unused */ 156 IPROC_PCIE_IB_MAP_INVALID 157}; 158 159/** 160 * iProc PCIe inbound mapping controller specific parameters 161 * 162 * @type: inbound mapping region type 163 * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or 164 * SZ_1G 165 * @region_sizes: list of supported inbound mapping region sizes in KB, MB, or 166 * GB, depending on the size unit 167 * @nr_sizes: number of supported inbound mapping region sizes 168 * @nr_windows: number of supported inbound mapping windows for the region 169 * @imap_addr_offset: register offset between the upper and lower 32-bit 170 * IMAP address registers 171 * @imap_window_offset: register offset between each IMAP window 172 */ 173struct iproc_pcie_ib_map { 174 enum iproc_pcie_ib_map_type type; 175 unsigned int size_unit; 176 resource_size_t region_sizes[MAX_NUM_IB_REGION_SIZES]; 177 unsigned int nr_sizes; 178 unsigned int nr_windows; 179 u16 imap_addr_offset; 180 u16 imap_window_offset; 181}; 182 183static const struct iproc_pcie_ib_map paxb_v2_ib_map[] = { 184 { 185 /* IARR0/IMAP0 */ 186 .type = IPROC_PCIE_IB_MAP_IO, 187 .size_unit = SZ_1K, 188 .region_sizes = { 32 }, 189 .nr_sizes = 1, 190 .nr_windows = 8, 191 .imap_addr_offset = 0x40, 192 .imap_window_offset = 0x4, 193 }, 194 { 195 /* IARR1/IMAP1 */ 196 .type = IPROC_PCIE_IB_MAP_MEM, 197 .size_unit = SZ_1M, 198 .region_sizes = { 8 }, 199 .nr_sizes = 1, 200 .nr_windows = 8, 201 .imap_addr_offset = 0x4, 202 .imap_window_offset = 0x8, 203 204 }, 205 { 206 /* IARR2/IMAP2 */ 207 .type = IPROC_PCIE_IB_MAP_MEM, 208 .size_unit = SZ_1M, 209 .region_sizes = { 64, 128, 256, 512, 1024, 2048, 4096, 8192, 210 16384 }, 211 .nr_sizes = 9, 212 .nr_windows = 1, 213 .imap_addr_offset = 0x4, 214 .imap_window_offset = 0x8, 215 }, 216 { 217 /* IARR3/IMAP3 */ 218 .type = IPROC_PCIE_IB_MAP_MEM, 219 .size_unit = SZ_1G, 220 .region_sizes = { 1, 2, 4, 8, 16, 32 }, 221 .nr_sizes = 6, 222 .nr_windows = 8, 223 .imap_addr_offset = 0x4, 224 .imap_window_offset = 0x8, 225 }, 226 { 227 /* IARR4/IMAP4 */ 228 .type = IPROC_PCIE_IB_MAP_MEM, 229 .size_unit = SZ_1G, 230 .region_sizes = { 32, 64, 128, 256, 512 }, 231 .nr_sizes = 5, 232 .nr_windows = 8, 233 .imap_addr_offset = 0x4, 234 .imap_window_offset = 0x8, 235 }, 236}; 237 238/* 239 * iProc PCIe host registers 240 */ 241enum iproc_pcie_reg { 242 /* clock/reset signal control */ 243 IPROC_PCIE_CLK_CTRL = 0, 244 245 /* 246 * To allow MSI to be steered to an external MSI controller (e.g., ARM 247 * GICv3 ITS) 248 */ 249 IPROC_PCIE_MSI_GIC_MODE, 250 251 /* 252 * IPROC_PCIE_MSI_BASE_ADDR and IPROC_PCIE_MSI_WINDOW_SIZE define the 253 * window where the MSI posted writes are written, for the writes to be 254 * interpreted as MSI writes. 255 */ 256 IPROC_PCIE_MSI_BASE_ADDR, 257 IPROC_PCIE_MSI_WINDOW_SIZE, 258 259 /* 260 * To hold the address of the register where the MSI writes are 261 * programed. When ARM GICv3 ITS is used, this should be programmed 262 * with the address of the GITS_TRANSLATER register. 263 */ 264 IPROC_PCIE_MSI_ADDR_LO, 265 IPROC_PCIE_MSI_ADDR_HI, 266 267 /* enable MSI */ 268 IPROC_PCIE_MSI_EN_CFG, 269 270 /* allow access to root complex configuration space */ 271 IPROC_PCIE_CFG_IND_ADDR, 272 IPROC_PCIE_CFG_IND_DATA, 273 274 /* allow access to device configuration space */ 275 IPROC_PCIE_CFG_ADDR, 276 IPROC_PCIE_CFG_DATA, 277 278 /* enable INTx */ 279 IPROC_PCIE_INTX_EN, 280 281 /* outbound address mapping */ 282 IPROC_PCIE_OARR0, 283 IPROC_PCIE_OMAP0, 284 IPROC_PCIE_OARR1, 285 IPROC_PCIE_OMAP1, 286 IPROC_PCIE_OARR2, 287 IPROC_PCIE_OMAP2, 288 IPROC_PCIE_OARR3, 289 IPROC_PCIE_OMAP3, 290 291 /* inbound address mapping */ 292 IPROC_PCIE_IARR0, 293 IPROC_PCIE_IMAP0, 294 IPROC_PCIE_IARR1, 295 IPROC_PCIE_IMAP1, 296 IPROC_PCIE_IARR2, 297 IPROC_PCIE_IMAP2, 298 IPROC_PCIE_IARR3, 299 IPROC_PCIE_IMAP3, 300 IPROC_PCIE_IARR4, 301 IPROC_PCIE_IMAP4, 302 303 /* config read status */ 304 IPROC_PCIE_CFG_RD_STATUS, 305 306 /* link status */ 307 IPROC_PCIE_LINK_STATUS, 308 309 /* enable APB error for unsupported requests */ 310 IPROC_PCIE_APB_ERR_EN, 311 312 /* total number of core registers */ 313 IPROC_PCIE_MAX_NUM_REG, 314}; 315 316/* iProc PCIe PAXB BCMA registers */ 317static const u16 iproc_pcie_reg_paxb_bcma[IPROC_PCIE_MAX_NUM_REG] = { 318 [IPROC_PCIE_CLK_CTRL] = 0x000, 319 [IPROC_PCIE_CFG_IND_ADDR] = 0x120, 320 [IPROC_PCIE_CFG_IND_DATA] = 0x124, 321 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 322 [IPROC_PCIE_CFG_DATA] = 0x1fc, 323 [IPROC_PCIE_INTX_EN] = 0x330, 324 [IPROC_PCIE_LINK_STATUS] = 0xf0c, 325}; 326 327/* iProc PCIe PAXB registers */ 328static const u16 iproc_pcie_reg_paxb[IPROC_PCIE_MAX_NUM_REG] = { 329 [IPROC_PCIE_CLK_CTRL] = 0x000, 330 [IPROC_PCIE_CFG_IND_ADDR] = 0x120, 331 [IPROC_PCIE_CFG_IND_DATA] = 0x124, 332 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 333 [IPROC_PCIE_CFG_DATA] = 0x1fc, 334 [IPROC_PCIE_INTX_EN] = 0x330, 335 [IPROC_PCIE_OARR0] = 0xd20, 336 [IPROC_PCIE_OMAP0] = 0xd40, 337 [IPROC_PCIE_OARR1] = 0xd28, 338 [IPROC_PCIE_OMAP1] = 0xd48, 339 [IPROC_PCIE_LINK_STATUS] = 0xf0c, 340 [IPROC_PCIE_APB_ERR_EN] = 0xf40, 341}; 342 343/* iProc PCIe PAXB v2 registers */ 344static const u16 iproc_pcie_reg_paxb_v2[IPROC_PCIE_MAX_NUM_REG] = { 345 [IPROC_PCIE_CLK_CTRL] = 0x000, 346 [IPROC_PCIE_CFG_IND_ADDR] = 0x120, 347 [IPROC_PCIE_CFG_IND_DATA] = 0x124, 348 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 349 [IPROC_PCIE_CFG_DATA] = 0x1fc, 350 [IPROC_PCIE_INTX_EN] = 0x330, 351 [IPROC_PCIE_OARR0] = 0xd20, 352 [IPROC_PCIE_OMAP0] = 0xd40, 353 [IPROC_PCIE_OARR1] = 0xd28, 354 [IPROC_PCIE_OMAP1] = 0xd48, 355 [IPROC_PCIE_OARR2] = 0xd60, 356 [IPROC_PCIE_OMAP2] = 0xd68, 357 [IPROC_PCIE_OARR3] = 0xdf0, 358 [IPROC_PCIE_OMAP3] = 0xdf8, 359 [IPROC_PCIE_IARR0] = 0xd00, 360 [IPROC_PCIE_IMAP0] = 0xc00, 361 [IPROC_PCIE_IARR1] = 0xd08, 362 [IPROC_PCIE_IMAP1] = 0xd70, 363 [IPROC_PCIE_IARR2] = 0xd10, 364 [IPROC_PCIE_IMAP2] = 0xcc0, 365 [IPROC_PCIE_IARR3] = 0xe00, 366 [IPROC_PCIE_IMAP3] = 0xe08, 367 [IPROC_PCIE_IARR4] = 0xe68, 368 [IPROC_PCIE_IMAP4] = 0xe70, 369 [IPROC_PCIE_CFG_RD_STATUS] = 0xee0, 370 [IPROC_PCIE_LINK_STATUS] = 0xf0c, 371 [IPROC_PCIE_APB_ERR_EN] = 0xf40, 372}; 373 374/* iProc PCIe PAXC v1 registers */ 375static const u16 iproc_pcie_reg_paxc[IPROC_PCIE_MAX_NUM_REG] = { 376 [IPROC_PCIE_CLK_CTRL] = 0x000, 377 [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0, 378 [IPROC_PCIE_CFG_IND_DATA] = 0x1f4, 379 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 380 [IPROC_PCIE_CFG_DATA] = 0x1fc, 381}; 382 383/* iProc PCIe PAXC v2 registers */ 384static const u16 iproc_pcie_reg_paxc_v2[IPROC_PCIE_MAX_NUM_REG] = { 385 [IPROC_PCIE_MSI_GIC_MODE] = 0x050, 386 [IPROC_PCIE_MSI_BASE_ADDR] = 0x074, 387 [IPROC_PCIE_MSI_WINDOW_SIZE] = 0x078, 388 [IPROC_PCIE_MSI_ADDR_LO] = 0x07c, 389 [IPROC_PCIE_MSI_ADDR_HI] = 0x080, 390 [IPROC_PCIE_MSI_EN_CFG] = 0x09c, 391 [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0, 392 [IPROC_PCIE_CFG_IND_DATA] = 0x1f4, 393 [IPROC_PCIE_CFG_ADDR] = 0x1f8, 394 [IPROC_PCIE_CFG_DATA] = 0x1fc, 395}; 396 397/* 398 * List of device IDs of controllers that have corrupted capability list that 399 * require SW fixup 400 */ 401static const u16 iproc_pcie_corrupt_cap_did[] = { 402 0x16cd, 403 0x16f0, 404 0xd802, 405 0xd804 406}; 407 408static inline struct iproc_pcie *iproc_data(struct pci_bus *bus) 409{ 410 struct iproc_pcie *pcie = bus->sysdata; 411 return pcie; 412} 413 414static inline bool iproc_pcie_reg_is_invalid(u16 reg_offset) 415{ 416 return !!(reg_offset == IPROC_PCIE_REG_INVALID); 417} 418 419static inline u16 iproc_pcie_reg_offset(struct iproc_pcie *pcie, 420 enum iproc_pcie_reg reg) 421{ 422 return pcie->reg_offsets[reg]; 423} 424 425static inline u32 iproc_pcie_read_reg(struct iproc_pcie *pcie, 426 enum iproc_pcie_reg reg) 427{ 428 u16 offset = iproc_pcie_reg_offset(pcie, reg); 429 430 if (iproc_pcie_reg_is_invalid(offset)) 431 return 0; 432 433 return readl(pcie->base + offset); 434} 435 436static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie, 437 enum iproc_pcie_reg reg, u32 val) 438{ 439 u16 offset = iproc_pcie_reg_offset(pcie, reg); 440 441 if (iproc_pcie_reg_is_invalid(offset)) 442 return; 443 444 writel(val, pcie->base + offset); 445} 446 447/** 448 * APB error forwarding can be disabled during access of configuration 449 * registers of the endpoint device, to prevent unsupported requests 450 * (typically seen during enumeration with multi-function devices) from 451 * triggering a system exception. 452 */ 453static inline void iproc_pcie_apb_err_disable(struct pci_bus *bus, 454 bool disable) 455{ 456 struct iproc_pcie *pcie = iproc_data(bus); 457 u32 val; 458 459 if (bus->number && pcie->has_apb_err_disable) { 460 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_APB_ERR_EN); 461 if (disable) 462 val &= ~APB_ERR_EN; 463 else 464 val |= APB_ERR_EN; 465 iproc_pcie_write_reg(pcie, IPROC_PCIE_APB_ERR_EN, val); 466 } 467} 468 469static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie, 470 unsigned int busno, 471 unsigned int slot, 472 unsigned int fn, 473 int where) 474{ 475 u16 offset; 476 u32 val; 477 478 /* EP device access */ 479 val = (busno << CFG_ADDR_BUS_NUM_SHIFT) | 480 (slot << CFG_ADDR_DEV_NUM_SHIFT) | 481 (fn << CFG_ADDR_FUNC_NUM_SHIFT) | 482 (where & CFG_ADDR_REG_NUM_MASK) | 483 (1 & CFG_ADDR_CFG_TYPE_MASK); 484 485 iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val); 486 offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA); 487 488 if (iproc_pcie_reg_is_invalid(offset)) 489 return NULL; 490 491 return (pcie->base + offset); 492} 493 494static unsigned int iproc_pcie_cfg_retry(struct iproc_pcie *pcie, 495 void __iomem *cfg_data_p) 496{ 497 int timeout = CFG_RETRY_STATUS_TIMEOUT_US; 498 unsigned int data; 499 u32 status; 500 501 /* 502 * As per PCIe spec r3.1, sec 2.3.2, CRS Software Visibility only 503 * affects config reads of the Vendor ID. For config writes or any 504 * other config reads, the Root may automatically reissue the 505 * configuration request again as a new request. 506 * 507 * For config reads, this hardware returns CFG_RETRY_STATUS data 508 * when it receives a CRS completion, regardless of the address of 509 * the read or the CRS Software Visibility Enable bit. As a 510 * partial workaround for this, we retry in software any read that 511 * returns CFG_RETRY_STATUS. 512 * 513 * Note that a non-Vendor ID config register may have a value of 514 * CFG_RETRY_STATUS. If we read that, we can't distinguish it from 515 * a CRS completion, so we will incorrectly retry the read and 516 * eventually return the wrong data (0xffffffff). 517 */ 518 data = readl(cfg_data_p); 519 while (data == CFG_RETRY_STATUS && timeout--) { 520 /* 521 * CRS state is set in CFG_RD status register 522 * This will handle the case where CFG_RETRY_STATUS is 523 * valid config data. 524 */ 525 status = iproc_pcie_read_reg(pcie, IPROC_PCIE_CFG_RD_STATUS); 526 if (status != CFG_RD_CRS) 527 return data; 528 529 udelay(1); 530 data = readl(cfg_data_p); 531 } 532 533 if (data == CFG_RETRY_STATUS) 534 data = 0xffffffff; 535 536 return data; 537} 538 539static void iproc_pcie_fix_cap(struct iproc_pcie *pcie, int where, u32 *val) 540{ 541 u32 i, dev_id; 542 543 switch (where & ~0x3) { 544 case PCI_VENDOR_ID: 545 dev_id = *val >> 16; 546 547 /* 548 * Activate fixup for those controllers that have corrupted 549 * capability list registers 550 */ 551 for (i = 0; i < ARRAY_SIZE(iproc_pcie_corrupt_cap_did); i++) 552 if (dev_id == iproc_pcie_corrupt_cap_did[i]) 553 pcie->fix_paxc_cap = true; 554 break; 555 556 case IPROC_PCI_PM_CAP: 557 if (pcie->fix_paxc_cap) { 558 /* advertise PM, force next capability to PCIe */ 559 *val &= ~IPROC_PCI_PM_CAP_MASK; 560 *val |= IPROC_PCI_EXP_CAP << 8 | PCI_CAP_ID_PM; 561 } 562 break; 563 564 case IPROC_PCI_EXP_CAP: 565 if (pcie->fix_paxc_cap) { 566 /* advertise root port, version 2, terminate here */ 567 *val = (PCI_EXP_TYPE_ROOT_PORT << 4 | 2) << 16 | 568 PCI_CAP_ID_EXP; 569 } 570 break; 571 572 case IPROC_PCI_EXP_CAP + PCI_EXP_RTCTL: 573 /* Don't advertise CRS SV support */ 574 *val &= ~(PCI_EXP_RTCAP_CRSVIS << 16); 575 break; 576 577 default: 578 break; 579 } 580} 581 582static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn, 583 int where, int size, u32 *val) 584{ 585 struct iproc_pcie *pcie = iproc_data(bus); 586 unsigned int slot = PCI_SLOT(devfn); 587 unsigned int fn = PCI_FUNC(devfn); 588 unsigned int busno = bus->number; 589 void __iomem *cfg_data_p; 590 unsigned int data; 591 int ret; 592 593 /* root complex access */ 594 if (busno == 0) { 595 ret = pci_generic_config_read32(bus, devfn, where, size, val); 596 if (ret == PCIBIOS_SUCCESSFUL) 597 iproc_pcie_fix_cap(pcie, where, val); 598 599 return ret; 600 } 601 602 cfg_data_p = iproc_pcie_map_ep_cfg_reg(pcie, busno, slot, fn, where); 603 604 if (!cfg_data_p) 605 return PCIBIOS_DEVICE_NOT_FOUND; 606 607 data = iproc_pcie_cfg_retry(pcie, cfg_data_p); 608 609 *val = data; 610 if (size <= 2) 611 *val = (data >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 612 613 /* 614 * For PAXC and PAXCv2, the total number of PFs that one can enumerate 615 * depends on the firmware configuration. Unfortunately, due to an ASIC 616 * bug, unconfigured PFs cannot be properly hidden from the root 617 * complex. As a result, write access to these PFs will cause bus lock 618 * up on the embedded processor 619 * 620 * Since all unconfigured PFs are left with an incorrect, staled device 621 * ID of 0x168e (PCI_DEVICE_ID_NX2_57810), we try to catch those access 622 * early here and reject them all 623 */ 624#define DEVICE_ID_MASK 0xffff0000 625#define DEVICE_ID_SHIFT 16 626 if (pcie->rej_unconfig_pf && 627 (where & CFG_ADDR_REG_NUM_MASK) == PCI_VENDOR_ID) 628 if ((*val & DEVICE_ID_MASK) == 629 (PCI_DEVICE_ID_NX2_57810 << DEVICE_ID_SHIFT)) 630 return PCIBIOS_FUNC_NOT_SUPPORTED; 631 632 return PCIBIOS_SUCCESSFUL; 633} 634 635/** 636 * Note access to the configuration registers are protected at the higher layer 637 * by 'pci_lock' in drivers/pci/access.c 638 */ 639static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie, 640 int busno, unsigned int devfn, 641 int where) 642{ 643 unsigned slot = PCI_SLOT(devfn); 644 unsigned fn = PCI_FUNC(devfn); 645 u16 offset; 646 647 /* root complex access */ 648 if (busno == 0) { 649 if (slot > 0 || fn > 0) 650 return NULL; 651 652 iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_IND_ADDR, 653 where & CFG_IND_ADDR_MASK); 654 offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_IND_DATA); 655 if (iproc_pcie_reg_is_invalid(offset)) 656 return NULL; 657 else 658 return (pcie->base + offset); 659 } 660 661 return iproc_pcie_map_ep_cfg_reg(pcie, busno, slot, fn, where); 662} 663 664static void __iomem *iproc_pcie_bus_map_cfg_bus(struct pci_bus *bus, 665 unsigned int devfn, 666 int where) 667{ 668 return iproc_pcie_map_cfg_bus(iproc_data(bus), bus->number, devfn, 669 where); 670} 671 672static int iproc_pci_raw_config_read32(struct iproc_pcie *pcie, 673 unsigned int devfn, int where, 674 int size, u32 *val) 675{ 676 void __iomem *addr; 677 678 addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3); 679 if (!addr) { 680 *val = ~0; 681 return PCIBIOS_DEVICE_NOT_FOUND; 682 } 683 684 *val = readl(addr); 685 686 if (size <= 2) 687 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 688 689 return PCIBIOS_SUCCESSFUL; 690} 691 692static int iproc_pci_raw_config_write32(struct iproc_pcie *pcie, 693 unsigned int devfn, int where, 694 int size, u32 val) 695{ 696 void __iomem *addr; 697 u32 mask, tmp; 698 699 addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3); 700 if (!addr) 701 return PCIBIOS_DEVICE_NOT_FOUND; 702 703 if (size == 4) { 704 writel(val, addr); 705 return PCIBIOS_SUCCESSFUL; 706 } 707 708 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 709 tmp = readl(addr) & mask; 710 tmp |= val << ((where & 0x3) * 8); 711 writel(tmp, addr); 712 713 return PCIBIOS_SUCCESSFUL; 714} 715 716static int iproc_pcie_config_read32(struct pci_bus *bus, unsigned int devfn, 717 int where, int size, u32 *val) 718{ 719 int ret; 720 struct iproc_pcie *pcie = iproc_data(bus); 721 722 iproc_pcie_apb_err_disable(bus, true); 723 if (pcie->iproc_cfg_read) 724 ret = iproc_pcie_config_read(bus, devfn, where, size, val); 725 else 726 ret = pci_generic_config_read32(bus, devfn, where, size, val); 727 iproc_pcie_apb_err_disable(bus, false); 728 729 return ret; 730} 731 732static int iproc_pcie_config_write32(struct pci_bus *bus, unsigned int devfn, 733 int where, int size, u32 val) 734{ 735 int ret; 736 737 iproc_pcie_apb_err_disable(bus, true); 738 ret = pci_generic_config_write32(bus, devfn, where, size, val); 739 iproc_pcie_apb_err_disable(bus, false); 740 741 return ret; 742} 743 744static struct pci_ops iproc_pcie_ops = { 745 .map_bus = iproc_pcie_bus_map_cfg_bus, 746 .read = iproc_pcie_config_read32, 747 .write = iproc_pcie_config_write32, 748}; 749 750static void iproc_pcie_perst_ctrl(struct iproc_pcie *pcie, bool assert) 751{ 752 u32 val; 753 754 /* 755 * PAXC and the internal emulated endpoint device downstream should not 756 * be reset. If firmware has been loaded on the endpoint device at an 757 * earlier boot stage, reset here causes issues. 758 */ 759 if (pcie->ep_is_internal) 760 return; 761 762 if (assert) { 763 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL); 764 val &= ~EP_PERST_SOURCE_SELECT & ~EP_MODE_SURVIVE_PERST & 765 ~RC_PCIE_RST_OUTPUT; 766 iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val); 767 udelay(250); 768 } else { 769 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL); 770 val |= RC_PCIE_RST_OUTPUT; 771 iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val); 772 msleep(100); 773 } 774} 775 776int iproc_pcie_shutdown(struct iproc_pcie *pcie) 777{ 778 iproc_pcie_perst_ctrl(pcie, true); 779 msleep(500); 780 781 return 0; 782} 783EXPORT_SYMBOL_GPL(iproc_pcie_shutdown); 784 785static int iproc_pcie_check_link(struct iproc_pcie *pcie) 786{ 787 struct device *dev = pcie->dev; 788 u32 hdr_type, link_ctrl, link_status, class, val; 789 bool link_is_active = false; 790 791 /* 792 * PAXC connects to emulated endpoint devices directly and does not 793 * have a Serdes. Therefore skip the link detection logic here. 794 */ 795 if (pcie->ep_is_internal) 796 return 0; 797 798 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS); 799 if (!(val & PCIE_PHYLINKUP) || !(val & PCIE_DL_ACTIVE)) { 800 dev_err(dev, "PHY or data link is INACTIVE!\n"); 801 return -ENODEV; 802 } 803 804 /* make sure we are not in EP mode */ 805 iproc_pci_raw_config_read32(pcie, 0, PCI_HEADER_TYPE, 1, &hdr_type); 806 if ((hdr_type & 0x7f) != PCI_HEADER_TYPE_BRIDGE) { 807 dev_err(dev, "in EP mode, hdr=%#02x\n", hdr_type); 808 return -EFAULT; 809 } 810 811 /* force class to PCI_CLASS_BRIDGE_PCI (0x0604) */ 812#define PCI_BRIDGE_CTRL_REG_OFFSET 0x43c 813#define PCI_CLASS_BRIDGE_MASK 0xffff00 814#define PCI_CLASS_BRIDGE_SHIFT 8 815 iproc_pci_raw_config_read32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET, 816 4, &class); 817 class &= ~PCI_CLASS_BRIDGE_MASK; 818 class |= (PCI_CLASS_BRIDGE_PCI << PCI_CLASS_BRIDGE_SHIFT); 819 iproc_pci_raw_config_write32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET, 820 4, class); 821 822 /* check link status to see if link is active */ 823 iproc_pci_raw_config_read32(pcie, 0, IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA, 824 2, &link_status); 825 if (link_status & PCI_EXP_LNKSTA_NLW) 826 link_is_active = true; 827 828 if (!link_is_active) { 829 /* try GEN 1 link speed */ 830#define PCI_TARGET_LINK_SPEED_MASK 0xf 831#define PCI_TARGET_LINK_SPEED_GEN2 0x2 832#define PCI_TARGET_LINK_SPEED_GEN1 0x1 833 iproc_pci_raw_config_read32(pcie, 0, 834 IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2, 835 4, &link_ctrl); 836 if ((link_ctrl & PCI_TARGET_LINK_SPEED_MASK) == 837 PCI_TARGET_LINK_SPEED_GEN2) { 838 link_ctrl &= ~PCI_TARGET_LINK_SPEED_MASK; 839 link_ctrl |= PCI_TARGET_LINK_SPEED_GEN1; 840 iproc_pci_raw_config_write32(pcie, 0, 841 IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2, 842 4, link_ctrl); 843 msleep(100); 844 845 iproc_pci_raw_config_read32(pcie, 0, 846 IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA, 847 2, &link_status); 848 if (link_status & PCI_EXP_LNKSTA_NLW) 849 link_is_active = true; 850 } 851 } 852 853 dev_info(dev, "link: %s\n", link_is_active ? "UP" : "DOWN"); 854 855 return link_is_active ? 0 : -ENODEV; 856} 857 858static void iproc_pcie_enable(struct iproc_pcie *pcie) 859{ 860 iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK); 861} 862 863static inline bool iproc_pcie_ob_is_valid(struct iproc_pcie *pcie, 864 int window_idx) 865{ 866 u32 val; 867 868 val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_OARR0, window_idx)); 869 870 return !!(val & OARR_VALID); 871} 872 873static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx, 874 int size_idx, u64 axi_addr, u64 pci_addr) 875{ 876 struct device *dev = pcie->dev; 877 u16 oarr_offset, omap_offset; 878 879 /* 880 * Derive the OARR/OMAP offset from the first pair (OARR0/OMAP0) based 881 * on window index. 882 */ 883 oarr_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OARR0, 884 window_idx)); 885 omap_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OMAP0, 886 window_idx)); 887 if (iproc_pcie_reg_is_invalid(oarr_offset) || 888 iproc_pcie_reg_is_invalid(omap_offset)) 889 return -EINVAL; 890 891 /* 892 * Program the OARR registers. The upper 32-bit OARR register is 893 * always right after the lower 32-bit OARR register. 894 */ 895 writel(lower_32_bits(axi_addr) | (size_idx << OARR_SIZE_CFG_SHIFT) | 896 OARR_VALID, pcie->base + oarr_offset); 897 writel(upper_32_bits(axi_addr), pcie->base + oarr_offset + 4); 898 899 /* now program the OMAP registers */ 900 writel(lower_32_bits(pci_addr), pcie->base + omap_offset); 901 writel(upper_32_bits(pci_addr), pcie->base + omap_offset + 4); 902 903 dev_dbg(dev, "ob window [%d]: offset 0x%x axi %pap pci %pap\n", 904 window_idx, oarr_offset, &axi_addr, &pci_addr); 905 dev_dbg(dev, "oarr lo 0x%x oarr hi 0x%x\n", 906 readl(pcie->base + oarr_offset), 907 readl(pcie->base + oarr_offset + 4)); 908 dev_dbg(dev, "omap lo 0x%x omap hi 0x%x\n", 909 readl(pcie->base + omap_offset), 910 readl(pcie->base + omap_offset + 4)); 911 912 return 0; 913} 914 915/** 916 * Some iProc SoCs require the SW to configure the outbound address mapping 917 * 918 * Outbound address translation: 919 * 920 * iproc_pcie_address = axi_address - axi_offset 921 * OARR = iproc_pcie_address 922 * OMAP = pci_addr 923 * 924 * axi_addr -> iproc_pcie_address -> OARR -> OMAP -> pci_address 925 */ 926static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr, 927 u64 pci_addr, resource_size_t size) 928{ 929 struct iproc_pcie_ob *ob = &pcie->ob; 930 struct device *dev = pcie->dev; 931 int ret = -EINVAL, window_idx, size_idx; 932 933 if (axi_addr < ob->axi_offset) { 934 dev_err(dev, "axi address %pap less than offset %pap\n", 935 &axi_addr, &ob->axi_offset); 936 return -EINVAL; 937 } 938 939 /* 940 * Translate the AXI address to the internal address used by the iProc 941 * PCIe core before programming the OARR 942 */ 943 axi_addr -= ob->axi_offset; 944 945 /* iterate through all OARR/OMAP mapping windows */ 946 for (window_idx = ob->nr_windows - 1; window_idx >= 0; window_idx--) { 947 const struct iproc_pcie_ob_map *ob_map = 948 &pcie->ob_map[window_idx]; 949 950 /* 951 * If current outbound window is already in use, move on to the 952 * next one. 953 */ 954 if (iproc_pcie_ob_is_valid(pcie, window_idx)) 955 continue; 956 957 /* 958 * Iterate through all supported window sizes within the 959 * OARR/OMAP pair to find a match. Go through the window sizes 960 * in a descending order. 961 */ 962 for (size_idx = ob_map->nr_sizes - 1; size_idx >= 0; 963 size_idx--) { 964 resource_size_t window_size = 965 ob_map->window_sizes[size_idx] * SZ_1M; 966 967 /* 968 * Keep iterating until we reach the last window and 969 * with the minimal window size at index zero. In this 970 * case, we take a compromise by mapping it using the 971 * minimum window size that can be supported 972 */ 973 if (size < window_size) { 974 if (size_idx > 0 || window_idx > 0) 975 continue; 976 977 /* 978 * For the corner case of reaching the minimal 979 * window size that can be supported on the 980 * last window 981 */ 982 axi_addr = ALIGN_DOWN(axi_addr, window_size); 983 pci_addr = ALIGN_DOWN(pci_addr, window_size); 984 size = window_size; 985 } 986 987 if (!IS_ALIGNED(axi_addr, window_size) || 988 !IS_ALIGNED(pci_addr, window_size)) { 989 dev_err(dev, 990 "axi %pap or pci %pap not aligned\n", 991 &axi_addr, &pci_addr); 992 return -EINVAL; 993 } 994 995 /* 996 * Match found! Program both OARR and OMAP and mark 997 * them as a valid entry. 998 */ 999 ret = iproc_pcie_ob_write(pcie, window_idx, size_idx, 1000 axi_addr, pci_addr); 1001 if (ret) 1002 goto err_ob; 1003 1004 size -= window_size; 1005 if (size == 0) 1006 return 0; 1007 1008 /* 1009 * If we are here, we are done with the current window, 1010 * but not yet finished all mappings. Need to move on 1011 * to the next window. 1012 */ 1013 axi_addr += window_size; 1014 pci_addr += window_size; 1015 break; 1016 } 1017 } 1018 1019err_ob: 1020 dev_err(dev, "unable to configure outbound mapping\n"); 1021 dev_err(dev, 1022 "axi %pap, axi offset %pap, pci %pap, res size %pap\n", 1023 &axi_addr, &ob->axi_offset, &pci_addr, &size); 1024 1025 return ret; 1026} 1027 1028static int iproc_pcie_map_ranges(struct iproc_pcie *pcie, 1029 struct list_head *resources) 1030{ 1031 struct device *dev = pcie->dev; 1032 struct resource_entry *window; 1033 int ret; 1034 1035 resource_list_for_each_entry(window, resources) { 1036 struct resource *res = window->res; 1037 u64 res_type = resource_type(res); 1038 1039 switch (res_type) { 1040 case IORESOURCE_IO: 1041 case IORESOURCE_BUS: 1042 break; 1043 case IORESOURCE_MEM: 1044 ret = iproc_pcie_setup_ob(pcie, res->start, 1045 res->start - window->offset, 1046 resource_size(res)); 1047 if (ret) 1048 return ret; 1049 break; 1050 default: 1051 dev_err(dev, "invalid resource %pR\n", res); 1052 return -EINVAL; 1053 } 1054 } 1055 1056 return 0; 1057} 1058 1059static inline bool iproc_pcie_ib_is_in_use(struct iproc_pcie *pcie, 1060 int region_idx) 1061{ 1062 const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx]; 1063 u32 val; 1064 1065 val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_IARR0, region_idx)); 1066 1067 return !!(val & (BIT(ib_map->nr_sizes) - 1)); 1068} 1069 1070static inline bool iproc_pcie_ib_check_type(const struct iproc_pcie_ib_map *ib_map, 1071 enum iproc_pcie_ib_map_type type) 1072{ 1073 return !!(ib_map->type == type); 1074} 1075 1076static int iproc_pcie_ib_write(struct iproc_pcie *pcie, int region_idx, 1077 int size_idx, int nr_windows, u64 axi_addr, 1078 u64 pci_addr, resource_size_t size) 1079{ 1080 struct device *dev = pcie->dev; 1081 const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx]; 1082 u16 iarr_offset, imap_offset; 1083 u32 val; 1084 int window_idx; 1085 1086 iarr_offset = iproc_pcie_reg_offset(pcie, 1087 MAP_REG(IPROC_PCIE_IARR0, region_idx)); 1088 imap_offset = iproc_pcie_reg_offset(pcie, 1089 MAP_REG(IPROC_PCIE_IMAP0, region_idx)); 1090 if (iproc_pcie_reg_is_invalid(iarr_offset) || 1091 iproc_pcie_reg_is_invalid(imap_offset)) 1092 return -EINVAL; 1093 1094 dev_dbg(dev, "ib region [%d]: offset 0x%x axi %pap pci %pap\n", 1095 region_idx, iarr_offset, &axi_addr, &pci_addr); 1096 1097 /* 1098 * Program the IARR registers. The upper 32-bit IARR register is 1099 * always right after the lower 32-bit IARR register. 1100 */ 1101 writel(lower_32_bits(pci_addr) | BIT(size_idx), 1102 pcie->base + iarr_offset); 1103 writel(upper_32_bits(pci_addr), pcie->base + iarr_offset + 4); 1104 1105 dev_dbg(dev, "iarr lo 0x%x iarr hi 0x%x\n", 1106 readl(pcie->base + iarr_offset), 1107 readl(pcie->base + iarr_offset + 4)); 1108 1109 /* 1110 * Now program the IMAP registers. Each IARR region may have one or 1111 * more IMAP windows. 1112 */ 1113 size >>= ilog2(nr_windows); 1114 for (window_idx = 0; window_idx < nr_windows; window_idx++) { 1115 val = readl(pcie->base + imap_offset); 1116 val |= lower_32_bits(axi_addr) | IMAP_VALID; 1117 writel(val, pcie->base + imap_offset); 1118 writel(upper_32_bits(axi_addr), 1119 pcie->base + imap_offset + ib_map->imap_addr_offset); 1120 1121 dev_dbg(dev, "imap window [%d] lo 0x%x hi 0x%x\n", 1122 window_idx, readl(pcie->base + imap_offset), 1123 readl(pcie->base + imap_offset + 1124 ib_map->imap_addr_offset)); 1125 1126 imap_offset += ib_map->imap_window_offset; 1127 axi_addr += size; 1128 } 1129 1130 return 0; 1131} 1132 1133static int iproc_pcie_setup_ib(struct iproc_pcie *pcie, 1134 struct resource_entry *entry, 1135 enum iproc_pcie_ib_map_type type) 1136{ 1137 struct device *dev = pcie->dev; 1138 struct iproc_pcie_ib *ib = &pcie->ib; 1139 int ret; 1140 unsigned int region_idx, size_idx; 1141 u64 axi_addr = entry->res->start; 1142 u64 pci_addr = entry->res->start - entry->offset; 1143 resource_size_t size = resource_size(entry->res); 1144 1145 /* iterate through all IARR mapping regions */ 1146 for (region_idx = 0; region_idx < ib->nr_regions; region_idx++) { 1147 const struct iproc_pcie_ib_map *ib_map = 1148 &pcie->ib_map[region_idx]; 1149 1150 /* 1151 * If current inbound region is already in use or not a 1152 * compatible type, move on to the next. 1153 */ 1154 if (iproc_pcie_ib_is_in_use(pcie, region_idx) || 1155 !iproc_pcie_ib_check_type(ib_map, type)) 1156 continue; 1157 1158 /* iterate through all supported region sizes to find a match */ 1159 for (size_idx = 0; size_idx < ib_map->nr_sizes; size_idx++) { 1160 resource_size_t region_size = 1161 ib_map->region_sizes[size_idx] * ib_map->size_unit; 1162 1163 if (size != region_size) 1164 continue; 1165 1166 if (!IS_ALIGNED(axi_addr, region_size) || 1167 !IS_ALIGNED(pci_addr, region_size)) { 1168 dev_err(dev, 1169 "axi %pap or pci %pap not aligned\n", 1170 &axi_addr, &pci_addr); 1171 return -EINVAL; 1172 } 1173 1174 /* Match found! Program IARR and all IMAP windows. */ 1175 ret = iproc_pcie_ib_write(pcie, region_idx, size_idx, 1176 ib_map->nr_windows, axi_addr, 1177 pci_addr, size); 1178 if (ret) 1179 goto err_ib; 1180 else 1181 return 0; 1182 1183 } 1184 } 1185 ret = -EINVAL; 1186 1187err_ib: 1188 dev_err(dev, "unable to configure inbound mapping\n"); 1189 dev_err(dev, "axi %pap, pci %pap, res size %pap\n", 1190 &axi_addr, &pci_addr, &size); 1191 1192 return ret; 1193} 1194 1195static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie) 1196{ 1197 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 1198 struct resource_entry *entry; 1199 int ret = 0; 1200 1201 resource_list_for_each_entry(entry, &host->dma_ranges) { 1202 /* Each range entry corresponds to an inbound mapping region */ 1203 ret = iproc_pcie_setup_ib(pcie, entry, IPROC_PCIE_IB_MAP_MEM); 1204 if (ret) 1205 break; 1206 } 1207 1208 return ret; 1209} 1210 1211static void iproc_pcie_invalidate_mapping(struct iproc_pcie *pcie) 1212{ 1213 struct iproc_pcie_ib *ib = &pcie->ib; 1214 struct iproc_pcie_ob *ob = &pcie->ob; 1215 int idx; 1216 1217 if (pcie->ep_is_internal) 1218 return; 1219 1220 if (pcie->need_ob_cfg) { 1221 /* iterate through all OARR mapping regions */ 1222 for (idx = ob->nr_windows - 1; idx >= 0; idx--) { 1223 iproc_pcie_write_reg(pcie, 1224 MAP_REG(IPROC_PCIE_OARR0, idx), 0); 1225 } 1226 } 1227 1228 if (pcie->need_ib_cfg) { 1229 /* iterate through all IARR mapping regions */ 1230 for (idx = 0; idx < ib->nr_regions; idx++) { 1231 iproc_pcie_write_reg(pcie, 1232 MAP_REG(IPROC_PCIE_IARR0, idx), 0); 1233 } 1234 } 1235} 1236 1237static int iproce_pcie_get_msi(struct iproc_pcie *pcie, 1238 struct device_node *msi_node, 1239 u64 *msi_addr) 1240{ 1241 struct device *dev = pcie->dev; 1242 int ret; 1243 struct resource res; 1244 1245 /* 1246 * Check if 'msi-map' points to ARM GICv3 ITS, which is the only 1247 * supported external MSI controller that requires steering. 1248 */ 1249 if (!of_device_is_compatible(msi_node, "arm,gic-v3-its")) { 1250 dev_err(dev, "unable to find compatible MSI controller\n"); 1251 return -ENODEV; 1252 } 1253 1254 /* derive GITS_TRANSLATER address from GICv3 */ 1255 ret = of_address_to_resource(msi_node, 0, &res); 1256 if (ret < 0) { 1257 dev_err(dev, "unable to obtain MSI controller resources\n"); 1258 return ret; 1259 } 1260 1261 *msi_addr = res.start + GITS_TRANSLATER; 1262 return 0; 1263} 1264 1265static int iproc_pcie_paxb_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr) 1266{ 1267 int ret; 1268 struct resource_entry entry; 1269 1270 memset(&entry, 0, sizeof(entry)); 1271 entry.res = &entry.__res; 1272 1273 msi_addr &= ~(SZ_32K - 1); 1274 entry.res->start = msi_addr; 1275 entry.res->end = msi_addr + SZ_32K - 1; 1276 1277 ret = iproc_pcie_setup_ib(pcie, &entry, IPROC_PCIE_IB_MAP_IO); 1278 return ret; 1279} 1280 1281static void iproc_pcie_paxc_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr, 1282 bool enable) 1283{ 1284 u32 val; 1285 1286 if (!enable) { 1287 /* 1288 * Disable PAXC MSI steering. All write transfers will be 1289 * treated as non-MSI transfers 1290 */ 1291 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG); 1292 val &= ~MSI_ENABLE_CFG; 1293 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val); 1294 return; 1295 } 1296 1297 /* 1298 * Program bits [43:13] of address of GITS_TRANSLATER register into 1299 * bits [30:0] of the MSI base address register. In fact, in all iProc 1300 * based SoCs, all I/O register bases are well below the 32-bit 1301 * boundary, so we can safely assume bits [43:32] are always zeros. 1302 */ 1303 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_BASE_ADDR, 1304 (u32)(msi_addr >> 13)); 1305 1306 /* use a default 8K window size */ 1307 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_WINDOW_SIZE, 0); 1308 1309 /* steering MSI to GICv3 ITS */ 1310 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_GIC_MODE); 1311 val |= GIC_V3_CFG; 1312 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_GIC_MODE, val); 1313 1314 /* 1315 * Program bits [43:2] of address of GITS_TRANSLATER register into the 1316 * iProc MSI address registers. 1317 */ 1318 msi_addr >>= 2; 1319 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_HI, 1320 upper_32_bits(msi_addr)); 1321 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_LO, 1322 lower_32_bits(msi_addr)); 1323 1324 /* enable MSI */ 1325 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG); 1326 val |= MSI_ENABLE_CFG; 1327 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val); 1328} 1329 1330static int iproc_pcie_msi_steer(struct iproc_pcie *pcie, 1331 struct device_node *msi_node) 1332{ 1333 struct device *dev = pcie->dev; 1334 int ret; 1335 u64 msi_addr; 1336 1337 ret = iproce_pcie_get_msi(pcie, msi_node, &msi_addr); 1338 if (ret < 0) { 1339 dev_err(dev, "msi steering failed\n"); 1340 return ret; 1341 } 1342 1343 switch (pcie->type) { 1344 case IPROC_PCIE_PAXB_V2: 1345 ret = iproc_pcie_paxb_v2_msi_steer(pcie, msi_addr); 1346 if (ret) 1347 return ret; 1348 break; 1349 case IPROC_PCIE_PAXC_V2: 1350 iproc_pcie_paxc_v2_msi_steer(pcie, msi_addr, true); 1351 break; 1352 default: 1353 return -EINVAL; 1354 } 1355 1356 return 0; 1357} 1358 1359static int iproc_pcie_msi_enable(struct iproc_pcie *pcie) 1360{ 1361 struct device_node *msi_node; 1362 int ret; 1363 1364 /* 1365 * Either the "msi-parent" or the "msi-map" phandle needs to exist 1366 * for us to obtain the MSI node. 1367 */ 1368 1369 msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0); 1370 if (!msi_node) { 1371 const __be32 *msi_map = NULL; 1372 int len; 1373 u32 phandle; 1374 1375 msi_map = of_get_property(pcie->dev->of_node, "msi-map", &len); 1376 if (!msi_map) 1377 return -ENODEV; 1378 1379 phandle = be32_to_cpup(msi_map + 1); 1380 msi_node = of_find_node_by_phandle(phandle); 1381 if (!msi_node) 1382 return -ENODEV; 1383 } 1384 1385 /* 1386 * Certain revisions of the iProc PCIe controller require additional 1387 * configurations to steer the MSI writes towards an external MSI 1388 * controller. 1389 */ 1390 if (pcie->need_msi_steer) { 1391 ret = iproc_pcie_msi_steer(pcie, msi_node); 1392 if (ret) 1393 goto out_put_node; 1394 } 1395 1396 /* 1397 * If another MSI controller is being used, the call below should fail 1398 * but that is okay 1399 */ 1400 ret = iproc_msi_init(pcie, msi_node); 1401 1402out_put_node: 1403 of_node_put(msi_node); 1404 return ret; 1405} 1406 1407static void iproc_pcie_msi_disable(struct iproc_pcie *pcie) 1408{ 1409 iproc_msi_exit(pcie); 1410} 1411 1412static int iproc_pcie_rev_init(struct iproc_pcie *pcie) 1413{ 1414 struct device *dev = pcie->dev; 1415 unsigned int reg_idx; 1416 const u16 *regs; 1417 1418 switch (pcie->type) { 1419 case IPROC_PCIE_PAXB_BCMA: 1420 regs = iproc_pcie_reg_paxb_bcma; 1421 break; 1422 case IPROC_PCIE_PAXB: 1423 regs = iproc_pcie_reg_paxb; 1424 pcie->has_apb_err_disable = true; 1425 if (pcie->need_ob_cfg) { 1426 pcie->ob_map = paxb_ob_map; 1427 pcie->ob.nr_windows = ARRAY_SIZE(paxb_ob_map); 1428 } 1429 break; 1430 case IPROC_PCIE_PAXB_V2: 1431 regs = iproc_pcie_reg_paxb_v2; 1432 pcie->iproc_cfg_read = true; 1433 pcie->has_apb_err_disable = true; 1434 if (pcie->need_ob_cfg) { 1435 pcie->ob_map = paxb_v2_ob_map; 1436 pcie->ob.nr_windows = ARRAY_SIZE(paxb_v2_ob_map); 1437 } 1438 pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map); 1439 pcie->ib_map = paxb_v2_ib_map; 1440 pcie->need_msi_steer = true; 1441 dev_warn(dev, "reads of config registers that contain %#x return incorrect data\n", 1442 CFG_RETRY_STATUS); 1443 break; 1444 case IPROC_PCIE_PAXC: 1445 regs = iproc_pcie_reg_paxc; 1446 pcie->ep_is_internal = true; 1447 pcie->iproc_cfg_read = true; 1448 pcie->rej_unconfig_pf = true; 1449 break; 1450 case IPROC_PCIE_PAXC_V2: 1451 regs = iproc_pcie_reg_paxc_v2; 1452 pcie->ep_is_internal = true; 1453 pcie->iproc_cfg_read = true; 1454 pcie->rej_unconfig_pf = true; 1455 pcie->need_msi_steer = true; 1456 break; 1457 default: 1458 dev_err(dev, "incompatible iProc PCIe interface\n"); 1459 return -EINVAL; 1460 } 1461 1462 pcie->reg_offsets = devm_kcalloc(dev, IPROC_PCIE_MAX_NUM_REG, 1463 sizeof(*pcie->reg_offsets), 1464 GFP_KERNEL); 1465 if (!pcie->reg_offsets) 1466 return -ENOMEM; 1467 1468 /* go through the register table and populate all valid registers */ 1469 pcie->reg_offsets[0] = (pcie->type == IPROC_PCIE_PAXC_V2) ? 1470 IPROC_PCIE_REG_INVALID : regs[0]; 1471 for (reg_idx = 1; reg_idx < IPROC_PCIE_MAX_NUM_REG; reg_idx++) 1472 pcie->reg_offsets[reg_idx] = regs[reg_idx] ? 1473 regs[reg_idx] : IPROC_PCIE_REG_INVALID; 1474 1475 return 0; 1476} 1477 1478int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res) 1479{ 1480 struct device *dev; 1481 int ret; 1482 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 1483 1484 dev = pcie->dev; 1485 1486 ret = iproc_pcie_rev_init(pcie); 1487 if (ret) { 1488 dev_err(dev, "unable to initialize controller parameters\n"); 1489 return ret; 1490 } 1491 1492 ret = phy_init(pcie->phy); 1493 if (ret) { 1494 dev_err(dev, "unable to initialize PCIe PHY\n"); 1495 return ret; 1496 } 1497 1498 ret = phy_power_on(pcie->phy); 1499 if (ret) { 1500 dev_err(dev, "unable to power on PCIe PHY\n"); 1501 goto err_exit_phy; 1502 } 1503 1504 iproc_pcie_perst_ctrl(pcie, true); 1505 iproc_pcie_perst_ctrl(pcie, false); 1506 1507 iproc_pcie_invalidate_mapping(pcie); 1508 1509 if (pcie->need_ob_cfg) { 1510 ret = iproc_pcie_map_ranges(pcie, res); 1511 if (ret) { 1512 dev_err(dev, "map failed\n"); 1513 goto err_power_off_phy; 1514 } 1515 } 1516 1517 if (pcie->need_ib_cfg) { 1518 ret = iproc_pcie_map_dma_ranges(pcie); 1519 if (ret && ret != -ENOENT) 1520 goto err_power_off_phy; 1521 } 1522 1523 ret = iproc_pcie_check_link(pcie); 1524 if (ret) { 1525 dev_err(dev, "no PCIe EP device detected\n"); 1526 goto err_power_off_phy; 1527 } 1528 1529 iproc_pcie_enable(pcie); 1530 1531 if (IS_ENABLED(CONFIG_PCI_MSI)) 1532 if (iproc_pcie_msi_enable(pcie)) 1533 dev_info(dev, "not using iProc MSI\n"); 1534 1535 host->ops = &iproc_pcie_ops; 1536 host->sysdata = pcie; 1537 host->map_irq = pcie->map_irq; 1538 1539 ret = pci_host_probe(host); 1540 if (ret < 0) { 1541 dev_err(dev, "failed to scan host: %d\n", ret); 1542 goto err_power_off_phy; 1543 } 1544 1545 return 0; 1546 1547err_power_off_phy: 1548 phy_power_off(pcie->phy); 1549err_exit_phy: 1550 phy_exit(pcie->phy); 1551 return ret; 1552} 1553EXPORT_SYMBOL(iproc_pcie_setup); 1554 1555int iproc_pcie_remove(struct iproc_pcie *pcie) 1556{ 1557 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 1558 1559 pci_stop_root_bus(host->bus); 1560 pci_remove_root_bus(host->bus); 1561 1562 iproc_pcie_msi_disable(pcie); 1563 1564 phy_power_off(pcie->phy); 1565 phy_exit(pcie->phy); 1566 1567 return 0; 1568} 1569EXPORT_SYMBOL(iproc_pcie_remove); 1570 1571/* 1572 * The MSI parsing logic in certain revisions of Broadcom PAXC based root 1573 * complex does not work and needs to be disabled 1574 */ 1575static void quirk_paxc_disable_msi_parsing(struct pci_dev *pdev) 1576{ 1577 struct iproc_pcie *pcie = iproc_data(pdev->bus); 1578 1579 if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 1580 iproc_pcie_paxc_v2_msi_steer(pcie, 0, false); 1581} 1582DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, 1583 quirk_paxc_disable_msi_parsing); 1584DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, 1585 quirk_paxc_disable_msi_parsing); 1586DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, 1587 quirk_paxc_disable_msi_parsing); 1588 1589static void quirk_paxc_bridge(struct pci_dev *pdev) 1590{ 1591 /* 1592 * The PCI config space is shared with the PAXC root port and the first 1593 * Ethernet device. So, we need to workaround this by telling the PCI 1594 * code that the bridge is not an Ethernet device. 1595 */ 1596 if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 1597 pdev->class = PCI_CLASS_BRIDGE_PCI << 8; 1598 1599 /* 1600 * MPSS is not being set properly (as it is currently 0). This is 1601 * because that area of the PCI config space is hard coded to zero, and 1602 * is not modifiable by firmware. Set this to 2 (e.g., 512 byte MPS) 1603 * so that the MPS can be set to the real max value. 1604 */ 1605 pdev->pcie_mpss = 2; 1606} 1607DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16cd, quirk_paxc_bridge); 1608DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, quirk_paxc_bridge); 1609DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd750, quirk_paxc_bridge); 1610DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, quirk_paxc_bridge); 1611DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, quirk_paxc_bridge); 1612 1613MODULE_AUTHOR("Ray Jui <rjui@broadcom.com>"); 1614MODULE_DESCRIPTION("Broadcom iPROC PCIe common driver"); 1615MODULE_LICENSE("GPL v2"); 1616