1// SPDX-License-Identifier: GPL-2.0+ 2/* 3 * Rockchip AXI PCIe host controller driver 4 * 5 * Copyright (c) 2016 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Wenrui Li <wenrui.li@rock-chips.com> 9 * 10 * Bits taken from Synopsys DesignWare Host controller driver and 11 * ARM PCI Host generic driver. 12 */ 13 14#include <linux/bitrev.h> 15#include <linux/clk.h> 16#include <linux/delay.h> 17#include <linux/gpio/consumer.h> 18#include <linux/init.h> 19#include <linux/interrupt.h> 20#include <linux/iopoll.h> 21#include <linux/irq.h> 22#include <linux/irqchip/chained_irq.h> 23#include <linux/irqdomain.h> 24#include <linux/kernel.h> 25#include <linux/mfd/syscon.h> 26#include <linux/module.h> 27#include <linux/of_address.h> 28#include <linux/of_device.h> 29#include <linux/of_pci.h> 30#include <linux/of_platform.h> 31#include <linux/of_irq.h> 32#include <linux/pci.h> 33#include <linux/pci_ids.h> 34#include <linux/phy/phy.h> 35#include <linux/platform_device.h> 36#include <linux/reset.h> 37#include <linux/regmap.h> 38 39#include "../pci.h" 40#include "pcie-rockchip.h" 41 42static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) 43{ 44 u32 status; 45 46 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 47 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); 48 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 49} 50 51static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) 52{ 53 u32 status; 54 55 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 56 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; 57 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 58} 59 60static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) 61{ 62 u32 val; 63 64 /* Update Tx credit maximum update interval */ 65 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1); 66 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK; 67 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */ 68 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1); 69} 70 71static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip, 72 struct pci_bus *bus, int dev) 73{ 74 /* 75 * Access only one slot on each root port. 76 * Do not read more than one device on the bus directly attached 77 * to RC's downstream side. 78 */ 79 if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent)) 80 return dev == 0; 81 82 return 1; 83} 84 85static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip) 86{ 87 u32 val; 88 u8 map; 89 90 if (rockchip->legacy_phy) 91 return GENMASK(MAX_LANE_NUM - 1, 0); 92 93 val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP); 94 map = val & PCIE_CORE_LANE_MAP_MASK; 95 96 /* The link may be using a reverse-indexed mapping. */ 97 if (val & PCIE_CORE_LANE_MAP_REVERSE) 98 map = bitrev8(map) >> 4; 99 100 return map; 101} 102 103static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip, 104 int where, int size, u32 *val) 105{ 106 void __iomem *addr; 107 108 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where; 109 110 if (!IS_ALIGNED((uintptr_t)addr, size)) { 111 *val = 0; 112 return PCIBIOS_BAD_REGISTER_NUMBER; 113 } 114 115 if (size == 4) { 116 *val = readl(addr); 117 } else if (size == 2) { 118 *val = readw(addr); 119 } else if (size == 1) { 120 *val = readb(addr); 121 } else { 122 *val = 0; 123 return PCIBIOS_BAD_REGISTER_NUMBER; 124 } 125 return PCIBIOS_SUCCESSFUL; 126} 127 128static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip, 129 int where, int size, u32 val) 130{ 131 u32 mask, tmp, offset; 132 void __iomem *addr; 133 134 offset = where & ~0x3; 135 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset; 136 137 if (size == 4) { 138 writel(val, addr); 139 return PCIBIOS_SUCCESSFUL; 140 } 141 142 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 143 144 /* 145 * N.B. This read/modify/write isn't safe in general because it can 146 * corrupt RW1C bits in adjacent registers. But the hardware 147 * doesn't support smaller writes. 148 */ 149 tmp = readl(addr) & mask; 150 tmp |= val << ((where & 0x3) * 8); 151 writel(tmp, addr); 152 153 return PCIBIOS_SUCCESSFUL; 154} 155 156static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip, 157 struct pci_bus *bus, u32 devfn, 158 int where, int size, u32 *val) 159{ 160 u32 busdev; 161 162 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn), 163 PCI_FUNC(devfn), where); 164 165 if (!IS_ALIGNED(busdev, size)) { 166 *val = 0; 167 return PCIBIOS_BAD_REGISTER_NUMBER; 168 } 169 170 if (pci_is_root_bus(bus->parent)) 171 rockchip_pcie_cfg_configuration_accesses(rockchip, 172 AXI_WRAPPER_TYPE0_CFG); 173 else 174 rockchip_pcie_cfg_configuration_accesses(rockchip, 175 AXI_WRAPPER_TYPE1_CFG); 176 177 if (size == 4) { 178 *val = readl(rockchip->reg_base + busdev); 179 } else if (size == 2) { 180 *val = readw(rockchip->reg_base + busdev); 181 } else if (size == 1) { 182 *val = readb(rockchip->reg_base + busdev); 183 } else { 184 *val = 0; 185 return PCIBIOS_BAD_REGISTER_NUMBER; 186 } 187 return PCIBIOS_SUCCESSFUL; 188} 189 190static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip, 191 struct pci_bus *bus, u32 devfn, 192 int where, int size, u32 val) 193{ 194 u32 busdev; 195 196 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn), 197 PCI_FUNC(devfn), where); 198 if (!IS_ALIGNED(busdev, size)) 199 return PCIBIOS_BAD_REGISTER_NUMBER; 200 201 if (pci_is_root_bus(bus->parent)) 202 rockchip_pcie_cfg_configuration_accesses(rockchip, 203 AXI_WRAPPER_TYPE0_CFG); 204 else 205 rockchip_pcie_cfg_configuration_accesses(rockchip, 206 AXI_WRAPPER_TYPE1_CFG); 207 208 if (size == 4) 209 writel(val, rockchip->reg_base + busdev); 210 else if (size == 2) 211 writew(val, rockchip->reg_base + busdev); 212 else if (size == 1) 213 writeb(val, rockchip->reg_base + busdev); 214 else 215 return PCIBIOS_BAD_REGISTER_NUMBER; 216 217 return PCIBIOS_SUCCESSFUL; 218} 219 220static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, 221 int size, u32 *val) 222{ 223 struct rockchip_pcie *rockchip = bus->sysdata; 224 225 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) { 226 *val = 0xffffffff; 227 return PCIBIOS_DEVICE_NOT_FOUND; 228 } 229 230 if (pci_is_root_bus(bus)) 231 return rockchip_pcie_rd_own_conf(rockchip, where, size, val); 232 233 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, 234 val); 235} 236 237static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn, 238 int where, int size, u32 val) 239{ 240 struct rockchip_pcie *rockchip = bus->sysdata; 241 242 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 243 return PCIBIOS_DEVICE_NOT_FOUND; 244 245 if (pci_is_root_bus(bus)) 246 return rockchip_pcie_wr_own_conf(rockchip, where, size, val); 247 248 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, 249 val); 250} 251 252static struct pci_ops rockchip_pcie_ops = { 253 .read = rockchip_pcie_rd_conf, 254 .write = rockchip_pcie_wr_conf, 255}; 256 257static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) 258{ 259 int curr; 260 u32 status, scale, power; 261 262 if (IS_ERR(rockchip->vpcie3v3)) 263 return; 264 265 /* 266 * Set RC's captured slot power limit and scale if 267 * vpcie3v3 available. The default values are both zero 268 * which means the software should set these two according 269 * to the actual power supply. 270 */ 271 curr = regulator_get_current_limit(rockchip->vpcie3v3); 272 if (curr <= 0) 273 return; 274 275 scale = 3; /* 0.001x */ 276 curr = curr / 1000; /* convert to mA */ 277 power = (curr * 3300) / 1000; /* milliwatt */ 278 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) { 279 if (!scale) { 280 dev_warn(rockchip->dev, "invalid power supply\n"); 281 return; 282 } 283 scale--; 284 power = power / 10; 285 } 286 287 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR); 288 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) | 289 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT); 290 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR); 291} 292 293/** 294 * rockchip_pcie_host_init_port - Initialize hardware 295 * @rockchip: PCIe port information 296 */ 297static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) 298{ 299 struct device *dev = rockchip->dev; 300 int err, i = MAX_LANE_NUM; 301 u32 status; 302 303 gpiod_set_value_cansleep(rockchip->ep_gpio, 0); 304 305 err = rockchip_pcie_init_port(rockchip); 306 if (err) 307 return err; 308 309 /* Fix the transmitted FTS count desired to exit from L0s. */ 310 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); 311 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | 312 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); 313 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1); 314 315 rockchip_pcie_set_power_limit(rockchip); 316 317 /* Set RC's clock architecture as common clock */ 318 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 319 status |= PCI_EXP_LNKSTA_SLC << 16; 320 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 321 322 /* Set RC's RCB to 128 */ 323 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 324 status |= PCI_EXP_LNKCTL_RCB; 325 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 326 327 /* Enable Gen1 training */ 328 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 329 PCIE_CLIENT_CONFIG); 330 331 gpiod_set_value_cansleep(rockchip->ep_gpio, 1); 332 333 /* 500ms timeout value should be enough for Gen1/2 training */ 334 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, 335 status, PCIE_LINK_UP(status), 20, 336 500 * USEC_PER_MSEC); 337 if (err) { 338 dev_err(dev, "PCIe link training gen1 timeout!\n"); 339 goto err_power_off_phy; 340 } 341 342 if (rockchip->link_gen == 2) { 343 /* 344 * Enable retrain for gen2. This should be configured only after 345 * gen1 finished. 346 */ 347 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 348 status |= PCI_EXP_LNKCTL_RL; 349 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 350 351 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, 352 status, PCIE_LINK_IS_GEN2(status), 20, 353 500 * USEC_PER_MSEC); 354 if (err) 355 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n"); 356 } 357 358 /* Check the final link width from negotiated lane counter from MGMT */ 359 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); 360 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> 361 PCIE_CORE_PL_CONF_LANE_SHIFT); 362 dev_dbg(dev, "current link width is x%d\n", status); 363 364 /* Power off unused lane(s) */ 365 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip); 366 for (i = 0; i < MAX_LANE_NUM; i++) { 367 if (!(rockchip->lanes_map & BIT(i))) { 368 dev_dbg(dev, "idling lane %d\n", i); 369 phy_power_off(rockchip->phys[i]); 370 } 371 } 372 373 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID, 374 PCIE_CORE_CONFIG_VENDOR); 375 rockchip_pcie_write(rockchip, 376 PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT, 377 PCIE_RC_CONFIG_RID_CCR); 378 379 /* Clear THP cap's next cap pointer to remove L1 substate cap */ 380 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP); 381 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK; 382 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP); 383 384 /* Clear L0s from RC's link cap */ 385 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) { 386 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP); 387 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S; 388 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP); 389 } 390 391 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR); 392 status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK; 393 status |= PCIE_RC_CONFIG_DCSR_MPS_256; 394 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR); 395 396 return 0; 397err_power_off_phy: 398 while (i--) 399 phy_power_off(rockchip->phys[i]); 400 i = MAX_LANE_NUM; 401 while (i--) 402 phy_exit(rockchip->phys[i]); 403 return err; 404} 405 406static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) 407{ 408 struct rockchip_pcie *rockchip = arg; 409 struct device *dev = rockchip->dev; 410 u32 reg; 411 u32 sub_reg; 412 413 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 414 if (reg & PCIE_CLIENT_INT_LOCAL) { 415 dev_dbg(dev, "local interrupt received\n"); 416 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS); 417 if (sub_reg & PCIE_CORE_INT_PRFPE) 418 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n"); 419 420 if (sub_reg & PCIE_CORE_INT_CRFPE) 421 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n"); 422 423 if (sub_reg & PCIE_CORE_INT_RRPE) 424 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n"); 425 426 if (sub_reg & PCIE_CORE_INT_PRFO) 427 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n"); 428 429 if (sub_reg & PCIE_CORE_INT_CRFO) 430 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n"); 431 432 if (sub_reg & PCIE_CORE_INT_RT) 433 dev_dbg(dev, "replay timer timed out\n"); 434 435 if (sub_reg & PCIE_CORE_INT_RTR) 436 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n"); 437 438 if (sub_reg & PCIE_CORE_INT_PE) 439 dev_dbg(dev, "phy error detected on receive side\n"); 440 441 if (sub_reg & PCIE_CORE_INT_MTR) 442 dev_dbg(dev, "malformed TLP received from the link\n"); 443 444 if (sub_reg & PCIE_CORE_INT_UCR) 445 dev_dbg(dev, "malformed TLP received from the link\n"); 446 447 if (sub_reg & PCIE_CORE_INT_FCE) 448 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n"); 449 450 if (sub_reg & PCIE_CORE_INT_CT) 451 dev_dbg(dev, "a request timed out waiting for completion\n"); 452 453 if (sub_reg & PCIE_CORE_INT_UTC) 454 dev_dbg(dev, "unmapped TC error\n"); 455 456 if (sub_reg & PCIE_CORE_INT_MMVC) 457 dev_dbg(dev, "MSI mask register changes\n"); 458 459 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS); 460 } else if (reg & PCIE_CLIENT_INT_PHY) { 461 dev_dbg(dev, "phy link changes\n"); 462 rockchip_pcie_update_txcredit_mui(rockchip); 463 rockchip_pcie_clr_bw_int(rockchip); 464 } 465 466 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL, 467 PCIE_CLIENT_INT_STATUS); 468 469 return IRQ_HANDLED; 470} 471 472static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg) 473{ 474 struct rockchip_pcie *rockchip = arg; 475 struct device *dev = rockchip->dev; 476 u32 reg; 477 478 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 479 if (reg & PCIE_CLIENT_INT_LEGACY_DONE) 480 dev_dbg(dev, "legacy done interrupt received\n"); 481 482 if (reg & PCIE_CLIENT_INT_MSG) 483 dev_dbg(dev, "message done interrupt received\n"); 484 485 if (reg & PCIE_CLIENT_INT_HOT_RST) 486 dev_dbg(dev, "hot reset interrupt received\n"); 487 488 if (reg & PCIE_CLIENT_INT_DPA) 489 dev_dbg(dev, "dpa interrupt received\n"); 490 491 if (reg & PCIE_CLIENT_INT_FATAL_ERR) 492 dev_dbg(dev, "fatal error interrupt received\n"); 493 494 if (reg & PCIE_CLIENT_INT_NFATAL_ERR) 495 dev_dbg(dev, "no fatal error interrupt received\n"); 496 497 if (reg & PCIE_CLIENT_INT_CORR_ERR) 498 dev_dbg(dev, "correctable error interrupt received\n"); 499 500 if (reg & PCIE_CLIENT_INT_PHY) 501 dev_dbg(dev, "phy interrupt received\n"); 502 503 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE | 504 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST | 505 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR | 506 PCIE_CLIENT_INT_NFATAL_ERR | 507 PCIE_CLIENT_INT_CORR_ERR | 508 PCIE_CLIENT_INT_PHY), 509 PCIE_CLIENT_INT_STATUS); 510 511 return IRQ_HANDLED; 512} 513 514static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc) 515{ 516 struct irq_chip *chip = irq_desc_get_chip(desc); 517 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); 518 struct device *dev = rockchip->dev; 519 u32 reg; 520 u32 hwirq; 521 u32 virq; 522 523 chained_irq_enter(chip, desc); 524 525 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 526 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT; 527 528 while (reg) { 529 hwirq = ffs(reg) - 1; 530 reg &= ~BIT(hwirq); 531 532 virq = irq_find_mapping(rockchip->irq_domain, hwirq); 533 if (virq) 534 generic_handle_irq(virq); 535 else 536 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq); 537 } 538 539 chained_irq_exit(chip, desc); 540} 541 542static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip) 543{ 544 int irq, err; 545 struct device *dev = rockchip->dev; 546 struct platform_device *pdev = to_platform_device(dev); 547 548 irq = platform_get_irq_byname(pdev, "sys"); 549 if (irq < 0) 550 return irq; 551 552 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler, 553 IRQF_SHARED, "pcie-sys", rockchip); 554 if (err) { 555 dev_err(dev, "failed to request PCIe subsystem IRQ\n"); 556 return err; 557 } 558 559 irq = platform_get_irq_byname(pdev, "legacy"); 560 if (irq < 0) 561 return irq; 562 563 irq_set_chained_handler_and_data(irq, 564 rockchip_pcie_legacy_int_handler, 565 rockchip); 566 567 irq = platform_get_irq_byname(pdev, "client"); 568 if (irq < 0) 569 return irq; 570 571 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler, 572 IRQF_SHARED, "pcie-client", rockchip); 573 if (err) { 574 dev_err(dev, "failed to request PCIe client IRQ\n"); 575 return err; 576 } 577 578 return 0; 579} 580 581/** 582 * rockchip_pcie_parse_host_dt - Parse Device Tree 583 * @rockchip: PCIe port information 584 * 585 * Return: '0' on success and error value on failure 586 */ 587static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) 588{ 589 struct device *dev = rockchip->dev; 590 int err; 591 592 err = rockchip_pcie_parse_dt(rockchip); 593 if (err) 594 return err; 595 596 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); 597 if (IS_ERR(rockchip->vpcie12v)) { 598 if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) 599 return PTR_ERR(rockchip->vpcie12v); 600 dev_info(dev, "no vpcie12v regulator found\n"); 601 } 602 603 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); 604 if (IS_ERR(rockchip->vpcie3v3)) { 605 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) 606 return PTR_ERR(rockchip->vpcie3v3); 607 dev_info(dev, "no vpcie3v3 regulator found\n"); 608 } 609 610 rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8"); 611 if (IS_ERR(rockchip->vpcie1v8)) 612 return PTR_ERR(rockchip->vpcie1v8); 613 614 rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9"); 615 if (IS_ERR(rockchip->vpcie0v9)) 616 return PTR_ERR(rockchip->vpcie0v9); 617 618 return 0; 619} 620 621static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip) 622{ 623 struct device *dev = rockchip->dev; 624 int err; 625 626 if (!IS_ERR(rockchip->vpcie12v)) { 627 err = regulator_enable(rockchip->vpcie12v); 628 if (err) { 629 dev_err(dev, "fail to enable vpcie12v regulator\n"); 630 goto err_out; 631 } 632 } 633 634 if (!IS_ERR(rockchip->vpcie3v3)) { 635 err = regulator_enable(rockchip->vpcie3v3); 636 if (err) { 637 dev_err(dev, "fail to enable vpcie3v3 regulator\n"); 638 goto err_disable_12v; 639 } 640 } 641 642 err = regulator_enable(rockchip->vpcie1v8); 643 if (err) { 644 dev_err(dev, "fail to enable vpcie1v8 regulator\n"); 645 goto err_disable_3v3; 646 } 647 648 err = regulator_enable(rockchip->vpcie0v9); 649 if (err) { 650 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 651 goto err_disable_1v8; 652 } 653 654 return 0; 655 656err_disable_1v8: 657 regulator_disable(rockchip->vpcie1v8); 658err_disable_3v3: 659 if (!IS_ERR(rockchip->vpcie3v3)) 660 regulator_disable(rockchip->vpcie3v3); 661err_disable_12v: 662 if (!IS_ERR(rockchip->vpcie12v)) 663 regulator_disable(rockchip->vpcie12v); 664err_out: 665 return err; 666} 667 668static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip) 669{ 670 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) & 671 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK); 672 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT), 673 PCIE_CORE_INT_MASK); 674 675 rockchip_pcie_enable_bw_int(rockchip); 676} 677 678static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, 679 irq_hw_number_t hwirq) 680{ 681 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); 682 irq_set_chip_data(irq, domain->host_data); 683 684 return 0; 685} 686 687static const struct irq_domain_ops intx_domain_ops = { 688 .map = rockchip_pcie_intx_map, 689}; 690 691static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) 692{ 693 struct device *dev = rockchip->dev; 694 struct device_node *intc = of_get_next_child(dev->of_node, NULL); 695 696 if (!intc) { 697 dev_err(dev, "missing child interrupt-controller node\n"); 698 return -EINVAL; 699 } 700 701 rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX, 702 &intx_domain_ops, rockchip); 703 of_node_put(intc); 704 if (!rockchip->irq_domain) { 705 dev_err(dev, "failed to get a INTx IRQ domain\n"); 706 return -EINVAL; 707 } 708 709 return 0; 710} 711 712static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip, 713 int region_no, int type, u8 num_pass_bits, 714 u32 lower_addr, u32 upper_addr) 715{ 716 u32 ob_addr_0; 717 u32 ob_addr_1; 718 u32 ob_desc_0; 719 u32 aw_offset; 720 721 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM) 722 return -EINVAL; 723 if (num_pass_bits + 1 < 8) 724 return -EINVAL; 725 if (num_pass_bits > 63) 726 return -EINVAL; 727 if (region_no == 0) { 728 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits)) 729 return -EINVAL; 730 } 731 if (region_no != 0) { 732 if (AXI_REGION_SIZE < (2ULL << num_pass_bits)) 733 return -EINVAL; 734 } 735 736 aw_offset = (region_no << OB_REG_SIZE_SHIFT); 737 738 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS; 739 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR; 740 ob_addr_1 = upper_addr; 741 ob_desc_0 = (1 << 23 | type); 742 743 rockchip_pcie_write(rockchip, ob_addr_0, 744 PCIE_CORE_OB_REGION_ADDR0 + aw_offset); 745 rockchip_pcie_write(rockchip, ob_addr_1, 746 PCIE_CORE_OB_REGION_ADDR1 + aw_offset); 747 rockchip_pcie_write(rockchip, ob_desc_0, 748 PCIE_CORE_OB_REGION_DESC0 + aw_offset); 749 rockchip_pcie_write(rockchip, 0, 750 PCIE_CORE_OB_REGION_DESC1 + aw_offset); 751 752 return 0; 753} 754 755static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip, 756 int region_no, u8 num_pass_bits, 757 u32 lower_addr, u32 upper_addr) 758{ 759 u32 ib_addr_0; 760 u32 ib_addr_1; 761 u32 aw_offset; 762 763 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM) 764 return -EINVAL; 765 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED) 766 return -EINVAL; 767 if (num_pass_bits > 63) 768 return -EINVAL; 769 770 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT); 771 772 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS; 773 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR; 774 ib_addr_1 = upper_addr; 775 776 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset); 777 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset); 778 779 return 0; 780} 781 782static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip) 783{ 784 struct device *dev = rockchip->dev; 785 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 786 struct resource_entry *entry; 787 u64 pci_addr, size; 788 int offset; 789 int err; 790 int reg_no; 791 792 rockchip_pcie_cfg_configuration_accesses(rockchip, 793 AXI_WRAPPER_TYPE0_CFG); 794 entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); 795 if (!entry) 796 return -ENODEV; 797 798 size = resource_size(entry->res); 799 pci_addr = entry->res->start - entry->offset; 800 rockchip->msg_bus_addr = pci_addr; 801 802 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 803 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1, 804 AXI_WRAPPER_MEM_WRITE, 805 20 - 1, 806 pci_addr + (reg_no << 20), 807 0); 808 if (err) { 809 dev_err(dev, "program RC mem outbound ATU failed\n"); 810 return err; 811 } 812 } 813 814 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0); 815 if (err) { 816 dev_err(dev, "program RC mem inbound ATU failed\n"); 817 return err; 818 } 819 820 entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO); 821 if (!entry) 822 return -ENODEV; 823 824 /* store the register number offset to program RC io outbound ATU */ 825 offset = size >> 20; 826 827 size = resource_size(entry->res); 828 pci_addr = entry->res->start - entry->offset; 829 830 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 831 err = rockchip_pcie_prog_ob_atu(rockchip, 832 reg_no + 1 + offset, 833 AXI_WRAPPER_IO_WRITE, 834 20 - 1, 835 pci_addr + (reg_no << 20), 836 0); 837 if (err) { 838 dev_err(dev, "program RC io outbound ATU failed\n"); 839 return err; 840 } 841 } 842 843 /* assign message regions */ 844 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset, 845 AXI_WRAPPER_NOR_MSG, 846 20 - 1, 0, 0); 847 848 rockchip->msg_bus_addr += ((reg_no + offset) << 20); 849 return err; 850} 851 852static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip) 853{ 854 u32 value; 855 int err; 856 857 /* send PME_TURN_OFF message */ 858 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF); 859 860 /* read LTSSM and wait for falling into L2 link state */ 861 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0, 862 value, PCIE_LINK_IS_L2(value), 20, 863 jiffies_to_usecs(5 * HZ)); 864 if (err) { 865 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n"); 866 return err; 867 } 868 869 return 0; 870} 871 872static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev) 873{ 874 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 875 int ret; 876 877 /* disable core and cli int since we don't need to ack PME_ACK */ 878 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) | 879 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK); 880 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK); 881 882 ret = rockchip_pcie_wait_l2(rockchip); 883 if (ret) { 884 rockchip_pcie_enable_interrupts(rockchip); 885 return ret; 886 } 887 888 rockchip_pcie_deinit_phys(rockchip); 889 890 rockchip_pcie_disable_clocks(rockchip); 891 892 regulator_disable(rockchip->vpcie0v9); 893 894 return ret; 895} 896 897static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev) 898{ 899 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 900 int err; 901 902 err = regulator_enable(rockchip->vpcie0v9); 903 if (err) { 904 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 905 return err; 906 } 907 908 err = rockchip_pcie_enable_clocks(rockchip); 909 if (err) 910 goto err_disable_0v9; 911 912 err = rockchip_pcie_host_init_port(rockchip); 913 if (err) 914 goto err_pcie_resume; 915 916 err = rockchip_pcie_cfg_atu(rockchip); 917 if (err) 918 goto err_err_deinit_port; 919 920 /* Need this to enter L1 again */ 921 rockchip_pcie_update_txcredit_mui(rockchip); 922 rockchip_pcie_enable_interrupts(rockchip); 923 924 return 0; 925 926err_err_deinit_port: 927 rockchip_pcie_deinit_phys(rockchip); 928err_pcie_resume: 929 rockchip_pcie_disable_clocks(rockchip); 930err_disable_0v9: 931 regulator_disable(rockchip->vpcie0v9); 932 return err; 933} 934 935static int rockchip_pcie_probe(struct platform_device *pdev) 936{ 937 struct rockchip_pcie *rockchip; 938 struct device *dev = &pdev->dev; 939 struct pci_host_bridge *bridge; 940 int err; 941 942 if (!dev->of_node) 943 return -ENODEV; 944 945 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip)); 946 if (!bridge) 947 return -ENOMEM; 948 949 rockchip = pci_host_bridge_priv(bridge); 950 951 platform_set_drvdata(pdev, rockchip); 952 rockchip->dev = dev; 953 rockchip->is_rc = true; 954 955 err = rockchip_pcie_parse_host_dt(rockchip); 956 if (err) 957 return err; 958 959 err = rockchip_pcie_enable_clocks(rockchip); 960 if (err) 961 return err; 962 963 err = rockchip_pcie_set_vpcie(rockchip); 964 if (err) { 965 dev_err(dev, "failed to set vpcie regulator\n"); 966 goto err_set_vpcie; 967 } 968 969 err = rockchip_pcie_host_init_port(rockchip); 970 if (err) 971 goto err_vpcie; 972 973 err = rockchip_pcie_init_irq_domain(rockchip); 974 if (err < 0) 975 goto err_deinit_port; 976 977 err = rockchip_pcie_cfg_atu(rockchip); 978 if (err) 979 goto err_remove_irq_domain; 980 981 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M); 982 if (!rockchip->msg_region) { 983 err = -ENOMEM; 984 goto err_remove_irq_domain; 985 } 986 987 bridge->sysdata = rockchip; 988 bridge->ops = &rockchip_pcie_ops; 989 990 err = rockchip_pcie_setup_irq(rockchip); 991 if (err) 992 goto err_remove_irq_domain; 993 994 rockchip_pcie_enable_interrupts(rockchip); 995 996 err = pci_host_probe(bridge); 997 if (err < 0) 998 goto err_remove_irq_domain; 999 1000 return 0; 1001 1002err_remove_irq_domain: 1003 irq_domain_remove(rockchip->irq_domain); 1004err_deinit_port: 1005 rockchip_pcie_deinit_phys(rockchip); 1006err_vpcie: 1007 if (!IS_ERR(rockchip->vpcie12v)) 1008 regulator_disable(rockchip->vpcie12v); 1009 if (!IS_ERR(rockchip->vpcie3v3)) 1010 regulator_disable(rockchip->vpcie3v3); 1011 regulator_disable(rockchip->vpcie1v8); 1012 regulator_disable(rockchip->vpcie0v9); 1013err_set_vpcie: 1014 rockchip_pcie_disable_clocks(rockchip); 1015 return err; 1016} 1017 1018static int rockchip_pcie_remove(struct platform_device *pdev) 1019{ 1020 struct device *dev = &pdev->dev; 1021 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 1022 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 1023 1024 pci_stop_root_bus(bridge->bus); 1025 pci_remove_root_bus(bridge->bus); 1026 irq_domain_remove(rockchip->irq_domain); 1027 1028 rockchip_pcie_deinit_phys(rockchip); 1029 1030 rockchip_pcie_disable_clocks(rockchip); 1031 1032 if (!IS_ERR(rockchip->vpcie12v)) 1033 regulator_disable(rockchip->vpcie12v); 1034 if (!IS_ERR(rockchip->vpcie3v3)) 1035 regulator_disable(rockchip->vpcie3v3); 1036 regulator_disable(rockchip->vpcie1v8); 1037 regulator_disable(rockchip->vpcie0v9); 1038 1039 return 0; 1040} 1041 1042static const struct dev_pm_ops rockchip_pcie_pm_ops = { 1043 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq, 1044 rockchip_pcie_resume_noirq) 1045}; 1046 1047static const struct of_device_id rockchip_pcie_of_match[] = { 1048 { .compatible = "rockchip,rk3399-pcie", }, 1049 {} 1050}; 1051MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match); 1052 1053static struct platform_driver rockchip_pcie_driver = { 1054 .driver = { 1055 .name = "rockchip-pcie", 1056 .of_match_table = rockchip_pcie_of_match, 1057 .pm = &rockchip_pcie_pm_ops, 1058 }, 1059 .probe = rockchip_pcie_probe, 1060 .remove = rockchip_pcie_remove, 1061}; 1062module_platform_driver(rockchip_pcie_driver); 1063 1064MODULE_AUTHOR("Rockchip Inc"); 1065MODULE_DESCRIPTION("Rockchip AXI PCIe driver"); 1066MODULE_LICENSE("GPL v2"); 1067