1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. 4 */ 5 6#define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME 7 8#include <linux/atomic.h> 9#include <linux/cpu_pm.h> 10#include <linux/delay.h> 11#include <linux/interrupt.h> 12#include <linux/io.h> 13#include <linux/iopoll.h> 14#include <linux/kernel.h> 15#include <linux/list.h> 16#include <linux/of.h> 17#include <linux/of_irq.h> 18#include <linux/of_platform.h> 19#include <linux/platform_device.h> 20#include <linux/slab.h> 21#include <linux/spinlock.h> 22#include <linux/wait.h> 23 24#include <soc/qcom/cmd-db.h> 25#include <soc/qcom/tcs.h> 26#include <dt-bindings/soc/qcom,rpmh-rsc.h> 27 28#include "rpmh-internal.h" 29 30#define CREATE_TRACE_POINTS 31#include "trace-rpmh.h" 32 33#define RSC_DRV_TCS_OFFSET 672 34#define RSC_DRV_CMD_OFFSET 20 35 36/* DRV HW Solver Configuration Information Register */ 37#define DRV_SOLVER_CONFIG 0x04 38#define DRV_HW_SOLVER_MASK 1 39#define DRV_HW_SOLVER_SHIFT 24 40 41/* DRV TCS Configuration Information Register */ 42#define DRV_PRNT_CHLD_CONFIG 0x0C 43#define DRV_NUM_TCS_MASK 0x3F 44#define DRV_NUM_TCS_SHIFT 6 45#define DRV_NCPT_MASK 0x1F 46#define DRV_NCPT_SHIFT 27 47 48/* Offsets for common TCS Registers, one bit per TCS */ 49#define RSC_DRV_IRQ_ENABLE 0x00 50#define RSC_DRV_IRQ_STATUS 0x04 51#define RSC_DRV_IRQ_CLEAR 0x08 /* w/o; write 1 to clear */ 52 53/* 54 * Offsets for per TCS Registers. 55 * 56 * TCSes start at 0x10 from tcs_base and are stored one after another. 57 * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one 58 * of the below to find a register. 59 */ 60#define RSC_DRV_CMD_WAIT_FOR_CMPL 0x10 /* 1 bit per command */ 61#define RSC_DRV_CONTROL 0x14 62#define RSC_DRV_STATUS 0x18 /* zero if tcs is busy */ 63#define RSC_DRV_CMD_ENABLE 0x1C /* 1 bit per command */ 64 65/* 66 * Offsets for per command in a TCS. 67 * 68 * Commands (up to 16) start at 0x30 in a TCS; multiply command index 69 * by RSC_DRV_CMD_OFFSET and add one of the below to find a register. 70 */ 71#define RSC_DRV_CMD_MSGID 0x30 72#define RSC_DRV_CMD_ADDR 0x34 73#define RSC_DRV_CMD_DATA 0x38 74#define RSC_DRV_CMD_STATUS 0x3C 75#define RSC_DRV_CMD_RESP_DATA 0x40 76 77#define TCS_AMC_MODE_ENABLE BIT(16) 78#define TCS_AMC_MODE_TRIGGER BIT(24) 79 80/* TCS CMD register bit mask */ 81#define CMD_MSGID_LEN 8 82#define CMD_MSGID_RESP_REQ BIT(8) 83#define CMD_MSGID_WRITE BIT(16) 84#define CMD_STATUS_ISSUED BIT(8) 85#define CMD_STATUS_COMPL BIT(16) 86 87/* 88 * Here's a high level overview of how all the registers in RPMH work 89 * together: 90 * 91 * - The main rpmh-rsc address is the base of a register space that can 92 * be used to find overall configuration of the hardware 93 * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register 94 * space are all the TCS blocks. The offset of the TCS blocks is 95 * specified in the device tree by "qcom,tcs-offset" and used to 96 * compute tcs_base. 97 * - TCS blocks come one after another. Type, count, and order are 98 * specified by the device tree as "qcom,tcs-config". 99 * - Each TCS block has some registers, then space for up to 16 commands. 100 * Note that though address space is reserved for 16 commands, fewer 101 * might be present. See ncpt (num cmds per TCS). 102 * 103 * Here's a picture: 104 * 105 * +---------------------------------------------------+ 106 * |RSC | 107 * | ctrl | 108 * | | 109 * | Drvs: | 110 * | +-----------------------------------------------+ | 111 * | |DRV0 | | 112 * | | ctrl/config | | 113 * | | IRQ | | 114 * | | | | 115 * | | TCSes: | | 116 * | | +------------------------------------------+ | | 117 * | | |TCS0 | | | | | | | | | | | | | | | 118 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 119 * | | | | | | | | | | | | | | | | | | 120 * | | +------------------------------------------+ | | 121 * | | +------------------------------------------+ | | 122 * | | |TCS1 | | | | | | | | | | | | | | | 123 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 124 * | | | | | | | | | | | | | | | | | | 125 * | | +------------------------------------------+ | | 126 * | | +------------------------------------------+ | | 127 * | | |TCS2 | | | | | | | | | | | | | | | 128 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | 129 * | | | | | | | | | | | | | | | | | | 130 * | | +------------------------------------------+ | | 131 * | | ...... | | 132 * | +-----------------------------------------------+ | 133 * | +-----------------------------------------------+ | 134 * | |DRV1 | | 135 * | | (same as DRV0) | | 136 * | +-----------------------------------------------+ | 137 * | ...... | 138 * +---------------------------------------------------+ 139 */ 140 141static inline void __iomem * 142tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) 143{ 144 return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg; 145} 146 147static inline void __iomem * 148tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) 149{ 150 return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id; 151} 152 153static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 154 int cmd_id) 155{ 156 return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 157} 158 159static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id) 160{ 161 return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id)); 162} 163 164static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, 165 int cmd_id, u32 data) 166{ 167 writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); 168} 169 170static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id, 171 u32 data) 172{ 173 writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id)); 174} 175 176static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, 177 u32 data) 178{ 179 int i; 180 181 writel(data, tcs_reg_addr(drv, reg, tcs_id)); 182 183 /* 184 * Wait until we read back the same value. Use a counter rather than 185 * ktime for timeout since this may be called after timekeeping stops. 186 */ 187 for (i = 0; i < USEC_PER_SEC; i++) { 188 if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data) 189 return; 190 udelay(1); 191 } 192 pr_err("%s: error writing %#x to %d:%#x\n", drv->name, 193 data, tcs_id, reg); 194} 195 196/** 197 * tcs_is_free() - Return if a TCS is totally free. 198 * @drv: The RSC controller. 199 * @tcs_id: The global ID of this TCS. 200 * 201 * Returns true if nobody has claimed this TCS (by setting tcs_in_use). 202 * 203 * Context: Must be called with the drv->lock held. 204 * 205 * Return: true if the given TCS is free. 206 */ 207static bool tcs_is_free(struct rsc_drv *drv, int tcs_id) 208{ 209 return !test_bit(tcs_id, drv->tcs_in_use); 210} 211 212/** 213 * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). 214 * @drv: The RSC controller. 215 * @type: SLEEP_TCS or WAKE_TCS 216 * 217 * This will clear the "slots" variable of the given tcs_group and also 218 * tell the hardware to forget about all entries. 219 * 220 * The caller must ensure that no other RPMH actions are happening when this 221 * function is called, since otherwise the device may immediately become 222 * used again even before this function exits. 223 */ 224static void tcs_invalidate(struct rsc_drv *drv, int type) 225{ 226 int m; 227 struct tcs_group *tcs = &drv->tcs[type]; 228 229 /* Caller ensures nobody else is running so no lock */ 230 if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) 231 return; 232 233 for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { 234 write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0); 235 write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, m, 0); 236 } 237 bitmap_zero(tcs->slots, MAX_TCS_SLOTS); 238} 239 240/** 241 * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. 242 * @drv: The RSC controller. 243 * 244 * The caller must ensure that no other RPMH actions are happening when this 245 * function is called, since otherwise the device may immediately become 246 * used again even before this function exits. 247 */ 248void rpmh_rsc_invalidate(struct rsc_drv *drv) 249{ 250 tcs_invalidate(drv, SLEEP_TCS); 251 tcs_invalidate(drv, WAKE_TCS); 252} 253 254/** 255 * get_tcs_for_msg() - Get the tcs_group used to send the given message. 256 * @drv: The RSC controller. 257 * @msg: The message we want to send. 258 * 259 * This is normally pretty straightforward except if we are trying to send 260 * an ACTIVE_ONLY message but don't have any active_only TCSes. 261 * 262 * Return: A pointer to a tcs_group or an ERR_PTR. 263 */ 264static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, 265 const struct tcs_request *msg) 266{ 267 int type; 268 struct tcs_group *tcs; 269 270 switch (msg->state) { 271 case RPMH_ACTIVE_ONLY_STATE: 272 type = ACTIVE_TCS; 273 break; 274 case RPMH_WAKE_ONLY_STATE: 275 type = WAKE_TCS; 276 break; 277 case RPMH_SLEEP_STATE: 278 type = SLEEP_TCS; 279 break; 280 default: 281 return ERR_PTR(-EINVAL); 282 } 283 284 /* 285 * If we are making an active request on a RSC that does not have a 286 * dedicated TCS for active state use, then re-purpose a wake TCS to 287 * send active votes. This is safe because we ensure any active-only 288 * transfers have finished before we use it (maybe by running from 289 * the last CPU in PM code). 290 */ 291 tcs = &drv->tcs[type]; 292 if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) 293 tcs = &drv->tcs[WAKE_TCS]; 294 295 return tcs; 296} 297 298/** 299 * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS. 300 * @drv: The RSC controller. 301 * @tcs_id: The global ID of this TCS. 302 * 303 * For ACTIVE_ONLY transfers we want to call back into the client when the 304 * transfer finishes. To do this we need the "request" that the client 305 * originally provided us. This function grabs the request that we stashed 306 * when we started the transfer. 307 * 308 * This only makes sense for ACTIVE_ONLY transfers since those are the only 309 * ones we track sending (the only ones we enable interrupts for and the only 310 * ones we call back to the client for). 311 * 312 * Return: The stashed request. 313 */ 314static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, 315 int tcs_id) 316{ 317 struct tcs_group *tcs; 318 int i; 319 320 for (i = 0; i < TCS_TYPE_NR; i++) { 321 tcs = &drv->tcs[i]; 322 if (tcs->mask & BIT(tcs_id)) 323 return tcs->req[tcs_id - tcs->offset]; 324 } 325 326 return NULL; 327} 328 329/** 330 * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS 331 * @drv: The controller. 332 * @tcs_id: The global ID of this TCS. 333 * @trigger: If true then untrigger/retrigger. If false then just untrigger. 334 * 335 * In the normal case we only ever call with "trigger=true" to start a 336 * transfer. That will un-trigger/disable the TCS from the last transfer 337 * then trigger/enable for this transfer. 338 * 339 * If we borrowed a wake TCS for an active-only transfer we'll also call 340 * this function with "trigger=false" to just do the un-trigger/disable 341 * before using the TCS for wake purposes again. 342 * 343 * Note that the AP is only in charge of triggering active-only transfers. 344 * The AP never triggers sleep/wake values using this function. 345 */ 346static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) 347{ 348 u32 enable; 349 350 /* 351 * HW req: Clear the DRV_CONTROL and enable TCS again 352 * While clearing ensure that the AMC mode trigger is cleared 353 * and then the mode enable is cleared. 354 */ 355 enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); 356 enable &= ~TCS_AMC_MODE_TRIGGER; 357 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 358 enable &= ~TCS_AMC_MODE_ENABLE; 359 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 360 361 if (trigger) { 362 /* Enable the AMC mode on the TCS and then trigger the TCS */ 363 enable = TCS_AMC_MODE_ENABLE; 364 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 365 enable |= TCS_AMC_MODE_TRIGGER; 366 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); 367 } 368} 369 370/** 371 * enable_tcs_irq() - Enable or disable interrupts on the given TCS. 372 * @drv: The controller. 373 * @tcs_id: The global ID of this TCS. 374 * @enable: If true then enable; if false then disable 375 * 376 * We only ever call this when we borrow a wake TCS for an active-only 377 * transfer. For active-only TCSes interrupts are always left enabled. 378 */ 379static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) 380{ 381 u32 data; 382 383 data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE); 384 if (enable) 385 data |= BIT(tcs_id); 386 else 387 data &= ~BIT(tcs_id); 388 writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE); 389} 390 391/** 392 * tcs_tx_done() - TX Done interrupt handler. 393 * @irq: The IRQ number (ignored). 394 * @p: Pointer to "struct rsc_drv". 395 * 396 * Called for ACTIVE_ONLY transfers (those are the only ones we enable the 397 * IRQ for) when a transfer is done. 398 * 399 * Return: IRQ_HANDLED 400 */ 401static irqreturn_t tcs_tx_done(int irq, void *p) 402{ 403 struct rsc_drv *drv = p; 404 int i, j, err = 0; 405 unsigned long irq_status; 406 const struct tcs_request *req; 407 struct tcs_cmd *cmd; 408 409 irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); 410 411 for_each_set_bit(i, &irq_status, BITS_PER_LONG) { 412 req = get_req_from_tcs(drv, i); 413 if (!req) { 414 WARN_ON(1); 415 goto skip; 416 } 417 418 err = 0; 419 for (j = 0; j < req->num_cmds; j++) { 420 u32 sts; 421 422 cmd = &req->cmds[j]; 423 sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j); 424 if (!(sts & CMD_STATUS_ISSUED) || 425 ((req->wait_for_compl || cmd->wait) && 426 !(sts & CMD_STATUS_COMPL))) { 427 pr_err("Incomplete request: %s: addr=%#x data=%#x", 428 drv->name, cmd->addr, cmd->data); 429 err = -EIO; 430 } 431 } 432 433 trace_rpmh_tx_done(drv, i, req, err); 434 435 /* 436 * If wake tcs was re-purposed for sending active 437 * votes, clear AMC trigger & enable modes and 438 * disable interrupt for this TCS 439 */ 440 if (!drv->tcs[ACTIVE_TCS].num_tcs) 441 __tcs_set_trigger(drv, i, false); 442skip: 443 /* Reclaim the TCS */ 444 write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0); 445 write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, i, 0); 446 writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR); 447 spin_lock(&drv->lock); 448 clear_bit(i, drv->tcs_in_use); 449 /* 450 * Disable interrupt for WAKE TCS to avoid being 451 * spammed with interrupts coming when the solver 452 * sends its wake votes. 453 */ 454 if (!drv->tcs[ACTIVE_TCS].num_tcs) 455 enable_tcs_irq(drv, i, false); 456 spin_unlock(&drv->lock); 457 wake_up(&drv->tcs_wait); 458 if (req) 459 rpmh_tx_done(req, err); 460 } 461 462 return IRQ_HANDLED; 463} 464 465/** 466 * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger. 467 * @drv: The controller. 468 * @tcs_id: The global ID of this TCS. 469 * @cmd_id: The index within the TCS to start writing. 470 * @msg: The message we want to send, which will contain several addr/data 471 * pairs to program (but few enough that they all fit in one TCS). 472 * 473 * This is used for all types of transfers (active, sleep, and wake). 474 */ 475static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, 476 const struct tcs_request *msg) 477{ 478 u32 msgid, cmd_msgid; 479 u32 cmd_enable = 0; 480 u32 cmd_complete; 481 struct tcs_cmd *cmd; 482 int i, j; 483 484 cmd_msgid = CMD_MSGID_LEN; 485 cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; 486 cmd_msgid |= CMD_MSGID_WRITE; 487 488 cmd_complete = read_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id); 489 490 for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { 491 cmd = &msg->cmds[i]; 492 cmd_enable |= BIT(j); 493 cmd_complete |= cmd->wait << j; 494 msgid = cmd_msgid; 495 msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; 496 497 write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid); 498 write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr); 499 write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data); 500 trace_rpmh_send_msg_rcuidle(drv, tcs_id, j, msgid, cmd); 501 } 502 503 write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, cmd_complete); 504 cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); 505 write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable); 506} 507 508/** 509 * check_for_req_inflight() - Look to see if conflicting cmds are in flight. 510 * @drv: The controller. 511 * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers. 512 * @msg: The message we want to send, which will contain several addr/data 513 * pairs to program (but few enough that they all fit in one TCS). 514 * 515 * This will walk through the TCSes in the group and check if any of them 516 * appear to be sending to addresses referenced in the message. If it finds 517 * one it'll return -EBUSY. 518 * 519 * Only for use for active-only transfers. 520 * 521 * Must be called with the drv->lock held since that protects tcs_in_use. 522 * 523 * Return: 0 if nothing in flight or -EBUSY if we should try again later. 524 * The caller must re-enable interrupts between tries since that's 525 * the only way tcs_is_free() will ever return true and the only way 526 * RSC_DRV_CMD_ENABLE will ever be cleared. 527 */ 528static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, 529 const struct tcs_request *msg) 530{ 531 unsigned long curr_enabled; 532 u32 addr; 533 int i, j, k; 534 int tcs_id = tcs->offset; 535 536 for (i = 0; i < tcs->num_tcs; i++, tcs_id++) { 537 if (tcs_is_free(drv, tcs_id)) 538 continue; 539 540 curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); 541 542 for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { 543 addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j); 544 for (k = 0; k < msg->num_cmds; k++) { 545 if (addr == msg->cmds[k].addr) 546 return -EBUSY; 547 } 548 } 549 } 550 551 return 0; 552} 553 554/** 555 * find_free_tcs() - Find free tcs in the given tcs_group; only for active. 556 * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if 557 * we borrowed it because there are zero active-only ones). 558 * 559 * Must be called with the drv->lock held since that protects tcs_in_use. 560 * 561 * Return: The first tcs that's free. 562 */ 563static int find_free_tcs(struct tcs_group *tcs) 564{ 565 int i; 566 567 for (i = 0; i < tcs->num_tcs; i++) { 568 if (tcs_is_free(tcs->drv, tcs->offset + i)) 569 return tcs->offset + i; 570 } 571 572 return -EBUSY; 573} 574 575/** 576 * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active. 577 * @drv: The controller. 578 * @tcs: The tcs_group used for ACTIVE_ONLY transfers. 579 * @msg: The data to be sent. 580 * 581 * Claims a tcs in the given tcs_group while making sure that no existing cmd 582 * is in flight that would conflict with the one in @msg. 583 * 584 * Context: Must be called with the drv->lock held since that protects 585 * tcs_in_use. 586 * 587 * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight 588 * or the tcs_group is full. 589 */ 590static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, 591 const struct tcs_request *msg) 592{ 593 int ret; 594 595 /* 596 * The h/w does not like if we send a request to the same address, 597 * when one is already in-flight or being processed. 598 */ 599 ret = check_for_req_inflight(drv, tcs, msg); 600 if (ret) 601 return ret; 602 603 return find_free_tcs(tcs); 604} 605 606/** 607 * rpmh_rsc_send_data() - Write / trigger active-only message. 608 * @drv: The controller. 609 * @msg: The data to be sent. 610 * 611 * NOTES: 612 * - This is only used for "ACTIVE_ONLY" since the limitations of this 613 * function don't make sense for sleep/wake cases. 614 * - To do the transfer, we will grab a whole TCS for ourselves--we don't 615 * try to share. If there are none available we'll wait indefinitely 616 * for a free one. 617 * - This function will not wait for the commands to be finished, only for 618 * data to be programmed into the RPMh. See rpmh_tx_done() which will 619 * be called when the transfer is fully complete. 620 * - This function must be called with interrupts enabled. If the hardware 621 * is busy doing someone else's transfer we need that transfer to fully 622 * finish so that we can have the hardware, and to fully finish it needs 623 * the interrupt handler to run. If the interrupts is set to run on the 624 * active CPU this can never happen if interrupts are disabled. 625 * 626 * Return: 0 on success, -EINVAL on error. 627 */ 628int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) 629{ 630 struct tcs_group *tcs; 631 int tcs_id; 632 unsigned long flags; 633 634 tcs = get_tcs_for_msg(drv, msg); 635 if (IS_ERR(tcs)) 636 return PTR_ERR(tcs); 637 638 spin_lock_irqsave(&drv->lock, flags); 639 640 /* Wait forever for a free tcs. It better be there eventually! */ 641 wait_event_lock_irq(drv->tcs_wait, 642 (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0, 643 drv->lock); 644 645 tcs->req[tcs_id - tcs->offset] = msg; 646 set_bit(tcs_id, drv->tcs_in_use); 647 if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { 648 /* 649 * Clear previously programmed WAKE commands in selected 650 * repurposed TCS to avoid triggering them. tcs->slots will be 651 * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() 652 */ 653 write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); 654 write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0); 655 enable_tcs_irq(drv, tcs_id, true); 656 } 657 spin_unlock_irqrestore(&drv->lock, flags); 658 659 /* 660 * These two can be done after the lock is released because: 661 * - We marked "tcs_in_use" under lock. 662 * - Once "tcs_in_use" has been marked nobody else could be writing 663 * to these registers until the interrupt goes off. 664 * - The interrupt can't go off until we trigger w/ the last line 665 * of __tcs_set_trigger() below. 666 */ 667 __tcs_buffer_write(drv, tcs_id, 0, msg); 668 __tcs_set_trigger(drv, tcs_id, true); 669 670 return 0; 671} 672 673/** 674 * find_slots() - Find a place to write the given message. 675 * @tcs: The tcs group to search. 676 * @msg: The message we want to find room for. 677 * @tcs_id: If we return 0 from the function, we return the global ID of the 678 * TCS to write to here. 679 * @cmd_id: If we return 0 from the function, we return the index of 680 * the command array of the returned TCS where the client should 681 * start writing the message. 682 * 683 * Only for use on sleep/wake TCSes since those are the only ones we maintain 684 * tcs->slots for. 685 * 686 * Return: -ENOMEM if there was no room, else 0. 687 */ 688static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, 689 int *tcs_id, int *cmd_id) 690{ 691 int slot, offset; 692 int i = 0; 693 694 /* Do over, until we can fit the full payload in a single TCS */ 695 do { 696 slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, 697 i, msg->num_cmds, 0); 698 if (slot >= tcs->num_tcs * tcs->ncpt) 699 return -ENOMEM; 700 i += tcs->ncpt; 701 } while (slot + msg->num_cmds - 1 >= i); 702 703 bitmap_set(tcs->slots, slot, msg->num_cmds); 704 705 offset = slot / tcs->ncpt; 706 *tcs_id = offset + tcs->offset; 707 *cmd_id = slot % tcs->ncpt; 708 709 return 0; 710} 711 712/** 713 * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. 714 * @drv: The controller. 715 * @msg: The data to be written to the controller. 716 * 717 * This should only be called for for sleep/wake state, never active-only 718 * state. 719 * 720 * The caller must ensure that no other RPMH actions are happening and the 721 * controller is idle when this function is called since it runs lockless. 722 * 723 * Return: 0 if no error; else -error. 724 */ 725int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) 726{ 727 struct tcs_group *tcs; 728 int tcs_id = 0, cmd_id = 0; 729 int ret; 730 731 tcs = get_tcs_for_msg(drv, msg); 732 if (IS_ERR(tcs)) 733 return PTR_ERR(tcs); 734 735 /* find the TCS id and the command in the TCS to write to */ 736 ret = find_slots(tcs, msg, &tcs_id, &cmd_id); 737 if (!ret) 738 __tcs_buffer_write(drv, tcs_id, cmd_id, msg); 739 740 return ret; 741} 742 743/** 744 * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy. 745 * @drv: The controller 746 * 747 * Checks if any of the AMCs are busy in handling ACTIVE sets. 748 * This is called from the last cpu powering down before flushing 749 * SLEEP and WAKE sets. If AMCs are busy, controller can not enter 750 * power collapse, so deny from the last cpu's pm notification. 751 * 752 * Context: Must be called with the drv->lock held. 753 * 754 * Return: 755 * * False - AMCs are idle 756 * * True - AMCs are busy 757 */ 758static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) 759{ 760 int m; 761 struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; 762 763 /* 764 * If we made an active request on a RSC that does not have a 765 * dedicated TCS for active state use, then re-purposed wake TCSes 766 * should be checked for not busy, because we used wake TCSes for 767 * active requests in this case. 768 */ 769 if (!tcs->num_tcs) 770 tcs = &drv->tcs[WAKE_TCS]; 771 772 for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { 773 if (!tcs_is_free(drv, m)) 774 return true; 775 } 776 777 return false; 778} 779 780/** 781 * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy. 782 * @nfb: Pointer to the notifier block in struct rsc_drv. 783 * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT. 784 * @v: Unused 785 * 786 * This function is given to cpu_pm_register_notifier so we can be informed 787 * about when CPUs go down. When all CPUs go down we know no more active 788 * transfers will be started so we write sleep/wake sets. This function gets 789 * called from cpuidle code paths and also at system suspend time. 790 * 791 * If its last CPU going down and AMCs are not busy then writes cached sleep 792 * and wake messages to TCSes. The firmware then takes care of triggering 793 * them when entering deepest low power modes. 794 * 795 * Return: See cpu_pm_register_notifier() 796 */ 797static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, 798 unsigned long action, void *v) 799{ 800 struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); 801 int ret = NOTIFY_OK; 802 int cpus_in_pm; 803 804 switch (action) { 805 case CPU_PM_ENTER: 806 cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm); 807 /* 808 * NOTE: comments for num_online_cpus() point out that it's 809 * only a snapshot so we need to be careful. It should be OK 810 * for us to use, though. It's important for us not to miss 811 * if we're the last CPU going down so it would only be a 812 * problem if a CPU went offline right after we did the check 813 * AND that CPU was not idle AND that CPU was the last non-idle 814 * CPU. That can't happen. CPUs would have to come out of idle 815 * before the CPU could go offline. 816 */ 817 if (cpus_in_pm < num_online_cpus()) 818 return NOTIFY_OK; 819 break; 820 case CPU_PM_ENTER_FAILED: 821 case CPU_PM_EXIT: 822 atomic_dec(&drv->cpus_in_pm); 823 return NOTIFY_OK; 824 default: 825 return NOTIFY_DONE; 826 } 827 828 /* 829 * It's likely we're on the last CPU. Grab the drv->lock and write 830 * out the sleep/wake commands to RPMH hardware. Grabbing the lock 831 * means that if we race with another CPU coming up we are still 832 * guaranteed to be safe. If another CPU came up just after we checked 833 * and has grabbed the lock or started an active transfer then we'll 834 * notice we're busy and abort. If another CPU comes up after we start 835 * flushing it will be blocked from starting an active transfer until 836 * we're done flushing. If another CPU starts an active transfer after 837 * we release the lock we're still OK because we're no longer the last 838 * CPU. 839 */ 840 if (spin_trylock(&drv->lock)) { 841 if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)) 842 ret = NOTIFY_BAD; 843 spin_unlock(&drv->lock); 844 } else { 845 /* Another CPU must be up */ 846 return NOTIFY_OK; 847 } 848 849 if (ret == NOTIFY_BAD) { 850 /* Double-check if we're here because someone else is up */ 851 if (cpus_in_pm < num_online_cpus()) 852 ret = NOTIFY_OK; 853 else 854 /* We won't be called w/ CPU_PM_ENTER_FAILED */ 855 atomic_dec(&drv->cpus_in_pm); 856 } 857 858 return ret; 859} 860 861static int rpmh_probe_tcs_config(struct platform_device *pdev, 862 struct rsc_drv *drv, void __iomem *base) 863{ 864 struct tcs_type_config { 865 u32 type; 866 u32 n; 867 } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; 868 struct device_node *dn = pdev->dev.of_node; 869 u32 config, max_tcs, ncpt, offset; 870 int i, ret, n, st = 0; 871 struct tcs_group *tcs; 872 873 ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); 874 if (ret) 875 return ret; 876 drv->tcs_base = base + offset; 877 878 config = readl_relaxed(base + DRV_PRNT_CHLD_CONFIG); 879 880 max_tcs = config; 881 max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); 882 max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); 883 884 ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); 885 ncpt = ncpt >> DRV_NCPT_SHIFT; 886 887 n = of_property_count_u32_elems(dn, "qcom,tcs-config"); 888 if (n != 2 * TCS_TYPE_NR) 889 return -EINVAL; 890 891 for (i = 0; i < TCS_TYPE_NR; i++) { 892 ret = of_property_read_u32_index(dn, "qcom,tcs-config", 893 i * 2, &tcs_cfg[i].type); 894 if (ret) 895 return ret; 896 if (tcs_cfg[i].type >= TCS_TYPE_NR) 897 return -EINVAL; 898 899 ret = of_property_read_u32_index(dn, "qcom,tcs-config", 900 i * 2 + 1, &tcs_cfg[i].n); 901 if (ret) 902 return ret; 903 if (tcs_cfg[i].n > MAX_TCS_PER_TYPE) 904 return -EINVAL; 905 } 906 907 for (i = 0; i < TCS_TYPE_NR; i++) { 908 tcs = &drv->tcs[tcs_cfg[i].type]; 909 if (tcs->drv) 910 return -EINVAL; 911 tcs->drv = drv; 912 tcs->type = tcs_cfg[i].type; 913 tcs->num_tcs = tcs_cfg[i].n; 914 tcs->ncpt = ncpt; 915 916 if (!tcs->num_tcs || tcs->type == CONTROL_TCS) 917 continue; 918 919 if (st + tcs->num_tcs > max_tcs || 920 st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) 921 return -EINVAL; 922 923 tcs->mask = ((1 << tcs->num_tcs) - 1) << st; 924 tcs->offset = st; 925 st += tcs->num_tcs; 926 } 927 928 drv->num_tcs = st; 929 930 return 0; 931} 932 933static int rpmh_rsc_probe(struct platform_device *pdev) 934{ 935 struct device_node *dn = pdev->dev.of_node; 936 struct rsc_drv *drv; 937 struct resource *res; 938 char drv_id[10] = {0}; 939 int ret, irq; 940 u32 solver_config; 941 void __iomem *base; 942 943 /* 944 * Even though RPMh doesn't directly use cmd-db, all of its children 945 * do. To avoid adding this check to our children we'll do it now. 946 */ 947 ret = cmd_db_ready(); 948 if (ret) { 949 if (ret != -EPROBE_DEFER) 950 dev_err(&pdev->dev, "Command DB not available (%d)\n", 951 ret); 952 return ret; 953 } 954 955 drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); 956 if (!drv) 957 return -ENOMEM; 958 959 ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); 960 if (ret) 961 return ret; 962 963 drv->name = of_get_property(dn, "label", NULL); 964 if (!drv->name) 965 drv->name = dev_name(&pdev->dev); 966 967 snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); 968 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id); 969 base = devm_ioremap_resource(&pdev->dev, res); 970 if (IS_ERR(base)) 971 return PTR_ERR(base); 972 973 ret = rpmh_probe_tcs_config(pdev, drv, base); 974 if (ret) 975 return ret; 976 977 spin_lock_init(&drv->lock); 978 init_waitqueue_head(&drv->tcs_wait); 979 bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); 980 981 irq = platform_get_irq(pdev, drv->id); 982 if (irq < 0) 983 return irq; 984 985 ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, 986 IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, 987 drv->name, drv); 988 if (ret) 989 return ret; 990 991 /* 992 * CPU PM notification are not required for controllers that support 993 * 'HW solver' mode where they can be in autonomous mode executing low 994 * power mode to power down. 995 */ 996 solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG); 997 solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; 998 solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; 999 if (!solver_config) { 1000 drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; 1001 cpu_pm_register_notifier(&drv->rsc_pm); 1002 } 1003 1004 /* Enable the active TCS to send requests immediately */ 1005 writel_relaxed(drv->tcs[ACTIVE_TCS].mask, 1006 drv->tcs_base + RSC_DRV_IRQ_ENABLE); 1007 1008 spin_lock_init(&drv->client.cache_lock); 1009 INIT_LIST_HEAD(&drv->client.cache); 1010 INIT_LIST_HEAD(&drv->client.batch_cache); 1011 1012 dev_set_drvdata(&pdev->dev, drv); 1013 1014 return devm_of_platform_populate(&pdev->dev); 1015} 1016 1017static const struct of_device_id rpmh_drv_match[] = { 1018 { .compatible = "qcom,rpmh-rsc", }, 1019 { } 1020}; 1021 1022static struct platform_driver rpmh_driver = { 1023 .probe = rpmh_rsc_probe, 1024 .driver = { 1025 .name = "rpmh", 1026 .of_match_table = rpmh_drv_match, 1027 .suppress_bind_attrs = true, 1028 }, 1029}; 1030 1031static int __init rpmh_driver_init(void) 1032{ 1033 return platform_driver_register(&rpmh_driver); 1034} 1035arch_initcall(rpmh_driver_init); 1036