1// SPDX-License-Identifier: GPL-2.0 2/* 3 * TI K3 NAVSS Ring Accelerator subsystem driver 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 */ 7 8#include <linux/dma-mapping.h> 9#include <linux/io.h> 10#include <linux/init.h> 11#include <linux/of.h> 12#include <linux/platform_device.h> 13#include <linux/sys_soc.h> 14#include <linux/soc/ti/k3-ringacc.h> 15#include <linux/soc/ti/ti_sci_protocol.h> 16#include <linux/soc/ti/ti_sci_inta_msi.h> 17#include <linux/of_irq.h> 18#include <linux/irqdomain.h> 19 20static LIST_HEAD(k3_ringacc_list); 21static DEFINE_MUTEX(k3_ringacc_list_lock); 22 23#define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) 24 25/** 26 * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region 27 * 28 * @resv_16: Reserved 29 * @db: Ring Doorbell Register 30 * @resv_4: Reserved 31 * @occ: Ring Occupancy Register 32 * @indx: Ring Current Index Register 33 * @hwocc: Ring Hardware Occupancy Register 34 * @hwindx: Ring Hardware Current Index Register 35 */ 36struct k3_ring_rt_regs { 37 u32 resv_16[4]; 38 u32 db; 39 u32 resv_4[1]; 40 u32 occ; 41 u32 indx; 42 u32 hwocc; 43 u32 hwindx; 44}; 45 46#define K3_RINGACC_RT_REGS_STEP 0x1000 47 48/** 49 * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region 50 * 51 * @head_data: Ring Head Entry Data Registers 52 * @tail_data: Ring Tail Entry Data Registers 53 * @peek_head_data: Ring Peek Head Entry Data Regs 54 * @peek_tail_data: Ring Peek Tail Entry Data Regs 55 */ 56struct k3_ring_fifo_regs { 57 u32 head_data[128]; 58 u32 tail_data[128]; 59 u32 peek_head_data[128]; 60 u32 peek_tail_data[128]; 61}; 62 63/** 64 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region 65 * 66 * @revision: Revision Register 67 * @config: Config Register 68 */ 69struct k3_ringacc_proxy_gcfg_regs { 70 u32 revision; 71 u32 config; 72}; 73 74#define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) 75 76/** 77 * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region 78 * 79 * @control: Proxy Control Register 80 * @status: Proxy Status Register 81 * @resv_512: Reserved 82 * @data: Proxy Data Register 83 */ 84struct k3_ringacc_proxy_target_regs { 85 u32 control; 86 u32 status; 87 u8 resv_512[504]; 88 u32 data[128]; 89}; 90 91#define K3_RINGACC_PROXY_TARGET_STEP 0x1000 92#define K3_RINGACC_PROXY_NOT_USED (-1) 93 94enum k3_ringacc_proxy_access_mode { 95 PROXY_ACCESS_MODE_HEAD = 0, 96 PROXY_ACCESS_MODE_TAIL = 1, 97 PROXY_ACCESS_MODE_PEEK_HEAD = 2, 98 PROXY_ACCESS_MODE_PEEK_TAIL = 3, 99}; 100 101#define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) 102#define K3_RINGACC_FIFO_REGS_STEP 0x1000 103#define K3_RINGACC_MAX_DB_RING_CNT (127U) 104 105struct k3_ring_ops { 106 int (*push_tail)(struct k3_ring *ring, void *elm); 107 int (*push_head)(struct k3_ring *ring, void *elm); 108 int (*pop_tail)(struct k3_ring *ring, void *elm); 109 int (*pop_head)(struct k3_ring *ring, void *elm); 110}; 111 112/** 113 * struct k3_ring_state - Internal state tracking structure 114 * 115 * @free: Number of free entries 116 * @occ: Occupancy 117 * @windex: Write index 118 * @rindex: Read index 119 */ 120struct k3_ring_state { 121 u32 free; 122 u32 occ; 123 u32 windex; 124 u32 rindex; 125}; 126 127/** 128 * struct k3_ring - RA Ring descriptor 129 * 130 * @rt: Ring control/status registers 131 * @fifos: Ring queues registers 132 * @proxy: Ring Proxy Datapath registers 133 * @ring_mem_dma: Ring buffer dma address 134 * @ring_mem_virt: Ring buffer virt address 135 * @ops: Ring operations 136 * @size: Ring size in elements 137 * @elm_size: Size of the ring element 138 * @mode: Ring mode 139 * @flags: flags 140 * @ring_id: Ring Id 141 * @parent: Pointer on struct @k3_ringacc 142 * @use_count: Use count for shared rings 143 * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) 144 */ 145struct k3_ring { 146 struct k3_ring_rt_regs __iomem *rt; 147 struct k3_ring_fifo_regs __iomem *fifos; 148 struct k3_ringacc_proxy_target_regs __iomem *proxy; 149 dma_addr_t ring_mem_dma; 150 void *ring_mem_virt; 151 struct k3_ring_ops *ops; 152 u32 size; 153 enum k3_ring_size elm_size; 154 enum k3_ring_mode mode; 155 u32 flags; 156#define K3_RING_FLAG_BUSY BIT(1) 157#define K3_RING_FLAG_SHARED BIT(2) 158 struct k3_ring_state state; 159 u32 ring_id; 160 struct k3_ringacc *parent; 161 u32 use_count; 162 int proxy_id; 163}; 164 165struct k3_ringacc_ops { 166 int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc); 167}; 168 169/** 170 * struct k3_ringacc - Rings accelerator descriptor 171 * 172 * @dev: pointer on RA device 173 * @proxy_gcfg: RA proxy global config registers 174 * @proxy_target_base: RA proxy datapath region 175 * @num_rings: number of ring in RA 176 * @rings_inuse: bitfield for ring usage tracking 177 * @rm_gp_range: general purpose rings range from tisci 178 * @dma_ring_reset_quirk: DMA reset w/a enable 179 * @num_proxies: number of RA proxies 180 * @proxy_inuse: bitfield for proxy usage tracking 181 * @rings: array of rings descriptors (struct @k3_ring) 182 * @list: list of RAs in the system 183 * @req_lock: protect rings allocation 184 * @tisci: pointer ti-sci handle 185 * @tisci_ring_ops: ti-sci rings ops 186 * @tisci_dev_id: ti-sci device id 187 * @ops: SoC specific ringacc operation 188 */ 189struct k3_ringacc { 190 struct device *dev; 191 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; 192 void __iomem *proxy_target_base; 193 u32 num_rings; /* number of rings in Ringacc module */ 194 unsigned long *rings_inuse; 195 struct ti_sci_resource *rm_gp_range; 196 197 bool dma_ring_reset_quirk; 198 u32 num_proxies; 199 unsigned long *proxy_inuse; 200 201 struct k3_ring *rings; 202 struct list_head list; 203 struct mutex req_lock; /* protect rings allocation */ 204 205 const struct ti_sci_handle *tisci; 206 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; 207 u32 tisci_dev_id; 208 209 const struct k3_ringacc_ops *ops; 210}; 211 212/** 213 * struct k3_ringacc - Rings accelerator SoC data 214 * 215 * @dma_ring_reset_quirk: DMA reset w/a enable 216 */ 217struct k3_ringacc_soc_data { 218 unsigned dma_ring_reset_quirk:1; 219}; 220 221static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) 222{ 223 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - 224 (4 << ring->elm_size); 225} 226 227static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) 228{ 229 return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); 230} 231 232static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); 233static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); 234 235static struct k3_ring_ops k3_ring_mode_ring_ops = { 236 .push_tail = k3_ringacc_ring_push_mem, 237 .pop_head = k3_ringacc_ring_pop_mem, 238}; 239 240static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); 241static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); 242static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); 243static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); 244 245static struct k3_ring_ops k3_ring_mode_msg_ops = { 246 .push_tail = k3_ringacc_ring_push_io, 247 .push_head = k3_ringacc_ring_push_head_io, 248 .pop_tail = k3_ringacc_ring_pop_tail_io, 249 .pop_head = k3_ringacc_ring_pop_io, 250}; 251 252static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); 253static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); 254static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); 255static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); 256 257static struct k3_ring_ops k3_ring_mode_proxy_ops = { 258 .push_tail = k3_ringacc_ring_push_tail_proxy, 259 .push_head = k3_ringacc_ring_push_head_proxy, 260 .pop_tail = k3_ringacc_ring_pop_tail_proxy, 261 .pop_head = k3_ringacc_ring_pop_head_proxy, 262}; 263 264static void k3_ringacc_ring_dump(struct k3_ring *ring) 265{ 266 struct device *dev = ring->parent->dev; 267 268 dev_dbg(dev, "dump ring: %d\n", ring->ring_id); 269 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, 270 &ring->ring_mem_dma); 271 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", 272 ring->elm_size, ring->size, ring->mode, ring->proxy_id); 273 dev_dbg(dev, "dump flags %08X\n", ring->flags); 274 275 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); 276 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); 277 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); 278 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); 279 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); 280 281 if (ring->ring_mem_virt) 282 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, 283 16, 1, ring->ring_mem_virt, 16 * 8, false); 284} 285 286struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, 287 int id, u32 flags) 288{ 289 int proxy_id = K3_RINGACC_PROXY_NOT_USED; 290 291 mutex_lock(&ringacc->req_lock); 292 293 if (id == K3_RINGACC_RING_ID_ANY) { 294 /* Request for any general purpose ring */ 295 struct ti_sci_resource_desc *gp_rings = 296 &ringacc->rm_gp_range->desc[0]; 297 unsigned long size; 298 299 size = gp_rings->start + gp_rings->num; 300 id = find_next_zero_bit(ringacc->rings_inuse, size, 301 gp_rings->start); 302 if (id == size) 303 goto error; 304 } else if (id < 0) { 305 goto error; 306 } 307 308 if (test_bit(id, ringacc->rings_inuse) && 309 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) 310 goto error; 311 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) 312 goto out; 313 314 if (flags & K3_RINGACC_RING_USE_PROXY) { 315 proxy_id = find_next_zero_bit(ringacc->proxy_inuse, 316 ringacc->num_proxies, 0); 317 if (proxy_id == ringacc->num_proxies) 318 goto error; 319 } 320 321 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { 322 set_bit(proxy_id, ringacc->proxy_inuse); 323 ringacc->rings[id].proxy_id = proxy_id; 324 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, 325 proxy_id); 326 } else { 327 dev_dbg(ringacc->dev, "Giving ring#%d\n", id); 328 } 329 330 set_bit(id, ringacc->rings_inuse); 331out: 332 ringacc->rings[id].use_count++; 333 mutex_unlock(&ringacc->req_lock); 334 return &ringacc->rings[id]; 335 336error: 337 mutex_unlock(&ringacc->req_lock); 338 return NULL; 339} 340EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); 341 342int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, 343 int fwd_id, int compl_id, 344 struct k3_ring **fwd_ring, 345 struct k3_ring **compl_ring) 346{ 347 int ret = 0; 348 349 if (!fwd_ring || !compl_ring) 350 return -EINVAL; 351 352 *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); 353 if (!(*fwd_ring)) 354 return -ENODEV; 355 356 *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0); 357 if (!(*compl_ring)) { 358 k3_ringacc_ring_free(*fwd_ring); 359 ret = -ENODEV; 360 } 361 362 return ret; 363} 364EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair); 365 366static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) 367{ 368 struct k3_ringacc *ringacc = ring->parent; 369 int ret; 370 371 ret = ringacc->tisci_ring_ops->config( 372 ringacc->tisci, 373 TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID, 374 ringacc->tisci_dev_id, 375 ring->ring_id, 376 0, 377 0, 378 ring->size, 379 0, 380 0, 381 0); 382 if (ret) 383 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", 384 ret, ring->ring_id); 385} 386 387void k3_ringacc_ring_reset(struct k3_ring *ring) 388{ 389 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 390 return; 391 392 memset(&ring->state, 0, sizeof(ring->state)); 393 394 k3_ringacc_ring_reset_sci(ring); 395} 396EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); 397 398static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, 399 enum k3_ring_mode mode) 400{ 401 struct k3_ringacc *ringacc = ring->parent; 402 int ret; 403 404 ret = ringacc->tisci_ring_ops->config( 405 ringacc->tisci, 406 TI_SCI_MSG_VALUE_RM_RING_MODE_VALID, 407 ringacc->tisci_dev_id, 408 ring->ring_id, 409 0, 410 0, 411 0, 412 mode, 413 0, 414 0); 415 if (ret) 416 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", 417 ret, ring->ring_id); 418} 419 420void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) 421{ 422 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 423 return; 424 425 if (!ring->parent->dma_ring_reset_quirk) 426 goto reset; 427 428 if (!occ) 429 occ = readl(&ring->rt->occ); 430 431 if (occ) { 432 u32 db_ring_cnt, db_ring_cnt_cur; 433 434 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, 435 ring->ring_id, occ); 436 /* TI-SCI ring reset */ 437 k3_ringacc_ring_reset_sci(ring); 438 439 /* 440 * Setup the ring in ring/doorbell mode (if not already in this 441 * mode) 442 */ 443 if (ring->mode != K3_RINGACC_RING_MODE_RING) 444 k3_ringacc_ring_reconfig_qmode_sci( 445 ring, K3_RINGACC_RING_MODE_RING); 446 /* 447 * Ring the doorbell 2**22 – ringOcc times. 448 * This will wrap the internal UDMAP ring state occupancy 449 * counter (which is 21-bits wide) to 0. 450 */ 451 db_ring_cnt = (1U << 22) - occ; 452 453 while (db_ring_cnt != 0) { 454 /* 455 * Ring the doorbell with the maximum count each 456 * iteration if possible to minimize the total 457 * of writes 458 */ 459 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) 460 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; 461 else 462 db_ring_cnt_cur = db_ring_cnt; 463 464 writel(db_ring_cnt_cur, &ring->rt->db); 465 db_ring_cnt -= db_ring_cnt_cur; 466 } 467 468 /* Restore the original ring mode (if not ring mode) */ 469 if (ring->mode != K3_RINGACC_RING_MODE_RING) 470 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); 471 } 472 473reset: 474 /* Reset the ring */ 475 k3_ringacc_ring_reset(ring); 476} 477EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); 478 479static void k3_ringacc_ring_free_sci(struct k3_ring *ring) 480{ 481 struct k3_ringacc *ringacc = ring->parent; 482 int ret; 483 484 ret = ringacc->tisci_ring_ops->config( 485 ringacc->tisci, 486 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER, 487 ringacc->tisci_dev_id, 488 ring->ring_id, 489 0, 490 0, 491 0, 492 0, 493 0, 494 0); 495 if (ret) 496 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", 497 ret, ring->ring_id); 498} 499 500int k3_ringacc_ring_free(struct k3_ring *ring) 501{ 502 struct k3_ringacc *ringacc; 503 504 if (!ring) 505 return -EINVAL; 506 507 ringacc = ring->parent; 508 509 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); 510 511 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 512 return -EINVAL; 513 514 mutex_lock(&ringacc->req_lock); 515 516 if (--ring->use_count) 517 goto out; 518 519 if (!(ring->flags & K3_RING_FLAG_BUSY)) 520 goto no_init; 521 522 k3_ringacc_ring_free_sci(ring); 523 524 dma_free_coherent(ringacc->dev, 525 ring->size * (4 << ring->elm_size), 526 ring->ring_mem_virt, ring->ring_mem_dma); 527 ring->flags = 0; 528 ring->ops = NULL; 529 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { 530 clear_bit(ring->proxy_id, ringacc->proxy_inuse); 531 ring->proxy = NULL; 532 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 533 } 534 535no_init: 536 clear_bit(ring->ring_id, ringacc->rings_inuse); 537 538out: 539 mutex_unlock(&ringacc->req_lock); 540 return 0; 541} 542EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); 543 544u32 k3_ringacc_get_ring_id(struct k3_ring *ring) 545{ 546 if (!ring) 547 return -EINVAL; 548 549 return ring->ring_id; 550} 551EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); 552 553u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) 554{ 555 if (!ring) 556 return -EINVAL; 557 558 return ring->parent->tisci_dev_id; 559} 560EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); 561 562int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) 563{ 564 int irq_num; 565 566 if (!ring) 567 return -EINVAL; 568 569 irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); 570 if (irq_num <= 0) 571 irq_num = -EINVAL; 572 return irq_num; 573} 574EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); 575 576static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) 577{ 578 struct k3_ringacc *ringacc = ring->parent; 579 u32 ring_idx; 580 int ret; 581 582 if (!ringacc->tisci) 583 return -EINVAL; 584 585 ring_idx = ring->ring_id; 586 ret = ringacc->tisci_ring_ops->config( 587 ringacc->tisci, 588 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER, 589 ringacc->tisci_dev_id, 590 ring_idx, 591 lower_32_bits(ring->ring_mem_dma), 592 upper_32_bits(ring->ring_mem_dma), 593 ring->size, 594 ring->mode, 595 ring->elm_size, 596 0); 597 if (ret) 598 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", 599 ret, ring_idx); 600 601 return ret; 602} 603 604int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 605{ 606 struct k3_ringacc *ringacc; 607 int ret = 0; 608 609 if (!ring || !cfg) 610 return -EINVAL; 611 ringacc = ring->parent; 612 613 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || 614 cfg->mode >= K3_RINGACC_RING_MODE_INVALID || 615 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || 616 !test_bit(ring->ring_id, ringacc->rings_inuse)) 617 return -EINVAL; 618 619 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && 620 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && 621 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { 622 dev_err(ringacc->dev, 623 "Message mode must use proxy for %u element size\n", 624 4 << ring->elm_size); 625 return -EINVAL; 626 } 627 628 /* 629 * In case of shared ring only the first user (master user) can 630 * configure the ring. The sequence should be by the client: 631 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user 632 * k3_ringacc_ring_cfg(ring, cfg); # master configuration 633 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 634 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 635 */ 636 if (ring->use_count != 1) 637 return 0; 638 639 ring->size = cfg->size; 640 ring->elm_size = cfg->elm_size; 641 ring->mode = cfg->mode; 642 memset(&ring->state, 0, sizeof(ring->state)); 643 644 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) 645 ring->proxy = ringacc->proxy_target_base + 646 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; 647 648 switch (ring->mode) { 649 case K3_RINGACC_RING_MODE_RING: 650 ring->ops = &k3_ring_mode_ring_ops; 651 break; 652 case K3_RINGACC_RING_MODE_MESSAGE: 653 if (ring->proxy) 654 ring->ops = &k3_ring_mode_proxy_ops; 655 else 656 ring->ops = &k3_ring_mode_msg_ops; 657 break; 658 default: 659 ring->ops = NULL; 660 ret = -EINVAL; 661 goto err_free_proxy; 662 } 663 664 ring->ring_mem_virt = dma_alloc_coherent(ringacc->dev, 665 ring->size * (4 << ring->elm_size), 666 &ring->ring_mem_dma, GFP_KERNEL); 667 if (!ring->ring_mem_virt) { 668 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 669 ret = -ENOMEM; 670 goto err_free_ops; 671 } 672 673 ret = k3_ringacc_ring_cfg_sci(ring); 674 675 if (ret) 676 goto err_free_mem; 677 678 ring->flags |= K3_RING_FLAG_BUSY; 679 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? 680 K3_RING_FLAG_SHARED : 0; 681 682 k3_ringacc_ring_dump(ring); 683 684 return 0; 685 686err_free_mem: 687 dma_free_coherent(ringacc->dev, 688 ring->size * (4 << ring->elm_size), 689 ring->ring_mem_virt, 690 ring->ring_mem_dma); 691err_free_ops: 692 ring->ops = NULL; 693err_free_proxy: 694 ring->proxy = NULL; 695 return ret; 696} 697EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); 698 699u32 k3_ringacc_ring_get_size(struct k3_ring *ring) 700{ 701 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 702 return -EINVAL; 703 704 return ring->size; 705} 706EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); 707 708u32 k3_ringacc_ring_get_free(struct k3_ring *ring) 709{ 710 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 711 return -EINVAL; 712 713 if (!ring->state.free) 714 ring->state.free = ring->size - readl(&ring->rt->occ); 715 716 return ring->state.free; 717} 718EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); 719 720u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) 721{ 722 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 723 return -EINVAL; 724 725 return readl(&ring->rt->occ); 726} 727EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); 728 729u32 k3_ringacc_ring_is_full(struct k3_ring *ring) 730{ 731 return !k3_ringacc_ring_get_free(ring); 732} 733EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); 734 735enum k3_ringacc_access_mode { 736 K3_RINGACC_ACCESS_MODE_PUSH_HEAD, 737 K3_RINGACC_ACCESS_MODE_POP_HEAD, 738 K3_RINGACC_ACCESS_MODE_PUSH_TAIL, 739 K3_RINGACC_ACCESS_MODE_POP_TAIL, 740 K3_RINGACC_ACCESS_MODE_PEEK_HEAD, 741 K3_RINGACC_ACCESS_MODE_PEEK_TAIL, 742}; 743 744#define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) 745#define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) 746static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, 747 enum k3_ringacc_proxy_access_mode mode) 748{ 749 u32 val; 750 751 val = ring->ring_id; 752 val |= K3_RINGACC_PROXY_MODE(mode); 753 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); 754 writel(val, &ring->proxy->control); 755 return 0; 756} 757 758static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, 759 enum k3_ringacc_access_mode access_mode) 760{ 761 void __iomem *ptr; 762 763 ptr = (void __iomem *)&ring->proxy->data; 764 765 switch (access_mode) { 766 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 767 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 768 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); 769 break; 770 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 771 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 772 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); 773 break; 774 default: 775 return -EINVAL; 776 } 777 778 ptr += k3_ringacc_ring_get_fifo_pos(ring); 779 780 switch (access_mode) { 781 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 782 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 783 dev_dbg(ring->parent->dev, 784 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 785 access_mode); 786 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 787 ring->state.occ--; 788 break; 789 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 790 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 791 dev_dbg(ring->parent->dev, 792 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 793 access_mode); 794 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 795 ring->state.free--; 796 break; 797 default: 798 return -EINVAL; 799 } 800 801 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, 802 ring->state.occ); 803 return 0; 804} 805 806static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) 807{ 808 return k3_ringacc_ring_access_proxy(ring, elem, 809 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 810} 811 812static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) 813{ 814 return k3_ringacc_ring_access_proxy(ring, elem, 815 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 816} 817 818static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) 819{ 820 return k3_ringacc_ring_access_proxy(ring, elem, 821 K3_RINGACC_ACCESS_MODE_POP_HEAD); 822} 823 824static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) 825{ 826 return k3_ringacc_ring_access_proxy(ring, elem, 827 K3_RINGACC_ACCESS_MODE_POP_HEAD); 828} 829 830static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, 831 enum k3_ringacc_access_mode access_mode) 832{ 833 void __iomem *ptr; 834 835 switch (access_mode) { 836 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 837 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 838 ptr = (void __iomem *)&ring->fifos->head_data; 839 break; 840 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 841 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 842 ptr = (void __iomem *)&ring->fifos->tail_data; 843 break; 844 default: 845 return -EINVAL; 846 } 847 848 ptr += k3_ringacc_ring_get_fifo_pos(ring); 849 850 switch (access_mode) { 851 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 852 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 853 dev_dbg(ring->parent->dev, 854 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 855 access_mode); 856 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 857 ring->state.occ--; 858 break; 859 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 860 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 861 dev_dbg(ring->parent->dev, 862 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 863 access_mode); 864 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 865 ring->state.free--; 866 break; 867 default: 868 return -EINVAL; 869 } 870 871 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", 872 ring->state.free, ring->state.windex, ring->state.occ, 873 ring->state.rindex); 874 return 0; 875} 876 877static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) 878{ 879 return k3_ringacc_ring_access_io(ring, elem, 880 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 881} 882 883static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) 884{ 885 return k3_ringacc_ring_access_io(ring, elem, 886 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 887} 888 889static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) 890{ 891 return k3_ringacc_ring_access_io(ring, elem, 892 K3_RINGACC_ACCESS_MODE_POP_HEAD); 893} 894 895static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) 896{ 897 return k3_ringacc_ring_access_io(ring, elem, 898 K3_RINGACC_ACCESS_MODE_POP_HEAD); 899} 900 901static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) 902{ 903 void *elem_ptr; 904 905 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); 906 907 memcpy(elem_ptr, elem, (4 << ring->elm_size)); 908 909 ring->state.windex = (ring->state.windex + 1) % ring->size; 910 ring->state.free--; 911 writel(1, &ring->rt->db); 912 913 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", 914 ring->state.free, ring->state.windex); 915 916 return 0; 917} 918 919static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) 920{ 921 void *elem_ptr; 922 923 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 924 925 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 926 927 ring->state.rindex = (ring->state.rindex + 1) % ring->size; 928 ring->state.occ--; 929 writel(-1, &ring->rt->db); 930 931 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", 932 ring->state.occ, ring->state.rindex, elem_ptr); 933 return 0; 934} 935 936int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) 937{ 938 int ret = -EOPNOTSUPP; 939 940 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 941 return -EINVAL; 942 943 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", 944 ring->state.free, ring->state.windex); 945 946 if (k3_ringacc_ring_is_full(ring)) 947 return -ENOMEM; 948 949 if (ring->ops && ring->ops->push_tail) 950 ret = ring->ops->push_tail(ring, elem); 951 952 return ret; 953} 954EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); 955 956int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) 957{ 958 int ret = -EOPNOTSUPP; 959 960 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 961 return -EINVAL; 962 963 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", 964 ring->state.free, ring->state.windex); 965 966 if (k3_ringacc_ring_is_full(ring)) 967 return -ENOMEM; 968 969 if (ring->ops && ring->ops->push_head) 970 ret = ring->ops->push_head(ring, elem); 971 972 return ret; 973} 974EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); 975 976int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) 977{ 978 int ret = -EOPNOTSUPP; 979 980 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 981 return -EINVAL; 982 983 if (!ring->state.occ) 984 ring->state.occ = k3_ringacc_ring_get_occ(ring); 985 986 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, 987 ring->state.rindex); 988 989 if (!ring->state.occ) 990 return -ENODATA; 991 992 if (ring->ops && ring->ops->pop_head) 993 ret = ring->ops->pop_head(ring, elem); 994 995 return ret; 996} 997EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); 998 999int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) 1000{ 1001 int ret = -EOPNOTSUPP; 1002 1003 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1004 return -EINVAL; 1005 1006 if (!ring->state.occ) 1007 ring->state.occ = k3_ringacc_ring_get_occ(ring); 1008 1009 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", 1010 ring->state.occ, ring->state.rindex); 1011 1012 if (!ring->state.occ) 1013 return -ENODATA; 1014 1015 if (ring->ops && ring->ops->pop_tail) 1016 ret = ring->ops->pop_tail(ring, elem); 1017 1018 return ret; 1019} 1020EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); 1021 1022struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, 1023 const char *property) 1024{ 1025 struct device_node *ringacc_np; 1026 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); 1027 struct k3_ringacc *entry; 1028 1029 ringacc_np = of_parse_phandle(np, property, 0); 1030 if (!ringacc_np) 1031 return ERR_PTR(-ENODEV); 1032 1033 mutex_lock(&k3_ringacc_list_lock); 1034 list_for_each_entry(entry, &k3_ringacc_list, list) 1035 if (entry->dev->of_node == ringacc_np) { 1036 ringacc = entry; 1037 break; 1038 } 1039 mutex_unlock(&k3_ringacc_list_lock); 1040 of_node_put(ringacc_np); 1041 1042 return ringacc; 1043} 1044EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); 1045 1046static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) 1047{ 1048 struct device_node *node = ringacc->dev->of_node; 1049 struct device *dev = ringacc->dev; 1050 struct platform_device *pdev = to_platform_device(dev); 1051 int ret; 1052 1053 if (!node) { 1054 dev_err(dev, "device tree info unavailable\n"); 1055 return -ENODEV; 1056 } 1057 1058 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); 1059 if (ret) { 1060 dev_err(dev, "ti,num-rings read failure %d\n", ret); 1061 return ret; 1062 } 1063 1064 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); 1065 if (IS_ERR(ringacc->tisci)) { 1066 ret = PTR_ERR(ringacc->tisci); 1067 if (ret != -EPROBE_DEFER) 1068 dev_err(dev, "ti,sci read fail %d\n", ret); 1069 ringacc->tisci = NULL; 1070 return ret; 1071 } 1072 1073 ret = of_property_read_u32(node, "ti,sci-dev-id", 1074 &ringacc->tisci_dev_id); 1075 if (ret) { 1076 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); 1077 return ret; 1078 } 1079 1080 pdev->id = ringacc->tisci_dev_id; 1081 1082 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, 1083 ringacc->tisci_dev_id, 1084 "ti,sci-rm-range-gp-rings"); 1085 if (IS_ERR(ringacc->rm_gp_range)) { 1086 dev_err(dev, "Failed to allocate MSI interrupts\n"); 1087 return PTR_ERR(ringacc->rm_gp_range); 1088 } 1089 1090 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, 1091 ringacc->rm_gp_range); 1092} 1093 1094static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = { 1095 .dma_ring_reset_quirk = 1, 1096}; 1097 1098static const struct soc_device_attribute k3_ringacc_socinfo[] = { 1099 { .family = "AM65X", 1100 .revision = "SR1.0", 1101 .data = &k3_ringacc_soc_data_sr1 1102 }, 1103 {/* sentinel */} 1104}; 1105 1106static int k3_ringacc_init(struct platform_device *pdev, 1107 struct k3_ringacc *ringacc) 1108{ 1109 const struct soc_device_attribute *soc; 1110 void __iomem *base_fifo, *base_rt; 1111 struct device *dev = &pdev->dev; 1112 struct resource *res; 1113 int ret, i; 1114 1115 dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 1116 DOMAIN_BUS_TI_SCI_INTA_MSI); 1117 if (!dev->msi_domain) { 1118 dev_err(dev, "Failed to get MSI domain\n"); 1119 return -EPROBE_DEFER; 1120 } 1121 1122 ret = k3_ringacc_probe_dt(ringacc); 1123 if (ret) 1124 return ret; 1125 1126 soc = soc_device_match(k3_ringacc_socinfo); 1127 if (soc && soc->data) { 1128 const struct k3_ringacc_soc_data *soc_data = soc->data; 1129 1130 ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk; 1131 } 1132 1133 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); 1134 base_rt = devm_ioremap_resource(dev, res); 1135 if (IS_ERR(base_rt)) 1136 return PTR_ERR(base_rt); 1137 1138 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos"); 1139 base_fifo = devm_ioremap_resource(dev, res); 1140 if (IS_ERR(base_fifo)) 1141 return PTR_ERR(base_fifo); 1142 1143 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg"); 1144 ringacc->proxy_gcfg = devm_ioremap_resource(dev, res); 1145 if (IS_ERR(ringacc->proxy_gcfg)) 1146 return PTR_ERR(ringacc->proxy_gcfg); 1147 1148 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 1149 "proxy_target"); 1150 ringacc->proxy_target_base = devm_ioremap_resource(dev, res); 1151 if (IS_ERR(ringacc->proxy_target_base)) 1152 return PTR_ERR(ringacc->proxy_target_base); 1153 1154 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & 1155 K3_RINGACC_PROXY_CFG_THREADS_MASK; 1156 1157 ringacc->rings = devm_kzalloc(dev, 1158 sizeof(*ringacc->rings) * 1159 ringacc->num_rings, 1160 GFP_KERNEL); 1161 ringacc->rings_inuse = devm_kcalloc(dev, 1162 BITS_TO_LONGS(ringacc->num_rings), 1163 sizeof(unsigned long), GFP_KERNEL); 1164 ringacc->proxy_inuse = devm_kcalloc(dev, 1165 BITS_TO_LONGS(ringacc->num_proxies), 1166 sizeof(unsigned long), GFP_KERNEL); 1167 1168 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) 1169 return -ENOMEM; 1170 1171 for (i = 0; i < ringacc->num_rings; i++) { 1172 ringacc->rings[i].rt = base_rt + 1173 K3_RINGACC_RT_REGS_STEP * i; 1174 ringacc->rings[i].fifos = base_fifo + 1175 K3_RINGACC_FIFO_REGS_STEP * i; 1176 ringacc->rings[i].parent = ringacc; 1177 ringacc->rings[i].ring_id = i; 1178 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; 1179 } 1180 1181 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1182 1183 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", 1184 ringacc->num_rings, 1185 ringacc->rm_gp_range->desc[0].start, 1186 ringacc->rm_gp_range->desc[0].num, 1187 ringacc->tisci_dev_id); 1188 dev_info(dev, "dma-ring-reset-quirk: %s\n", 1189 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled"); 1190 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", 1191 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); 1192 1193 return 0; 1194} 1195 1196struct ringacc_match_data { 1197 struct k3_ringacc_ops ops; 1198}; 1199 1200static struct ringacc_match_data k3_ringacc_data = { 1201 .ops = { 1202 .init = k3_ringacc_init, 1203 }, 1204}; 1205 1206/* Match table for of_platform binding */ 1207static const struct of_device_id k3_ringacc_of_match[] = { 1208 { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, }, 1209 {}, 1210}; 1211 1212static int k3_ringacc_probe(struct platform_device *pdev) 1213{ 1214 const struct ringacc_match_data *match_data; 1215 const struct of_device_id *match; 1216 struct device *dev = &pdev->dev; 1217 struct k3_ringacc *ringacc; 1218 int ret; 1219 1220 match = of_match_node(k3_ringacc_of_match, dev->of_node); 1221 if (!match) 1222 return -ENODEV; 1223 match_data = match->data; 1224 1225 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1226 if (!ringacc) 1227 return -ENOMEM; 1228 1229 ringacc->dev = dev; 1230 mutex_init(&ringacc->req_lock); 1231 ringacc->ops = &match_data->ops; 1232 1233 ret = ringacc->ops->init(pdev, ringacc); 1234 if (ret) 1235 return ret; 1236 1237 dev_set_drvdata(dev, ringacc); 1238 1239 mutex_lock(&k3_ringacc_list_lock); 1240 list_add_tail(&ringacc->list, &k3_ringacc_list); 1241 mutex_unlock(&k3_ringacc_list_lock); 1242 1243 return 0; 1244} 1245 1246static struct platform_driver k3_ringacc_driver = { 1247 .probe = k3_ringacc_probe, 1248 .driver = { 1249 .name = "k3-ringacc", 1250 .of_match_table = k3_ringacc_of_match, 1251 .suppress_bind_attrs = true, 1252 }, 1253}; 1254builtin_platform_driver(k3_ringacc_driver); 1255