1// SPDX-License-Identifier: GPL-2.0-only 2 3/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */ 4/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ 5 6#include <linux/bitfield.h> 7#include <linux/bits.h> 8#include <linux/completion.h> 9#include <linux/delay.h> 10#include <linux/dma-buf.h> 11#include <linux/dma-mapping.h> 12#include <linux/interrupt.h> 13#include <linux/kref.h> 14#include <linux/list.h> 15#include <linux/math64.h> 16#include <linux/mm.h> 17#include <linux/moduleparam.h> 18#include <linux/scatterlist.h> 19#include <linux/spinlock.h> 20#include <linux/srcu.h> 21#include <linux/types.h> 22#include <linux/uaccess.h> 23#include <linux/wait.h> 24#include <drm/drm_file.h> 25#include <drm/drm_gem.h> 26#include <drm/drm_prime.h> 27#include <drm/drm_print.h> 28#include <uapi/drm/qaic_accel.h> 29 30#include "qaic.h" 31 32#define SEM_VAL_MASK GENMASK_ULL(11, 0) 33#define SEM_INDEX_MASK GENMASK_ULL(4, 0) 34#define BULK_XFER BIT(3) 35#define GEN_COMPLETION BIT(4) 36#define INBOUND_XFER 1 37#define OUTBOUND_XFER 2 38#define REQHP_OFF 0x0 /* we read this */ 39#define REQTP_OFF 0x4 /* we write this */ 40#define RSPHP_OFF 0x8 /* we write this */ 41#define RSPTP_OFF 0xc /* we read this */ 42 43#define ENCODE_SEM(val, index, sync, cmd, flags) \ 44 ({ \ 45 FIELD_PREP(GENMASK(11, 0), (val)) | \ 46 FIELD_PREP(GENMASK(20, 16), (index)) | \ 47 FIELD_PREP(BIT(22), (sync)) | \ 48 FIELD_PREP(GENMASK(26, 24), (cmd)) | \ 49 FIELD_PREP(GENMASK(30, 29), (flags)) | \ 50 FIELD_PREP(BIT(31), (cmd) ? 1 : 0); \ 51 }) 52#define NUM_EVENTS 128 53#define NUM_DELAYS 10 54 55static unsigned int wait_exec_default_timeout_ms = 5000; /* 5 sec default */ 56module_param(wait_exec_default_timeout_ms, uint, 0600); 57MODULE_PARM_DESC(wait_exec_default_timeout_ms, "Default timeout for DRM_IOCTL_QAIC_WAIT_BO"); 58 59static unsigned int datapath_poll_interval_us = 100; /* 100 usec default */ 60module_param(datapath_poll_interval_us, uint, 0600); 61MODULE_PARM_DESC(datapath_poll_interval_us, 62 "Amount of time to sleep between activity when datapath polling is enabled"); 63 64struct dbc_req { 65 /* 66 * A request ID is assigned to each memory handle going in DMA queue. 67 * As a single memory handle can enqueue multiple elements in DMA queue 68 * all of them will have the same request ID. 69 */ 70 __le16 req_id; 71 /* Future use */ 72 __u8 seq_id; 73 /* 74 * Special encoded variable 75 * 7 0 - Do not force to generate MSI after DMA is completed 76 * 1 - Force to generate MSI after DMA is completed 77 * 6:5 Reserved 78 * 4 1 - Generate completion element in the response queue 79 * 0 - No Completion Code 80 * 3 0 - DMA request is a Link list transfer 81 * 1 - DMA request is a Bulk transfer 82 * 2 Reserved 83 * 1:0 00 - No DMA transfer involved 84 * 01 - DMA transfer is part of inbound transfer 85 * 10 - DMA transfer has outbound transfer 86 * 11 - NA 87 */ 88 __u8 cmd; 89 __le32 resv; 90 /* Source address for the transfer */ 91 __le64 src_addr; 92 /* Destination address for the transfer */ 93 __le64 dest_addr; 94 /* Length of transfer request */ 95 __le32 len; 96 __le32 resv2; 97 /* Doorbell address */ 98 __le64 db_addr; 99 /* 100 * Special encoded variable 101 * 7 1 - Doorbell(db) write 102 * 0 - No doorbell write 103 * 6:2 Reserved 104 * 1:0 00 - 32 bit access, db address must be aligned to 32bit-boundary 105 * 01 - 16 bit access, db address must be aligned to 16bit-boundary 106 * 10 - 8 bit access, db address must be aligned to 8bit-boundary 107 * 11 - Reserved 108 */ 109 __u8 db_len; 110 __u8 resv3; 111 __le16 resv4; 112 /* 32 bit data written to doorbell address */ 113 __le32 db_data; 114 /* 115 * Special encoded variable 116 * All the fields of sem_cmdX are passed from user and all are ORed 117 * together to form sem_cmd. 118 * 0:11 Semaphore value 119 * 15:12 Reserved 120 * 20:16 Semaphore index 121 * 21 Reserved 122 * 22 Semaphore Sync 123 * 23 Reserved 124 * 26:24 Semaphore command 125 * 28:27 Reserved 126 * 29 Semaphore DMA out bound sync fence 127 * 30 Semaphore DMA in bound sync fence 128 * 31 Enable semaphore command 129 */ 130 __le32 sem_cmd0; 131 __le32 sem_cmd1; 132 __le32 sem_cmd2; 133 __le32 sem_cmd3; 134} __packed; 135 136struct dbc_rsp { 137 /* Request ID of the memory handle whose DMA transaction is completed */ 138 __le16 req_id; 139 /* Status of the DMA transaction. 0 : Success otherwise failure */ 140 __le16 status; 141} __packed; 142 143inline int get_dbc_req_elem_size(void) 144{ 145 return sizeof(struct dbc_req); 146} 147 148inline int get_dbc_rsp_elem_size(void) 149{ 150 return sizeof(struct dbc_rsp); 151} 152 153static void free_slice(struct kref *kref) 154{ 155 struct bo_slice *slice = container_of(kref, struct bo_slice, ref_count); 156 157 list_del(&slice->slice); 158 drm_gem_object_put(&slice->bo->base); 159 sg_free_table(slice->sgt); 160 kfree(slice->sgt); 161 kfree(slice->reqs); 162 kfree(slice); 163} 164 165static int clone_range_of_sgt_for_slice(struct qaic_device *qdev, struct sg_table **sgt_out, 166 struct sg_table *sgt_in, u64 size, u64 offset) 167{ 168 int total_len, len, nents, offf = 0, offl = 0; 169 struct scatterlist *sg, *sgn, *sgf, *sgl; 170 struct sg_table *sgt; 171 int ret, j; 172 173 /* find out number of relevant nents needed for this mem */ 174 total_len = 0; 175 sgf = NULL; 176 sgl = NULL; 177 nents = 0; 178 179 size = size ? size : PAGE_SIZE; 180 for (sg = sgt_in->sgl; sg; sg = sg_next(sg)) { 181 len = sg_dma_len(sg); 182 183 if (!len) 184 continue; 185 if (offset >= total_len && offset < total_len + len) { 186 sgf = sg; 187 offf = offset - total_len; 188 } 189 if (sgf) 190 nents++; 191 if (offset + size >= total_len && 192 offset + size <= total_len + len) { 193 sgl = sg; 194 offl = offset + size - total_len; 195 break; 196 } 197 total_len += len; 198 } 199 200 if (!sgf || !sgl) { 201 ret = -EINVAL; 202 goto out; 203 } 204 205 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); 206 if (!sgt) { 207 ret = -ENOMEM; 208 goto out; 209 } 210 211 ret = sg_alloc_table(sgt, nents, GFP_KERNEL); 212 if (ret) 213 goto free_sgt; 214 215 /* copy relevant sg node and fix page and length */ 216 sgn = sgf; 217 for_each_sgtable_sg(sgt, sg, j) { 218 memcpy(sg, sgn, sizeof(*sg)); 219 if (sgn == sgf) { 220 sg_dma_address(sg) += offf; 221 sg_dma_len(sg) -= offf; 222 sg_set_page(sg, sg_page(sgn), sg_dma_len(sg), offf); 223 } else { 224 offf = 0; 225 } 226 if (sgn == sgl) { 227 sg_dma_len(sg) = offl - offf; 228 sg_set_page(sg, sg_page(sgn), offl - offf, offf); 229 sg_mark_end(sg); 230 break; 231 } 232 sgn = sg_next(sgn); 233 } 234 235 *sgt_out = sgt; 236 return ret; 237 238free_sgt: 239 kfree(sgt); 240out: 241 *sgt_out = NULL; 242 return ret; 243} 244 245static int encode_reqs(struct qaic_device *qdev, struct bo_slice *slice, 246 struct qaic_attach_slice_entry *req) 247{ 248 __le64 db_addr = cpu_to_le64(req->db_addr); 249 __le32 db_data = cpu_to_le32(req->db_data); 250 struct scatterlist *sg; 251 __u8 cmd = BULK_XFER; 252 int presync_sem; 253 u64 dev_addr; 254 __u8 db_len; 255 int i; 256 257 if (!slice->no_xfer) 258 cmd |= (slice->dir == DMA_TO_DEVICE ? INBOUND_XFER : OUTBOUND_XFER); 259 260 if (req->db_len && !IS_ALIGNED(req->db_addr, req->db_len / 8)) 261 return -EINVAL; 262 263 presync_sem = req->sem0.presync + req->sem1.presync + req->sem2.presync + req->sem3.presync; 264 if (presync_sem > 1) 265 return -EINVAL; 266 267 presync_sem = req->sem0.presync << 0 | req->sem1.presync << 1 | 268 req->sem2.presync << 2 | req->sem3.presync << 3; 269 270 switch (req->db_len) { 271 case 32: 272 db_len = BIT(7); 273 break; 274 case 16: 275 db_len = BIT(7) | 1; 276 break; 277 case 8: 278 db_len = BIT(7) | 2; 279 break; 280 case 0: 281 db_len = 0; /* doorbell is not active for this command */ 282 break; 283 default: 284 return -EINVAL; /* should never hit this */ 285 } 286 287 /* 288 * When we end up splitting up a single request (ie a buf slice) into 289 * multiple DMA requests, we have to manage the sync data carefully. 290 * There can only be one presync sem. That needs to be on every xfer 291 * so that the DMA engine doesn't transfer data before the receiver is 292 * ready. We only do the doorbell and postsync sems after the xfer. 293 * To guarantee previous xfers for the request are complete, we use a 294 * fence. 295 */ 296 dev_addr = req->dev_addr; 297 for_each_sgtable_sg(slice->sgt, sg, i) { 298 slice->reqs[i].cmd = cmd; 299 slice->reqs[i].src_addr = cpu_to_le64(slice->dir == DMA_TO_DEVICE ? 300 sg_dma_address(sg) : dev_addr); 301 slice->reqs[i].dest_addr = cpu_to_le64(slice->dir == DMA_TO_DEVICE ? 302 dev_addr : sg_dma_address(sg)); 303 /* 304 * sg_dma_len(sg) returns size of a DMA segment, maximum DMA 305 * segment size is set to UINT_MAX by qaic and hence return 306 * values of sg_dma_len(sg) can never exceed u32 range. So, 307 * by down sizing we are not corrupting the value. 308 */ 309 slice->reqs[i].len = cpu_to_le32((u32)sg_dma_len(sg)); 310 switch (presync_sem) { 311 case BIT(0): 312 slice->reqs[i].sem_cmd0 = cpu_to_le32(ENCODE_SEM(req->sem0.val, 313 req->sem0.index, 314 req->sem0.presync, 315 req->sem0.cmd, 316 req->sem0.flags)); 317 break; 318 case BIT(1): 319 slice->reqs[i].sem_cmd1 = cpu_to_le32(ENCODE_SEM(req->sem1.val, 320 req->sem1.index, 321 req->sem1.presync, 322 req->sem1.cmd, 323 req->sem1.flags)); 324 break; 325 case BIT(2): 326 slice->reqs[i].sem_cmd2 = cpu_to_le32(ENCODE_SEM(req->sem2.val, 327 req->sem2.index, 328 req->sem2.presync, 329 req->sem2.cmd, 330 req->sem2.flags)); 331 break; 332 case BIT(3): 333 slice->reqs[i].sem_cmd3 = cpu_to_le32(ENCODE_SEM(req->sem3.val, 334 req->sem3.index, 335 req->sem3.presync, 336 req->sem3.cmd, 337 req->sem3.flags)); 338 break; 339 } 340 dev_addr += sg_dma_len(sg); 341 } 342 /* add post transfer stuff to last segment */ 343 i--; 344 slice->reqs[i].cmd |= GEN_COMPLETION; 345 slice->reqs[i].db_addr = db_addr; 346 slice->reqs[i].db_len = db_len; 347 slice->reqs[i].db_data = db_data; 348 /* 349 * Add a fence if we have more than one request going to the hardware 350 * representing the entirety of the user request, and the user request 351 * has no presync condition. 352 * Fences are expensive, so we try to avoid them. We rely on the 353 * hardware behavior to avoid needing one when there is a presync 354 * condition. When a presync exists, all requests for that same 355 * presync will be queued into a fifo. Thus, since we queue the 356 * post xfer activity only on the last request we queue, the hardware 357 * will ensure that the last queued request is processed last, thus 358 * making sure the post xfer activity happens at the right time without 359 * a fence. 360 */ 361 if (i && !presync_sem) 362 req->sem0.flags |= (slice->dir == DMA_TO_DEVICE ? 363 QAIC_SEM_INSYNCFENCE : QAIC_SEM_OUTSYNCFENCE); 364 slice->reqs[i].sem_cmd0 = cpu_to_le32(ENCODE_SEM(req->sem0.val, req->sem0.index, 365 req->sem0.presync, req->sem0.cmd, 366 req->sem0.flags)); 367 slice->reqs[i].sem_cmd1 = cpu_to_le32(ENCODE_SEM(req->sem1.val, req->sem1.index, 368 req->sem1.presync, req->sem1.cmd, 369 req->sem1.flags)); 370 slice->reqs[i].sem_cmd2 = cpu_to_le32(ENCODE_SEM(req->sem2.val, req->sem2.index, 371 req->sem2.presync, req->sem2.cmd, 372 req->sem2.flags)); 373 slice->reqs[i].sem_cmd3 = cpu_to_le32(ENCODE_SEM(req->sem3.val, req->sem3.index, 374 req->sem3.presync, req->sem3.cmd, 375 req->sem3.flags)); 376 377 return 0; 378} 379 380static int qaic_map_one_slice(struct qaic_device *qdev, struct qaic_bo *bo, 381 struct qaic_attach_slice_entry *slice_ent) 382{ 383 struct sg_table *sgt = NULL; 384 struct bo_slice *slice; 385 int ret; 386 387 ret = clone_range_of_sgt_for_slice(qdev, &sgt, bo->sgt, slice_ent->size, slice_ent->offset); 388 if (ret) 389 goto out; 390 391 slice = kmalloc(sizeof(*slice), GFP_KERNEL); 392 if (!slice) { 393 ret = -ENOMEM; 394 goto free_sgt; 395 } 396 397 slice->reqs = kcalloc(sgt->nents, sizeof(*slice->reqs), GFP_KERNEL); 398 if (!slice->reqs) { 399 ret = -ENOMEM; 400 goto free_slice; 401 } 402 403 slice->no_xfer = !slice_ent->size; 404 slice->sgt = sgt; 405 slice->nents = sgt->nents; 406 slice->dir = bo->dir; 407 slice->bo = bo; 408 slice->size = slice_ent->size; 409 slice->offset = slice_ent->offset; 410 411 ret = encode_reqs(qdev, slice, slice_ent); 412 if (ret) 413 goto free_req; 414 415 bo->total_slice_nents += sgt->nents; 416 kref_init(&slice->ref_count); 417 drm_gem_object_get(&bo->base); 418 list_add_tail(&slice->slice, &bo->slices); 419 420 return 0; 421 422free_req: 423 kfree(slice->reqs); 424free_slice: 425 kfree(slice); 426free_sgt: 427 sg_free_table(sgt); 428 kfree(sgt); 429out: 430 return ret; 431} 432 433static int create_sgt(struct qaic_device *qdev, struct sg_table **sgt_out, u64 size) 434{ 435 struct scatterlist *sg; 436 struct sg_table *sgt; 437 struct page **pages; 438 int *pages_order; 439 int buf_extra; 440 int max_order; 441 int nr_pages; 442 int ret = 0; 443 int i, j, k; 444 int order; 445 446 if (size) { 447 nr_pages = DIV_ROUND_UP(size, PAGE_SIZE); 448 /* 449 * calculate how much extra we are going to allocate, to remove 450 * later 451 */ 452 buf_extra = (PAGE_SIZE - size % PAGE_SIZE) % PAGE_SIZE; 453 max_order = min(MAX_ORDER - 1, get_order(size)); 454 } else { 455 /* allocate a single page for book keeping */ 456 nr_pages = 1; 457 buf_extra = 0; 458 max_order = 0; 459 } 460 461 pages = kvmalloc_array(nr_pages, sizeof(*pages) + sizeof(*pages_order), GFP_KERNEL); 462 if (!pages) { 463 ret = -ENOMEM; 464 goto out; 465 } 466 pages_order = (void *)pages + sizeof(*pages) * nr_pages; 467 468 /* 469 * Allocate requested memory using alloc_pages. It is possible to allocate 470 * the requested memory in multiple chunks by calling alloc_pages 471 * multiple times. Use SG table to handle multiple allocated pages. 472 */ 473 i = 0; 474 while (nr_pages > 0) { 475 order = min(get_order(nr_pages * PAGE_SIZE), max_order); 476 while (1) { 477 pages[i] = alloc_pages(GFP_KERNEL | GFP_HIGHUSER | 478 __GFP_NOWARN | __GFP_ZERO | 479 (order ? __GFP_NORETRY : __GFP_RETRY_MAYFAIL), 480 order); 481 if (pages[i]) 482 break; 483 if (!order--) { 484 ret = -ENOMEM; 485 goto free_partial_alloc; 486 } 487 } 488 489 max_order = order; 490 pages_order[i] = order; 491 492 nr_pages -= 1 << order; 493 if (nr_pages <= 0) 494 /* account for over allocation */ 495 buf_extra += abs(nr_pages) * PAGE_SIZE; 496 i++; 497 } 498 499 sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); 500 if (!sgt) { 501 ret = -ENOMEM; 502 goto free_partial_alloc; 503 } 504 505 if (sg_alloc_table(sgt, i, GFP_KERNEL)) { 506 ret = -ENOMEM; 507 goto free_sgt; 508 } 509 510 /* Populate the SG table with the allocated memory pages */ 511 sg = sgt->sgl; 512 for (k = 0; k < i; k++, sg = sg_next(sg)) { 513 /* Last entry requires special handling */ 514 if (k < i - 1) { 515 sg_set_page(sg, pages[k], PAGE_SIZE << pages_order[k], 0); 516 } else { 517 sg_set_page(sg, pages[k], (PAGE_SIZE << pages_order[k]) - buf_extra, 0); 518 sg_mark_end(sg); 519 } 520 } 521 522 kvfree(pages); 523 *sgt_out = sgt; 524 return ret; 525 526free_sgt: 527 kfree(sgt); 528free_partial_alloc: 529 for (j = 0; j < i; j++) 530 __free_pages(pages[j], pages_order[j]); 531 kvfree(pages); 532out: 533 *sgt_out = NULL; 534 return ret; 535} 536 537static bool invalid_sem(struct qaic_sem *sem) 538{ 539 if (sem->val & ~SEM_VAL_MASK || sem->index & ~SEM_INDEX_MASK || 540 !(sem->presync == 0 || sem->presync == 1) || sem->pad || 541 sem->flags & ~(QAIC_SEM_INSYNCFENCE | QAIC_SEM_OUTSYNCFENCE) || 542 sem->cmd > QAIC_SEM_WAIT_GT_0) 543 return true; 544 return false; 545} 546 547static int qaic_validate_req(struct qaic_device *qdev, struct qaic_attach_slice_entry *slice_ent, 548 u32 count, u64 total_size) 549{ 550 int i; 551 552 for (i = 0; i < count; i++) { 553 if (!(slice_ent[i].db_len == 32 || slice_ent[i].db_len == 16 || 554 slice_ent[i].db_len == 8 || slice_ent[i].db_len == 0) || 555 invalid_sem(&slice_ent[i].sem0) || invalid_sem(&slice_ent[i].sem1) || 556 invalid_sem(&slice_ent[i].sem2) || invalid_sem(&slice_ent[i].sem3)) 557 return -EINVAL; 558 559 if (slice_ent[i].offset + slice_ent[i].size > total_size) 560 return -EINVAL; 561 } 562 563 return 0; 564} 565 566static void qaic_free_sgt(struct sg_table *sgt) 567{ 568 struct scatterlist *sg; 569 570 for (sg = sgt->sgl; sg; sg = sg_next(sg)) 571 if (sg_page(sg)) 572 __free_pages(sg_page(sg), get_order(sg->length)); 573 sg_free_table(sgt); 574 kfree(sgt); 575} 576 577static void qaic_gem_print_info(struct drm_printer *p, unsigned int indent, 578 const struct drm_gem_object *obj) 579{ 580 struct qaic_bo *bo = to_qaic_bo(obj); 581 582 drm_printf_indent(p, indent, "user requested size=%llu\n", bo->size); 583} 584 585static const struct vm_operations_struct drm_vm_ops = { 586 .open = drm_gem_vm_open, 587 .close = drm_gem_vm_close, 588}; 589 590static int qaic_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 591{ 592 struct qaic_bo *bo = to_qaic_bo(obj); 593 unsigned long offset = 0; 594 struct scatterlist *sg; 595 int ret = 0; 596 597 if (obj->import_attach) 598 return -EINVAL; 599 600 for (sg = bo->sgt->sgl; sg; sg = sg_next(sg)) { 601 if (sg_page(sg)) { 602 ret = remap_pfn_range(vma, vma->vm_start + offset, page_to_pfn(sg_page(sg)), 603 sg->length, vma->vm_page_prot); 604 if (ret) 605 goto out; 606 offset += sg->length; 607 } 608 } 609 610out: 611 return ret; 612} 613 614static void qaic_free_object(struct drm_gem_object *obj) 615{ 616 struct qaic_bo *bo = to_qaic_bo(obj); 617 618 if (obj->import_attach) { 619 /* DMABUF/PRIME Path */ 620 drm_prime_gem_destroy(obj, NULL); 621 } else { 622 /* Private buffer allocation path */ 623 qaic_free_sgt(bo->sgt); 624 } 625 626 drm_gem_object_release(obj); 627 kfree(bo); 628} 629 630static const struct drm_gem_object_funcs qaic_gem_funcs = { 631 .free = qaic_free_object, 632 .print_info = qaic_gem_print_info, 633 .mmap = qaic_gem_object_mmap, 634 .vm_ops = &drm_vm_ops, 635}; 636 637static struct qaic_bo *qaic_alloc_init_bo(void) 638{ 639 struct qaic_bo *bo; 640 641 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 642 if (!bo) 643 return ERR_PTR(-ENOMEM); 644 645 INIT_LIST_HEAD(&bo->slices); 646 init_completion(&bo->xfer_done); 647 complete_all(&bo->xfer_done); 648 649 return bo; 650} 651 652int qaic_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 653{ 654 struct qaic_create_bo *args = data; 655 int usr_rcu_id, qdev_rcu_id; 656 struct drm_gem_object *obj; 657 struct qaic_device *qdev; 658 struct qaic_user *usr; 659 struct qaic_bo *bo; 660 size_t size; 661 int ret; 662 663 if (args->pad) 664 return -EINVAL; 665 666 size = PAGE_ALIGN(args->size); 667 if (size == 0) 668 return -EINVAL; 669 670 usr = file_priv->driver_priv; 671 usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 672 if (!usr->qddev) { 673 ret = -ENODEV; 674 goto unlock_usr_srcu; 675 } 676 677 qdev = usr->qddev->qdev; 678 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 679 if (qdev->in_reset) { 680 ret = -ENODEV; 681 goto unlock_dev_srcu; 682 } 683 684 bo = qaic_alloc_init_bo(); 685 if (IS_ERR(bo)) { 686 ret = PTR_ERR(bo); 687 goto unlock_dev_srcu; 688 } 689 obj = &bo->base; 690 691 drm_gem_private_object_init(dev, obj, size); 692 693 obj->funcs = &qaic_gem_funcs; 694 ret = create_sgt(qdev, &bo->sgt, size); 695 if (ret) 696 goto free_bo; 697 698 bo->size = args->size; 699 700 ret = drm_gem_handle_create(file_priv, obj, &args->handle); 701 if (ret) 702 goto free_sgt; 703 704 bo->handle = args->handle; 705 drm_gem_object_put(obj); 706 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 707 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 708 709 return 0; 710 711free_sgt: 712 qaic_free_sgt(bo->sgt); 713free_bo: 714 kfree(bo); 715unlock_dev_srcu: 716 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 717unlock_usr_srcu: 718 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 719 return ret; 720} 721 722int qaic_mmap_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 723{ 724 struct qaic_mmap_bo *args = data; 725 int usr_rcu_id, qdev_rcu_id; 726 struct drm_gem_object *obj; 727 struct qaic_device *qdev; 728 struct qaic_user *usr; 729 int ret; 730 731 usr = file_priv->driver_priv; 732 usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 733 if (!usr->qddev) { 734 ret = -ENODEV; 735 goto unlock_usr_srcu; 736 } 737 738 qdev = usr->qddev->qdev; 739 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 740 if (qdev->in_reset) { 741 ret = -ENODEV; 742 goto unlock_dev_srcu; 743 } 744 745 obj = drm_gem_object_lookup(file_priv, args->handle); 746 if (!obj) { 747 ret = -ENOENT; 748 goto unlock_dev_srcu; 749 } 750 751 ret = drm_gem_create_mmap_offset(obj); 752 if (ret == 0) 753 args->offset = drm_vma_node_offset_addr(&obj->vma_node); 754 755 drm_gem_object_put(obj); 756 757unlock_dev_srcu: 758 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 759unlock_usr_srcu: 760 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 761 return ret; 762} 763 764struct drm_gem_object *qaic_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf) 765{ 766 struct dma_buf_attachment *attach; 767 struct drm_gem_object *obj; 768 struct qaic_bo *bo; 769 int ret; 770 771 bo = qaic_alloc_init_bo(); 772 if (IS_ERR(bo)) { 773 ret = PTR_ERR(bo); 774 goto out; 775 } 776 777 obj = &bo->base; 778 get_dma_buf(dma_buf); 779 780 attach = dma_buf_attach(dma_buf, dev->dev); 781 if (IS_ERR(attach)) { 782 ret = PTR_ERR(attach); 783 goto attach_fail; 784 } 785 786 if (!attach->dmabuf->size) { 787 ret = -EINVAL; 788 goto size_align_fail; 789 } 790 791 drm_gem_private_object_init(dev, obj, attach->dmabuf->size); 792 /* 793 * skipping dma_buf_map_attachment() as we do not know the direction 794 * just yet. Once the direction is known in the subsequent IOCTL to 795 * attach slicing, we can do it then. 796 */ 797 798 obj->funcs = &qaic_gem_funcs; 799 obj->import_attach = attach; 800 obj->resv = dma_buf->resv; 801 802 return obj; 803 804size_align_fail: 805 dma_buf_detach(dma_buf, attach); 806attach_fail: 807 dma_buf_put(dma_buf); 808 kfree(bo); 809out: 810 return ERR_PTR(ret); 811} 812 813static int qaic_prepare_import_bo(struct qaic_bo *bo, struct qaic_attach_slice_hdr *hdr) 814{ 815 struct drm_gem_object *obj = &bo->base; 816 struct sg_table *sgt; 817 int ret; 818 819 if (obj->import_attach->dmabuf->size < hdr->size) 820 return -EINVAL; 821 822 sgt = dma_buf_map_attachment(obj->import_attach, hdr->dir); 823 if (IS_ERR(sgt)) { 824 ret = PTR_ERR(sgt); 825 return ret; 826 } 827 828 bo->sgt = sgt; 829 bo->size = hdr->size; 830 831 return 0; 832} 833 834static int qaic_prepare_export_bo(struct qaic_device *qdev, struct qaic_bo *bo, 835 struct qaic_attach_slice_hdr *hdr) 836{ 837 int ret; 838 839 if (bo->size != hdr->size) 840 return -EINVAL; 841 842 ret = dma_map_sgtable(&qdev->pdev->dev, bo->sgt, hdr->dir, 0); 843 if (ret) 844 return -EFAULT; 845 846 return 0; 847} 848 849static int qaic_prepare_bo(struct qaic_device *qdev, struct qaic_bo *bo, 850 struct qaic_attach_slice_hdr *hdr) 851{ 852 int ret; 853 854 if (bo->base.import_attach) 855 ret = qaic_prepare_import_bo(bo, hdr); 856 else 857 ret = qaic_prepare_export_bo(qdev, bo, hdr); 858 859 if (ret == 0) 860 bo->dir = hdr->dir; 861 862 return ret; 863} 864 865static void qaic_unprepare_import_bo(struct qaic_bo *bo) 866{ 867 dma_buf_unmap_attachment(bo->base.import_attach, bo->sgt, bo->dir); 868 bo->sgt = NULL; 869 bo->size = 0; 870} 871 872static void qaic_unprepare_export_bo(struct qaic_device *qdev, struct qaic_bo *bo) 873{ 874 dma_unmap_sgtable(&qdev->pdev->dev, bo->sgt, bo->dir, 0); 875} 876 877static void qaic_unprepare_bo(struct qaic_device *qdev, struct qaic_bo *bo) 878{ 879 if (bo->base.import_attach) 880 qaic_unprepare_import_bo(bo); 881 else 882 qaic_unprepare_export_bo(qdev, bo); 883 884 bo->dir = 0; 885} 886 887static void qaic_free_slices_bo(struct qaic_bo *bo) 888{ 889 struct bo_slice *slice, *temp; 890 891 list_for_each_entry_safe(slice, temp, &bo->slices, slice) 892 kref_put(&slice->ref_count, free_slice); 893} 894 895static int qaic_attach_slicing_bo(struct qaic_device *qdev, struct qaic_bo *bo, 896 struct qaic_attach_slice_hdr *hdr, 897 struct qaic_attach_slice_entry *slice_ent) 898{ 899 int ret, i; 900 901 for (i = 0; i < hdr->count; i++) { 902 ret = qaic_map_one_slice(qdev, bo, &slice_ent[i]); 903 if (ret) { 904 qaic_free_slices_bo(bo); 905 return ret; 906 } 907 } 908 909 if (bo->total_slice_nents > qdev->dbc[hdr->dbc_id].nelem) { 910 qaic_free_slices_bo(bo); 911 return -ENOSPC; 912 } 913 914 bo->sliced = true; 915 bo->nr_slice = hdr->count; 916 list_add_tail(&bo->bo_list, &qdev->dbc[hdr->dbc_id].bo_lists); 917 918 return 0; 919} 920 921int qaic_attach_slice_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 922{ 923 struct qaic_attach_slice_entry *slice_ent; 924 struct qaic_attach_slice *args = data; 925 int rcu_id, usr_rcu_id, qdev_rcu_id; 926 struct dma_bridge_chan *dbc; 927 struct drm_gem_object *obj; 928 struct qaic_device *qdev; 929 unsigned long arg_size; 930 struct qaic_user *usr; 931 u8 __user *user_data; 932 struct qaic_bo *bo; 933 int ret; 934 935 if (args->hdr.count == 0) 936 return -EINVAL; 937 938 arg_size = args->hdr.count * sizeof(*slice_ent); 939 if (arg_size / args->hdr.count != sizeof(*slice_ent)) 940 return -EINVAL; 941 942 if (args->hdr.size == 0) 943 return -EINVAL; 944 945 if (!(args->hdr.dir == DMA_TO_DEVICE || args->hdr.dir == DMA_FROM_DEVICE)) 946 return -EINVAL; 947 948 if (args->data == 0) 949 return -EINVAL; 950 951 usr = file_priv->driver_priv; 952 usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 953 if (!usr->qddev) { 954 ret = -ENODEV; 955 goto unlock_usr_srcu; 956 } 957 958 qdev = usr->qddev->qdev; 959 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 960 if (qdev->in_reset) { 961 ret = -ENODEV; 962 goto unlock_dev_srcu; 963 } 964 965 if (args->hdr.dbc_id >= qdev->num_dbc) { 966 ret = -EINVAL; 967 goto unlock_dev_srcu; 968 } 969 970 user_data = u64_to_user_ptr(args->data); 971 972 slice_ent = kzalloc(arg_size, GFP_KERNEL); 973 if (!slice_ent) { 974 ret = -EINVAL; 975 goto unlock_dev_srcu; 976 } 977 978 ret = copy_from_user(slice_ent, user_data, arg_size); 979 if (ret) { 980 ret = -EFAULT; 981 goto free_slice_ent; 982 } 983 984 ret = qaic_validate_req(qdev, slice_ent, args->hdr.count, args->hdr.size); 985 if (ret) 986 goto free_slice_ent; 987 988 obj = drm_gem_object_lookup(file_priv, args->hdr.handle); 989 if (!obj) { 990 ret = -ENOENT; 991 goto free_slice_ent; 992 } 993 994 bo = to_qaic_bo(obj); 995 996 if (bo->sliced) { 997 ret = -EINVAL; 998 goto put_bo; 999 } 1000 1001 dbc = &qdev->dbc[args->hdr.dbc_id]; 1002 rcu_id = srcu_read_lock(&dbc->ch_lock); 1003 if (dbc->usr != usr) { 1004 ret = -EINVAL; 1005 goto unlock_ch_srcu; 1006 } 1007 1008 ret = qaic_prepare_bo(qdev, bo, &args->hdr); 1009 if (ret) 1010 goto unlock_ch_srcu; 1011 1012 ret = qaic_attach_slicing_bo(qdev, bo, &args->hdr, slice_ent); 1013 if (ret) 1014 goto unprepare_bo; 1015 1016 if (args->hdr.dir == DMA_TO_DEVICE) 1017 dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, args->hdr.dir); 1018 1019 bo->dbc = dbc; 1020 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1021 drm_gem_object_put(obj); 1022 kfree(slice_ent); 1023 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 1024 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 1025 1026 return 0; 1027 1028unprepare_bo: 1029 qaic_unprepare_bo(qdev, bo); 1030unlock_ch_srcu: 1031 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1032put_bo: 1033 drm_gem_object_put(obj); 1034free_slice_ent: 1035 kfree(slice_ent); 1036unlock_dev_srcu: 1037 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 1038unlock_usr_srcu: 1039 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 1040 return ret; 1041} 1042 1043static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, u32 dbc_id, 1044 u32 head, u32 *ptail) 1045{ 1046 struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id]; 1047 struct dbc_req *reqs = slice->reqs; 1048 u32 tail = *ptail; 1049 u32 avail; 1050 1051 avail = head - tail; 1052 if (head <= tail) 1053 avail += dbc->nelem; 1054 1055 --avail; 1056 1057 if (avail < slice->nents) 1058 return -EAGAIN; 1059 1060 if (tail + slice->nents > dbc->nelem) { 1061 avail = dbc->nelem - tail; 1062 avail = min_t(u32, avail, slice->nents); 1063 memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, 1064 sizeof(*reqs) * avail); 1065 reqs += avail; 1066 avail = slice->nents - avail; 1067 if (avail) 1068 memcpy(dbc->req_q_base, reqs, sizeof(*reqs) * avail); 1069 } else { 1070 memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, 1071 sizeof(*reqs) * slice->nents); 1072 } 1073 1074 *ptail = (tail + slice->nents) % dbc->nelem; 1075 1076 return 0; 1077} 1078 1079/* 1080 * Based on the value of resize we may only need to transmit first_n 1081 * entries and the last entry, with last_bytes to send from the last entry. 1082 * Note that first_n could be 0. 1083 */ 1084static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, 1085 u64 resize, u32 dbc_id, u32 head, u32 *ptail) 1086{ 1087 struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id]; 1088 struct dbc_req *reqs = slice->reqs; 1089 struct dbc_req *last_req; 1090 u32 tail = *ptail; 1091 u64 total_bytes; 1092 u64 last_bytes; 1093 u32 first_n; 1094 u32 avail; 1095 int ret; 1096 int i; 1097 1098 avail = head - tail; 1099 if (head <= tail) 1100 avail += dbc->nelem; 1101 1102 --avail; 1103 1104 total_bytes = 0; 1105 for (i = 0; i < slice->nents; i++) { 1106 total_bytes += le32_to_cpu(reqs[i].len); 1107 if (total_bytes >= resize) 1108 break; 1109 } 1110 1111 if (total_bytes < resize) { 1112 /* User space should have used the full buffer path. */ 1113 ret = -EINVAL; 1114 return ret; 1115 } 1116 1117 first_n = i; 1118 last_bytes = i ? resize + le32_to_cpu(reqs[i].len) - total_bytes : resize; 1119 1120 if (avail < (first_n + 1)) 1121 return -EAGAIN; 1122 1123 if (first_n) { 1124 if (tail + first_n > dbc->nelem) { 1125 avail = dbc->nelem - tail; 1126 avail = min_t(u32, avail, first_n); 1127 memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, 1128 sizeof(*reqs) * avail); 1129 last_req = reqs + avail; 1130 avail = first_n - avail; 1131 if (avail) 1132 memcpy(dbc->req_q_base, last_req, sizeof(*reqs) * avail); 1133 } else { 1134 memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, 1135 sizeof(*reqs) * first_n); 1136 } 1137 } 1138 1139 /* Copy over the last entry. Here we need to adjust len to the left over 1140 * size, and set src and dst to the entry it is copied to. 1141 */ 1142 last_req = dbc->req_q_base + (tail + first_n) % dbc->nelem * get_dbc_req_elem_size(); 1143 memcpy(last_req, reqs + slice->nents - 1, sizeof(*reqs)); 1144 1145 /* 1146 * last_bytes holds size of a DMA segment, maximum DMA segment size is 1147 * set to UINT_MAX by qaic and hence last_bytes can never exceed u32 1148 * range. So, by down sizing we are not corrupting the value. 1149 */ 1150 last_req->len = cpu_to_le32((u32)last_bytes); 1151 last_req->src_addr = reqs[first_n].src_addr; 1152 last_req->dest_addr = reqs[first_n].dest_addr; 1153 1154 *ptail = (tail + first_n + 1) % dbc->nelem; 1155 1156 return 0; 1157} 1158 1159static int send_bo_list_to_device(struct qaic_device *qdev, struct drm_file *file_priv, 1160 struct qaic_execute_entry *exec, unsigned int count, 1161 bool is_partial, struct dma_bridge_chan *dbc, u32 head, 1162 u32 *tail) 1163{ 1164 struct qaic_partial_execute_entry *pexec = (struct qaic_partial_execute_entry *)exec; 1165 struct drm_gem_object *obj; 1166 struct bo_slice *slice; 1167 unsigned long flags; 1168 struct qaic_bo *bo; 1169 bool queued; 1170 int i, j; 1171 int ret; 1172 1173 for (i = 0; i < count; i++) { 1174 /* 1175 * ref count will be decremented when the transfer of this 1176 * buffer is complete. It is inside dbc_irq_threaded_fn(). 1177 */ 1178 obj = drm_gem_object_lookup(file_priv, 1179 is_partial ? pexec[i].handle : exec[i].handle); 1180 if (!obj) { 1181 ret = -ENOENT; 1182 goto failed_to_send_bo; 1183 } 1184 1185 bo = to_qaic_bo(obj); 1186 1187 if (!bo->sliced) { 1188 ret = -EINVAL; 1189 goto failed_to_send_bo; 1190 } 1191 1192 if (is_partial && pexec[i].resize > bo->size) { 1193 ret = -EINVAL; 1194 goto failed_to_send_bo; 1195 } 1196 1197 spin_lock_irqsave(&dbc->xfer_lock, flags); 1198 queued = bo->queued; 1199 bo->queued = true; 1200 if (queued) { 1201 spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1202 ret = -EINVAL; 1203 goto failed_to_send_bo; 1204 } 1205 1206 bo->req_id = dbc->next_req_id++; 1207 1208 list_for_each_entry(slice, &bo->slices, slice) { 1209 /* 1210 * If this slice does not fall under the given 1211 * resize then skip this slice and continue the loop 1212 */ 1213 if (is_partial && pexec[i].resize && pexec[i].resize <= slice->offset) 1214 continue; 1215 1216 for (j = 0; j < slice->nents; j++) 1217 slice->reqs[j].req_id = cpu_to_le16(bo->req_id); 1218 1219 /* 1220 * If it is a partial execute ioctl call then check if 1221 * resize has cut this slice short then do a partial copy 1222 * else do complete copy 1223 */ 1224 if (is_partial && pexec[i].resize && 1225 pexec[i].resize < slice->offset + slice->size) 1226 ret = copy_partial_exec_reqs(qdev, slice, 1227 pexec[i].resize - slice->offset, 1228 dbc->id, head, tail); 1229 else 1230 ret = copy_exec_reqs(qdev, slice, dbc->id, head, tail); 1231 if (ret) { 1232 bo->queued = false; 1233 spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1234 goto failed_to_send_bo; 1235 } 1236 } 1237 reinit_completion(&bo->xfer_done); 1238 list_add_tail(&bo->xfer_list, &dbc->xfer_list); 1239 spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1240 dma_sync_sgtable_for_device(&qdev->pdev->dev, bo->sgt, bo->dir); 1241 } 1242 1243 return 0; 1244 1245failed_to_send_bo: 1246 if (likely(obj)) 1247 drm_gem_object_put(obj); 1248 for (j = 0; j < i; j++) { 1249 spin_lock_irqsave(&dbc->xfer_lock, flags); 1250 bo = list_last_entry(&dbc->xfer_list, struct qaic_bo, xfer_list); 1251 obj = &bo->base; 1252 bo->queued = false; 1253 list_del(&bo->xfer_list); 1254 spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1255 dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir); 1256 drm_gem_object_put(obj); 1257 } 1258 return ret; 1259} 1260 1261static void update_profiling_data(struct drm_file *file_priv, 1262 struct qaic_execute_entry *exec, unsigned int count, 1263 bool is_partial, u64 received_ts, u64 submit_ts, u32 queue_level) 1264{ 1265 struct qaic_partial_execute_entry *pexec = (struct qaic_partial_execute_entry *)exec; 1266 struct drm_gem_object *obj; 1267 struct qaic_bo *bo; 1268 int i; 1269 1270 for (i = 0; i < count; i++) { 1271 /* 1272 * Since we already committed the BO to hardware, the only way 1273 * this should fail is a pending signal. We can't cancel the 1274 * submit to hardware, so we have to just skip the profiling 1275 * data. In case the signal is not fatal to the process, we 1276 * return success so that the user doesn't try to resubmit. 1277 */ 1278 obj = drm_gem_object_lookup(file_priv, 1279 is_partial ? pexec[i].handle : exec[i].handle); 1280 if (!obj) 1281 break; 1282 bo = to_qaic_bo(obj); 1283 bo->perf_stats.req_received_ts = received_ts; 1284 bo->perf_stats.req_submit_ts = submit_ts; 1285 bo->perf_stats.queue_level_before = queue_level; 1286 queue_level += bo->total_slice_nents; 1287 drm_gem_object_put(obj); 1288 } 1289} 1290 1291static int __qaic_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv, 1292 bool is_partial) 1293{ 1294 struct qaic_execute *args = data; 1295 struct qaic_execute_entry *exec; 1296 struct dma_bridge_chan *dbc; 1297 int usr_rcu_id, qdev_rcu_id; 1298 struct qaic_device *qdev; 1299 struct qaic_user *usr; 1300 u8 __user *user_data; 1301 unsigned long n; 1302 u64 received_ts; 1303 u32 queue_level; 1304 u64 submit_ts; 1305 int rcu_id; 1306 u32 head; 1307 u32 tail; 1308 u64 size; 1309 int ret; 1310 1311 received_ts = ktime_get_ns(); 1312 1313 size = is_partial ? sizeof(struct qaic_partial_execute_entry) : sizeof(*exec); 1314 n = (unsigned long)size * args->hdr.count; 1315 if (args->hdr.count == 0 || n / args->hdr.count != size) 1316 return -EINVAL; 1317 1318 user_data = u64_to_user_ptr(args->data); 1319 1320 exec = kcalloc(args->hdr.count, size, GFP_KERNEL); 1321 if (!exec) 1322 return -ENOMEM; 1323 1324 if (copy_from_user(exec, user_data, n)) { 1325 ret = -EFAULT; 1326 goto free_exec; 1327 } 1328 1329 usr = file_priv->driver_priv; 1330 usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 1331 if (!usr->qddev) { 1332 ret = -ENODEV; 1333 goto unlock_usr_srcu; 1334 } 1335 1336 qdev = usr->qddev->qdev; 1337 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 1338 if (qdev->in_reset) { 1339 ret = -ENODEV; 1340 goto unlock_dev_srcu; 1341 } 1342 1343 if (args->hdr.dbc_id >= qdev->num_dbc) { 1344 ret = -EINVAL; 1345 goto unlock_dev_srcu; 1346 } 1347 1348 dbc = &qdev->dbc[args->hdr.dbc_id]; 1349 1350 rcu_id = srcu_read_lock(&dbc->ch_lock); 1351 if (!dbc->usr || dbc->usr->handle != usr->handle) { 1352 ret = -EPERM; 1353 goto release_ch_rcu; 1354 } 1355 1356 head = readl(dbc->dbc_base + REQHP_OFF); 1357 tail = readl(dbc->dbc_base + REQTP_OFF); 1358 1359 if (head == U32_MAX || tail == U32_MAX) { 1360 /* PCI link error */ 1361 ret = -ENODEV; 1362 goto release_ch_rcu; 1363 } 1364 1365 queue_level = head <= tail ? tail - head : dbc->nelem - (head - tail); 1366 1367 ret = send_bo_list_to_device(qdev, file_priv, exec, args->hdr.count, is_partial, dbc, 1368 head, &tail); 1369 if (ret) 1370 goto release_ch_rcu; 1371 1372 /* Finalize commit to hardware */ 1373 submit_ts = ktime_get_ns(); 1374 writel(tail, dbc->dbc_base + REQTP_OFF); 1375 1376 update_profiling_data(file_priv, exec, args->hdr.count, is_partial, received_ts, 1377 submit_ts, queue_level); 1378 1379 if (datapath_polling) 1380 schedule_work(&dbc->poll_work); 1381 1382release_ch_rcu: 1383 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1384unlock_dev_srcu: 1385 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 1386unlock_usr_srcu: 1387 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 1388free_exec: 1389 kfree(exec); 1390 return ret; 1391} 1392 1393int qaic_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 1394{ 1395 return __qaic_execute_bo_ioctl(dev, data, file_priv, false); 1396} 1397 1398int qaic_partial_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 1399{ 1400 return __qaic_execute_bo_ioctl(dev, data, file_priv, true); 1401} 1402 1403/* 1404 * Our interrupt handling is a bit more complicated than a simple ideal, but 1405 * sadly necessary. 1406 * 1407 * Each dbc has a completion queue. Entries in the queue correspond to DMA 1408 * requests which the device has processed. The hardware already has a built 1409 * in irq mitigation. When the device puts an entry into the queue, it will 1410 * only trigger an interrupt if the queue was empty. Therefore, when adding 1411 * the Nth event to a non-empty queue, the hardware doesn't trigger an 1412 * interrupt. This means the host doesn't get additional interrupts signaling 1413 * the same thing - the queue has something to process. 1414 * This behavior can be overridden in the DMA request. 1415 * This means that when the host receives an interrupt, it is required to 1416 * drain the queue. 1417 * 1418 * This behavior is what NAPI attempts to accomplish, although we can't use 1419 * NAPI as we don't have a netdev. We use threaded irqs instead. 1420 * 1421 * However, there is a situation where the host drains the queue fast enough 1422 * that every event causes an interrupt. Typically this is not a problem as 1423 * the rate of events would be low. However, that is not the case with 1424 * lprnet for example. On an Intel Xeon D-2191 where we run 8 instances of 1425 * lprnet, the host receives roughly 80k interrupts per second from the device 1426 * (per /proc/interrupts). While NAPI documentation indicates the host should 1427 * just chug along, sadly that behavior causes instability in some hosts. 1428 * 1429 * Therefore, we implement an interrupt disable scheme similar to NAPI. The 1430 * key difference is that we will delay after draining the queue for a small 1431 * time to allow additional events to come in via polling. Using the above 1432 * lprnet workload, this reduces the number of interrupts processed from 1433 * ~80k/sec to about 64 in 5 minutes and appears to solve the system 1434 * instability. 1435 */ 1436irqreturn_t dbc_irq_handler(int irq, void *data) 1437{ 1438 struct dma_bridge_chan *dbc = data; 1439 int rcu_id; 1440 u32 head; 1441 u32 tail; 1442 1443 rcu_id = srcu_read_lock(&dbc->ch_lock); 1444 1445 if (!dbc->usr) { 1446 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1447 return IRQ_HANDLED; 1448 } 1449 1450 head = readl(dbc->dbc_base + RSPHP_OFF); 1451 if (head == U32_MAX) { /* PCI link error */ 1452 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1453 return IRQ_NONE; 1454 } 1455 1456 tail = readl(dbc->dbc_base + RSPTP_OFF); 1457 if (tail == U32_MAX) { /* PCI link error */ 1458 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1459 return IRQ_NONE; 1460 } 1461 1462 if (head == tail) { /* queue empty */ 1463 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1464 return IRQ_NONE; 1465 } 1466 1467 disable_irq_nosync(irq); 1468 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1469 return IRQ_WAKE_THREAD; 1470} 1471 1472void irq_polling_work(struct work_struct *work) 1473{ 1474 struct dma_bridge_chan *dbc = container_of(work, struct dma_bridge_chan, poll_work); 1475 unsigned long flags; 1476 int rcu_id; 1477 u32 head; 1478 u32 tail; 1479 1480 rcu_id = srcu_read_lock(&dbc->ch_lock); 1481 1482 while (1) { 1483 if (dbc->qdev->in_reset) { 1484 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1485 return; 1486 } 1487 if (!dbc->usr) { 1488 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1489 return; 1490 } 1491 spin_lock_irqsave(&dbc->xfer_lock, flags); 1492 if (list_empty(&dbc->xfer_list)) { 1493 spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1494 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1495 return; 1496 } 1497 spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1498 1499 head = readl(dbc->dbc_base + RSPHP_OFF); 1500 if (head == U32_MAX) { /* PCI link error */ 1501 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1502 return; 1503 } 1504 1505 tail = readl(dbc->dbc_base + RSPTP_OFF); 1506 if (tail == U32_MAX) { /* PCI link error */ 1507 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1508 return; 1509 } 1510 1511 if (head != tail) { 1512 irq_wake_thread(dbc->irq, dbc); 1513 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1514 return; 1515 } 1516 1517 cond_resched(); 1518 usleep_range(datapath_poll_interval_us, 2 * datapath_poll_interval_us); 1519 } 1520} 1521 1522irqreturn_t dbc_irq_threaded_fn(int irq, void *data) 1523{ 1524 struct dma_bridge_chan *dbc = data; 1525 int event_count = NUM_EVENTS; 1526 int delay_count = NUM_DELAYS; 1527 struct qaic_device *qdev; 1528 struct qaic_bo *bo, *i; 1529 struct dbc_rsp *rsp; 1530 unsigned long flags; 1531 int rcu_id; 1532 u16 status; 1533 u16 req_id; 1534 u32 head; 1535 u32 tail; 1536 1537 rcu_id = srcu_read_lock(&dbc->ch_lock); 1538 1539 head = readl(dbc->dbc_base + RSPHP_OFF); 1540 if (head == U32_MAX) /* PCI link error */ 1541 goto error_out; 1542 1543 qdev = dbc->qdev; 1544read_fifo: 1545 1546 if (!event_count) { 1547 event_count = NUM_EVENTS; 1548 cond_resched(); 1549 } 1550 1551 /* 1552 * if this channel isn't assigned or gets unassigned during processing 1553 * we have nothing further to do 1554 */ 1555 if (!dbc->usr) 1556 goto error_out; 1557 1558 tail = readl(dbc->dbc_base + RSPTP_OFF); 1559 if (tail == U32_MAX) /* PCI link error */ 1560 goto error_out; 1561 1562 if (head == tail) { /* queue empty */ 1563 if (delay_count) { 1564 --delay_count; 1565 usleep_range(100, 200); 1566 goto read_fifo; /* check for a new event */ 1567 } 1568 goto normal_out; 1569 } 1570 1571 delay_count = NUM_DELAYS; 1572 while (head != tail) { 1573 if (!event_count) 1574 break; 1575 --event_count; 1576 rsp = dbc->rsp_q_base + head * sizeof(*rsp); 1577 req_id = le16_to_cpu(rsp->req_id); 1578 status = le16_to_cpu(rsp->status); 1579 if (status) 1580 pci_dbg(qdev->pdev, "req_id %d failed with status %d\n", req_id, status); 1581 spin_lock_irqsave(&dbc->xfer_lock, flags); 1582 /* 1583 * A BO can receive multiple interrupts, since a BO can be 1584 * divided into multiple slices and a buffer receives as many 1585 * interrupts as slices. So until it receives interrupts for 1586 * all the slices we cannot mark that buffer complete. 1587 */ 1588 list_for_each_entry_safe(bo, i, &dbc->xfer_list, xfer_list) { 1589 if (bo->req_id == req_id) 1590 bo->nr_slice_xfer_done++; 1591 else 1592 continue; 1593 1594 if (bo->nr_slice_xfer_done < bo->nr_slice) 1595 break; 1596 1597 /* 1598 * At this point we have received all the interrupts for 1599 * BO, which means BO execution is complete. 1600 */ 1601 dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir); 1602 bo->nr_slice_xfer_done = 0; 1603 bo->queued = false; 1604 list_del(&bo->xfer_list); 1605 bo->perf_stats.req_processed_ts = ktime_get_ns(); 1606 complete_all(&bo->xfer_done); 1607 drm_gem_object_put(&bo->base); 1608 break; 1609 } 1610 spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1611 head = (head + 1) % dbc->nelem; 1612 } 1613 1614 /* 1615 * Update the head pointer of response queue and let the device know 1616 * that we have consumed elements from the queue. 1617 */ 1618 writel(head, dbc->dbc_base + RSPHP_OFF); 1619 1620 /* elements might have been put in the queue while we were processing */ 1621 goto read_fifo; 1622 1623normal_out: 1624 if (likely(!datapath_polling)) 1625 enable_irq(irq); 1626 else 1627 schedule_work(&dbc->poll_work); 1628 /* checking the fifo and enabling irqs is a race, missed event check */ 1629 tail = readl(dbc->dbc_base + RSPTP_OFF); 1630 if (tail != U32_MAX && head != tail) { 1631 if (likely(!datapath_polling)) 1632 disable_irq_nosync(irq); 1633 goto read_fifo; 1634 } 1635 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1636 return IRQ_HANDLED; 1637 1638error_out: 1639 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1640 if (likely(!datapath_polling)) 1641 enable_irq(irq); 1642 else 1643 schedule_work(&dbc->poll_work); 1644 1645 return IRQ_HANDLED; 1646} 1647 1648int qaic_wait_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 1649{ 1650 struct qaic_wait *args = data; 1651 int usr_rcu_id, qdev_rcu_id; 1652 struct dma_bridge_chan *dbc; 1653 struct drm_gem_object *obj; 1654 struct qaic_device *qdev; 1655 unsigned long timeout; 1656 struct qaic_user *usr; 1657 struct qaic_bo *bo; 1658 int rcu_id; 1659 int ret; 1660 1661 if (args->pad != 0) 1662 return -EINVAL; 1663 1664 usr = file_priv->driver_priv; 1665 usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 1666 if (!usr->qddev) { 1667 ret = -ENODEV; 1668 goto unlock_usr_srcu; 1669 } 1670 1671 qdev = usr->qddev->qdev; 1672 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 1673 if (qdev->in_reset) { 1674 ret = -ENODEV; 1675 goto unlock_dev_srcu; 1676 } 1677 1678 if (args->dbc_id >= qdev->num_dbc) { 1679 ret = -EINVAL; 1680 goto unlock_dev_srcu; 1681 } 1682 1683 dbc = &qdev->dbc[args->dbc_id]; 1684 1685 rcu_id = srcu_read_lock(&dbc->ch_lock); 1686 if (dbc->usr != usr) { 1687 ret = -EPERM; 1688 goto unlock_ch_srcu; 1689 } 1690 1691 obj = drm_gem_object_lookup(file_priv, args->handle); 1692 if (!obj) { 1693 ret = -ENOENT; 1694 goto unlock_ch_srcu; 1695 } 1696 1697 bo = to_qaic_bo(obj); 1698 timeout = args->timeout ? args->timeout : wait_exec_default_timeout_ms; 1699 timeout = msecs_to_jiffies(timeout); 1700 ret = wait_for_completion_interruptible_timeout(&bo->xfer_done, timeout); 1701 if (!ret) { 1702 ret = -ETIMEDOUT; 1703 goto put_obj; 1704 } 1705 if (ret > 0) 1706 ret = 0; 1707 1708 if (!dbc->usr) 1709 ret = -EPERM; 1710 1711put_obj: 1712 drm_gem_object_put(obj); 1713unlock_ch_srcu: 1714 srcu_read_unlock(&dbc->ch_lock, rcu_id); 1715unlock_dev_srcu: 1716 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 1717unlock_usr_srcu: 1718 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 1719 return ret; 1720} 1721 1722int qaic_perf_stats_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 1723{ 1724 struct qaic_perf_stats_entry *ent = NULL; 1725 struct qaic_perf_stats *args = data; 1726 int usr_rcu_id, qdev_rcu_id; 1727 struct drm_gem_object *obj; 1728 struct qaic_device *qdev; 1729 struct qaic_user *usr; 1730 struct qaic_bo *bo; 1731 int ret, i; 1732 1733 usr = file_priv->driver_priv; 1734 usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 1735 if (!usr->qddev) { 1736 ret = -ENODEV; 1737 goto unlock_usr_srcu; 1738 } 1739 1740 qdev = usr->qddev->qdev; 1741 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 1742 if (qdev->in_reset) { 1743 ret = -ENODEV; 1744 goto unlock_dev_srcu; 1745 } 1746 1747 if (args->hdr.dbc_id >= qdev->num_dbc) { 1748 ret = -EINVAL; 1749 goto unlock_dev_srcu; 1750 } 1751 1752 ent = kcalloc(args->hdr.count, sizeof(*ent), GFP_KERNEL); 1753 if (!ent) { 1754 ret = -EINVAL; 1755 goto unlock_dev_srcu; 1756 } 1757 1758 ret = copy_from_user(ent, u64_to_user_ptr(args->data), args->hdr.count * sizeof(*ent)); 1759 if (ret) { 1760 ret = -EFAULT; 1761 goto free_ent; 1762 } 1763 1764 for (i = 0; i < args->hdr.count; i++) { 1765 obj = drm_gem_object_lookup(file_priv, ent[i].handle); 1766 if (!obj) { 1767 ret = -ENOENT; 1768 goto free_ent; 1769 } 1770 bo = to_qaic_bo(obj); 1771 /* 1772 * perf stats ioctl is called before wait ioctl is complete then 1773 * the latency information is invalid. 1774 */ 1775 if (bo->perf_stats.req_processed_ts < bo->perf_stats.req_submit_ts) { 1776 ent[i].device_latency_us = 0; 1777 } else { 1778 ent[i].device_latency_us = div_u64((bo->perf_stats.req_processed_ts - 1779 bo->perf_stats.req_submit_ts), 1000); 1780 } 1781 ent[i].submit_latency_us = div_u64((bo->perf_stats.req_submit_ts - 1782 bo->perf_stats.req_received_ts), 1000); 1783 ent[i].queue_level_before = bo->perf_stats.queue_level_before; 1784 ent[i].num_queue_element = bo->total_slice_nents; 1785 drm_gem_object_put(obj); 1786 } 1787 1788 if (copy_to_user(u64_to_user_ptr(args->data), ent, args->hdr.count * sizeof(*ent))) 1789 ret = -EFAULT; 1790 1791free_ent: 1792 kfree(ent); 1793unlock_dev_srcu: 1794 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 1795unlock_usr_srcu: 1796 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 1797 return ret; 1798} 1799 1800static void empty_xfer_list(struct qaic_device *qdev, struct dma_bridge_chan *dbc) 1801{ 1802 unsigned long flags; 1803 struct qaic_bo *bo; 1804 1805 spin_lock_irqsave(&dbc->xfer_lock, flags); 1806 while (!list_empty(&dbc->xfer_list)) { 1807 bo = list_first_entry(&dbc->xfer_list, typeof(*bo), xfer_list); 1808 bo->queued = false; 1809 list_del(&bo->xfer_list); 1810 spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1811 dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir); 1812 complete_all(&bo->xfer_done); 1813 drm_gem_object_put(&bo->base); 1814 spin_lock_irqsave(&dbc->xfer_lock, flags); 1815 } 1816 spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1817} 1818 1819int disable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr) 1820{ 1821 if (!qdev->dbc[dbc_id].usr || qdev->dbc[dbc_id].usr->handle != usr->handle) 1822 return -EPERM; 1823 1824 qdev->dbc[dbc_id].usr = NULL; 1825 synchronize_srcu(&qdev->dbc[dbc_id].ch_lock); 1826 return 0; 1827} 1828 1829/** 1830 * enable_dbc - Enable the DBC. DBCs are disabled by removing the context of 1831 * user. Add user context back to DBC to enable it. This function trusts the 1832 * DBC ID passed and expects the DBC to be disabled. 1833 * @qdev: Qranium device handle 1834 * @dbc_id: ID of the DBC 1835 * @usr: User context 1836 */ 1837void enable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr) 1838{ 1839 qdev->dbc[dbc_id].usr = usr; 1840} 1841 1842void wakeup_dbc(struct qaic_device *qdev, u32 dbc_id) 1843{ 1844 struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id]; 1845 1846 dbc->usr = NULL; 1847 empty_xfer_list(qdev, dbc); 1848 synchronize_srcu(&dbc->ch_lock); 1849 /* 1850 * Threads holding channel lock, may add more elements in the xfer_list. 1851 * Flush out these elements from xfer_list. 1852 */ 1853 empty_xfer_list(qdev, dbc); 1854} 1855 1856void release_dbc(struct qaic_device *qdev, u32 dbc_id) 1857{ 1858 struct bo_slice *slice, *slice_temp; 1859 struct qaic_bo *bo, *bo_temp; 1860 struct dma_bridge_chan *dbc; 1861 1862 dbc = &qdev->dbc[dbc_id]; 1863 if (!dbc->in_use) 1864 return; 1865 1866 wakeup_dbc(qdev, dbc_id); 1867 1868 dma_free_coherent(&qdev->pdev->dev, dbc->total_size, dbc->req_q_base, dbc->dma_addr); 1869 dbc->total_size = 0; 1870 dbc->req_q_base = NULL; 1871 dbc->dma_addr = 0; 1872 dbc->nelem = 0; 1873 dbc->usr = NULL; 1874 1875 list_for_each_entry_safe(bo, bo_temp, &dbc->bo_lists, bo_list) { 1876 list_for_each_entry_safe(slice, slice_temp, &bo->slices, slice) 1877 kref_put(&slice->ref_count, free_slice); 1878 bo->sliced = false; 1879 INIT_LIST_HEAD(&bo->slices); 1880 bo->total_slice_nents = 0; 1881 bo->dir = 0; 1882 bo->dbc = NULL; 1883 bo->nr_slice = 0; 1884 bo->nr_slice_xfer_done = 0; 1885 bo->queued = false; 1886 bo->req_id = 0; 1887 init_completion(&bo->xfer_done); 1888 complete_all(&bo->xfer_done); 1889 list_del(&bo->bo_list); 1890 bo->perf_stats.req_received_ts = 0; 1891 bo->perf_stats.req_submit_ts = 0; 1892 bo->perf_stats.req_processed_ts = 0; 1893 bo->perf_stats.queue_level_before = 0; 1894 } 1895 1896 dbc->in_use = false; 1897 wake_up(&dbc->dbc_release); 1898} 1899