1/* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23#ifndef KFD_PRIV_H_INCLUDED 24#define KFD_PRIV_H_INCLUDED 25 26#include <linux/hashtable.h> 27#include <linux/mmu_notifier.h> 28#include <linux/mutex.h> 29#include <linux/types.h> 30#include <linux/atomic.h> 31#include <linux/workqueue.h> 32#include <linux/spinlock.h> 33#include <linux/kfd_ioctl.h> 34#include <linux/idr.h> 35#include <linux/kfifo.h> 36#include <linux/seq_file.h> 37#include <linux/kref.h> 38#include <linux/sysfs.h> 39#include <linux/device_cgroup.h> 40#include <drm/drm_file.h> 41#include <drm/drm_drv.h> 42#include <drm/drm_device.h> 43#include <drm/drm_ioctl.h> 44#include <kgd_kfd_interface.h> 45#include <linux/swap.h> 46 47#include "amd_shared.h" 48 49#define KFD_MAX_RING_ENTRY_SIZE 8 50 51#define KFD_SYSFS_FILE_MODE 0444 52 53/* GPU ID hash width in bits */ 54#define KFD_GPU_ID_HASH_WIDTH 16 55 56/* Use upper bits of mmap offset to store KFD driver specific information. 57 * BITS[63:62] - Encode MMAP type 58 * BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to 59 * BITS[45:0] - MMAP offset value 60 * 61 * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these 62 * defines are w.r.t to PAGE_SIZE 63 */ 64#define KFD_MMAP_TYPE_SHIFT 62 65#define KFD_MMAP_TYPE_MASK (0x3ULL << KFD_MMAP_TYPE_SHIFT) 66#define KFD_MMAP_TYPE_DOORBELL (0x3ULL << KFD_MMAP_TYPE_SHIFT) 67#define KFD_MMAP_TYPE_EVENTS (0x2ULL << KFD_MMAP_TYPE_SHIFT) 68#define KFD_MMAP_TYPE_RESERVED_MEM (0x1ULL << KFD_MMAP_TYPE_SHIFT) 69#define KFD_MMAP_TYPE_MMIO (0x0ULL << KFD_MMAP_TYPE_SHIFT) 70 71#define KFD_MMAP_GPU_ID_SHIFT 46 72#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \ 73 << KFD_MMAP_GPU_ID_SHIFT) 74#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\ 75 & KFD_MMAP_GPU_ID_MASK) 76#define KFD_MMAP_GET_GPU_ID(offset) ((offset & KFD_MMAP_GPU_ID_MASK) \ 77 >> KFD_MMAP_GPU_ID_SHIFT) 78 79/* 80 * When working with cp scheduler we should assign the HIQ manually or via 81 * the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot 82 * definitions for Kaveri. In Kaveri only the first ME queues participates 83 * in the cp scheduling taking that in mind we set the HIQ slot in the 84 * second ME. 85 */ 86#define KFD_CIK_HIQ_PIPE 4 87#define KFD_CIK_HIQ_QUEUE 0 88 89/* Macro for allocating structures */ 90#define kfd_alloc_struct(ptr_to_struct) \ 91 ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL)) 92 93#define KFD_MAX_NUM_OF_PROCESSES 512 94#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 95 96/* 97 * Size of the per-process TBA+TMA buffer: 2 pages 98 * 99 * The first page is the TBA used for the CWSR ISA code. The second 100 * page is used as TMA for user-mode trap handler setup in daisy-chain mode. 101 */ 102#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2) 103#define KFD_CWSR_TMA_OFFSET PAGE_SIZE 104 105#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \ 106 (KFD_MAX_NUM_OF_PROCESSES * \ 107 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) 108 109#define KFD_KERNEL_QUEUE_SIZE 2048 110 111#define KFD_UNMAP_LATENCY_MS (4000) 112 113/* 114 * 512 = 0x200 115 * The doorbell index distance between SDMA RLC (2*i) and (2*i+1) in the 116 * same SDMA engine on SOC15, which has 8-byte doorbells for SDMA. 117 * 512 8-byte doorbell distance (i.e. one page away) ensures that SDMA RLC 118 * (2*i+1) doorbells (in terms of the lower 12 bit address) lie exactly in 119 * the OFFSET and SIZE set in registers like BIF_SDMA0_DOORBELL_RANGE. 120 */ 121#define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512 122 123 124/* 125 * Kernel module parameter to specify maximum number of supported queues per 126 * device 127 */ 128extern int max_num_of_queues_per_device; 129 130 131/* Kernel module parameter to specify the scheduling policy */ 132extern int sched_policy; 133 134/* 135 * Kernel module parameter to specify the maximum process 136 * number per HW scheduler 137 */ 138extern int hws_max_conc_proc; 139 140extern int cwsr_enable; 141 142/* 143 * Kernel module parameter to specify whether to send sigterm to HSA process on 144 * unhandled exception 145 */ 146extern int send_sigterm; 147 148/* 149 * This kernel module is used to simulate large bar machine on non-large bar 150 * enabled machines. 151 */ 152extern int debug_largebar; 153 154/* 155 * Ignore CRAT table during KFD initialization, can be used to work around 156 * broken CRAT tables on some AMD systems 157 */ 158extern int ignore_crat; 159 160/* Set sh_mem_config.retry_disable on GFX v9 */ 161extern int amdgpu_noretry; 162 163/* Halt if HWS hang is detected */ 164extern int halt_if_hws_hang; 165 166/* Whether MEC FW support GWS barriers */ 167extern bool hws_gws_support; 168 169/* Queue preemption timeout in ms */ 170extern int queue_preemption_timeout_ms; 171 172/* Enable eviction debug messages */ 173extern bool debug_evictions; 174 175enum cache_policy { 176 cache_policy_coherent, 177 cache_policy_noncoherent 178}; 179 180#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10) 181 182struct kfd_event_interrupt_class { 183 bool (*interrupt_isr)(struct kfd_dev *dev, 184 const uint32_t *ih_ring_entry, uint32_t *patched_ihre, 185 bool *patched_flag); 186 void (*interrupt_wq)(struct kfd_dev *dev, 187 const uint32_t *ih_ring_entry); 188}; 189 190struct kfd_device_info { 191 enum amd_asic_type asic_family; 192 const char *asic_name; 193 const struct kfd_event_interrupt_class *event_interrupt_class; 194 unsigned int max_pasid_bits; 195 unsigned int max_no_of_hqd; 196 unsigned int doorbell_size; 197 size_t ih_ring_entry_size; 198 uint8_t num_of_watch_points; 199 uint16_t mqd_size_aligned; 200 bool supports_cwsr; 201 bool needs_iommu_device; 202 bool needs_pci_atomics; 203 unsigned int num_sdma_engines; 204 unsigned int num_xgmi_sdma_engines; 205 unsigned int num_sdma_queues_per_engine; 206}; 207 208struct kfd_mem_obj { 209 uint32_t range_start; 210 uint32_t range_end; 211 uint64_t gpu_addr; 212 uint32_t *cpu_ptr; 213 void *gtt_mem; 214}; 215 216struct kfd_vmid_info { 217 uint32_t first_vmid_kfd; 218 uint32_t last_vmid_kfd; 219 uint32_t vmid_num_kfd; 220}; 221 222struct kfd_dev { 223 struct kgd_dev *kgd; 224 225 const struct kfd_device_info *device_info; 226 struct pci_dev *pdev; 227 struct drm_device *ddev; 228 229 unsigned int id; /* topology stub index */ 230 231 phys_addr_t doorbell_base; /* Start of actual doorbells used by 232 * KFD. It is aligned for mapping 233 * into user mode 234 */ 235 size_t doorbell_base_dw_offset; /* Offset from the start of the PCI 236 * doorbell BAR to the first KFD 237 * doorbell in dwords. GFX reserves 238 * the segment before this offset. 239 */ 240 u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells 241 * page used by kernel queue 242 */ 243 244 struct kgd2kfd_shared_resources shared_resources; 245 struct kfd_vmid_info vm_info; 246 247 const struct kfd2kgd_calls *kfd2kgd; 248 struct mutex doorbell_mutex; 249 DECLARE_BITMAP(doorbell_available_index, 250 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); 251 252 void *gtt_mem; 253 uint64_t gtt_start_gpu_addr; 254 void *gtt_start_cpu_ptr; 255 void *gtt_sa_bitmap; 256 struct mutex gtt_sa_lock; 257 unsigned int gtt_sa_chunk_size; 258 unsigned int gtt_sa_num_of_chunks; 259 260 /* Interrupts */ 261 struct kfifo ih_fifo; 262 struct workqueue_struct *ih_wq; 263 struct work_struct interrupt_work; 264 spinlock_t interrupt_lock; 265 266 /* QCM Device instance */ 267 struct device_queue_manager *dqm; 268 269 bool init_complete; 270 /* 271 * Interrupts of interest to KFD are copied 272 * from the HW ring into a SW ring. 273 */ 274 bool interrupts_active; 275 276 /* Debug manager */ 277 struct kfd_dbgmgr *dbgmgr; 278 279 /* Firmware versions */ 280 uint16_t mec_fw_version; 281 uint16_t mec2_fw_version; 282 uint16_t sdma_fw_version; 283 284 /* Maximum process number mapped to HW scheduler */ 285 unsigned int max_proc_per_quantum; 286 287 /* CWSR */ 288 bool cwsr_enabled; 289 const void *cwsr_isa; 290 unsigned int cwsr_isa_size; 291 292 /* xGMI */ 293 uint64_t hive_id; 294 295 /* UUID */ 296 uint64_t unique_id; 297 298 bool pci_atomic_requested; 299 300 /* Use IOMMU v2 flag */ 301 bool use_iommu_v2; 302 303 /* SRAM ECC flag */ 304 atomic_t sram_ecc_flag; 305 306 /* Compute Profile ref. count */ 307 atomic_t compute_profile; 308 309 /* Global GWS resource shared between processes */ 310 void *gws; 311 312 /* Clients watching SMI events */ 313 struct list_head smi_clients; 314 spinlock_t smi_lock; 315 316 uint32_t reset_seq_num; 317 318 struct ida doorbell_ida; 319 unsigned int max_doorbell_slices; 320 321 int noretry; 322}; 323 324enum kfd_mempool { 325 KFD_MEMPOOL_SYSTEM_CACHEABLE = 1, 326 KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2, 327 KFD_MEMPOOL_FRAMEBUFFER = 3, 328}; 329 330/* Character device interface */ 331int kfd_chardev_init(void); 332void kfd_chardev_exit(void); 333struct device *kfd_chardev(void); 334 335/** 336 * enum kfd_unmap_queues_filter - Enum for queue filters. 337 * 338 * @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue. 339 * 340 * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the 341 * running queues list. 342 * 343 * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to 344 * specific process. 345 * 346 */ 347enum kfd_unmap_queues_filter { 348 KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE, 349 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 350 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 351 KFD_UNMAP_QUEUES_FILTER_BY_PASID 352}; 353 354/** 355 * enum kfd_queue_type - Enum for various queue types. 356 * 357 * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type. 358 * 359 * @KFD_QUEUE_TYPE_SDMA: SDMA user mode queue type. 360 * 361 * @KFD_QUEUE_TYPE_HIQ: HIQ queue type. 362 * 363 * @KFD_QUEUE_TYPE_DIQ: DIQ queue type. 364 * 365 * @KFD_QUEUE_TYPE_SDMA_XGMI: Special SDMA queue for XGMI interface. 366 */ 367enum kfd_queue_type { 368 KFD_QUEUE_TYPE_COMPUTE, 369 KFD_QUEUE_TYPE_SDMA, 370 KFD_QUEUE_TYPE_HIQ, 371 KFD_QUEUE_TYPE_DIQ, 372 KFD_QUEUE_TYPE_SDMA_XGMI 373}; 374 375enum kfd_queue_format { 376 KFD_QUEUE_FORMAT_PM4, 377 KFD_QUEUE_FORMAT_AQL 378}; 379 380enum KFD_QUEUE_PRIORITY { 381 KFD_QUEUE_PRIORITY_MINIMUM = 0, 382 KFD_QUEUE_PRIORITY_MAXIMUM = 15 383}; 384 385/** 386 * struct queue_properties 387 * 388 * @type: The queue type. 389 * 390 * @queue_id: Queue identifier. 391 * 392 * @queue_address: Queue ring buffer address. 393 * 394 * @queue_size: Queue ring buffer size. 395 * 396 * @priority: Defines the queue priority relative to other queues in the 397 * process. 398 * This is just an indication and HW scheduling may override the priority as 399 * necessary while keeping the relative prioritization. 400 * the priority granularity is from 0 to f which f is the highest priority. 401 * currently all queues are initialized with the highest priority. 402 * 403 * @queue_percent: This field is partially implemented and currently a zero in 404 * this field defines that the queue is non active. 405 * 406 * @read_ptr: User space address which points to the number of dwords the 407 * cp read from the ring buffer. This field updates automatically by the H/W. 408 * 409 * @write_ptr: Defines the number of dwords written to the ring buffer. 410 * 411 * @doorbell_ptr: Notifies the H/W of new packet written to the queue ring 412 * buffer. This field should be similar to write_ptr and the user should 413 * update this field after updating the write_ptr. 414 * 415 * @doorbell_off: The doorbell offset in the doorbell pci-bar. 416 * 417 * @is_interop: Defines if this is a interop queue. Interop queue means that 418 * the queue can access both graphics and compute resources. 419 * 420 * @is_evicted: Defines if the queue is evicted. Only active queues 421 * are evicted, rendering them inactive. 422 * 423 * @is_active: Defines if the queue is active or not. @is_active and 424 * @is_evicted are protected by the DQM lock. 425 * 426 * @is_gws: Defines if the queue has been updated to be GWS-capable or not. 427 * @is_gws should be protected by the DQM lock, since changing it can yield the 428 * possibility of updating DQM state on number of GWS queues. 429 * 430 * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid 431 * of the queue. 432 * 433 * This structure represents the queue properties for each queue no matter if 434 * it's user mode or kernel mode queue. 435 * 436 */ 437struct queue_properties { 438 enum kfd_queue_type type; 439 enum kfd_queue_format format; 440 unsigned int queue_id; 441 uint64_t queue_address; 442 uint64_t queue_size; 443 uint32_t priority; 444 uint32_t queue_percent; 445 uint32_t *read_ptr; 446 uint32_t *write_ptr; 447 void __iomem *doorbell_ptr; 448 uint32_t doorbell_off; 449 bool is_interop; 450 bool is_evicted; 451 bool is_active; 452 bool is_gws; 453 /* Not relevant for user mode queues in cp scheduling */ 454 unsigned int vmid; 455 /* Relevant only for sdma queues*/ 456 uint32_t sdma_engine_id; 457 uint32_t sdma_queue_id; 458 uint32_t sdma_vm_addr; 459 /* Relevant only for VI */ 460 uint64_t eop_ring_buffer_address; 461 uint32_t eop_ring_buffer_size; 462 uint64_t ctx_save_restore_area_address; 463 uint32_t ctx_save_restore_area_size; 464 uint32_t ctl_stack_size; 465 uint64_t tba_addr; 466 uint64_t tma_addr; 467 /* Relevant for CU */ 468 uint32_t cu_mask_count; /* Must be a multiple of 32 */ 469 uint32_t *cu_mask; 470}; 471 472#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 && \ 473 (q).queue_address != 0 && \ 474 (q).queue_percent > 0 && \ 475 !(q).is_evicted) 476 477/** 478 * struct queue 479 * 480 * @list: Queue linked list. 481 * 482 * @mqd: The queue MQD (memory queue descriptor). 483 * 484 * @mqd_mem_obj: The MQD local gpu memory object. 485 * 486 * @gart_mqd_addr: The MQD gart mc address. 487 * 488 * @properties: The queue properties. 489 * 490 * @mec: Used only in no cp scheduling mode and identifies to micro engine id 491 * that the queue should be executed on. 492 * 493 * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe 494 * id. 495 * 496 * @queue: Used only in no cp scheduliong mode and identifies the queue's slot. 497 * 498 * @process: The kfd process that created this queue. 499 * 500 * @device: The kfd device that created this queue. 501 * 502 * @gws: Pointing to gws kgd_mem if this is a gws control queue; NULL 503 * otherwise. 504 * 505 * This structure represents user mode compute queues. 506 * It contains all the necessary data to handle such queues. 507 * 508 */ 509 510struct queue { 511 struct list_head list; 512 void *mqd; 513 struct kfd_mem_obj *mqd_mem_obj; 514 uint64_t gart_mqd_addr; 515 struct queue_properties properties; 516 517 uint32_t mec; 518 uint32_t pipe; 519 uint32_t queue; 520 521 unsigned int sdma_id; 522 unsigned int doorbell_id; 523 524 struct kfd_process *process; 525 struct kfd_dev *device; 526 void *gws; 527 528 /* procfs */ 529 struct kobject kobj; 530}; 531 532enum KFD_MQD_TYPE { 533 KFD_MQD_TYPE_HIQ = 0, /* for hiq */ 534 KFD_MQD_TYPE_CP, /* for cp queues and diq */ 535 KFD_MQD_TYPE_SDMA, /* for sdma queues */ 536 KFD_MQD_TYPE_DIQ, /* for diq */ 537 KFD_MQD_TYPE_MAX 538}; 539 540enum KFD_PIPE_PRIORITY { 541 KFD_PIPE_PRIORITY_CS_LOW = 0, 542 KFD_PIPE_PRIORITY_CS_MEDIUM, 543 KFD_PIPE_PRIORITY_CS_HIGH 544}; 545 546struct scheduling_resources { 547 unsigned int vmid_mask; 548 enum kfd_queue_type type; 549 uint64_t queue_mask; 550 uint64_t gws_mask; 551 uint32_t oac_mask; 552 uint32_t gds_heap_base; 553 uint32_t gds_heap_size; 554}; 555 556struct process_queue_manager { 557 /* data */ 558 struct kfd_process *process; 559 struct list_head queues; 560 unsigned long *queue_slot_bitmap; 561}; 562 563struct qcm_process_device { 564 /* The Device Queue Manager that owns this data */ 565 struct device_queue_manager *dqm; 566 struct process_queue_manager *pqm; 567 /* Queues list */ 568 struct list_head queues_list; 569 struct list_head priv_queue_list; 570 571 unsigned int queue_count; 572 unsigned int vmid; 573 bool is_debug; 574 unsigned int evicted; /* eviction counter, 0=active */ 575 576 /* This flag tells if we should reset all wavefronts on 577 * process termination 578 */ 579 bool reset_wavefronts; 580 581 /* This flag tells us if this process has a GWS-capable 582 * queue that will be mapped into the runlist. It's 583 * possible to request a GWS BO, but not have the queue 584 * currently mapped, and this changes how the MAP_PROCESS 585 * PM4 packet is configured. 586 */ 587 bool mapped_gws_queue; 588 589 /* All the memory management data should be here too */ 590 uint64_t gds_context_area; 591 /* Contains page table flags such as AMDGPU_PTE_VALID since gfx9 */ 592 uint64_t page_table_base; 593 uint32_t sh_mem_config; 594 uint32_t sh_mem_bases; 595 uint32_t sh_mem_ape1_base; 596 uint32_t sh_mem_ape1_limit; 597 uint32_t gds_size; 598 uint32_t num_gws; 599 uint32_t num_oac; 600 uint32_t sh_hidden_private_base; 601 602 /* CWSR memory */ 603 void *cwsr_kaddr; 604 uint64_t cwsr_base; 605 uint64_t tba_addr; 606 uint64_t tma_addr; 607 608 /* IB memory */ 609 uint64_t ib_base; 610 void *ib_kaddr; 611 612 /* doorbell resources per process per device */ 613 unsigned long *doorbell_bitmap; 614}; 615 616/* KFD Memory Eviction */ 617 618/* Approx. wait time before attempting to restore evicted BOs */ 619#define PROCESS_RESTORE_TIME_MS 100 620/* Approx. back off time if restore fails due to lack of memory */ 621#define PROCESS_BACK_OFF_TIME_MS 100 622/* Approx. time before evicting the process again */ 623#define PROCESS_ACTIVE_TIME_MS 10 624 625/* 8 byte handle containing GPU ID in the most significant 4 bytes and 626 * idr_handle in the least significant 4 bytes 627 */ 628#define MAKE_HANDLE(gpu_id, idr_handle) \ 629 (((uint64_t)(gpu_id) << 32) + idr_handle) 630#define GET_GPU_ID(handle) (handle >> 32) 631#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF) 632 633enum kfd_pdd_bound { 634 PDD_UNBOUND = 0, 635 PDD_BOUND, 636 PDD_BOUND_SUSPENDED, 637}; 638 639#define MAX_SYSFS_FILENAME_LEN 15 640 641/* 642 * SDMA counter runs at 100MHz frequency. 643 * We display SDMA activity in microsecond granularity in sysfs. 644 * As a result, the divisor is 100. 645 */ 646#define SDMA_ACTIVITY_DIVISOR 100 647 648/* Data that is per-process-per device. */ 649struct kfd_process_device { 650 /* 651 * List of all per-device data for a process. 652 * Starts from kfd_process.per_device_data. 653 */ 654 struct list_head per_device_list; 655 656 /* The device that owns this data. */ 657 struct kfd_dev *dev; 658 659 /* The process that owns this kfd_process_device. */ 660 struct kfd_process *process; 661 662 /* per-process-per device QCM data structure */ 663 struct qcm_process_device qpd; 664 665 /*Apertures*/ 666 uint64_t lds_base; 667 uint64_t lds_limit; 668 uint64_t gpuvm_base; 669 uint64_t gpuvm_limit; 670 uint64_t scratch_base; 671 uint64_t scratch_limit; 672 673 /* VM context for GPUVM allocations */ 674 struct file *drm_file; 675 void *vm; 676 677 /* GPUVM allocations storage */ 678 struct idr alloc_idr; 679 680 /* Flag used to tell the pdd has dequeued from the dqm. 681 * This is used to prevent dev->dqm->ops.process_termination() from 682 * being called twice when it is already called in IOMMU callback 683 * function. 684 */ 685 bool already_dequeued; 686 bool runtime_inuse; 687 688 /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */ 689 enum kfd_pdd_bound bound; 690 691 /* VRAM usage */ 692 uint64_t vram_usage; 693 struct attribute attr_vram; 694 char vram_filename[MAX_SYSFS_FILENAME_LEN]; 695 696 /* SDMA activity tracking */ 697 uint64_t sdma_past_activity_counter; 698 struct attribute attr_sdma; 699 char sdma_filename[MAX_SYSFS_FILENAME_LEN]; 700 701 /* Eviction activity tracking */ 702 uint64_t last_evict_timestamp; 703 atomic64_t evict_duration_counter; 704 struct attribute attr_evict; 705 706 struct kobject *kobj_stats; 707 unsigned int doorbell_index; 708 709 /* 710 * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process 711 * that is associated with device encoded by "this" struct instance. The 712 * value reflects CU usage by all of the waves launched by this process 713 * on this device. A very important property of occupancy parameter is 714 * that its value is a snapshot of current use. 715 * 716 * Following is to be noted regarding how this parameter is reported: 717 * 718 * The number of waves that a CU can launch is limited by couple of 719 * parameters. These are encoded by struct amdgpu_cu_info instance 720 * that is part of every device definition. For GFX9 devices this 721 * translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves 722 * do not use scratch memory and 32 waves (max_scratch_slots_per_cu) 723 * when they do use scratch memory. This could change for future 724 * devices and therefore this example should be considered as a guide. 725 * 726 * All CU's of a device are available for the process. This may not be true 727 * under certain conditions - e.g. CU masking. 728 * 729 * Finally number of CU's that are occupied by a process is affected by both 730 * number of CU's a device has along with number of other competing processes 731 */ 732 struct attribute attr_cu_occupancy; 733}; 734 735#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd) 736 737/* Process data */ 738struct kfd_process { 739 /* 740 * kfd_process are stored in an mm_struct*->kfd_process* 741 * hash table (kfd_processes in kfd_process.c) 742 */ 743 struct hlist_node kfd_processes; 744 745 /* 746 * Opaque pointer to mm_struct. We don't hold a reference to 747 * it so it should never be dereferenced from here. This is 748 * only used for looking up processes by their mm. 749 */ 750 void *mm; 751 752 struct kref ref; 753 struct work_struct release_work; 754 755 struct mutex mutex; 756 757 /* 758 * In any process, the thread that started main() is the lead 759 * thread and outlives the rest. 760 * It is here because amd_iommu_bind_pasid wants a task_struct. 761 * It can also be used for safely getting a reference to the 762 * mm_struct of the process. 763 */ 764 struct task_struct *lead_thread; 765 766 /* We want to receive a notification when the mm_struct is destroyed */ 767 struct mmu_notifier mmu_notifier; 768 769 u32 pasid; 770 771 /* 772 * List of kfd_process_device structures, 773 * one for each device the process is using. 774 */ 775 struct list_head per_device_data; 776 777 struct process_queue_manager pqm; 778 779 /*Is the user space process 32 bit?*/ 780 bool is_32bit_user_mode; 781 782 /* Event-related data */ 783 struct mutex event_mutex; 784 /* Event ID allocator and lookup */ 785 struct idr event_idr; 786 /* Event page */ 787 struct kfd_signal_page *signal_page; 788 size_t signal_mapped_size; 789 size_t signal_event_count; 790 bool signal_event_limit_reached; 791 792 /* Information used for memory eviction */ 793 void *kgd_process_info; 794 /* Eviction fence that is attached to all the BOs of this process. The 795 * fence will be triggered during eviction and new one will be created 796 * during restore 797 */ 798 struct dma_fence *ef; 799 800 /* Work items for evicting and restoring BOs */ 801 struct delayed_work eviction_work; 802 struct delayed_work restore_work; 803 /* seqno of the last scheduled eviction */ 804 unsigned int last_eviction_seqno; 805 /* Approx. the last timestamp (in jiffies) when the process was 806 * restored after an eviction 807 */ 808 unsigned long last_restore_timestamp; 809 810 /* Kobj for our procfs */ 811 struct kobject *kobj; 812 struct kobject *kobj_queues; 813 struct attribute attr_pasid; 814}; 815 816#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */ 817extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); 818extern struct srcu_struct kfd_processes_srcu; 819 820/** 821 * typedef amdkfd_ioctl_t - typedef for ioctl function pointer. 822 * 823 * @filep: pointer to file structure. 824 * @p: amdkfd process pointer. 825 * @data: pointer to arg that was copied from user. 826 * 827 * Return: returns ioctl completion code. 828 */ 829typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p, 830 void *data); 831 832struct amdkfd_ioctl_desc { 833 unsigned int cmd; 834 int flags; 835 amdkfd_ioctl_t *func; 836 unsigned int cmd_drv; 837 const char *name; 838}; 839bool kfd_dev_is_large_bar(struct kfd_dev *dev); 840 841int kfd_process_create_wq(void); 842void kfd_process_destroy_wq(void); 843struct kfd_process *kfd_create_process(struct file *filep); 844struct kfd_process *kfd_get_process(const struct task_struct *); 845struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid); 846struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm); 847void kfd_unref_process(struct kfd_process *p); 848int kfd_process_evict_queues(struct kfd_process *p); 849int kfd_process_restore_queues(struct kfd_process *p); 850void kfd_suspend_all_processes(void); 851int kfd_resume_all_processes(void); 852 853int kfd_process_device_init_vm(struct kfd_process_device *pdd, 854 struct file *drm_file); 855struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, 856 struct kfd_process *p); 857struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, 858 struct kfd_process *p); 859struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, 860 struct kfd_process *p); 861 862int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process, 863 struct vm_area_struct *vma); 864 865/* KFD process API for creating and translating handles */ 866int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, 867 void *mem); 868void *kfd_process_device_translate_handle(struct kfd_process_device *p, 869 int handle); 870void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, 871 int handle); 872 873/* Process device data iterator */ 874struct kfd_process_device *kfd_get_first_process_device_data( 875 struct kfd_process *p); 876struct kfd_process_device *kfd_get_next_process_device_data( 877 struct kfd_process *p, 878 struct kfd_process_device *pdd); 879bool kfd_has_process_device_data(struct kfd_process *p); 880 881/* PASIDs */ 882int kfd_pasid_init(void); 883void kfd_pasid_exit(void); 884bool kfd_set_pasid_limit(unsigned int new_limit); 885unsigned int kfd_get_pasid_limit(void); 886u32 kfd_pasid_alloc(void); 887void kfd_pasid_free(u32 pasid); 888 889/* Doorbells */ 890size_t kfd_doorbell_process_slice(struct kfd_dev *kfd); 891int kfd_doorbell_init(struct kfd_dev *kfd); 892void kfd_doorbell_fini(struct kfd_dev *kfd); 893int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process, 894 struct vm_area_struct *vma); 895void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, 896 unsigned int *doorbell_off); 897void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr); 898u32 read_kernel_doorbell(u32 __iomem *db); 899void write_kernel_doorbell(void __iomem *db, u32 value); 900void write_kernel_doorbell64(void __iomem *db, u64 value); 901unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd, 902 struct kfd_process_device *pdd, 903 unsigned int doorbell_id); 904phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd); 905int kfd_alloc_process_doorbells(struct kfd_dev *kfd, 906 unsigned int *doorbell_index); 907void kfd_free_process_doorbells(struct kfd_dev *kfd, 908 unsigned int doorbell_index); 909/* GTT Sub-Allocator */ 910 911int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, 912 struct kfd_mem_obj **mem_obj); 913 914int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj); 915 916extern struct device *kfd_device; 917 918/* KFD's procfs */ 919void kfd_procfs_init(void); 920void kfd_procfs_shutdown(void); 921int kfd_procfs_add_queue(struct queue *q); 922void kfd_procfs_del_queue(struct queue *q); 923 924/* Topology */ 925int kfd_topology_init(void); 926void kfd_topology_shutdown(void); 927int kfd_topology_add_device(struct kfd_dev *gpu); 928int kfd_topology_remove_device(struct kfd_dev *gpu); 929struct kfd_topology_device *kfd_topology_device_by_proximity_domain( 930 uint32_t proximity_domain); 931struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id); 932struct kfd_dev *kfd_device_by_id(uint32_t gpu_id); 933struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev); 934struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd); 935int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev); 936int kfd_numa_node_to_apic_id(int numa_node_id); 937void kfd_double_confirm_iommu_support(struct kfd_dev *gpu); 938 939/* Interrupts */ 940int kfd_interrupt_init(struct kfd_dev *dev); 941void kfd_interrupt_exit(struct kfd_dev *dev); 942bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry); 943bool interrupt_is_wanted(struct kfd_dev *dev, 944 const uint32_t *ih_ring_entry, 945 uint32_t *patched_ihre, bool *flag); 946 947/* amdkfd Apertures */ 948int kfd_init_apertures(struct kfd_process *process); 949 950/* Queue Context Management */ 951int init_queue(struct queue **q, const struct queue_properties *properties); 952void uninit_queue(struct queue *q); 953void print_queue_properties(struct queue_properties *q); 954void print_queue(struct queue *q); 955 956struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, 957 struct kfd_dev *dev); 958struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type, 959 struct kfd_dev *dev); 960struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type, 961 struct kfd_dev *dev); 962struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type, 963 struct kfd_dev *dev); 964struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, 965 struct kfd_dev *dev); 966struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type, 967 struct kfd_dev *dev); 968struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev); 969void device_queue_manager_uninit(struct device_queue_manager *dqm); 970struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, 971 enum kfd_queue_type type); 972void kernel_queue_uninit(struct kernel_queue *kq, bool hanging); 973int kfd_process_vm_fault(struct device_queue_manager *dqm, u32 pasid); 974 975/* Process Queue Manager */ 976struct process_queue_node { 977 struct queue *q; 978 struct kernel_queue *kq; 979 struct list_head process_queue_list; 980}; 981 982void kfd_process_dequeue_from_device(struct kfd_process_device *pdd); 983void kfd_process_dequeue_from_all_devices(struct kfd_process *p); 984int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p); 985void pqm_uninit(struct process_queue_manager *pqm); 986int pqm_create_queue(struct process_queue_manager *pqm, 987 struct kfd_dev *dev, 988 struct file *f, 989 struct queue_properties *properties, 990 unsigned int *qid, 991 uint32_t *p_doorbell_offset_in_process); 992int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid); 993int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid, 994 struct queue_properties *p); 995int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid, 996 struct queue_properties *p); 997int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid, 998 void *gws); 999struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm, 1000 unsigned int qid); 1001struct queue *pqm_get_user_queue(struct process_queue_manager *pqm, 1002 unsigned int qid); 1003int pqm_get_wave_state(struct process_queue_manager *pqm, 1004 unsigned int qid, 1005 void __user *ctl_stack, 1006 u32 *ctl_stack_used_size, 1007 u32 *save_area_used_size); 1008 1009int amdkfd_fence_wait_timeout(uint64_t *fence_addr, 1010 uint64_t fence_value, 1011 unsigned int timeout_ms); 1012 1013/* Packet Manager */ 1014 1015#define KFD_FENCE_COMPLETED (100) 1016#define KFD_FENCE_INIT (10) 1017 1018struct packet_manager { 1019 struct device_queue_manager *dqm; 1020 struct kernel_queue *priv_queue; 1021 struct mutex lock; 1022 bool allocated; 1023 struct kfd_mem_obj *ib_buffer_obj; 1024 unsigned int ib_size_bytes; 1025 bool is_over_subscription; 1026 1027 const struct packet_manager_funcs *pmf; 1028}; 1029 1030struct packet_manager_funcs { 1031 /* Support ASIC-specific packet formats for PM4 packets */ 1032 int (*map_process)(struct packet_manager *pm, uint32_t *buffer, 1033 struct qcm_process_device *qpd); 1034 int (*runlist)(struct packet_manager *pm, uint32_t *buffer, 1035 uint64_t ib, size_t ib_size_in_dwords, bool chain); 1036 int (*set_resources)(struct packet_manager *pm, uint32_t *buffer, 1037 struct scheduling_resources *res); 1038 int (*map_queues)(struct packet_manager *pm, uint32_t *buffer, 1039 struct queue *q, bool is_static); 1040 int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer, 1041 enum kfd_queue_type type, 1042 enum kfd_unmap_queues_filter mode, 1043 uint32_t filter_param, bool reset, 1044 unsigned int sdma_engine); 1045 int (*query_status)(struct packet_manager *pm, uint32_t *buffer, 1046 uint64_t fence_address, uint64_t fence_value); 1047 int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer); 1048 1049 /* Packet sizes */ 1050 int map_process_size; 1051 int runlist_size; 1052 int set_resources_size; 1053 int map_queues_size; 1054 int unmap_queues_size; 1055 int query_status_size; 1056 int release_mem_size; 1057}; 1058 1059extern const struct packet_manager_funcs kfd_vi_pm_funcs; 1060extern const struct packet_manager_funcs kfd_v9_pm_funcs; 1061 1062int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm); 1063void pm_uninit(struct packet_manager *pm, bool hanging); 1064int pm_send_set_resources(struct packet_manager *pm, 1065 struct scheduling_resources *res); 1066int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues); 1067int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, 1068 uint64_t fence_value); 1069 1070int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, 1071 enum kfd_unmap_queues_filter mode, 1072 uint32_t filter_param, bool reset, 1073 unsigned int sdma_engine); 1074 1075void pm_release_ib(struct packet_manager *pm); 1076 1077/* Following PM funcs can be shared among VI and AI */ 1078unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size); 1079 1080uint64_t kfd_get_number_elems(struct kfd_dev *kfd); 1081 1082/* Events */ 1083extern const struct kfd_event_interrupt_class event_interrupt_class_cik; 1084extern const struct kfd_event_interrupt_class event_interrupt_class_v9; 1085 1086extern const struct kfd_device_global_init_class device_global_init_class_cik; 1087 1088void kfd_event_init_process(struct kfd_process *p); 1089void kfd_event_free_process(struct kfd_process *p); 1090int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma); 1091int kfd_wait_on_events(struct kfd_process *p, 1092 uint32_t num_events, void __user *data, 1093 bool all, uint32_t user_timeout_ms, 1094 uint32_t *wait_result); 1095void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id, 1096 uint32_t valid_id_bits); 1097void kfd_signal_iommu_event(struct kfd_dev *dev, 1098 u32 pasid, unsigned long address, 1099 bool is_write_requested, bool is_execute_requested); 1100void kfd_signal_hw_exception_event(u32 pasid); 1101int kfd_set_event(struct kfd_process *p, uint32_t event_id); 1102int kfd_reset_event(struct kfd_process *p, uint32_t event_id); 1103int kfd_event_page_set(struct kfd_process *p, void *kernel_address, 1104 uint64_t size); 1105int kfd_event_create(struct file *devkfd, struct kfd_process *p, 1106 uint32_t event_type, bool auto_reset, uint32_t node_id, 1107 uint32_t *event_id, uint32_t *event_trigger_data, 1108 uint64_t *event_page_offset, uint32_t *event_slot_index); 1109int kfd_event_destroy(struct kfd_process *p, uint32_t event_id); 1110 1111void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid, 1112 struct kfd_vm_fault_info *info); 1113 1114void kfd_signal_reset_event(struct kfd_dev *dev); 1115 1116void kfd_flush_tlb(struct kfd_process_device *pdd); 1117 1118int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p); 1119 1120bool kfd_is_locked(void); 1121 1122/* Compute profile */ 1123void kfd_inc_compute_active(struct kfd_dev *dev); 1124void kfd_dec_compute_active(struct kfd_dev *dev); 1125 1126/* Cgroup Support */ 1127/* Check with device cgroup if @kfd device is accessible */ 1128static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd) 1129{ 1130#if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF) 1131 struct drm_device *ddev = kfd->ddev; 1132 1133 return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR, 1134 ddev->render->index, 1135 DEVCG_ACC_WRITE | DEVCG_ACC_READ); 1136#else 1137 return 0; 1138#endif 1139} 1140 1141/* Debugfs */ 1142#if defined(CONFIG_DEBUG_FS) 1143 1144void kfd_debugfs_init(void); 1145void kfd_debugfs_fini(void); 1146int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data); 1147int pqm_debugfs_mqds(struct seq_file *m, void *data); 1148int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data); 1149int dqm_debugfs_hqds(struct seq_file *m, void *data); 1150int kfd_debugfs_rls_by_device(struct seq_file *m, void *data); 1151int pm_debugfs_runlist(struct seq_file *m, void *data); 1152 1153int kfd_debugfs_hang_hws(struct kfd_dev *dev); 1154int pm_debugfs_hang_hws(struct packet_manager *pm); 1155int dqm_debugfs_execute_queues(struct device_queue_manager *dqm); 1156 1157#else 1158 1159static inline void kfd_debugfs_init(void) {} 1160static inline void kfd_debugfs_fini(void) {} 1161 1162#endif 1163 1164#endif 1165