1// SPDX-License-Identifier: GPL-2.0 OR MIT 2/* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24#include <linux/types.h> 25#include <linux/kernel.h> 26#include <linux/pci.h> 27#include <linux/errno.h> 28#include <linux/acpi.h> 29#include <linux/hash.h> 30#include <linux/cpufreq.h> 31#include <linux/log2.h> 32#include <linux/dmi.h> 33#include <linux/atomic.h> 34 35#include "kfd_priv.h" 36#include "kfd_crat.h" 37#include "kfd_topology.h" 38#include "kfd_device_queue_manager.h" 39#include "kfd_svm.h" 40#include "kfd_debug.h" 41#include "amdgpu_amdkfd.h" 42#include "amdgpu_ras.h" 43#include "amdgpu.h" 44 45/* topology_device_list - Master list of all topology devices */ 46static struct list_head topology_device_list; 47static struct kfd_system_properties sys_props; 48 49static DECLARE_RWSEM(topology_lock); 50static uint32_t topology_crat_proximity_domain; 51 52struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock( 53 uint32_t proximity_domain) 54{ 55 struct kfd_topology_device *top_dev; 56 struct kfd_topology_device *device = NULL; 57 58 list_for_each_entry(top_dev, &topology_device_list, list) 59 if (top_dev->proximity_domain == proximity_domain) { 60 device = top_dev; 61 break; 62 } 63 64 return device; 65} 66 67struct kfd_topology_device *kfd_topology_device_by_proximity_domain( 68 uint32_t proximity_domain) 69{ 70 struct kfd_topology_device *device = NULL; 71 72 down_read(&topology_lock); 73 74 device = kfd_topology_device_by_proximity_domain_no_lock( 75 proximity_domain); 76 up_read(&topology_lock); 77 78 return device; 79} 80 81struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id) 82{ 83 struct kfd_topology_device *top_dev = NULL; 84 struct kfd_topology_device *ret = NULL; 85 86 down_read(&topology_lock); 87 88 list_for_each_entry(top_dev, &topology_device_list, list) 89 if (top_dev->gpu_id == gpu_id) { 90 ret = top_dev; 91 break; 92 } 93 94 up_read(&topology_lock); 95 96 return ret; 97} 98 99struct kfd_node *kfd_device_by_id(uint32_t gpu_id) 100{ 101 struct kfd_topology_device *top_dev; 102 103 top_dev = kfd_topology_device_by_id(gpu_id); 104 if (!top_dev) 105 return NULL; 106 107 return top_dev->gpu; 108} 109 110struct kfd_node *kfd_device_by_pci_dev(const struct pci_dev *pdev) 111{ 112 struct kfd_topology_device *top_dev; 113 struct kfd_node *device = NULL; 114 115 down_read(&topology_lock); 116 117 list_for_each_entry(top_dev, &topology_device_list, list) 118 if (top_dev->gpu && top_dev->gpu->adev->pdev == pdev) { 119 device = top_dev->gpu; 120 break; 121 } 122 123 up_read(&topology_lock); 124 125 return device; 126} 127 128/* Called with write topology_lock acquired */ 129static void kfd_release_topology_device(struct kfd_topology_device *dev) 130{ 131 struct kfd_mem_properties *mem; 132 struct kfd_cache_properties *cache; 133 struct kfd_iolink_properties *iolink; 134 struct kfd_iolink_properties *p2plink; 135 struct kfd_perf_properties *perf; 136 137 list_del(&dev->list); 138 139 while (dev->mem_props.next != &dev->mem_props) { 140 mem = container_of(dev->mem_props.next, 141 struct kfd_mem_properties, list); 142 list_del(&mem->list); 143 kfree(mem); 144 } 145 146 while (dev->cache_props.next != &dev->cache_props) { 147 cache = container_of(dev->cache_props.next, 148 struct kfd_cache_properties, list); 149 list_del(&cache->list); 150 kfree(cache); 151 } 152 153 while (dev->io_link_props.next != &dev->io_link_props) { 154 iolink = container_of(dev->io_link_props.next, 155 struct kfd_iolink_properties, list); 156 list_del(&iolink->list); 157 kfree(iolink); 158 } 159 160 while (dev->p2p_link_props.next != &dev->p2p_link_props) { 161 p2plink = container_of(dev->p2p_link_props.next, 162 struct kfd_iolink_properties, list); 163 list_del(&p2plink->list); 164 kfree(p2plink); 165 } 166 167 while (dev->perf_props.next != &dev->perf_props) { 168 perf = container_of(dev->perf_props.next, 169 struct kfd_perf_properties, list); 170 list_del(&perf->list); 171 kfree(perf); 172 } 173 174 kfree(dev); 175} 176 177void kfd_release_topology_device_list(struct list_head *device_list) 178{ 179 struct kfd_topology_device *dev; 180 181 while (!list_empty(device_list)) { 182 dev = list_first_entry(device_list, 183 struct kfd_topology_device, list); 184 kfd_release_topology_device(dev); 185 } 186} 187 188static void kfd_release_live_view(void) 189{ 190 kfd_release_topology_device_list(&topology_device_list); 191 memset(&sys_props, 0, sizeof(sys_props)); 192} 193 194struct kfd_topology_device *kfd_create_topology_device( 195 struct list_head *device_list) 196{ 197 struct kfd_topology_device *dev; 198 199 dev = kfd_alloc_struct(dev); 200 if (!dev) { 201 pr_err("No memory to allocate a topology device"); 202 return NULL; 203 } 204 205 INIT_LIST_HEAD(&dev->mem_props); 206 INIT_LIST_HEAD(&dev->cache_props); 207 INIT_LIST_HEAD(&dev->io_link_props); 208 INIT_LIST_HEAD(&dev->p2p_link_props); 209 INIT_LIST_HEAD(&dev->perf_props); 210 211 list_add_tail(&dev->list, device_list); 212 213 return dev; 214} 215 216 217#define sysfs_show_gen_prop(buffer, offs, fmt, ...) \ 218 (offs += snprintf(buffer+offs, PAGE_SIZE-offs, \ 219 fmt, __VA_ARGS__)) 220#define sysfs_show_32bit_prop(buffer, offs, name, value) \ 221 sysfs_show_gen_prop(buffer, offs, "%s %u\n", name, value) 222#define sysfs_show_64bit_prop(buffer, offs, name, value) \ 223 sysfs_show_gen_prop(buffer, offs, "%s %llu\n", name, value) 224#define sysfs_show_32bit_val(buffer, offs, value) \ 225 sysfs_show_gen_prop(buffer, offs, "%u\n", value) 226#define sysfs_show_str_val(buffer, offs, value) \ 227 sysfs_show_gen_prop(buffer, offs, "%s\n", value) 228 229static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr, 230 char *buffer) 231{ 232 int offs = 0; 233 234 /* Making sure that the buffer is an empty string */ 235 buffer[0] = 0; 236 237 if (attr == &sys_props.attr_genid) { 238 sysfs_show_32bit_val(buffer, offs, 239 sys_props.generation_count); 240 } else if (attr == &sys_props.attr_props) { 241 sysfs_show_64bit_prop(buffer, offs, "platform_oem", 242 sys_props.platform_oem); 243 sysfs_show_64bit_prop(buffer, offs, "platform_id", 244 sys_props.platform_id); 245 sysfs_show_64bit_prop(buffer, offs, "platform_rev", 246 sys_props.platform_rev); 247 } else { 248 offs = -EINVAL; 249 } 250 251 return offs; 252} 253 254static void kfd_topology_kobj_release(struct kobject *kobj) 255{ 256 kfree(kobj); 257} 258 259static const struct sysfs_ops sysprops_ops = { 260 .show = sysprops_show, 261}; 262 263static const struct kobj_type sysprops_type = { 264 .release = kfd_topology_kobj_release, 265 .sysfs_ops = &sysprops_ops, 266}; 267 268static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr, 269 char *buffer) 270{ 271 int offs = 0; 272 struct kfd_iolink_properties *iolink; 273 274 /* Making sure that the buffer is an empty string */ 275 buffer[0] = 0; 276 277 iolink = container_of(attr, struct kfd_iolink_properties, attr); 278 if (iolink->gpu && kfd_devcgroup_check_permission(iolink->gpu)) 279 return -EPERM; 280 sysfs_show_32bit_prop(buffer, offs, "type", iolink->iolink_type); 281 sysfs_show_32bit_prop(buffer, offs, "version_major", iolink->ver_maj); 282 sysfs_show_32bit_prop(buffer, offs, "version_minor", iolink->ver_min); 283 sysfs_show_32bit_prop(buffer, offs, "node_from", iolink->node_from); 284 sysfs_show_32bit_prop(buffer, offs, "node_to", iolink->node_to); 285 sysfs_show_32bit_prop(buffer, offs, "weight", iolink->weight); 286 sysfs_show_32bit_prop(buffer, offs, "min_latency", iolink->min_latency); 287 sysfs_show_32bit_prop(buffer, offs, "max_latency", iolink->max_latency); 288 sysfs_show_32bit_prop(buffer, offs, "min_bandwidth", 289 iolink->min_bandwidth); 290 sysfs_show_32bit_prop(buffer, offs, "max_bandwidth", 291 iolink->max_bandwidth); 292 sysfs_show_32bit_prop(buffer, offs, "recommended_transfer_size", 293 iolink->rec_transfer_size); 294 sysfs_show_32bit_prop(buffer, offs, "flags", iolink->flags); 295 296 return offs; 297} 298 299static const struct sysfs_ops iolink_ops = { 300 .show = iolink_show, 301}; 302 303static const struct kobj_type iolink_type = { 304 .release = kfd_topology_kobj_release, 305 .sysfs_ops = &iolink_ops, 306}; 307 308static ssize_t mem_show(struct kobject *kobj, struct attribute *attr, 309 char *buffer) 310{ 311 int offs = 0; 312 struct kfd_mem_properties *mem; 313 314 /* Making sure that the buffer is an empty string */ 315 buffer[0] = 0; 316 317 mem = container_of(attr, struct kfd_mem_properties, attr); 318 if (mem->gpu && kfd_devcgroup_check_permission(mem->gpu)) 319 return -EPERM; 320 sysfs_show_32bit_prop(buffer, offs, "heap_type", mem->heap_type); 321 sysfs_show_64bit_prop(buffer, offs, "size_in_bytes", 322 mem->size_in_bytes); 323 sysfs_show_32bit_prop(buffer, offs, "flags", mem->flags); 324 sysfs_show_32bit_prop(buffer, offs, "width", mem->width); 325 sysfs_show_32bit_prop(buffer, offs, "mem_clk_max", 326 mem->mem_clk_max); 327 328 return offs; 329} 330 331static const struct sysfs_ops mem_ops = { 332 .show = mem_show, 333}; 334 335static const struct kobj_type mem_type = { 336 .release = kfd_topology_kobj_release, 337 .sysfs_ops = &mem_ops, 338}; 339 340static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr, 341 char *buffer) 342{ 343 int offs = 0; 344 uint32_t i, j; 345 struct kfd_cache_properties *cache; 346 347 /* Making sure that the buffer is an empty string */ 348 buffer[0] = 0; 349 cache = container_of(attr, struct kfd_cache_properties, attr); 350 if (cache->gpu && kfd_devcgroup_check_permission(cache->gpu)) 351 return -EPERM; 352 sysfs_show_32bit_prop(buffer, offs, "processor_id_low", 353 cache->processor_id_low); 354 sysfs_show_32bit_prop(buffer, offs, "level", cache->cache_level); 355 sysfs_show_32bit_prop(buffer, offs, "size", cache->cache_size); 356 sysfs_show_32bit_prop(buffer, offs, "cache_line_size", 357 cache->cacheline_size); 358 sysfs_show_32bit_prop(buffer, offs, "cache_lines_per_tag", 359 cache->cachelines_per_tag); 360 sysfs_show_32bit_prop(buffer, offs, "association", cache->cache_assoc); 361 sysfs_show_32bit_prop(buffer, offs, "latency", cache->cache_latency); 362 sysfs_show_32bit_prop(buffer, offs, "type", cache->cache_type); 363 364 offs += snprintf(buffer+offs, PAGE_SIZE-offs, "sibling_map "); 365 for (i = 0; i < cache->sibling_map_size; i++) 366 for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++) 367 /* Check each bit */ 368 offs += snprintf(buffer+offs, PAGE_SIZE-offs, "%d,", 369 (cache->sibling_map[i] >> j) & 1); 370 371 /* Replace the last "," with end of line */ 372 buffer[offs-1] = '\n'; 373 return offs; 374} 375 376static const struct sysfs_ops cache_ops = { 377 .show = kfd_cache_show, 378}; 379 380static const struct kobj_type cache_type = { 381 .release = kfd_topology_kobj_release, 382 .sysfs_ops = &cache_ops, 383}; 384 385/****** Sysfs of Performance Counters ******/ 386 387struct kfd_perf_attr { 388 struct kobj_attribute attr; 389 uint32_t data; 390}; 391 392static ssize_t perf_show(struct kobject *kobj, struct kobj_attribute *attrs, 393 char *buf) 394{ 395 int offs = 0; 396 struct kfd_perf_attr *attr; 397 398 buf[0] = 0; 399 attr = container_of(attrs, struct kfd_perf_attr, attr); 400 if (!attr->data) /* invalid data for PMC */ 401 return 0; 402 else 403 return sysfs_show_32bit_val(buf, offs, attr->data); 404} 405 406#define KFD_PERF_DESC(_name, _data) \ 407{ \ 408 .attr = __ATTR(_name, 0444, perf_show, NULL), \ 409 .data = _data, \ 410} 411 412static struct kfd_perf_attr perf_attr_iommu[] = { 413 KFD_PERF_DESC(max_concurrent, 0), 414 KFD_PERF_DESC(num_counters, 0), 415 KFD_PERF_DESC(counter_ids, 0), 416}; 417/****************************************/ 418 419static ssize_t node_show(struct kobject *kobj, struct attribute *attr, 420 char *buffer) 421{ 422 int offs = 0; 423 struct kfd_topology_device *dev; 424 uint32_t log_max_watch_addr; 425 426 /* Making sure that the buffer is an empty string */ 427 buffer[0] = 0; 428 429 if (strcmp(attr->name, "gpu_id") == 0) { 430 dev = container_of(attr, struct kfd_topology_device, 431 attr_gpuid); 432 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 433 return -EPERM; 434 return sysfs_show_32bit_val(buffer, offs, dev->gpu_id); 435 } 436 437 if (strcmp(attr->name, "name") == 0) { 438 dev = container_of(attr, struct kfd_topology_device, 439 attr_name); 440 441 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 442 return -EPERM; 443 return sysfs_show_str_val(buffer, offs, dev->node_props.name); 444 } 445 446 dev = container_of(attr, struct kfd_topology_device, 447 attr_props); 448 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 449 return -EPERM; 450 sysfs_show_32bit_prop(buffer, offs, "cpu_cores_count", 451 dev->node_props.cpu_cores_count); 452 sysfs_show_32bit_prop(buffer, offs, "simd_count", 453 dev->gpu ? dev->node_props.simd_count : 0); 454 sysfs_show_32bit_prop(buffer, offs, "mem_banks_count", 455 dev->node_props.mem_banks_count); 456 sysfs_show_32bit_prop(buffer, offs, "caches_count", 457 dev->node_props.caches_count); 458 sysfs_show_32bit_prop(buffer, offs, "io_links_count", 459 dev->node_props.io_links_count); 460 sysfs_show_32bit_prop(buffer, offs, "p2p_links_count", 461 dev->node_props.p2p_links_count); 462 sysfs_show_32bit_prop(buffer, offs, "cpu_core_id_base", 463 dev->node_props.cpu_core_id_base); 464 sysfs_show_32bit_prop(buffer, offs, "simd_id_base", 465 dev->node_props.simd_id_base); 466 sysfs_show_32bit_prop(buffer, offs, "max_waves_per_simd", 467 dev->node_props.max_waves_per_simd); 468 sysfs_show_32bit_prop(buffer, offs, "lds_size_in_kb", 469 dev->node_props.lds_size_in_kb); 470 sysfs_show_32bit_prop(buffer, offs, "gds_size_in_kb", 471 dev->node_props.gds_size_in_kb); 472 sysfs_show_32bit_prop(buffer, offs, "num_gws", 473 dev->node_props.num_gws); 474 sysfs_show_32bit_prop(buffer, offs, "wave_front_size", 475 dev->node_props.wave_front_size); 476 sysfs_show_32bit_prop(buffer, offs, "array_count", 477 dev->gpu ? (dev->node_props.array_count * 478 NUM_XCC(dev->gpu->xcc_mask)) : 0); 479 sysfs_show_32bit_prop(buffer, offs, "simd_arrays_per_engine", 480 dev->node_props.simd_arrays_per_engine); 481 sysfs_show_32bit_prop(buffer, offs, "cu_per_simd_array", 482 dev->node_props.cu_per_simd_array); 483 sysfs_show_32bit_prop(buffer, offs, "simd_per_cu", 484 dev->node_props.simd_per_cu); 485 sysfs_show_32bit_prop(buffer, offs, "max_slots_scratch_cu", 486 dev->node_props.max_slots_scratch_cu); 487 sysfs_show_32bit_prop(buffer, offs, "gfx_target_version", 488 dev->node_props.gfx_target_version); 489 sysfs_show_32bit_prop(buffer, offs, "vendor_id", 490 dev->node_props.vendor_id); 491 sysfs_show_32bit_prop(buffer, offs, "device_id", 492 dev->node_props.device_id); 493 sysfs_show_32bit_prop(buffer, offs, "location_id", 494 dev->node_props.location_id); 495 sysfs_show_32bit_prop(buffer, offs, "domain", 496 dev->node_props.domain); 497 sysfs_show_32bit_prop(buffer, offs, "drm_render_minor", 498 dev->node_props.drm_render_minor); 499 sysfs_show_64bit_prop(buffer, offs, "hive_id", 500 dev->node_props.hive_id); 501 sysfs_show_32bit_prop(buffer, offs, "num_sdma_engines", 502 dev->node_props.num_sdma_engines); 503 sysfs_show_32bit_prop(buffer, offs, "num_sdma_xgmi_engines", 504 dev->node_props.num_sdma_xgmi_engines); 505 sysfs_show_32bit_prop(buffer, offs, "num_sdma_queues_per_engine", 506 dev->node_props.num_sdma_queues_per_engine); 507 sysfs_show_32bit_prop(buffer, offs, "num_cp_queues", 508 dev->node_props.num_cp_queues); 509 510 if (dev->gpu) { 511 log_max_watch_addr = 512 __ilog2_u32(dev->gpu->kfd->device_info.num_of_watch_points); 513 514 if (log_max_watch_addr) { 515 dev->node_props.capability |= 516 HSA_CAP_WATCH_POINTS_SUPPORTED; 517 518 dev->node_props.capability |= 519 ((log_max_watch_addr << 520 HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT) & 521 HSA_CAP_WATCH_POINTS_TOTALBITS_MASK); 522 } 523 524 if (dev->gpu->adev->asic_type == CHIP_TONGA) 525 dev->node_props.capability |= 526 HSA_CAP_AQL_QUEUE_DOUBLE_MAP; 527 528 sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_fcompute", 529 dev->node_props.max_engine_clk_fcompute); 530 531 sysfs_show_64bit_prop(buffer, offs, "local_mem_size", 0ULL); 532 533 sysfs_show_32bit_prop(buffer, offs, "fw_version", 534 dev->gpu->kfd->mec_fw_version); 535 sysfs_show_32bit_prop(buffer, offs, "capability", 536 dev->node_props.capability); 537 sysfs_show_64bit_prop(buffer, offs, "debug_prop", 538 dev->node_props.debug_prop); 539 sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version", 540 dev->gpu->kfd->sdma_fw_version); 541 sysfs_show_64bit_prop(buffer, offs, "unique_id", 542 dev->gpu->adev->unique_id); 543 sysfs_show_32bit_prop(buffer, offs, "num_xcc", 544 NUM_XCC(dev->gpu->xcc_mask)); 545 } 546 547 return sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_ccompute", 548 cpufreq_quick_get_max(0)/1000); 549} 550 551static const struct sysfs_ops node_ops = { 552 .show = node_show, 553}; 554 555static const struct kobj_type node_type = { 556 .release = kfd_topology_kobj_release, 557 .sysfs_ops = &node_ops, 558}; 559 560static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr) 561{ 562 sysfs_remove_file(kobj, attr); 563 kobject_del(kobj); 564 kobject_put(kobj); 565} 566 567static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev) 568{ 569 struct kfd_iolink_properties *p2plink; 570 struct kfd_iolink_properties *iolink; 571 struct kfd_cache_properties *cache; 572 struct kfd_mem_properties *mem; 573 struct kfd_perf_properties *perf; 574 575 if (dev->kobj_iolink) { 576 list_for_each_entry(iolink, &dev->io_link_props, list) 577 if (iolink->kobj) { 578 kfd_remove_sysfs_file(iolink->kobj, 579 &iolink->attr); 580 iolink->kobj = NULL; 581 } 582 kobject_del(dev->kobj_iolink); 583 kobject_put(dev->kobj_iolink); 584 dev->kobj_iolink = NULL; 585 } 586 587 if (dev->kobj_p2plink) { 588 list_for_each_entry(p2plink, &dev->p2p_link_props, list) 589 if (p2plink->kobj) { 590 kfd_remove_sysfs_file(p2plink->kobj, 591 &p2plink->attr); 592 p2plink->kobj = NULL; 593 } 594 kobject_del(dev->kobj_p2plink); 595 kobject_put(dev->kobj_p2plink); 596 dev->kobj_p2plink = NULL; 597 } 598 599 if (dev->kobj_cache) { 600 list_for_each_entry(cache, &dev->cache_props, list) 601 if (cache->kobj) { 602 kfd_remove_sysfs_file(cache->kobj, 603 &cache->attr); 604 cache->kobj = NULL; 605 } 606 kobject_del(dev->kobj_cache); 607 kobject_put(dev->kobj_cache); 608 dev->kobj_cache = NULL; 609 } 610 611 if (dev->kobj_mem) { 612 list_for_each_entry(mem, &dev->mem_props, list) 613 if (mem->kobj) { 614 kfd_remove_sysfs_file(mem->kobj, &mem->attr); 615 mem->kobj = NULL; 616 } 617 kobject_del(dev->kobj_mem); 618 kobject_put(dev->kobj_mem); 619 dev->kobj_mem = NULL; 620 } 621 622 if (dev->kobj_perf) { 623 list_for_each_entry(perf, &dev->perf_props, list) { 624 kfree(perf->attr_group); 625 perf->attr_group = NULL; 626 } 627 kobject_del(dev->kobj_perf); 628 kobject_put(dev->kobj_perf); 629 dev->kobj_perf = NULL; 630 } 631 632 if (dev->kobj_node) { 633 sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid); 634 sysfs_remove_file(dev->kobj_node, &dev->attr_name); 635 sysfs_remove_file(dev->kobj_node, &dev->attr_props); 636 kobject_del(dev->kobj_node); 637 kobject_put(dev->kobj_node); 638 dev->kobj_node = NULL; 639 } 640} 641 642static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev, 643 uint32_t id) 644{ 645 struct kfd_iolink_properties *p2plink; 646 struct kfd_iolink_properties *iolink; 647 struct kfd_cache_properties *cache; 648 struct kfd_mem_properties *mem; 649 struct kfd_perf_properties *perf; 650 int ret; 651 uint32_t i, num_attrs; 652 struct attribute **attrs; 653 654 if (WARN_ON(dev->kobj_node)) 655 return -EEXIST; 656 657 /* 658 * Creating the sysfs folders 659 */ 660 dev->kobj_node = kfd_alloc_struct(dev->kobj_node); 661 if (!dev->kobj_node) 662 return -ENOMEM; 663 664 ret = kobject_init_and_add(dev->kobj_node, &node_type, 665 sys_props.kobj_nodes, "%d", id); 666 if (ret < 0) { 667 kobject_put(dev->kobj_node); 668 return ret; 669 } 670 671 dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node); 672 if (!dev->kobj_mem) 673 return -ENOMEM; 674 675 dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node); 676 if (!dev->kobj_cache) 677 return -ENOMEM; 678 679 dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node); 680 if (!dev->kobj_iolink) 681 return -ENOMEM; 682 683 dev->kobj_p2plink = kobject_create_and_add("p2p_links", dev->kobj_node); 684 if (!dev->kobj_p2plink) 685 return -ENOMEM; 686 687 dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node); 688 if (!dev->kobj_perf) 689 return -ENOMEM; 690 691 /* 692 * Creating sysfs files for node properties 693 */ 694 dev->attr_gpuid.name = "gpu_id"; 695 dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE; 696 sysfs_attr_init(&dev->attr_gpuid); 697 dev->attr_name.name = "name"; 698 dev->attr_name.mode = KFD_SYSFS_FILE_MODE; 699 sysfs_attr_init(&dev->attr_name); 700 dev->attr_props.name = "properties"; 701 dev->attr_props.mode = KFD_SYSFS_FILE_MODE; 702 sysfs_attr_init(&dev->attr_props); 703 ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid); 704 if (ret < 0) 705 return ret; 706 ret = sysfs_create_file(dev->kobj_node, &dev->attr_name); 707 if (ret < 0) 708 return ret; 709 ret = sysfs_create_file(dev->kobj_node, &dev->attr_props); 710 if (ret < 0) 711 return ret; 712 713 i = 0; 714 list_for_each_entry(mem, &dev->mem_props, list) { 715 mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 716 if (!mem->kobj) 717 return -ENOMEM; 718 ret = kobject_init_and_add(mem->kobj, &mem_type, 719 dev->kobj_mem, "%d", i); 720 if (ret < 0) { 721 kobject_put(mem->kobj); 722 return ret; 723 } 724 725 mem->attr.name = "properties"; 726 mem->attr.mode = KFD_SYSFS_FILE_MODE; 727 sysfs_attr_init(&mem->attr); 728 ret = sysfs_create_file(mem->kobj, &mem->attr); 729 if (ret < 0) 730 return ret; 731 i++; 732 } 733 734 i = 0; 735 list_for_each_entry(cache, &dev->cache_props, list) { 736 cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 737 if (!cache->kobj) 738 return -ENOMEM; 739 ret = kobject_init_and_add(cache->kobj, &cache_type, 740 dev->kobj_cache, "%d", i); 741 if (ret < 0) { 742 kobject_put(cache->kobj); 743 return ret; 744 } 745 746 cache->attr.name = "properties"; 747 cache->attr.mode = KFD_SYSFS_FILE_MODE; 748 sysfs_attr_init(&cache->attr); 749 ret = sysfs_create_file(cache->kobj, &cache->attr); 750 if (ret < 0) 751 return ret; 752 i++; 753 } 754 755 i = 0; 756 list_for_each_entry(iolink, &dev->io_link_props, list) { 757 iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 758 if (!iolink->kobj) 759 return -ENOMEM; 760 ret = kobject_init_and_add(iolink->kobj, &iolink_type, 761 dev->kobj_iolink, "%d", i); 762 if (ret < 0) { 763 kobject_put(iolink->kobj); 764 return ret; 765 } 766 767 iolink->attr.name = "properties"; 768 iolink->attr.mode = KFD_SYSFS_FILE_MODE; 769 sysfs_attr_init(&iolink->attr); 770 ret = sysfs_create_file(iolink->kobj, &iolink->attr); 771 if (ret < 0) 772 return ret; 773 i++; 774 } 775 776 i = 0; 777 list_for_each_entry(p2plink, &dev->p2p_link_props, list) { 778 p2plink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 779 if (!p2plink->kobj) 780 return -ENOMEM; 781 ret = kobject_init_and_add(p2plink->kobj, &iolink_type, 782 dev->kobj_p2plink, "%d", i); 783 if (ret < 0) { 784 kobject_put(p2plink->kobj); 785 return ret; 786 } 787 788 p2plink->attr.name = "properties"; 789 p2plink->attr.mode = KFD_SYSFS_FILE_MODE; 790 sysfs_attr_init(&p2plink->attr); 791 ret = sysfs_create_file(p2plink->kobj, &p2plink->attr); 792 if (ret < 0) 793 return ret; 794 i++; 795 } 796 797 /* All hardware blocks have the same number of attributes. */ 798 num_attrs = ARRAY_SIZE(perf_attr_iommu); 799 list_for_each_entry(perf, &dev->perf_props, list) { 800 perf->attr_group = kzalloc(sizeof(struct kfd_perf_attr) 801 * num_attrs + sizeof(struct attribute_group), 802 GFP_KERNEL); 803 if (!perf->attr_group) 804 return -ENOMEM; 805 806 attrs = (struct attribute **)(perf->attr_group + 1); 807 if (!strcmp(perf->block_name, "iommu")) { 808 /* Information of IOMMU's num_counters and counter_ids is shown 809 * under /sys/bus/event_source/devices/amd_iommu. We don't 810 * duplicate here. 811 */ 812 perf_attr_iommu[0].data = perf->max_concurrent; 813 for (i = 0; i < num_attrs; i++) 814 attrs[i] = &perf_attr_iommu[i].attr.attr; 815 } 816 perf->attr_group->name = perf->block_name; 817 perf->attr_group->attrs = attrs; 818 ret = sysfs_create_group(dev->kobj_perf, perf->attr_group); 819 if (ret < 0) 820 return ret; 821 } 822 823 return 0; 824} 825 826/* Called with write topology lock acquired */ 827static int kfd_build_sysfs_node_tree(void) 828{ 829 struct kfd_topology_device *dev; 830 int ret; 831 uint32_t i = 0; 832 833 list_for_each_entry(dev, &topology_device_list, list) { 834 ret = kfd_build_sysfs_node_entry(dev, i); 835 if (ret < 0) 836 return ret; 837 i++; 838 } 839 840 return 0; 841} 842 843/* Called with write topology lock acquired */ 844static void kfd_remove_sysfs_node_tree(void) 845{ 846 struct kfd_topology_device *dev; 847 848 list_for_each_entry(dev, &topology_device_list, list) 849 kfd_remove_sysfs_node_entry(dev); 850} 851 852static int kfd_topology_update_sysfs(void) 853{ 854 int ret; 855 856 if (!sys_props.kobj_topology) { 857 sys_props.kobj_topology = 858 kfd_alloc_struct(sys_props.kobj_topology); 859 if (!sys_props.kobj_topology) 860 return -ENOMEM; 861 862 ret = kobject_init_and_add(sys_props.kobj_topology, 863 &sysprops_type, &kfd_device->kobj, 864 "topology"); 865 if (ret < 0) { 866 kobject_put(sys_props.kobj_topology); 867 return ret; 868 } 869 870 sys_props.kobj_nodes = kobject_create_and_add("nodes", 871 sys_props.kobj_topology); 872 if (!sys_props.kobj_nodes) 873 return -ENOMEM; 874 875 sys_props.attr_genid.name = "generation_id"; 876 sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE; 877 sysfs_attr_init(&sys_props.attr_genid); 878 ret = sysfs_create_file(sys_props.kobj_topology, 879 &sys_props.attr_genid); 880 if (ret < 0) 881 return ret; 882 883 sys_props.attr_props.name = "system_properties"; 884 sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE; 885 sysfs_attr_init(&sys_props.attr_props); 886 ret = sysfs_create_file(sys_props.kobj_topology, 887 &sys_props.attr_props); 888 if (ret < 0) 889 return ret; 890 } 891 892 kfd_remove_sysfs_node_tree(); 893 894 return kfd_build_sysfs_node_tree(); 895} 896 897static void kfd_topology_release_sysfs(void) 898{ 899 kfd_remove_sysfs_node_tree(); 900 if (sys_props.kobj_topology) { 901 sysfs_remove_file(sys_props.kobj_topology, 902 &sys_props.attr_genid); 903 sysfs_remove_file(sys_props.kobj_topology, 904 &sys_props.attr_props); 905 if (sys_props.kobj_nodes) { 906 kobject_del(sys_props.kobj_nodes); 907 kobject_put(sys_props.kobj_nodes); 908 sys_props.kobj_nodes = NULL; 909 } 910 kobject_del(sys_props.kobj_topology); 911 kobject_put(sys_props.kobj_topology); 912 sys_props.kobj_topology = NULL; 913 } 914} 915 916/* Called with write topology_lock acquired */ 917static void kfd_topology_update_device_list(struct list_head *temp_list, 918 struct list_head *master_list) 919{ 920 while (!list_empty(temp_list)) { 921 list_move_tail(temp_list->next, master_list); 922 sys_props.num_devices++; 923 } 924} 925 926static void kfd_debug_print_topology(void) 927{ 928 struct kfd_topology_device *dev; 929 930 down_read(&topology_lock); 931 932 dev = list_last_entry(&topology_device_list, 933 struct kfd_topology_device, list); 934 if (dev) { 935 if (dev->node_props.cpu_cores_count && 936 dev->node_props.simd_count) { 937 pr_info("Topology: Add APU node [0x%0x:0x%0x]\n", 938 dev->node_props.device_id, 939 dev->node_props.vendor_id); 940 } else if (dev->node_props.cpu_cores_count) 941 pr_info("Topology: Add CPU node\n"); 942 else if (dev->node_props.simd_count) 943 pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n", 944 dev->node_props.device_id, 945 dev->node_props.vendor_id); 946 } 947 up_read(&topology_lock); 948} 949 950/* Helper function for intializing platform_xx members of 951 * kfd_system_properties. Uses OEM info from the last CPU/APU node. 952 */ 953static void kfd_update_system_properties(void) 954{ 955 struct kfd_topology_device *dev; 956 957 down_read(&topology_lock); 958 dev = list_last_entry(&topology_device_list, 959 struct kfd_topology_device, list); 960 if (dev) { 961 sys_props.platform_id = 962 (*((uint64_t *)dev->oem_id)) & CRAT_OEMID_64BIT_MASK; 963 sys_props.platform_oem = *((uint64_t *)dev->oem_table_id); 964 sys_props.platform_rev = dev->oem_revision; 965 } 966 up_read(&topology_lock); 967} 968 969static void find_system_memory(const struct dmi_header *dm, 970 void *private) 971{ 972 struct kfd_mem_properties *mem; 973 u16 mem_width, mem_clock; 974 struct kfd_topology_device *kdev = 975 (struct kfd_topology_device *)private; 976 const u8 *dmi_data = (const u8 *)(dm + 1); 977 978 if (dm->type == DMI_ENTRY_MEM_DEVICE && dm->length >= 0x15) { 979 mem_width = (u16)(*(const u16 *)(dmi_data + 0x6)); 980 mem_clock = (u16)(*(const u16 *)(dmi_data + 0x11)); 981 list_for_each_entry(mem, &kdev->mem_props, list) { 982 if (mem_width != 0xFFFF && mem_width != 0) 983 mem->width = mem_width; 984 if (mem_clock != 0) 985 mem->mem_clk_max = mem_clock; 986 } 987 } 988} 989 990/* kfd_add_non_crat_information - Add information that is not currently 991 * defined in CRAT but is necessary for KFD topology 992 * @dev - topology device to which addition info is added 993 */ 994static void kfd_add_non_crat_information(struct kfd_topology_device *kdev) 995{ 996 /* Check if CPU only node. */ 997 if (!kdev->gpu) { 998 /* Add system memory information */ 999 dmi_walk(find_system_memory, kdev); 1000 } 1001 /* TODO: For GPU node, rearrange code from kfd_topology_add_device */ 1002} 1003 1004int kfd_topology_init(void) 1005{ 1006 void *crat_image = NULL; 1007 size_t image_size = 0; 1008 int ret; 1009 struct list_head temp_topology_device_list; 1010 int cpu_only_node = 0; 1011 struct kfd_topology_device *kdev; 1012 int proximity_domain; 1013 1014 /* topology_device_list - Master list of all topology devices 1015 * temp_topology_device_list - temporary list created while parsing CRAT 1016 * or VCRAT. Once parsing is complete the contents of list is moved to 1017 * topology_device_list 1018 */ 1019 1020 /* Initialize the head for the both the lists */ 1021 INIT_LIST_HEAD(&topology_device_list); 1022 INIT_LIST_HEAD(&temp_topology_device_list); 1023 init_rwsem(&topology_lock); 1024 1025 memset(&sys_props, 0, sizeof(sys_props)); 1026 1027 /* Proximity domains in ACPI CRAT tables start counting at 1028 * 0. The same should be true for virtual CRAT tables created 1029 * at this stage. GPUs added later in kfd_topology_add_device 1030 * use a counter. 1031 */ 1032 proximity_domain = 0; 1033 1034 ret = kfd_create_crat_image_virtual(&crat_image, &image_size, 1035 COMPUTE_UNIT_CPU, NULL, 1036 proximity_domain); 1037 cpu_only_node = 1; 1038 if (ret) { 1039 pr_err("Error creating VCRAT table for CPU\n"); 1040 return ret; 1041 } 1042 1043 ret = kfd_parse_crat_table(crat_image, 1044 &temp_topology_device_list, 1045 proximity_domain); 1046 if (ret) { 1047 pr_err("Error parsing VCRAT table for CPU\n"); 1048 goto err; 1049 } 1050 1051 kdev = list_first_entry(&temp_topology_device_list, 1052 struct kfd_topology_device, list); 1053 1054 down_write(&topology_lock); 1055 kfd_topology_update_device_list(&temp_topology_device_list, 1056 &topology_device_list); 1057 topology_crat_proximity_domain = sys_props.num_devices-1; 1058 ret = kfd_topology_update_sysfs(); 1059 up_write(&topology_lock); 1060 1061 if (!ret) { 1062 sys_props.generation_count++; 1063 kfd_update_system_properties(); 1064 kfd_debug_print_topology(); 1065 } else 1066 pr_err("Failed to update topology in sysfs ret=%d\n", ret); 1067 1068 /* For nodes with GPU, this information gets added 1069 * when GPU is detected (kfd_topology_add_device). 1070 */ 1071 if (cpu_only_node) { 1072 /* Add additional information to CPU only node created above */ 1073 down_write(&topology_lock); 1074 kdev = list_first_entry(&topology_device_list, 1075 struct kfd_topology_device, list); 1076 up_write(&topology_lock); 1077 kfd_add_non_crat_information(kdev); 1078 } 1079 1080err: 1081 kfd_destroy_crat_image(crat_image); 1082 return ret; 1083} 1084 1085void kfd_topology_shutdown(void) 1086{ 1087 down_write(&topology_lock); 1088 kfd_topology_release_sysfs(); 1089 kfd_release_live_view(); 1090 up_write(&topology_lock); 1091} 1092 1093static uint32_t kfd_generate_gpu_id(struct kfd_node *gpu) 1094{ 1095 uint32_t hashout; 1096 uint32_t buf[8]; 1097 uint64_t local_mem_size; 1098 int i; 1099 1100 if (!gpu) 1101 return 0; 1102 1103 local_mem_size = gpu->local_mem_info.local_mem_size_private + 1104 gpu->local_mem_info.local_mem_size_public; 1105 buf[0] = gpu->adev->pdev->devfn; 1106 buf[1] = gpu->adev->pdev->subsystem_vendor | 1107 (gpu->adev->pdev->subsystem_device << 16); 1108 buf[2] = pci_domain_nr(gpu->adev->pdev->bus); 1109 buf[3] = gpu->adev->pdev->device; 1110 buf[4] = gpu->adev->pdev->bus->number; 1111 buf[5] = lower_32_bits(local_mem_size); 1112 buf[6] = upper_32_bits(local_mem_size); 1113 buf[7] = (ffs(gpu->xcc_mask) - 1) | (NUM_XCC(gpu->xcc_mask) << 16); 1114 1115 for (i = 0, hashout = 0; i < 8; i++) 1116 hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH); 1117 1118 return hashout; 1119} 1120/* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If 1121 * the GPU device is not already present in the topology device 1122 * list then return NULL. This means a new topology device has to 1123 * be created for this GPU. 1124 */ 1125static struct kfd_topology_device *kfd_assign_gpu(struct kfd_node *gpu) 1126{ 1127 struct kfd_topology_device *dev; 1128 struct kfd_topology_device *out_dev = NULL; 1129 struct kfd_mem_properties *mem; 1130 struct kfd_cache_properties *cache; 1131 struct kfd_iolink_properties *iolink; 1132 struct kfd_iolink_properties *p2plink; 1133 1134 list_for_each_entry(dev, &topology_device_list, list) { 1135 /* Discrete GPUs need their own topology device list 1136 * entries. Don't assign them to CPU/APU nodes. 1137 */ 1138 if (dev->node_props.cpu_cores_count) 1139 continue; 1140 1141 if (!dev->gpu && (dev->node_props.simd_count > 0)) { 1142 dev->gpu = gpu; 1143 out_dev = dev; 1144 1145 list_for_each_entry(mem, &dev->mem_props, list) 1146 mem->gpu = dev->gpu; 1147 list_for_each_entry(cache, &dev->cache_props, list) 1148 cache->gpu = dev->gpu; 1149 list_for_each_entry(iolink, &dev->io_link_props, list) 1150 iolink->gpu = dev->gpu; 1151 list_for_each_entry(p2plink, &dev->p2p_link_props, list) 1152 p2plink->gpu = dev->gpu; 1153 break; 1154 } 1155 } 1156 return out_dev; 1157} 1158 1159static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival) 1160{ 1161 /* 1162 * TODO: Generate an event for thunk about the arrival/removal 1163 * of the GPU 1164 */ 1165} 1166 1167/* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info, 1168 * patch this after CRAT parsing. 1169 */ 1170static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev) 1171{ 1172 struct kfd_mem_properties *mem; 1173 struct kfd_local_mem_info local_mem_info; 1174 1175 if (!dev) 1176 return; 1177 1178 /* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with 1179 * single bank of VRAM local memory. 1180 * for dGPUs - VCRAT reports only one bank of Local Memory 1181 * for APUs - If CRAT from ACPI reports more than one bank, then 1182 * all the banks will report the same mem_clk_max information 1183 */ 1184 amdgpu_amdkfd_get_local_mem_info(dev->gpu->adev, &local_mem_info, 1185 dev->gpu->xcp); 1186 1187 list_for_each_entry(mem, &dev->mem_props, list) 1188 mem->mem_clk_max = local_mem_info.mem_clk_max; 1189} 1190 1191static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev, 1192 struct kfd_topology_device *target_gpu_dev, 1193 struct kfd_iolink_properties *link) 1194{ 1195 /* xgmi always supports atomics between links. */ 1196 if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI) 1197 return; 1198 1199 /* check pcie support to set cpu(dev) flags for target_gpu_dev link. */ 1200 if (target_gpu_dev) { 1201 uint32_t cap; 1202 1203 pcie_capability_read_dword(target_gpu_dev->gpu->adev->pdev, 1204 PCI_EXP_DEVCAP2, &cap); 1205 1206 if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 1207 PCI_EXP_DEVCAP2_ATOMIC_COMP64))) 1208 link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT | 1209 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT; 1210 /* set gpu (dev) flags. */ 1211 } else { 1212 if (!dev->gpu->kfd->pci_atomic_requested || 1213 dev->gpu->adev->asic_type == CHIP_HAWAII) 1214 link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT | 1215 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT; 1216 } 1217} 1218 1219static void kfd_set_iolink_non_coherent(struct kfd_topology_device *to_dev, 1220 struct kfd_iolink_properties *outbound_link, 1221 struct kfd_iolink_properties *inbound_link) 1222{ 1223 /* CPU -> GPU with PCIe */ 1224 if (!to_dev->gpu && 1225 inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS) 1226 inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1227 1228 if (to_dev->gpu) { 1229 /* GPU <-> GPU with PCIe and 1230 * Vega20 with XGMI 1231 */ 1232 if (inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS || 1233 (inbound_link->iolink_type == CRAT_IOLINK_TYPE_XGMI && 1234 KFD_GC_VERSION(to_dev->gpu) == IP_VERSION(9, 4, 0))) { 1235 outbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1236 inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1237 } 1238 } 1239} 1240 1241static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev) 1242{ 1243 struct kfd_iolink_properties *link, *inbound_link; 1244 struct kfd_topology_device *peer_dev; 1245 1246 if (!dev || !dev->gpu) 1247 return; 1248 1249 /* GPU only creates direct links so apply flags setting to all */ 1250 list_for_each_entry(link, &dev->io_link_props, list) { 1251 link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1252 kfd_set_iolink_no_atomics(dev, NULL, link); 1253 peer_dev = kfd_topology_device_by_proximity_domain( 1254 link->node_to); 1255 1256 if (!peer_dev) 1257 continue; 1258 1259 /* Include the CPU peer in GPU hive if connected over xGMI. */ 1260 if (!peer_dev->gpu && 1261 link->iolink_type == CRAT_IOLINK_TYPE_XGMI) { 1262 /* 1263 * If the GPU is not part of a GPU hive, use its pci 1264 * device location as the hive ID to bind with the CPU. 1265 */ 1266 if (!dev->node_props.hive_id) 1267 dev->node_props.hive_id = pci_dev_id(dev->gpu->adev->pdev); 1268 peer_dev->node_props.hive_id = dev->node_props.hive_id; 1269 } 1270 1271 list_for_each_entry(inbound_link, &peer_dev->io_link_props, 1272 list) { 1273 if (inbound_link->node_to != link->node_from) 1274 continue; 1275 1276 inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1277 kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link); 1278 kfd_set_iolink_non_coherent(peer_dev, link, inbound_link); 1279 } 1280 } 1281 1282 /* Create indirect links so apply flags setting to all */ 1283 list_for_each_entry(link, &dev->p2p_link_props, list) { 1284 link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1285 kfd_set_iolink_no_atomics(dev, NULL, link); 1286 peer_dev = kfd_topology_device_by_proximity_domain( 1287 link->node_to); 1288 1289 if (!peer_dev) 1290 continue; 1291 1292 list_for_each_entry(inbound_link, &peer_dev->p2p_link_props, 1293 list) { 1294 if (inbound_link->node_to != link->node_from) 1295 continue; 1296 1297 inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1298 kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link); 1299 kfd_set_iolink_non_coherent(peer_dev, link, inbound_link); 1300 } 1301 } 1302} 1303 1304static int kfd_build_p2p_node_entry(struct kfd_topology_device *dev, 1305 struct kfd_iolink_properties *p2plink) 1306{ 1307 int ret; 1308 1309 p2plink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 1310 if (!p2plink->kobj) 1311 return -ENOMEM; 1312 1313 ret = kobject_init_and_add(p2plink->kobj, &iolink_type, 1314 dev->kobj_p2plink, "%d", dev->node_props.p2p_links_count - 1); 1315 if (ret < 0) { 1316 kobject_put(p2plink->kobj); 1317 return ret; 1318 } 1319 1320 p2plink->attr.name = "properties"; 1321 p2plink->attr.mode = KFD_SYSFS_FILE_MODE; 1322 sysfs_attr_init(&p2plink->attr); 1323 ret = sysfs_create_file(p2plink->kobj, &p2plink->attr); 1324 if (ret < 0) 1325 return ret; 1326 1327 return 0; 1328} 1329 1330static int kfd_create_indirect_link_prop(struct kfd_topology_device *kdev, int gpu_node) 1331{ 1332 struct kfd_iolink_properties *gpu_link, *tmp_link, *cpu_link; 1333 struct kfd_iolink_properties *props = NULL, *props2 = NULL; 1334 struct kfd_topology_device *cpu_dev; 1335 int ret = 0; 1336 int i, num_cpu; 1337 1338 num_cpu = 0; 1339 list_for_each_entry(cpu_dev, &topology_device_list, list) { 1340 if (cpu_dev->gpu) 1341 break; 1342 num_cpu++; 1343 } 1344 1345 if (list_empty(&kdev->io_link_props)) 1346 return -ENODATA; 1347 1348 gpu_link = list_first_entry(&kdev->io_link_props, 1349 struct kfd_iolink_properties, list); 1350 1351 for (i = 0; i < num_cpu; i++) { 1352 /* CPU <--> GPU */ 1353 if (gpu_link->node_to == i) 1354 continue; 1355 1356 /* find CPU <--> CPU links */ 1357 cpu_link = NULL; 1358 cpu_dev = kfd_topology_device_by_proximity_domain(i); 1359 if (cpu_dev) { 1360 list_for_each_entry(tmp_link, 1361 &cpu_dev->io_link_props, list) { 1362 if (tmp_link->node_to == gpu_link->node_to) { 1363 cpu_link = tmp_link; 1364 break; 1365 } 1366 } 1367 } 1368 1369 if (!cpu_link) 1370 return -ENOMEM; 1371 1372 /* CPU <--> CPU <--> GPU, GPU node*/ 1373 props = kfd_alloc_struct(props); 1374 if (!props) 1375 return -ENOMEM; 1376 1377 memcpy(props, gpu_link, sizeof(struct kfd_iolink_properties)); 1378 props->weight = gpu_link->weight + cpu_link->weight; 1379 props->min_latency = gpu_link->min_latency + cpu_link->min_latency; 1380 props->max_latency = gpu_link->max_latency + cpu_link->max_latency; 1381 props->min_bandwidth = min(gpu_link->min_bandwidth, cpu_link->min_bandwidth); 1382 props->max_bandwidth = min(gpu_link->max_bandwidth, cpu_link->max_bandwidth); 1383 1384 props->node_from = gpu_node; 1385 props->node_to = i; 1386 kdev->node_props.p2p_links_count++; 1387 list_add_tail(&props->list, &kdev->p2p_link_props); 1388 ret = kfd_build_p2p_node_entry(kdev, props); 1389 if (ret < 0) 1390 return ret; 1391 1392 /* for small Bar, no CPU --> GPU in-direct links */ 1393 if (kfd_dev_is_large_bar(kdev->gpu)) { 1394 /* CPU <--> CPU <--> GPU, CPU node*/ 1395 props2 = kfd_alloc_struct(props2); 1396 if (!props2) 1397 return -ENOMEM; 1398 1399 memcpy(props2, props, sizeof(struct kfd_iolink_properties)); 1400 props2->node_from = i; 1401 props2->node_to = gpu_node; 1402 props2->kobj = NULL; 1403 cpu_dev->node_props.p2p_links_count++; 1404 list_add_tail(&props2->list, &cpu_dev->p2p_link_props); 1405 ret = kfd_build_p2p_node_entry(cpu_dev, props2); 1406 if (ret < 0) 1407 return ret; 1408 } 1409 } 1410 return ret; 1411} 1412 1413#if defined(CONFIG_HSA_AMD_P2P) 1414static int kfd_add_peer_prop(struct kfd_topology_device *kdev, 1415 struct kfd_topology_device *peer, int from, int to) 1416{ 1417 struct kfd_iolink_properties *props = NULL; 1418 struct kfd_iolink_properties *iolink1, *iolink2, *iolink3; 1419 struct kfd_topology_device *cpu_dev; 1420 int ret = 0; 1421 1422 if (!amdgpu_device_is_peer_accessible( 1423 kdev->gpu->adev, 1424 peer->gpu->adev)) 1425 return ret; 1426 1427 if (list_empty(&kdev->io_link_props)) 1428 return -ENODATA; 1429 1430 iolink1 = list_first_entry(&kdev->io_link_props, 1431 struct kfd_iolink_properties, list); 1432 1433 if (list_empty(&peer->io_link_props)) 1434 return -ENODATA; 1435 1436 iolink2 = list_first_entry(&peer->io_link_props, 1437 struct kfd_iolink_properties, list); 1438 1439 props = kfd_alloc_struct(props); 1440 if (!props) 1441 return -ENOMEM; 1442 1443 memcpy(props, iolink1, sizeof(struct kfd_iolink_properties)); 1444 1445 props->weight = iolink1->weight + iolink2->weight; 1446 props->min_latency = iolink1->min_latency + iolink2->min_latency; 1447 props->max_latency = iolink1->max_latency + iolink2->max_latency; 1448 props->min_bandwidth = min(iolink1->min_bandwidth, iolink2->min_bandwidth); 1449 props->max_bandwidth = min(iolink2->max_bandwidth, iolink2->max_bandwidth); 1450 1451 if (iolink1->node_to != iolink2->node_to) { 1452 /* CPU->CPU link*/ 1453 cpu_dev = kfd_topology_device_by_proximity_domain(iolink1->node_to); 1454 if (cpu_dev) { 1455 list_for_each_entry(iolink3, &cpu_dev->io_link_props, list) { 1456 if (iolink3->node_to != iolink2->node_to) 1457 continue; 1458 1459 props->weight += iolink3->weight; 1460 props->min_latency += iolink3->min_latency; 1461 props->max_latency += iolink3->max_latency; 1462 props->min_bandwidth = min(props->min_bandwidth, 1463 iolink3->min_bandwidth); 1464 props->max_bandwidth = min(props->max_bandwidth, 1465 iolink3->max_bandwidth); 1466 break; 1467 } 1468 } else { 1469 WARN(1, "CPU node not found"); 1470 } 1471 } 1472 1473 props->node_from = from; 1474 props->node_to = to; 1475 peer->node_props.p2p_links_count++; 1476 list_add_tail(&props->list, &peer->p2p_link_props); 1477 ret = kfd_build_p2p_node_entry(peer, props); 1478 1479 return ret; 1480} 1481#endif 1482 1483static int kfd_dev_create_p2p_links(void) 1484{ 1485 struct kfd_topology_device *dev; 1486 struct kfd_topology_device *new_dev; 1487#if defined(CONFIG_HSA_AMD_P2P) 1488 uint32_t i; 1489#endif 1490 uint32_t k; 1491 int ret = 0; 1492 1493 k = 0; 1494 list_for_each_entry(dev, &topology_device_list, list) 1495 k++; 1496 if (k < 2) 1497 return 0; 1498 1499 new_dev = list_last_entry(&topology_device_list, struct kfd_topology_device, list); 1500 if (WARN_ON(!new_dev->gpu)) 1501 return 0; 1502 1503 k--; 1504 1505 /* create in-direct links */ 1506 ret = kfd_create_indirect_link_prop(new_dev, k); 1507 if (ret < 0) 1508 goto out; 1509 1510 /* create p2p links */ 1511#if defined(CONFIG_HSA_AMD_P2P) 1512 i = 0; 1513 list_for_each_entry(dev, &topology_device_list, list) { 1514 if (dev == new_dev) 1515 break; 1516 if (!dev->gpu || !dev->gpu->adev || 1517 (dev->gpu->kfd->hive_id && 1518 dev->gpu->kfd->hive_id == new_dev->gpu->kfd->hive_id)) 1519 goto next; 1520 1521 /* check if node(s) is/are peer accessible in one direction or bi-direction */ 1522 ret = kfd_add_peer_prop(new_dev, dev, i, k); 1523 if (ret < 0) 1524 goto out; 1525 1526 ret = kfd_add_peer_prop(dev, new_dev, k, i); 1527 if (ret < 0) 1528 goto out; 1529next: 1530 i++; 1531 } 1532#endif 1533 1534out: 1535 return ret; 1536} 1537 1538/* Helper function. See kfd_fill_gpu_cache_info for parameter description */ 1539static int fill_in_l1_pcache(struct kfd_cache_properties **props_ext, 1540 struct kfd_gpu_cache_info *pcache_info, 1541 struct kfd_cu_info *cu_info, 1542 int cu_bitmask, 1543 int cache_type, unsigned int cu_processor_id, 1544 int cu_block) 1545{ 1546 unsigned int cu_sibling_map_mask; 1547 int first_active_cu; 1548 struct kfd_cache_properties *pcache = NULL; 1549 1550 cu_sibling_map_mask = cu_bitmask; 1551 cu_sibling_map_mask >>= cu_block; 1552 cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1553 first_active_cu = ffs(cu_sibling_map_mask); 1554 1555 /* CU could be inactive. In case of shared cache find the first active 1556 * CU. and incase of non-shared cache check if the CU is inactive. If 1557 * inactive active skip it 1558 */ 1559 if (first_active_cu) { 1560 pcache = kfd_alloc_struct(pcache); 1561 if (!pcache) 1562 return -ENOMEM; 1563 1564 memset(pcache, 0, sizeof(struct kfd_cache_properties)); 1565 pcache->processor_id_low = cu_processor_id + (first_active_cu - 1); 1566 pcache->cache_level = pcache_info[cache_type].cache_level; 1567 pcache->cache_size = pcache_info[cache_type].cache_size; 1568 1569 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE) 1570 pcache->cache_type |= HSA_CACHE_TYPE_DATA; 1571 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE) 1572 pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; 1573 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE) 1574 pcache->cache_type |= HSA_CACHE_TYPE_CPU; 1575 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE) 1576 pcache->cache_type |= HSA_CACHE_TYPE_HSACU; 1577 1578 /* Sibling map is w.r.t processor_id_low, so shift out 1579 * inactive CU 1580 */ 1581 cu_sibling_map_mask = 1582 cu_sibling_map_mask >> (first_active_cu - 1); 1583 1584 pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF); 1585 pcache->sibling_map[1] = 1586 (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); 1587 pcache->sibling_map[2] = 1588 (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); 1589 pcache->sibling_map[3] = 1590 (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); 1591 1592 pcache->sibling_map_size = 4; 1593 *props_ext = pcache; 1594 1595 return 0; 1596 } 1597 return 1; 1598} 1599 1600/* Helper function. See kfd_fill_gpu_cache_info for parameter description */ 1601static int fill_in_l2_l3_pcache(struct kfd_cache_properties **props_ext, 1602 struct kfd_gpu_cache_info *pcache_info, 1603 struct kfd_cu_info *cu_info, 1604 int cache_type, unsigned int cu_processor_id, 1605 struct kfd_node *knode) 1606{ 1607 unsigned int cu_sibling_map_mask; 1608 int first_active_cu; 1609 int i, j, k, xcc, start, end; 1610 struct kfd_cache_properties *pcache = NULL; 1611 1612 start = ffs(knode->xcc_mask) - 1; 1613 end = start + NUM_XCC(knode->xcc_mask); 1614 cu_sibling_map_mask = cu_info->cu_bitmap[start][0][0]; 1615 cu_sibling_map_mask &= 1616 ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1617 first_active_cu = ffs(cu_sibling_map_mask); 1618 1619 /* CU could be inactive. In case of shared cache find the first active 1620 * CU. and incase of non-shared cache check if the CU is inactive. If 1621 * inactive active skip it 1622 */ 1623 if (first_active_cu) { 1624 pcache = kfd_alloc_struct(pcache); 1625 if (!pcache) 1626 return -ENOMEM; 1627 1628 memset(pcache, 0, sizeof(struct kfd_cache_properties)); 1629 pcache->processor_id_low = cu_processor_id 1630 + (first_active_cu - 1); 1631 pcache->cache_level = pcache_info[cache_type].cache_level; 1632 pcache->cache_size = pcache_info[cache_type].cache_size; 1633 1634 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE) 1635 pcache->cache_type |= HSA_CACHE_TYPE_DATA; 1636 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE) 1637 pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; 1638 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE) 1639 pcache->cache_type |= HSA_CACHE_TYPE_CPU; 1640 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE) 1641 pcache->cache_type |= HSA_CACHE_TYPE_HSACU; 1642 1643 /* Sibling map is w.r.t processor_id_low, so shift out 1644 * inactive CU 1645 */ 1646 cu_sibling_map_mask = cu_sibling_map_mask >> (first_active_cu - 1); 1647 k = 0; 1648 1649 for (xcc = start; xcc < end; xcc++) { 1650 for (i = 0; i < cu_info->num_shader_engines; i++) { 1651 for (j = 0; j < cu_info->num_shader_arrays_per_engine; j++) { 1652 pcache->sibling_map[k] = (uint8_t)(cu_sibling_map_mask & 0xFF); 1653 pcache->sibling_map[k+1] = (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); 1654 pcache->sibling_map[k+2] = (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); 1655 pcache->sibling_map[k+3] = (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); 1656 k += 4; 1657 1658 cu_sibling_map_mask = cu_info->cu_bitmap[xcc][i % 4][j + i / 4]; 1659 cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1660 } 1661 } 1662 } 1663 pcache->sibling_map_size = k; 1664 *props_ext = pcache; 1665 return 0; 1666 } 1667 return 1; 1668} 1669 1670#define KFD_MAX_CACHE_TYPES 6 1671 1672/* kfd_fill_cache_non_crat_info - Fill GPU cache info using kfd_gpu_cache_info 1673 * tables 1674 */ 1675static void kfd_fill_cache_non_crat_info(struct kfd_topology_device *dev, struct kfd_node *kdev) 1676{ 1677 struct kfd_gpu_cache_info *pcache_info = NULL; 1678 int i, j, k, xcc, start, end; 1679 int ct = 0; 1680 unsigned int cu_processor_id; 1681 int ret; 1682 unsigned int num_cu_shared; 1683 struct kfd_cu_info cu_info; 1684 struct kfd_cu_info *pcu_info; 1685 int gpu_processor_id; 1686 struct kfd_cache_properties *props_ext; 1687 int num_of_entries = 0; 1688 int num_of_cache_types = 0; 1689 struct kfd_gpu_cache_info cache_info[KFD_MAX_CACHE_TYPES]; 1690 1691 amdgpu_amdkfd_get_cu_info(kdev->adev, &cu_info); 1692 pcu_info = &cu_info; 1693 1694 gpu_processor_id = dev->node_props.simd_id_base; 1695 1696 pcache_info = cache_info; 1697 num_of_cache_types = kfd_get_gpu_cache_info(kdev, &pcache_info); 1698 if (!num_of_cache_types) { 1699 pr_warn("no cache info found\n"); 1700 return; 1701 } 1702 1703 /* For each type of cache listed in the kfd_gpu_cache_info table, 1704 * go through all available Compute Units. 1705 * The [i,j,k] loop will 1706 * if kfd_gpu_cache_info.num_cu_shared = 1 1707 * will parse through all available CU 1708 * If (kfd_gpu_cache_info.num_cu_shared != 1) 1709 * then it will consider only one CU from 1710 * the shared unit 1711 */ 1712 start = ffs(kdev->xcc_mask) - 1; 1713 end = start + NUM_XCC(kdev->xcc_mask); 1714 1715 for (ct = 0; ct < num_of_cache_types; ct++) { 1716 cu_processor_id = gpu_processor_id; 1717 if (pcache_info[ct].cache_level == 1) { 1718 for (xcc = start; xcc < end; xcc++) { 1719 for (i = 0; i < pcu_info->num_shader_engines; i++) { 1720 for (j = 0; j < pcu_info->num_shader_arrays_per_engine; j++) { 1721 for (k = 0; k < pcu_info->num_cu_per_sh; k += pcache_info[ct].num_cu_shared) { 1722 1723 ret = fill_in_l1_pcache(&props_ext, pcache_info, pcu_info, 1724 pcu_info->cu_bitmap[xcc][i % 4][j + i / 4], ct, 1725 cu_processor_id, k); 1726 1727 if (ret < 0) 1728 break; 1729 1730 if (!ret) { 1731 num_of_entries++; 1732 list_add_tail(&props_ext->list, &dev->cache_props); 1733 } 1734 1735 /* Move to next CU block */ 1736 num_cu_shared = ((k + pcache_info[ct].num_cu_shared) <= 1737 pcu_info->num_cu_per_sh) ? 1738 pcache_info[ct].num_cu_shared : 1739 (pcu_info->num_cu_per_sh - k); 1740 cu_processor_id += num_cu_shared; 1741 } 1742 } 1743 } 1744 } 1745 } else { 1746 ret = fill_in_l2_l3_pcache(&props_ext, pcache_info, 1747 pcu_info, ct, cu_processor_id, kdev); 1748 1749 if (ret < 0) 1750 break; 1751 1752 if (!ret) { 1753 num_of_entries++; 1754 list_add_tail(&props_ext->list, &dev->cache_props); 1755 } 1756 } 1757 } 1758 dev->node_props.caches_count += num_of_entries; 1759 pr_debug("Added [%d] GPU cache entries\n", num_of_entries); 1760} 1761 1762static int kfd_topology_add_device_locked(struct kfd_node *gpu, uint32_t gpu_id, 1763 struct kfd_topology_device **dev) 1764{ 1765 int proximity_domain = ++topology_crat_proximity_domain; 1766 struct list_head temp_topology_device_list; 1767 void *crat_image = NULL; 1768 size_t image_size = 0; 1769 int res; 1770 1771 res = kfd_create_crat_image_virtual(&crat_image, &image_size, 1772 COMPUTE_UNIT_GPU, gpu, 1773 proximity_domain); 1774 if (res) { 1775 pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n", 1776 gpu_id); 1777 topology_crat_proximity_domain--; 1778 goto err; 1779 } 1780 1781 INIT_LIST_HEAD(&temp_topology_device_list); 1782 1783 res = kfd_parse_crat_table(crat_image, 1784 &temp_topology_device_list, 1785 proximity_domain); 1786 if (res) { 1787 pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n", 1788 gpu_id); 1789 topology_crat_proximity_domain--; 1790 goto err; 1791 } 1792 1793 kfd_topology_update_device_list(&temp_topology_device_list, 1794 &topology_device_list); 1795 1796 *dev = kfd_assign_gpu(gpu); 1797 if (WARN_ON(!*dev)) { 1798 res = -ENODEV; 1799 goto err; 1800 } 1801 1802 /* Fill the cache affinity information here for the GPUs 1803 * using VCRAT 1804 */ 1805 kfd_fill_cache_non_crat_info(*dev, gpu); 1806 1807 /* Update the SYSFS tree, since we added another topology 1808 * device 1809 */ 1810 res = kfd_topology_update_sysfs(); 1811 if (!res) 1812 sys_props.generation_count++; 1813 else 1814 pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n", 1815 gpu_id, res); 1816 1817err: 1818 kfd_destroy_crat_image(crat_image); 1819 return res; 1820} 1821 1822static void kfd_topology_set_dbg_firmware_support(struct kfd_topology_device *dev) 1823{ 1824 bool firmware_supported = true; 1825 1826 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(11, 0, 0) && 1827 KFD_GC_VERSION(dev->gpu) < IP_VERSION(12, 0, 0)) { 1828 uint32_t mes_api_rev = (dev->gpu->adev->mes.sched_version & 1829 AMDGPU_MES_API_VERSION_MASK) >> 1830 AMDGPU_MES_API_VERSION_SHIFT; 1831 uint32_t mes_rev = dev->gpu->adev->mes.sched_version & 1832 AMDGPU_MES_VERSION_MASK; 1833 1834 firmware_supported = (mes_api_rev >= 14) && (mes_rev >= 64); 1835 goto out; 1836 } 1837 1838 /* 1839 * Note: Any unlisted devices here are assumed to support exception handling. 1840 * Add additional checks here as needed. 1841 */ 1842 switch (KFD_GC_VERSION(dev->gpu)) { 1843 case IP_VERSION(9, 0, 1): 1844 firmware_supported = dev->gpu->kfd->mec_fw_version >= 459 + 32768; 1845 break; 1846 case IP_VERSION(9, 1, 0): 1847 case IP_VERSION(9, 2, 1): 1848 case IP_VERSION(9, 2, 2): 1849 case IP_VERSION(9, 3, 0): 1850 case IP_VERSION(9, 4, 0): 1851 firmware_supported = dev->gpu->kfd->mec_fw_version >= 459; 1852 break; 1853 case IP_VERSION(9, 4, 1): 1854 firmware_supported = dev->gpu->kfd->mec_fw_version >= 60; 1855 break; 1856 case IP_VERSION(9, 4, 2): 1857 firmware_supported = dev->gpu->kfd->mec_fw_version >= 51; 1858 break; 1859 case IP_VERSION(10, 1, 10): 1860 case IP_VERSION(10, 1, 2): 1861 case IP_VERSION(10, 1, 1): 1862 firmware_supported = dev->gpu->kfd->mec_fw_version >= 144; 1863 break; 1864 case IP_VERSION(10, 3, 0): 1865 case IP_VERSION(10, 3, 2): 1866 case IP_VERSION(10, 3, 1): 1867 case IP_VERSION(10, 3, 4): 1868 case IP_VERSION(10, 3, 5): 1869 firmware_supported = dev->gpu->kfd->mec_fw_version >= 89; 1870 break; 1871 case IP_VERSION(10, 1, 3): 1872 case IP_VERSION(10, 3, 3): 1873 firmware_supported = false; 1874 break; 1875 default: 1876 break; 1877 } 1878 1879out: 1880 if (firmware_supported) 1881 dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_FIRMWARE_SUPPORTED; 1882} 1883 1884static void kfd_topology_set_capabilities(struct kfd_topology_device *dev) 1885{ 1886 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 << 1887 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 1888 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 1889 1890 dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_SUPPORT | 1891 HSA_CAP_TRAP_DEBUG_WAVE_LAUNCH_TRAP_OVERRIDE_SUPPORTED | 1892 HSA_CAP_TRAP_DEBUG_WAVE_LAUNCH_MODE_SUPPORTED; 1893 1894 if (kfd_dbg_has_ttmps_always_setup(dev->gpu)) 1895 dev->node_props.debug_prop |= HSA_DBG_DISPATCH_INFO_ALWAYS_VALID; 1896 1897 if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0)) { 1898 if (KFD_GC_VERSION(dev->gpu) == IP_VERSION(9, 4, 3)) 1899 dev->node_props.debug_prop |= 1900 HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX9_4_3 | 1901 HSA_DBG_WATCH_ADDR_MASK_HI_BIT_GFX9_4_3; 1902 else 1903 dev->node_props.debug_prop |= 1904 HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX9 | 1905 HSA_DBG_WATCH_ADDR_MASK_HI_BIT; 1906 1907 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(9, 4, 2)) 1908 dev->node_props.capability |= 1909 HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED; 1910 } else { 1911 dev->node_props.debug_prop |= HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX10 | 1912 HSA_DBG_WATCH_ADDR_MASK_HI_BIT; 1913 1914 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(11, 0, 0)) 1915 dev->node_props.capability |= 1916 HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED; 1917 } 1918 1919 kfd_topology_set_dbg_firmware_support(dev); 1920} 1921 1922int kfd_topology_add_device(struct kfd_node *gpu) 1923{ 1924 uint32_t gpu_id; 1925 struct kfd_topology_device *dev; 1926 struct kfd_cu_info cu_info; 1927 int res = 0; 1928 int i; 1929 const char *asic_name = amdgpu_asic_name[gpu->adev->asic_type]; 1930 1931 gpu_id = kfd_generate_gpu_id(gpu); 1932 if (gpu->xcp && !gpu->xcp->ddev) { 1933 dev_warn(gpu->adev->dev, 1934 "Won't add GPU (ID: 0x%x) to topology since it has no drm node assigned.", 1935 gpu_id); 1936 return 0; 1937 } else { 1938 pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id); 1939 } 1940 1941 /* Check to see if this gpu device exists in the topology_device_list. 1942 * If so, assign the gpu to that device, 1943 * else create a Virtual CRAT for this gpu device and then parse that 1944 * CRAT to create a new topology device. Once created assign the gpu to 1945 * that topology device 1946 */ 1947 down_write(&topology_lock); 1948 dev = kfd_assign_gpu(gpu); 1949 if (!dev) 1950 res = kfd_topology_add_device_locked(gpu, gpu_id, &dev); 1951 up_write(&topology_lock); 1952 if (res) 1953 return res; 1954 1955 dev->gpu_id = gpu_id; 1956 gpu->id = gpu_id; 1957 1958 kfd_dev_create_p2p_links(); 1959 1960 /* TODO: Move the following lines to function 1961 * kfd_add_non_crat_information 1962 */ 1963 1964 /* Fill-in additional information that is not available in CRAT but 1965 * needed for the topology 1966 */ 1967 1968 amdgpu_amdkfd_get_cu_info(dev->gpu->adev, &cu_info); 1969 1970 for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1; i++) { 1971 dev->node_props.name[i] = __tolower(asic_name[i]); 1972 if (asic_name[i] == '\0') 1973 break; 1974 } 1975 dev->node_props.name[i] = '\0'; 1976 1977 dev->node_props.simd_arrays_per_engine = 1978 cu_info.num_shader_arrays_per_engine; 1979 1980 dev->node_props.gfx_target_version = 1981 gpu->kfd->device_info.gfx_target_version; 1982 dev->node_props.vendor_id = gpu->adev->pdev->vendor; 1983 dev->node_props.device_id = gpu->adev->pdev->device; 1984 dev->node_props.capability |= 1985 ((dev->gpu->adev->rev_id << HSA_CAP_ASIC_REVISION_SHIFT) & 1986 HSA_CAP_ASIC_REVISION_MASK); 1987 1988 dev->node_props.location_id = pci_dev_id(gpu->adev->pdev); 1989 if (KFD_GC_VERSION(dev->gpu->kfd) == IP_VERSION(9, 4, 3)) 1990 dev->node_props.location_id |= dev->gpu->node_id; 1991 1992 dev->node_props.domain = pci_domain_nr(gpu->adev->pdev->bus); 1993 dev->node_props.max_engine_clk_fcompute = 1994 amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->adev); 1995 dev->node_props.max_engine_clk_ccompute = 1996 cpufreq_quick_get_max(0) / 1000; 1997 1998 if (gpu->xcp) 1999 dev->node_props.drm_render_minor = gpu->xcp->ddev->render->index; 2000 else 2001 dev->node_props.drm_render_minor = 2002 gpu->kfd->shared_resources.drm_render_minor; 2003 2004 dev->node_props.hive_id = gpu->kfd->hive_id; 2005 dev->node_props.num_sdma_engines = kfd_get_num_sdma_engines(gpu); 2006 dev->node_props.num_sdma_xgmi_engines = 2007 kfd_get_num_xgmi_sdma_engines(gpu); 2008 dev->node_props.num_sdma_queues_per_engine = 2009 gpu->kfd->device_info.num_sdma_queues_per_engine - 2010 gpu->kfd->device_info.num_reserved_sdma_queues_per_engine; 2011 dev->node_props.num_gws = (dev->gpu->gws && 2012 dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ? 2013 dev->gpu->adev->gds.gws_size : 0; 2014 dev->node_props.num_cp_queues = get_cp_queues_num(dev->gpu->dqm); 2015 2016 kfd_fill_mem_clk_max_info(dev); 2017 kfd_fill_iolink_non_crat_info(dev); 2018 2019 switch (dev->gpu->adev->asic_type) { 2020 case CHIP_KAVERI: 2021 case CHIP_HAWAII: 2022 case CHIP_TONGA: 2023 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 << 2024 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 2025 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 2026 break; 2027 case CHIP_CARRIZO: 2028 case CHIP_FIJI: 2029 case CHIP_POLARIS10: 2030 case CHIP_POLARIS11: 2031 case CHIP_POLARIS12: 2032 case CHIP_VEGAM: 2033 pr_debug("Adding doorbell packet type capability\n"); 2034 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 << 2035 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 2036 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 2037 break; 2038 default: 2039 if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(9, 0, 1)) 2040 WARN(1, "Unexpected ASIC family %u", 2041 dev->gpu->adev->asic_type); 2042 else 2043 kfd_topology_set_capabilities(dev); 2044 } 2045 2046 /* 2047 * Overwrite ATS capability according to needs_iommu_device to fix 2048 * potential missing corresponding bit in CRAT of BIOS. 2049 */ 2050 dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT; 2051 2052 /* Fix errors in CZ CRAT. 2053 * simd_count: Carrizo CRAT reports wrong simd_count, probably 2054 * because it doesn't consider masked out CUs 2055 * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd 2056 */ 2057 if (dev->gpu->adev->asic_type == CHIP_CARRIZO) { 2058 dev->node_props.simd_count = 2059 cu_info.simd_per_cu * cu_info.cu_active_number; 2060 dev->node_props.max_waves_per_simd = 10; 2061 } 2062 2063 /* kfd only concerns sram ecc on GFX and HBM ecc on UMC */ 2064 dev->node_props.capability |= 2065 ((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ? 2066 HSA_CAP_SRAM_EDCSUPPORTED : 0; 2067 dev->node_props.capability |= 2068 ((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ? 2069 HSA_CAP_MEM_EDCSUPPORTED : 0; 2070 2071 if (KFD_GC_VERSION(dev->gpu) != IP_VERSION(9, 0, 1)) 2072 dev->node_props.capability |= (dev->gpu->adev->ras_enabled != 0) ? 2073 HSA_CAP_RASEVENTNOTIFY : 0; 2074 2075 if (KFD_IS_SVM_API_SUPPORTED(dev->gpu->adev)) 2076 dev->node_props.capability |= HSA_CAP_SVMAPI_SUPPORTED; 2077 2078 if (dev->gpu->adev->gmc.is_app_apu || 2079 dev->gpu->adev->gmc.xgmi.connected_to_cpu) 2080 dev->node_props.capability |= HSA_CAP_FLAGS_COHERENTHOSTACCESS; 2081 2082 kfd_debug_print_topology(); 2083 2084 kfd_notify_gpu_change(gpu_id, 1); 2085 2086 return 0; 2087} 2088 2089/** 2090 * kfd_topology_update_io_links() - Update IO links after device removal. 2091 * @proximity_domain: Proximity domain value of the dev being removed. 2092 * 2093 * The topology list currently is arranged in increasing order of 2094 * proximity domain. 2095 * 2096 * Two things need to be done when a device is removed: 2097 * 1. All the IO links to this device need to be removed. 2098 * 2. All nodes after the current device node need to move 2099 * up once this device node is removed from the topology 2100 * list. As a result, the proximity domain values for 2101 * all nodes after the node being deleted reduce by 1. 2102 * This would also cause the proximity domain values for 2103 * io links to be updated based on new proximity domain 2104 * values. 2105 * 2106 * Context: The caller must hold write topology_lock. 2107 */ 2108static void kfd_topology_update_io_links(int proximity_domain) 2109{ 2110 struct kfd_topology_device *dev; 2111 struct kfd_iolink_properties *iolink, *p2plink, *tmp; 2112 2113 list_for_each_entry(dev, &topology_device_list, list) { 2114 if (dev->proximity_domain > proximity_domain) 2115 dev->proximity_domain--; 2116 2117 list_for_each_entry_safe(iolink, tmp, &dev->io_link_props, list) { 2118 /* 2119 * If there is an io link to the dev being deleted 2120 * then remove that IO link also. 2121 */ 2122 if (iolink->node_to == proximity_domain) { 2123 list_del(&iolink->list); 2124 dev->node_props.io_links_count--; 2125 } else { 2126 if (iolink->node_from > proximity_domain) 2127 iolink->node_from--; 2128 if (iolink->node_to > proximity_domain) 2129 iolink->node_to--; 2130 } 2131 } 2132 2133 list_for_each_entry_safe(p2plink, tmp, &dev->p2p_link_props, list) { 2134 /* 2135 * If there is a p2p link to the dev being deleted 2136 * then remove that p2p link also. 2137 */ 2138 if (p2plink->node_to == proximity_domain) { 2139 list_del(&p2plink->list); 2140 dev->node_props.p2p_links_count--; 2141 } else { 2142 if (p2plink->node_from > proximity_domain) 2143 p2plink->node_from--; 2144 if (p2plink->node_to > proximity_domain) 2145 p2plink->node_to--; 2146 } 2147 } 2148 } 2149} 2150 2151int kfd_topology_remove_device(struct kfd_node *gpu) 2152{ 2153 struct kfd_topology_device *dev, *tmp; 2154 uint32_t gpu_id; 2155 int res = -ENODEV; 2156 int i = 0; 2157 2158 down_write(&topology_lock); 2159 2160 list_for_each_entry_safe(dev, tmp, &topology_device_list, list) { 2161 if (dev->gpu == gpu) { 2162 gpu_id = dev->gpu_id; 2163 kfd_remove_sysfs_node_entry(dev); 2164 kfd_release_topology_device(dev); 2165 sys_props.num_devices--; 2166 kfd_topology_update_io_links(i); 2167 topology_crat_proximity_domain = sys_props.num_devices-1; 2168 sys_props.generation_count++; 2169 res = 0; 2170 if (kfd_topology_update_sysfs() < 0) 2171 kfd_topology_release_sysfs(); 2172 break; 2173 } 2174 i++; 2175 } 2176 2177 up_write(&topology_lock); 2178 2179 if (!res) 2180 kfd_notify_gpu_change(gpu_id, 0); 2181 2182 return res; 2183} 2184 2185/* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD 2186 * topology. If GPU device is found @idx, then valid kfd_dev pointer is 2187 * returned through @kdev 2188 * Return - 0: On success (@kdev will be NULL for non GPU nodes) 2189 * -1: If end of list 2190 */ 2191int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_node **kdev) 2192{ 2193 2194 struct kfd_topology_device *top_dev; 2195 uint8_t device_idx = 0; 2196 2197 *kdev = NULL; 2198 down_read(&topology_lock); 2199 2200 list_for_each_entry(top_dev, &topology_device_list, list) { 2201 if (device_idx == idx) { 2202 *kdev = top_dev->gpu; 2203 up_read(&topology_lock); 2204 return 0; 2205 } 2206 2207 device_idx++; 2208 } 2209 2210 up_read(&topology_lock); 2211 2212 return -1; 2213 2214} 2215 2216static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask) 2217{ 2218 int first_cpu_of_numa_node; 2219 2220 if (!cpumask || cpumask == cpu_none_mask) 2221 return -1; 2222 first_cpu_of_numa_node = cpumask_first(cpumask); 2223 if (first_cpu_of_numa_node >= nr_cpu_ids) 2224 return -1; 2225#ifdef CONFIG_X86_64 2226 return cpu_data(first_cpu_of_numa_node).apicid; 2227#else 2228 return first_cpu_of_numa_node; 2229#endif 2230} 2231 2232/* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor 2233 * of the given NUMA node (numa_node_id) 2234 * Return -1 on failure 2235 */ 2236int kfd_numa_node_to_apic_id(int numa_node_id) 2237{ 2238 if (numa_node_id == -1) { 2239 pr_warn("Invalid NUMA Node. Use online CPU mask\n"); 2240 return kfd_cpumask_to_apic_id(cpu_online_mask); 2241 } 2242 return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id)); 2243} 2244 2245#if defined(CONFIG_DEBUG_FS) 2246 2247int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data) 2248{ 2249 struct kfd_topology_device *dev; 2250 unsigned int i = 0; 2251 int r = 0; 2252 2253 down_read(&topology_lock); 2254 2255 list_for_each_entry(dev, &topology_device_list, list) { 2256 if (!dev->gpu) { 2257 i++; 2258 continue; 2259 } 2260 2261 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); 2262 r = dqm_debugfs_hqds(m, dev->gpu->dqm); 2263 if (r) 2264 break; 2265 } 2266 2267 up_read(&topology_lock); 2268 2269 return r; 2270} 2271 2272int kfd_debugfs_rls_by_device(struct seq_file *m, void *data) 2273{ 2274 struct kfd_topology_device *dev; 2275 unsigned int i = 0; 2276 int r = 0; 2277 2278 down_read(&topology_lock); 2279 2280 list_for_each_entry(dev, &topology_device_list, list) { 2281 if (!dev->gpu) { 2282 i++; 2283 continue; 2284 } 2285 2286 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); 2287 r = pm_debugfs_runlist(m, &dev->gpu->dqm->packet_mgr); 2288 if (r) 2289 break; 2290 } 2291 2292 up_read(&topology_lock); 2293 2294 return r; 2295} 2296 2297#endif 2298