1// SPDX-License-Identifier: GPL-2.0 2 3/* 4 * Copyright 2016-2019 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8#include <uapi/misc/habanalabs.h> 9#include "habanalabs.h" 10 11#include <linux/kernel.h> 12#include <linux/fs.h> 13#include <linux/uaccess.h> 14#include <linux/slab.h> 15 16static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = { 17 [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr), 18 [HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf), 19 [HL_DEBUG_OP_STM] = sizeof(struct hl_debug_params_stm), 20 [HL_DEBUG_OP_FUNNEL] = 0, 21 [HL_DEBUG_OP_BMON] = sizeof(struct hl_debug_params_bmon), 22 [HL_DEBUG_OP_SPMU] = sizeof(struct hl_debug_params_spmu), 23 [HL_DEBUG_OP_TIMESTAMP] = 0 24 25}; 26 27static int device_status_info(struct hl_device *hdev, struct hl_info_args *args) 28{ 29 struct hl_info_device_status dev_stat = {0}; 30 u32 size = args->return_size; 31 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 32 33 if ((!size) || (!out)) 34 return -EINVAL; 35 36 dev_stat.status = hl_device_status(hdev); 37 38 return copy_to_user(out, &dev_stat, 39 min((size_t)size, sizeof(dev_stat))) ? -EFAULT : 0; 40} 41 42static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) 43{ 44 struct hl_info_hw_ip_info hw_ip = {0}; 45 u32 size = args->return_size; 46 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 47 struct asic_fixed_properties *prop = &hdev->asic_prop; 48 u64 sram_kmd_size, dram_kmd_size; 49 50 if ((!size) || (!out)) 51 return -EINVAL; 52 53 sram_kmd_size = (prop->sram_user_base_address - 54 prop->sram_base_address); 55 dram_kmd_size = (prop->dram_user_base_address - 56 prop->dram_base_address); 57 58 hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev); 59 hw_ip.sram_base_address = prop->sram_user_base_address; 60 hw_ip.dram_base_address = prop->dram_user_base_address; 61 hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask; 62 hw_ip.sram_size = prop->sram_size - sram_kmd_size; 63 hw_ip.dram_size = prop->dram_size - dram_kmd_size; 64 if (hw_ip.dram_size > PAGE_SIZE) 65 hw_ip.dram_enabled = 1; 66 hw_ip.num_of_events = prop->num_of_events; 67 68 memcpy(hw_ip.cpucp_version, prop->cpucp_info.cpucp_version, 69 min(VERSION_MAX_LEN, HL_INFO_VERSION_MAX_LEN)); 70 71 memcpy(hw_ip.card_name, prop->cpucp_info.card_name, 72 min(CARD_NAME_MAX_LEN, HL_INFO_CARD_NAME_MAX_LEN)); 73 74 hw_ip.cpld_version = le32_to_cpu(prop->cpucp_info.cpld_version); 75 hw_ip.module_id = le32_to_cpu(prop->cpucp_info.card_location); 76 77 hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr; 78 hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf; 79 hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od; 80 hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor; 81 82 return copy_to_user(out, &hw_ip, 83 min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0; 84} 85 86static int hw_events_info(struct hl_device *hdev, bool aggregate, 87 struct hl_info_args *args) 88{ 89 u32 size, max_size = args->return_size; 90 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 91 void *arr; 92 93 if ((!max_size) || (!out)) 94 return -EINVAL; 95 96 arr = hdev->asic_funcs->get_events_stat(hdev, aggregate, &size); 97 98 return copy_to_user(out, arr, min(max_size, size)) ? -EFAULT : 0; 99} 100 101static int dram_usage_info(struct hl_fpriv *hpriv, struct hl_info_args *args) 102{ 103 struct hl_device *hdev = hpriv->hdev; 104 struct hl_info_dram_usage dram_usage = {0}; 105 u32 max_size = args->return_size; 106 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 107 struct asic_fixed_properties *prop = &hdev->asic_prop; 108 u64 dram_kmd_size; 109 110 if ((!max_size) || (!out)) 111 return -EINVAL; 112 113 dram_kmd_size = (prop->dram_user_base_address - 114 prop->dram_base_address); 115 dram_usage.dram_free_mem = (prop->dram_size - dram_kmd_size) - 116 atomic64_read(&hdev->dram_used_mem); 117 if (hpriv->ctx) 118 dram_usage.ctx_dram_mem = 119 atomic64_read(&hpriv->ctx->dram_phys_mem); 120 121 return copy_to_user(out, &dram_usage, 122 min((size_t) max_size, sizeof(dram_usage))) ? -EFAULT : 0; 123} 124 125static int hw_idle(struct hl_device *hdev, struct hl_info_args *args) 126{ 127 struct hl_info_hw_idle hw_idle = {0}; 128 u32 max_size = args->return_size; 129 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 130 131 if ((!max_size) || (!out)) 132 return -EINVAL; 133 134 hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev, 135 &hw_idle.busy_engines_mask_ext, NULL); 136 hw_idle.busy_engines_mask = 137 lower_32_bits(hw_idle.busy_engines_mask_ext); 138 139 return copy_to_user(out, &hw_idle, 140 min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0; 141} 142 143static int debug_coresight(struct hl_device *hdev, struct hl_debug_args *args) 144{ 145 struct hl_debug_params *params; 146 void *input = NULL, *output = NULL; 147 int rc; 148 149 params = kzalloc(sizeof(*params), GFP_KERNEL); 150 if (!params) 151 return -ENOMEM; 152 153 params->reg_idx = args->reg_idx; 154 params->enable = args->enable; 155 params->op = args->op; 156 157 if (args->input_ptr && args->input_size) { 158 input = kzalloc(hl_debug_struct_size[args->op], GFP_KERNEL); 159 if (!input) { 160 rc = -ENOMEM; 161 goto out; 162 } 163 164 if (copy_from_user(input, u64_to_user_ptr(args->input_ptr), 165 args->input_size)) { 166 rc = -EFAULT; 167 dev_err(hdev->dev, "failed to copy input debug data\n"); 168 goto out; 169 } 170 171 params->input = input; 172 } 173 174 if (args->output_ptr && args->output_size) { 175 output = kzalloc(args->output_size, GFP_KERNEL); 176 if (!output) { 177 rc = -ENOMEM; 178 goto out; 179 } 180 181 params->output = output; 182 params->output_size = args->output_size; 183 } 184 185 rc = hdev->asic_funcs->debug_coresight(hdev, params); 186 if (rc) { 187 dev_err(hdev->dev, 188 "debug coresight operation failed %d\n", rc); 189 goto out; 190 } 191 192 if (output && copy_to_user((void __user *) (uintptr_t) args->output_ptr, 193 output, args->output_size)) { 194 dev_err(hdev->dev, "copy to user failed in debug ioctl\n"); 195 rc = -EFAULT; 196 goto out; 197 } 198 199 200out: 201 kfree(params); 202 kfree(output); 203 kfree(input); 204 205 return rc; 206} 207 208static int device_utilization(struct hl_device *hdev, struct hl_info_args *args) 209{ 210 struct hl_info_device_utilization device_util = {0}; 211 u32 max_size = args->return_size; 212 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 213 214 if ((!max_size) || (!out)) 215 return -EINVAL; 216 217 if ((args->period_ms < 100) || (args->period_ms > 1000) || 218 (args->period_ms % 100)) { 219 dev_err(hdev->dev, 220 "period %u must be between 100 - 1000 and must be divisible by 100\n", 221 args->period_ms); 222 return -EINVAL; 223 } 224 225 device_util.utilization = hl_device_utilization(hdev, args->period_ms); 226 227 return copy_to_user(out, &device_util, 228 min((size_t) max_size, sizeof(device_util))) ? -EFAULT : 0; 229} 230 231static int get_clk_rate(struct hl_device *hdev, struct hl_info_args *args) 232{ 233 struct hl_info_clk_rate clk_rate = {0}; 234 u32 max_size = args->return_size; 235 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 236 int rc; 237 238 if ((!max_size) || (!out)) 239 return -EINVAL; 240 241 rc = hdev->asic_funcs->get_clk_rate(hdev, &clk_rate.cur_clk_rate_mhz, 242 &clk_rate.max_clk_rate_mhz); 243 if (rc) 244 return rc; 245 246 return copy_to_user(out, &clk_rate, 247 min((size_t) max_size, sizeof(clk_rate))) ? -EFAULT : 0; 248} 249 250static int get_reset_count(struct hl_device *hdev, struct hl_info_args *args) 251{ 252 struct hl_info_reset_count reset_count = {0}; 253 u32 max_size = args->return_size; 254 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 255 256 if ((!max_size) || (!out)) 257 return -EINVAL; 258 259 reset_count.hard_reset_cnt = hdev->hard_reset_cnt; 260 reset_count.soft_reset_cnt = hdev->soft_reset_cnt; 261 262 return copy_to_user(out, &reset_count, 263 min((size_t) max_size, sizeof(reset_count))) ? -EFAULT : 0; 264} 265 266static int time_sync_info(struct hl_device *hdev, struct hl_info_args *args) 267{ 268 struct hl_info_time_sync time_sync = {0}; 269 u32 max_size = args->return_size; 270 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 271 272 if ((!max_size) || (!out)) 273 return -EINVAL; 274 275 time_sync.device_time = hdev->asic_funcs->get_device_time(hdev); 276 time_sync.host_time = ktime_get_raw_ns(); 277 278 return copy_to_user(out, &time_sync, 279 min((size_t) max_size, sizeof(time_sync))) ? -EFAULT : 0; 280} 281 282static int pci_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args) 283{ 284 struct hl_device *hdev = hpriv->hdev; 285 struct hl_info_pci_counters pci_counters = {0}; 286 u32 max_size = args->return_size; 287 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 288 int rc; 289 290 if ((!max_size) || (!out)) 291 return -EINVAL; 292 293 rc = hl_fw_cpucp_pci_counters_get(hdev, &pci_counters); 294 if (rc) 295 return rc; 296 297 return copy_to_user(out, &pci_counters, 298 min((size_t) max_size, sizeof(pci_counters))) ? -EFAULT : 0; 299} 300 301static int clk_throttle_info(struct hl_fpriv *hpriv, struct hl_info_args *args) 302{ 303 struct hl_device *hdev = hpriv->hdev; 304 struct hl_info_clk_throttle clk_throttle = {0}; 305 u32 max_size = args->return_size; 306 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 307 308 if ((!max_size) || (!out)) 309 return -EINVAL; 310 311 clk_throttle.clk_throttling_reason = hdev->clk_throttling_reason; 312 313 return copy_to_user(out, &clk_throttle, 314 min((size_t) max_size, sizeof(clk_throttle))) ? -EFAULT : 0; 315} 316 317static int cs_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args) 318{ 319 struct hl_device *hdev = hpriv->hdev; 320 struct hl_info_cs_counters cs_counters = { {0} }; 321 u32 max_size = args->return_size; 322 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 323 324 if ((!max_size) || (!out)) 325 return -EINVAL; 326 327 memcpy(&cs_counters.cs_counters, &hdev->aggregated_cs_counters, 328 sizeof(struct hl_cs_counters)); 329 330 if (hpriv->ctx) 331 memcpy(&cs_counters.ctx_cs_counters, &hpriv->ctx->cs_counters, 332 sizeof(struct hl_cs_counters)); 333 334 return copy_to_user(out, &cs_counters, 335 min((size_t) max_size, sizeof(cs_counters))) ? -EFAULT : 0; 336} 337 338static int sync_manager_info(struct hl_fpriv *hpriv, struct hl_info_args *args) 339{ 340 struct hl_device *hdev = hpriv->hdev; 341 struct asic_fixed_properties *prop = &hdev->asic_prop; 342 struct hl_info_sync_manager sm_info = {0}; 343 u32 max_size = args->return_size; 344 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 345 346 if ((!max_size) || (!out)) 347 return -EINVAL; 348 349 if (args->dcore_id >= HL_MAX_DCORES) 350 return -EINVAL; 351 352 sm_info.first_available_sync_object = 353 prop->first_available_user_sob[args->dcore_id]; 354 sm_info.first_available_monitor = 355 prop->first_available_user_mon[args->dcore_id]; 356 357 358 return copy_to_user(out, &sm_info, min_t(size_t, (size_t) max_size, 359 sizeof(sm_info))) ? -EFAULT : 0; 360} 361 362static int total_energy_consumption_info(struct hl_fpriv *hpriv, 363 struct hl_info_args *args) 364{ 365 struct hl_device *hdev = hpriv->hdev; 366 struct hl_info_energy total_energy = {0}; 367 u32 max_size = args->return_size; 368 void __user *out = (void __user *) (uintptr_t) args->return_pointer; 369 int rc; 370 371 if ((!max_size) || (!out)) 372 return -EINVAL; 373 374 rc = hl_fw_cpucp_total_energy_get(hdev, 375 &total_energy.total_energy_consumption); 376 if (rc) 377 return rc; 378 379 return copy_to_user(out, &total_energy, 380 min((size_t) max_size, sizeof(total_energy))) ? -EFAULT : 0; 381} 382 383static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, 384 struct device *dev) 385{ 386 struct hl_info_args *args = data; 387 struct hl_device *hdev = hpriv->hdev; 388 int rc; 389 390 /* 391 * Information is returned for the following opcodes even if the device 392 * is disabled or in reset. 393 */ 394 switch (args->op) { 395 case HL_INFO_HW_IP_INFO: 396 return hw_ip_info(hdev, args); 397 398 case HL_INFO_DEVICE_STATUS: 399 return device_status_info(hdev, args); 400 401 case HL_INFO_RESET_COUNT: 402 return get_reset_count(hdev, args); 403 404 default: 405 break; 406 } 407 408 if (hl_device_disabled_or_in_reset(hdev)) { 409 dev_warn_ratelimited(dev, 410 "Device is %s. Can't execute INFO IOCTL\n", 411 atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); 412 return -EBUSY; 413 } 414 415 switch (args->op) { 416 case HL_INFO_HW_EVENTS: 417 rc = hw_events_info(hdev, false, args); 418 break; 419 420 case HL_INFO_DRAM_USAGE: 421 rc = dram_usage_info(hpriv, args); 422 break; 423 424 case HL_INFO_HW_IDLE: 425 rc = hw_idle(hdev, args); 426 break; 427 428 case HL_INFO_DEVICE_UTILIZATION: 429 rc = device_utilization(hdev, args); 430 break; 431 432 case HL_INFO_HW_EVENTS_AGGREGATE: 433 rc = hw_events_info(hdev, true, args); 434 break; 435 436 case HL_INFO_CLK_RATE: 437 rc = get_clk_rate(hdev, args); 438 break; 439 440 case HL_INFO_TIME_SYNC: 441 return time_sync_info(hdev, args); 442 443 case HL_INFO_CS_COUNTERS: 444 return cs_counters_info(hpriv, args); 445 446 case HL_INFO_PCI_COUNTERS: 447 return pci_counters_info(hpriv, args); 448 449 case HL_INFO_CLK_THROTTLE_REASON: 450 return clk_throttle_info(hpriv, args); 451 452 case HL_INFO_SYNC_MANAGER: 453 return sync_manager_info(hpriv, args); 454 455 case HL_INFO_TOTAL_ENERGY: 456 return total_energy_consumption_info(hpriv, args); 457 458 default: 459 dev_err(dev, "Invalid request %d\n", args->op); 460 rc = -ENOTTY; 461 break; 462 } 463 464 return rc; 465} 466 467static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data) 468{ 469 return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev); 470} 471 472static int hl_info_ioctl_control(struct hl_fpriv *hpriv, void *data) 473{ 474 return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev_ctrl); 475} 476 477static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data) 478{ 479 struct hl_debug_args *args = data; 480 struct hl_device *hdev = hpriv->hdev; 481 int rc = 0; 482 483 if (hl_device_disabled_or_in_reset(hdev)) { 484 dev_warn_ratelimited(hdev->dev, 485 "Device is %s. Can't execute DEBUG IOCTL\n", 486 atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); 487 return -EBUSY; 488 } 489 490 switch (args->op) { 491 case HL_DEBUG_OP_ETR: 492 case HL_DEBUG_OP_ETF: 493 case HL_DEBUG_OP_STM: 494 case HL_DEBUG_OP_FUNNEL: 495 case HL_DEBUG_OP_BMON: 496 case HL_DEBUG_OP_SPMU: 497 case HL_DEBUG_OP_TIMESTAMP: 498 if (!hdev->in_debug) { 499 dev_err_ratelimited(hdev->dev, 500 "Rejecting debug configuration request because device not in debug mode\n"); 501 return -EFAULT; 502 } 503 args->input_size = 504 min(args->input_size, hl_debug_struct_size[args->op]); 505 rc = debug_coresight(hdev, args); 506 break; 507 case HL_DEBUG_OP_SET_MODE: 508 rc = hl_device_set_debug_mode(hdev, (bool) args->enable); 509 break; 510 default: 511 dev_err(hdev->dev, "Invalid request %d\n", args->op); 512 rc = -ENOTTY; 513 break; 514 } 515 516 return rc; 517} 518 519#define HL_IOCTL_DEF(ioctl, _func) \ 520 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func} 521 522static const struct hl_ioctl_desc hl_ioctls[] = { 523 HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl), 524 HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl), 525 HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl), 526 HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_cs_wait_ioctl), 527 HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl), 528 HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl) 529}; 530 531static const struct hl_ioctl_desc hl_ioctls_control[] = { 532 HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl_control) 533}; 534 535static long _hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg, 536 const struct hl_ioctl_desc *ioctl, struct device *dev) 537{ 538 struct hl_fpriv *hpriv = filep->private_data; 539 struct hl_device *hdev = hpriv->hdev; 540 unsigned int nr = _IOC_NR(cmd); 541 char stack_kdata[128] = {0}; 542 char *kdata = NULL; 543 unsigned int usize, asize; 544 hl_ioctl_t *func; 545 u32 hl_size; 546 int retcode; 547 548 if (hdev->hard_reset_pending) { 549 dev_crit_ratelimited(hdev->dev_ctrl, 550 "Device HARD reset pending! Please close FD\n"); 551 return -ENODEV; 552 } 553 554 /* Do not trust userspace, use our own definition */ 555 func = ioctl->func; 556 557 if (unlikely(!func)) { 558 dev_dbg(dev, "no function\n"); 559 retcode = -ENOTTY; 560 goto out_err; 561 } 562 563 hl_size = _IOC_SIZE(ioctl->cmd); 564 usize = asize = _IOC_SIZE(cmd); 565 if (hl_size > asize) 566 asize = hl_size; 567 568 cmd = ioctl->cmd; 569 570 if (cmd & (IOC_IN | IOC_OUT)) { 571 if (asize <= sizeof(stack_kdata)) { 572 kdata = stack_kdata; 573 } else { 574 kdata = kzalloc(asize, GFP_KERNEL); 575 if (!kdata) { 576 retcode = -ENOMEM; 577 goto out_err; 578 } 579 } 580 } 581 582 if (cmd & IOC_IN) { 583 if (copy_from_user(kdata, (void __user *)arg, usize)) { 584 retcode = -EFAULT; 585 goto out_err; 586 } 587 } else if (cmd & IOC_OUT) { 588 memset(kdata, 0, usize); 589 } 590 591 retcode = func(hpriv, kdata); 592 593 if ((cmd & IOC_OUT) && copy_to_user((void __user *)arg, kdata, usize)) 594 retcode = -EFAULT; 595 596out_err: 597 if (retcode) 598 dev_dbg(dev, "error in ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n", 599 task_pid_nr(current), cmd, nr); 600 601 if (kdata != stack_kdata) 602 kfree(kdata); 603 604 return retcode; 605} 606 607long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) 608{ 609 struct hl_fpriv *hpriv = filep->private_data; 610 struct hl_device *hdev = hpriv->hdev; 611 const struct hl_ioctl_desc *ioctl = NULL; 612 unsigned int nr = _IOC_NR(cmd); 613 614 if ((nr >= HL_COMMAND_START) && (nr < HL_COMMAND_END)) { 615 ioctl = &hl_ioctls[nr]; 616 } else { 617 dev_err(hdev->dev, "invalid ioctl: pid=%d, nr=0x%02x\n", 618 task_pid_nr(current), nr); 619 return -ENOTTY; 620 } 621 622 return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev); 623} 624 625long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg) 626{ 627 struct hl_fpriv *hpriv = filep->private_data; 628 struct hl_device *hdev = hpriv->hdev; 629 const struct hl_ioctl_desc *ioctl = NULL; 630 unsigned int nr = _IOC_NR(cmd); 631 632 if (nr == _IOC_NR(HL_IOCTL_INFO)) { 633 ioctl = &hl_ioctls_control[nr]; 634 } else { 635 dev_err(hdev->dev_ctrl, "invalid ioctl: pid=%d, nr=0x%02x\n", 636 task_pid_nr(current), nr); 637 return -ENOTTY; 638 } 639 640 return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev_ctrl); 641} 642