1// SPDX-License-Identifier: GPL-2.0 2 3/* 4 * Copyright 2016-2019 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8#include "habanalabs.h" 9 10#include <linux/slab.h> 11 12/* 13 * hl_queue_add_ptr - add to pi or ci and checks if it wraps around 14 * 15 * @ptr: the current pi/ci value 16 * @val: the amount to add 17 * 18 * Add val to ptr. It can go until twice the queue length. 19 */ 20inline u32 hl_hw_queue_add_ptr(u32 ptr, u16 val) 21{ 22 ptr += val; 23 ptr &= ((HL_QUEUE_LENGTH << 1) - 1); 24 return ptr; 25} 26static inline int queue_ci_get(atomic_t *ci, u32 queue_len) 27{ 28 return atomic_read(ci) & ((queue_len << 1) - 1); 29} 30 31static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len) 32{ 33 int delta = (q->pi - queue_ci_get(&q->ci, queue_len)); 34 35 if (delta >= 0) 36 return (queue_len - delta); 37 else 38 return (abs(delta) - queue_len); 39} 40 41void hl_hw_queue_update_ci(struct hl_cs *cs) 42{ 43 struct hl_device *hdev = cs->ctx->hdev; 44 struct hl_hw_queue *q; 45 int i; 46 47 if (hdev->disabled) 48 return; 49 50 q = &hdev->kernel_queues[0]; 51 52 /* There are no internal queues if H/W queues are being used */ 53 if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW) 54 return; 55 56 /* We must increment CI for every queue that will never get a 57 * completion, there are 2 scenarios this can happen: 58 * 1. All queues of a non completion CS will never get a completion. 59 * 2. Internal queues never gets completion. 60 */ 61 for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) { 62 if (!cs_needs_completion(cs) || q->queue_type == QUEUE_TYPE_INT) 63 atomic_add(cs->jobs_in_queue_cnt[i], &q->ci); 64 } 65} 66 67/* 68 * hl_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a 69 * H/W queue. 70 * @hdev: pointer to habanalabs device structure 71 * @q: pointer to habanalabs queue structure 72 * @ctl: BD's control word 73 * @len: BD's length 74 * @ptr: BD's pointer 75 * 76 * This function assumes there is enough space on the queue to submit a new 77 * BD to it. It initializes the next BD and calls the device specific 78 * function to set the pi (and doorbell) 79 * 80 * This function must be called when the scheduler mutex is taken 81 * 82 */ 83void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q, 84 u32 ctl, u32 len, u64 ptr) 85{ 86 struct hl_bd *bd; 87 88 bd = q->kernel_address; 89 bd += hl_pi_2_offset(q->pi); 90 bd->ctl = cpu_to_le32(ctl); 91 bd->len = cpu_to_le32(len); 92 bd->ptr = cpu_to_le64(ptr); 93 94 q->pi = hl_queue_inc_ptr(q->pi); 95 hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi); 96} 97 98/* 99 * ext_queue_sanity_checks - perform some sanity checks on external queue 100 * 101 * @hdev : pointer to hl_device structure 102 * @q : pointer to hl_hw_queue structure 103 * @num_of_entries : how many entries to check for space 104 * @reserve_cq_entry : whether to reserve an entry in the cq 105 * 106 * H/W queues spinlock should be taken before calling this function 107 * 108 * Perform the following: 109 * - Make sure we have enough space in the h/w queue 110 * - Make sure we have enough space in the completion queue 111 * - Reserve space in the completion queue (needs to be reversed if there 112 * is a failure down the road before the actual submission of work). Only 113 * do this action if reserve_cq_entry is true 114 * 115 */ 116static int ext_queue_sanity_checks(struct hl_device *hdev, 117 struct hl_hw_queue *q, int num_of_entries, 118 bool reserve_cq_entry) 119{ 120 atomic_t *free_slots = 121 &hdev->completion_queue[q->cq_id].free_slots_cnt; 122 int free_slots_cnt; 123 124 /* Check we have enough space in the queue */ 125 free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH); 126 127 if (free_slots_cnt < num_of_entries) { 128 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n", 129 q->hw_queue_id, num_of_entries); 130 return -EAGAIN; 131 } 132 133 if (reserve_cq_entry) { 134 /* 135 * Check we have enough space in the completion queue 136 * Add -1 to counter (decrement) unless counter was already 0 137 * In that case, CQ is full so we can't submit a new CB because 138 * we won't get ack on its completion 139 * atomic_add_unless will return 0 if counter was already 0 140 */ 141 if (atomic_add_negative(num_of_entries * -1, free_slots)) { 142 dev_dbg(hdev->dev, "No space for %d on CQ %d\n", 143 num_of_entries, q->hw_queue_id); 144 atomic_add(num_of_entries, free_slots); 145 return -EAGAIN; 146 } 147 } 148 149 return 0; 150} 151 152/* 153 * int_queue_sanity_checks - perform some sanity checks on internal queue 154 * 155 * @hdev : pointer to hl_device structure 156 * @q : pointer to hl_hw_queue structure 157 * @num_of_entries : how many entries to check for space 158 * 159 * H/W queues spinlock should be taken before calling this function 160 * 161 * Perform the following: 162 * - Make sure we have enough space in the h/w queue 163 * 164 */ 165static int int_queue_sanity_checks(struct hl_device *hdev, 166 struct hl_hw_queue *q, 167 int num_of_entries) 168{ 169 int free_slots_cnt; 170 171 if (num_of_entries > q->int_queue_len) { 172 dev_err(hdev->dev, 173 "Cannot populate queue %u with %u jobs\n", 174 q->hw_queue_id, num_of_entries); 175 return -ENOMEM; 176 } 177 178 /* Check we have enough space in the queue */ 179 free_slots_cnt = queue_free_slots(q, q->int_queue_len); 180 181 if (free_slots_cnt < num_of_entries) { 182 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n", 183 q->hw_queue_id, num_of_entries); 184 return -EAGAIN; 185 } 186 187 return 0; 188} 189 190/* 191 * hw_queue_sanity_checks() - Make sure we have enough space in the h/w queue 192 * @hdev: Pointer to hl_device structure. 193 * @q: Pointer to hl_hw_queue structure. 194 * @num_of_entries: How many entries to check for space. 195 * 196 * Notice: We do not reserve queue entries so this function mustn't be called 197 * more than once per CS for the same queue 198 * 199 */ 200static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q, 201 int num_of_entries) 202{ 203 int free_slots_cnt; 204 205 /* Check we have enough space in the queue */ 206 free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH); 207 208 if (free_slots_cnt < num_of_entries) { 209 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n", 210 q->hw_queue_id, num_of_entries); 211 return -EAGAIN; 212 } 213 214 return 0; 215} 216 217/* 218 * hl_hw_queue_send_cb_no_cmpl - send a single CB (not a JOB) without completion 219 * 220 * @hdev: pointer to hl_device structure 221 * @hw_queue_id: Queue's type 222 * @cb_size: size of CB 223 * @cb_ptr: pointer to CB location 224 * 225 * This function sends a single CB, that must NOT generate a completion entry. 226 * Sending CPU messages can be done instead via 'hl_hw_queue_submit_bd()' 227 */ 228int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, 229 u32 cb_size, u64 cb_ptr) 230{ 231 struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id]; 232 int rc = 0; 233 234 hdev->asic_funcs->hw_queues_lock(hdev); 235 236 if (hdev->disabled) { 237 rc = -EPERM; 238 goto out; 239 } 240 241 /* 242 * hl_hw_queue_send_cb_no_cmpl() is called for queues of a H/W queue 243 * type only on init phase, when the queues are empty and being tested, 244 * so there is no need for sanity checks. 245 */ 246 if (q->queue_type != QUEUE_TYPE_HW) { 247 rc = ext_queue_sanity_checks(hdev, q, 1, false); 248 if (rc) 249 goto out; 250 } 251 252 hl_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr); 253 254out: 255 hdev->asic_funcs->hw_queues_unlock(hdev); 256 257 return rc; 258} 259 260/* 261 * ext_queue_schedule_job - submit a JOB to an external queue 262 * 263 * @job: pointer to the job that needs to be submitted to the queue 264 * 265 * This function must be called when the scheduler mutex is taken 266 * 267 */ 268static void ext_queue_schedule_job(struct hl_cs_job *job) 269{ 270 struct hl_device *hdev = job->cs->ctx->hdev; 271 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id]; 272 struct hl_cq_entry cq_pkt; 273 struct hl_cq *cq; 274 u64 cq_addr; 275 struct hl_cb *cb; 276 u32 ctl; 277 u32 len; 278 u64 ptr; 279 280 /* 281 * Update the JOB ID inside the BD CTL so the device would know what 282 * to write in the completion queue 283 */ 284 ctl = ((q->pi << BD_CTL_SHADOW_INDEX_SHIFT) & BD_CTL_SHADOW_INDEX_MASK); 285 286 cb = job->patched_cb; 287 len = job->job_cb_size; 288 ptr = cb->bus_address; 289 290 /* Skip completion flow in case this is a non completion CS */ 291 if (!cs_needs_completion(job->cs)) 292 goto submit_bd; 293 294 cq_pkt.data = cpu_to_le32( 295 ((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT) 296 & CQ_ENTRY_SHADOW_INDEX_MASK) | 297 FIELD_PREP(CQ_ENTRY_SHADOW_INDEX_VALID_MASK, 1) | 298 FIELD_PREP(CQ_ENTRY_READY_MASK, 1)); 299 300 /* 301 * No need to protect pi_offset because scheduling to the 302 * H/W queues is done under the scheduler mutex 303 * 304 * No need to check if CQ is full because it was already 305 * checked in ext_queue_sanity_checks 306 */ 307 cq = &hdev->completion_queue[q->cq_id]; 308 cq_addr = cq->bus_address + cq->pi * sizeof(struct hl_cq_entry); 309 310 hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len, 311 job->user_cb_size, 312 cq_addr, 313 le32_to_cpu(cq_pkt.data), 314 q->msi_vec, 315 job->contains_dma_pkt); 316 317 q->shadow_queue[hl_pi_2_offset(q->pi)] = job; 318 319 cq->pi = hl_cq_inc_ptr(cq->pi); 320 321submit_bd: 322 hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr); 323} 324 325/* 326 * int_queue_schedule_job - submit a JOB to an internal queue 327 * 328 * @job: pointer to the job that needs to be submitted to the queue 329 * 330 * This function must be called when the scheduler mutex is taken 331 * 332 */ 333static void int_queue_schedule_job(struct hl_cs_job *job) 334{ 335 struct hl_device *hdev = job->cs->ctx->hdev; 336 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id]; 337 struct hl_bd bd; 338 __le64 *pi; 339 340 bd.ctl = 0; 341 bd.len = cpu_to_le32(job->job_cb_size); 342 343 if (job->is_kernel_allocated_cb) 344 /* bus_address is actually a mmu mapped address 345 * allocated from an internal pool 346 */ 347 bd.ptr = cpu_to_le64(job->user_cb->bus_address); 348 else 349 bd.ptr = cpu_to_le64((u64) (uintptr_t) job->user_cb); 350 351 pi = q->kernel_address + (q->pi & (q->int_queue_len - 1)) * sizeof(bd); 352 353 q->pi++; 354 q->pi &= ((q->int_queue_len << 1) - 1); 355 356 hdev->asic_funcs->pqe_write(hdev, pi, &bd); 357 358 hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi); 359} 360 361/* 362 * hw_queue_schedule_job - submit a JOB to a H/W queue 363 * 364 * @job: pointer to the job that needs to be submitted to the queue 365 * 366 * This function must be called when the scheduler mutex is taken 367 * 368 */ 369static void hw_queue_schedule_job(struct hl_cs_job *job) 370{ 371 struct hl_device *hdev = job->cs->ctx->hdev; 372 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id]; 373 u64 ptr; 374 u32 offset, ctl, len; 375 376 /* 377 * Upon PQE completion, COMP_DATA is used as the write data to the 378 * completion queue (QMAN HBW message), and COMP_OFFSET is used as the 379 * write address offset in the SM block (QMAN LBW message). 380 * The write address offset is calculated as "COMP_OFFSET << 2". 381 */ 382 offset = job->cs->sequence & (hdev->asic_prop.max_pending_cs - 1); 383 ctl = ((offset << BD_CTL_COMP_OFFSET_SHIFT) & BD_CTL_COMP_OFFSET_MASK) | 384 ((q->pi << BD_CTL_COMP_DATA_SHIFT) & BD_CTL_COMP_DATA_MASK); 385 386 len = job->job_cb_size; 387 388 /* 389 * A patched CB is created only if a user CB was allocated by driver and 390 * MMU is disabled. If MMU is enabled, the user CB should be used 391 * instead. If the user CB wasn't allocated by driver, assume that it 392 * holds an address. 393 */ 394 if (job->patched_cb) 395 ptr = job->patched_cb->bus_address; 396 else if (job->is_kernel_allocated_cb) 397 ptr = job->user_cb->bus_address; 398 else 399 ptr = (u64) (uintptr_t) job->user_cb; 400 401 hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr); 402} 403 404static int init_signal_cs(struct hl_device *hdev, 405 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl) 406{ 407 struct hl_sync_stream_properties *prop; 408 struct hl_hw_sob *hw_sob; 409 u32 q_idx; 410 int rc = 0; 411 412 q_idx = job->hw_queue_id; 413 prop = &hdev->kernel_queues[q_idx].sync_stream_prop; 414 hw_sob = &prop->hw_sob[prop->curr_sob_offset]; 415 416 cs_cmpl->hw_sob = hw_sob; 417 cs_cmpl->sob_val = prop->next_sob_val; 418 419 dev_dbg(hdev->dev, 420 "generate signal CB, sob_id: %d, sob val: %u, q_idx: %d, seq: %llu\n", 421 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, q_idx, 422 cs_cmpl->cs_seq); 423 424 /* we set an EB since we must make sure all oeprations are done 425 * when sending the signal 426 */ 427 hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb, 428 cs_cmpl->hw_sob->sob_id, 0, true); 429 430 rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1, 431 false); 432 433 job->cs->sob_addr_offset = hw_sob->sob_addr; 434 job->cs->initial_sob_count = prop->next_sob_val - 1; 435 436 return rc; 437} 438 439void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev, 440 struct hl_cs *cs, struct hl_cs_job *job, 441 struct hl_cs_compl *cs_cmpl) 442{ 443 struct hl_cs_encaps_sig_handle *handle = cs->encaps_sig_hdl; 444 u32 offset = 0; 445 446 cs_cmpl->hw_sob = handle->hw_sob; 447 448 /* Note that encaps_sig_wait_offset was validated earlier in the flow 449 * for offset value which exceeds the max reserved signal count. 450 * always decrement 1 of the offset since when the user 451 * set offset 1 for example he mean to wait only for the first 452 * signal only, which will be pre_sob_val, and if he set offset 2 453 * then the value required is (pre_sob_val + 1) and so on... 454 * if user set wait offset to 0, then treat it as legacy wait cs, 455 * wait for the next signal. 456 */ 457 if (job->encaps_sig_wait_offset) 458 offset = job->encaps_sig_wait_offset - 1; 459 460 cs_cmpl->sob_val = handle->pre_sob_val + offset; 461} 462 463static int init_wait_cs(struct hl_device *hdev, struct hl_cs *cs, 464 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl) 465{ 466 struct hl_gen_wait_properties wait_prop; 467 struct hl_sync_stream_properties *prop; 468 struct hl_cs_compl *signal_cs_cmpl; 469 u32 q_idx; 470 471 q_idx = job->hw_queue_id; 472 prop = &hdev->kernel_queues[q_idx].sync_stream_prop; 473 474 signal_cs_cmpl = container_of(cs->signal_fence, 475 struct hl_cs_compl, 476 base_fence); 477 478 if (cs->encaps_signals) { 479 /* use the encaps signal handle stored earlier in the flow 480 * and set the SOB information from the encaps 481 * signals handle 482 */ 483 hl_hw_queue_encaps_sig_set_sob_info(hdev, cs, job, cs_cmpl); 484 485 dev_dbg(hdev->dev, "Wait for encaps signals handle, qidx(%u), CS sequence(%llu), sob val: 0x%x, offset: %u\n", 486 cs->encaps_sig_hdl->q_idx, 487 cs->encaps_sig_hdl->cs_seq, 488 cs_cmpl->sob_val, 489 job->encaps_sig_wait_offset); 490 } else { 491 /* Copy the SOB id and value of the signal CS */ 492 cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob; 493 cs_cmpl->sob_val = signal_cs_cmpl->sob_val; 494 } 495 496 /* check again if the signal cs already completed. 497 * if yes then don't send any wait cs since the hw_sob 498 * could be in reset already. if signal is not completed 499 * then get refcount to hw_sob to prevent resetting the sob 500 * while wait cs is not submitted. 501 * note that this check is protected by two locks, 502 * hw queue lock and completion object lock, 503 * and the same completion object lock also protects 504 * the hw_sob reset handler function. 505 * The hw_queue lock prevent out of sync of hw_sob 506 * refcount value, changed by signal/wait flows. 507 */ 508 spin_lock(&signal_cs_cmpl->lock); 509 510 if (completion_done(&cs->signal_fence->completion)) { 511 spin_unlock(&signal_cs_cmpl->lock); 512 return -EINVAL; 513 } 514 515 kref_get(&cs_cmpl->hw_sob->kref); 516 517 spin_unlock(&signal_cs_cmpl->lock); 518 519 dev_dbg(hdev->dev, 520 "generate wait CB, sob_id: %d, sob_val: 0x%x, mon_id: %d, q_idx: %d, seq: %llu\n", 521 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, 522 prop->base_mon_id, q_idx, cs->sequence); 523 524 wait_prop.data = (void *) job->patched_cb; 525 wait_prop.sob_base = cs_cmpl->hw_sob->sob_id; 526 wait_prop.sob_mask = 0x1; 527 wait_prop.sob_val = cs_cmpl->sob_val; 528 wait_prop.mon_id = prop->base_mon_id; 529 wait_prop.q_idx = q_idx; 530 wait_prop.size = 0; 531 532 hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop); 533 534 mb(); 535 hl_fence_put(cs->signal_fence); 536 cs->signal_fence = NULL; 537 538 return 0; 539} 540 541/* 542 * init_signal_wait_cs - initialize a signal/wait CS 543 * @cs: pointer to the signal/wait CS 544 * 545 * H/W queues spinlock should be taken before calling this function 546 */ 547static int init_signal_wait_cs(struct hl_cs *cs) 548{ 549 struct hl_ctx *ctx = cs->ctx; 550 struct hl_device *hdev = ctx->hdev; 551 struct hl_cs_job *job; 552 struct hl_cs_compl *cs_cmpl = 553 container_of(cs->fence, struct hl_cs_compl, base_fence); 554 int rc = 0; 555 556 /* There is only one job in a signal/wait CS */ 557 job = list_first_entry(&cs->job_list, struct hl_cs_job, 558 cs_node); 559 560 if (cs->type & CS_TYPE_SIGNAL) 561 rc = init_signal_cs(hdev, job, cs_cmpl); 562 else if (cs->type & CS_TYPE_WAIT) 563 rc = init_wait_cs(hdev, cs, job, cs_cmpl); 564 565 return rc; 566} 567 568static int encaps_sig_first_staged_cs_handler 569 (struct hl_device *hdev, struct hl_cs *cs) 570{ 571 struct hl_cs_compl *cs_cmpl = 572 container_of(cs->fence, 573 struct hl_cs_compl, base_fence); 574 struct hl_cs_encaps_sig_handle *encaps_sig_hdl; 575 struct hl_encaps_signals_mgr *mgr; 576 int rc = 0; 577 578 mgr = &cs->ctx->sig_mgr; 579 580 spin_lock(&mgr->lock); 581 encaps_sig_hdl = idr_find(&mgr->handles, cs->encaps_sig_hdl_id); 582 if (encaps_sig_hdl) { 583 /* 584 * Set handler CS sequence, 585 * the CS which contains the encapsulated signals. 586 */ 587 encaps_sig_hdl->cs_seq = cs->sequence; 588 /* store the handle and set encaps signal indication, 589 * to be used later in cs_do_release to put the last 590 * reference to encaps signals handlers. 591 */ 592 cs_cmpl->encaps_signals = true; 593 cs_cmpl->encaps_sig_hdl = encaps_sig_hdl; 594 595 /* set hw_sob pointer in completion object 596 * since it's used in cs_do_release flow to put 597 * refcount to sob 598 */ 599 cs_cmpl->hw_sob = encaps_sig_hdl->hw_sob; 600 cs_cmpl->sob_val = encaps_sig_hdl->pre_sob_val + 601 encaps_sig_hdl->count; 602 603 dev_dbg(hdev->dev, "CS seq (%llu) added to encaps signal handler id (%u), count(%u), qidx(%u), sob(%u), val(%u)\n", 604 cs->sequence, encaps_sig_hdl->id, 605 encaps_sig_hdl->count, 606 encaps_sig_hdl->q_idx, 607 cs_cmpl->hw_sob->sob_id, 608 cs_cmpl->sob_val); 609 610 } else { 611 dev_err(hdev->dev, "encaps handle id(%u) wasn't found!\n", 612 cs->encaps_sig_hdl_id); 613 rc = -EINVAL; 614 } 615 616 spin_unlock(&mgr->lock); 617 618 return rc; 619} 620 621/* 622 * hl_hw_queue_schedule_cs - schedule a command submission 623 * @cs: pointer to the CS 624 */ 625int hl_hw_queue_schedule_cs(struct hl_cs *cs) 626{ 627 enum hl_device_status status; 628 struct hl_cs_counters_atomic *cntr; 629 struct hl_ctx *ctx = cs->ctx; 630 struct hl_device *hdev = ctx->hdev; 631 struct hl_cs_job *job, *tmp; 632 struct hl_hw_queue *q; 633 int rc = 0, i, cq_cnt; 634 bool first_entry; 635 u32 max_queues; 636 637 cntr = &hdev->aggregated_cs_counters; 638 639 hdev->asic_funcs->hw_queues_lock(hdev); 640 641 if (!hl_device_operational(hdev, &status)) { 642 atomic64_inc(&cntr->device_in_reset_drop_cnt); 643 atomic64_inc(&ctx->cs_counters.device_in_reset_drop_cnt); 644 dev_err(hdev->dev, 645 "device is %s, CS rejected!\n", hdev->status[status]); 646 rc = -EPERM; 647 goto out; 648 } 649 650 max_queues = hdev->asic_prop.max_queues; 651 652 q = &hdev->kernel_queues[0]; 653 for (i = 0, cq_cnt = 0 ; i < max_queues ; i++, q++) { 654 if (cs->jobs_in_queue_cnt[i]) { 655 switch (q->queue_type) { 656 case QUEUE_TYPE_EXT: 657 rc = ext_queue_sanity_checks(hdev, q, 658 cs->jobs_in_queue_cnt[i], 659 cs_needs_completion(cs) ? 660 true : false); 661 break; 662 case QUEUE_TYPE_INT: 663 rc = int_queue_sanity_checks(hdev, q, 664 cs->jobs_in_queue_cnt[i]); 665 break; 666 case QUEUE_TYPE_HW: 667 rc = hw_queue_sanity_checks(hdev, q, 668 cs->jobs_in_queue_cnt[i]); 669 break; 670 default: 671 dev_err(hdev->dev, "Queue type %d is invalid\n", 672 q->queue_type); 673 rc = -EINVAL; 674 break; 675 } 676 677 if (rc) { 678 atomic64_inc( 679 &ctx->cs_counters.queue_full_drop_cnt); 680 atomic64_inc(&cntr->queue_full_drop_cnt); 681 goto unroll_cq_resv; 682 } 683 684 if (q->queue_type == QUEUE_TYPE_EXT) 685 cq_cnt++; 686 } 687 } 688 689 if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) { 690 rc = init_signal_wait_cs(cs); 691 if (rc) 692 goto unroll_cq_resv; 693 } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) { 694 rc = hdev->asic_funcs->collective_wait_init_cs(cs); 695 if (rc) 696 goto unroll_cq_resv; 697 } 698 699 rc = hdev->asic_funcs->pre_schedule_cs(cs); 700 if (rc) { 701 dev_err(hdev->dev, 702 "Failed in pre-submission operations of CS %d.%llu\n", 703 ctx->asid, cs->sequence); 704 goto unroll_cq_resv; 705 } 706 707 hdev->shadow_cs_queue[cs->sequence & 708 (hdev->asic_prop.max_pending_cs - 1)] = cs; 709 710 if (cs->encaps_signals && cs->staged_first) { 711 rc = encaps_sig_first_staged_cs_handler(hdev, cs); 712 if (rc) 713 goto unroll_cq_resv; 714 } 715 716 spin_lock(&hdev->cs_mirror_lock); 717 718 /* Verify staged CS exists and add to the staged list */ 719 if (cs->staged_cs && !cs->staged_first) { 720 struct hl_cs *staged_cs; 721 722 staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence); 723 if (!staged_cs) { 724 dev_err(hdev->dev, 725 "Cannot find staged submission sequence %llu", 726 cs->staged_sequence); 727 rc = -EINVAL; 728 goto unlock_cs_mirror; 729 } 730 731 if (is_staged_cs_last_exists(hdev, staged_cs)) { 732 dev_err(hdev->dev, 733 "Staged submission sequence %llu already submitted", 734 cs->staged_sequence); 735 rc = -EINVAL; 736 goto unlock_cs_mirror; 737 } 738 739 list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node); 740 741 /* update stream map of the first CS */ 742 if (hdev->supports_wait_for_multi_cs) 743 staged_cs->fence->stream_master_qid_map |= 744 cs->fence->stream_master_qid_map; 745 } 746 747 list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list); 748 749 /* Queue TDR if the CS is the first entry and if timeout is wanted */ 750 first_entry = list_first_entry(&hdev->cs_mirror_list, 751 struct hl_cs, mirror_node) == cs; 752 if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) && 753 first_entry && cs_needs_timeout(cs)) { 754 cs->tdr_active = true; 755 schedule_delayed_work(&cs->work_tdr, cs->timeout_jiffies); 756 757 } 758 759 spin_unlock(&hdev->cs_mirror_lock); 760 761 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node) 762 switch (job->queue_type) { 763 case QUEUE_TYPE_EXT: 764 ext_queue_schedule_job(job); 765 break; 766 case QUEUE_TYPE_INT: 767 int_queue_schedule_job(job); 768 break; 769 case QUEUE_TYPE_HW: 770 hw_queue_schedule_job(job); 771 break; 772 default: 773 break; 774 } 775 776 cs->submitted = true; 777 778 goto out; 779 780unlock_cs_mirror: 781 spin_unlock(&hdev->cs_mirror_lock); 782unroll_cq_resv: 783 q = &hdev->kernel_queues[0]; 784 for (i = 0 ; (i < max_queues) && (cq_cnt > 0) ; i++, q++) { 785 if ((q->queue_type == QUEUE_TYPE_EXT) && 786 (cs->jobs_in_queue_cnt[i])) { 787 atomic_t *free_slots = 788 &hdev->completion_queue[i].free_slots_cnt; 789 atomic_add(cs->jobs_in_queue_cnt[i], free_slots); 790 cq_cnt--; 791 } 792 } 793 794out: 795 hdev->asic_funcs->hw_queues_unlock(hdev); 796 797 return rc; 798} 799 800/* 801 * hl_hw_queue_inc_ci_kernel - increment ci for kernel's queue 802 * 803 * @hdev: pointer to hl_device structure 804 * @hw_queue_id: which queue to increment its ci 805 */ 806void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id) 807{ 808 struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id]; 809 810 atomic_inc(&q->ci); 811} 812 813static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q, 814 bool is_cpu_queue) 815{ 816 void *p; 817 int rc; 818 819 if (is_cpu_queue) 820 p = hl_cpu_accessible_dma_pool_alloc(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address); 821 else 822 p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address, 823 GFP_KERNEL | __GFP_ZERO); 824 if (!p) 825 return -ENOMEM; 826 827 q->kernel_address = p; 828 829 q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH, sizeof(struct hl_cs_job *), GFP_KERNEL); 830 if (!q->shadow_queue) { 831 dev_err(hdev->dev, 832 "Failed to allocate shadow queue for H/W queue %d\n", 833 q->hw_queue_id); 834 rc = -ENOMEM; 835 goto free_queue; 836 } 837 838 /* Make sure read/write pointers are initialized to start of queue */ 839 atomic_set(&q->ci, 0); 840 q->pi = 0; 841 842 return 0; 843 844free_queue: 845 if (is_cpu_queue) 846 hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address); 847 else 848 hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address, 849 q->bus_address); 850 851 return rc; 852} 853 854static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) 855{ 856 void *p; 857 858 p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id, 859 &q->bus_address, &q->int_queue_len); 860 if (!p) { 861 dev_err(hdev->dev, 862 "Failed to get base address for internal queue %d\n", 863 q->hw_queue_id); 864 return -EFAULT; 865 } 866 867 q->kernel_address = p; 868 q->pi = 0; 869 atomic_set(&q->ci, 0); 870 871 return 0; 872} 873 874static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) 875{ 876 return ext_and_cpu_queue_init(hdev, q, true); 877} 878 879static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) 880{ 881 return ext_and_cpu_queue_init(hdev, q, false); 882} 883 884static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) 885{ 886 void *p; 887 888 p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address, 889 GFP_KERNEL | __GFP_ZERO); 890 if (!p) 891 return -ENOMEM; 892 893 q->kernel_address = p; 894 895 /* Make sure read/write pointers are initialized to start of queue */ 896 atomic_set(&q->ci, 0); 897 q->pi = 0; 898 899 return 0; 900} 901 902static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx) 903{ 904 struct hl_sync_stream_properties *sync_stream_prop; 905 struct asic_fixed_properties *prop = &hdev->asic_prop; 906 struct hl_hw_sob *hw_sob; 907 int sob, reserved_mon_idx, queue_idx; 908 909 sync_stream_prop = &hdev->kernel_queues[q_idx].sync_stream_prop; 910 911 /* We use 'collective_mon_idx' as a running index in order to reserve 912 * monitors for collective master/slave queues. 913 * collective master queue gets 2 reserved monitors 914 * collective slave queue gets 1 reserved monitor 915 */ 916 if (hdev->kernel_queues[q_idx].collective_mode == 917 HL_COLLECTIVE_MASTER) { 918 reserved_mon_idx = hdev->collective_mon_idx; 919 920 /* reserve the first monitor for collective master queue */ 921 sync_stream_prop->collective_mstr_mon_id[0] = 922 prop->collective_first_mon + reserved_mon_idx; 923 924 /* reserve the second monitor for collective master queue */ 925 sync_stream_prop->collective_mstr_mon_id[1] = 926 prop->collective_first_mon + reserved_mon_idx + 1; 927 928 hdev->collective_mon_idx += HL_COLLECTIVE_RSVD_MSTR_MONS; 929 } else if (hdev->kernel_queues[q_idx].collective_mode == 930 HL_COLLECTIVE_SLAVE) { 931 reserved_mon_idx = hdev->collective_mon_idx++; 932 933 /* reserve a monitor for collective slave queue */ 934 sync_stream_prop->collective_slave_mon_id = 935 prop->collective_first_mon + reserved_mon_idx; 936 } 937 938 if (!hdev->kernel_queues[q_idx].supports_sync_stream) 939 return; 940 941 queue_idx = hdev->sync_stream_queue_idx++; 942 943 sync_stream_prop->base_sob_id = prop->sync_stream_first_sob + 944 (queue_idx * HL_RSVD_SOBS); 945 sync_stream_prop->base_mon_id = prop->sync_stream_first_mon + 946 (queue_idx * HL_RSVD_MONS); 947 sync_stream_prop->next_sob_val = 1; 948 sync_stream_prop->curr_sob_offset = 0; 949 950 for (sob = 0 ; sob < HL_RSVD_SOBS ; sob++) { 951 hw_sob = &sync_stream_prop->hw_sob[sob]; 952 hw_sob->hdev = hdev; 953 hw_sob->sob_id = sync_stream_prop->base_sob_id + sob; 954 hw_sob->sob_addr = 955 hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id); 956 hw_sob->q_idx = q_idx; 957 kref_init(&hw_sob->kref); 958 } 959} 960 961static void sync_stream_queue_reset(struct hl_device *hdev, u32 q_idx) 962{ 963 struct hl_sync_stream_properties *prop = 964 &hdev->kernel_queues[q_idx].sync_stream_prop; 965 966 /* 967 * In case we got here due to a stuck CS, the refcnt might be bigger 968 * than 1 and therefore we reset it. 969 */ 970 kref_init(&prop->hw_sob[prop->curr_sob_offset].kref); 971 prop->curr_sob_offset = 0; 972 prop->next_sob_val = 1; 973} 974 975/* 976 * queue_init - main initialization function for H/W queue object 977 * 978 * @hdev: pointer to hl_device device structure 979 * @q: pointer to hl_hw_queue queue structure 980 * @hw_queue_id: The id of the H/W queue 981 * 982 * Allocate dma-able memory for the queue and initialize fields 983 * Returns 0 on success 984 */ 985static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q, 986 u32 hw_queue_id) 987{ 988 int rc; 989 990 q->hw_queue_id = hw_queue_id; 991 992 switch (q->queue_type) { 993 case QUEUE_TYPE_EXT: 994 rc = ext_queue_init(hdev, q); 995 break; 996 case QUEUE_TYPE_INT: 997 rc = int_queue_init(hdev, q); 998 break; 999 case QUEUE_TYPE_CPU: 1000 rc = cpu_queue_init(hdev, q); 1001 break; 1002 case QUEUE_TYPE_HW: 1003 rc = hw_queue_init(hdev, q); 1004 break; 1005 case QUEUE_TYPE_NA: 1006 q->valid = 0; 1007 return 0; 1008 default: 1009 dev_crit(hdev->dev, "wrong queue type %d during init\n", 1010 q->queue_type); 1011 rc = -EINVAL; 1012 break; 1013 } 1014 1015 sync_stream_queue_init(hdev, q->hw_queue_id); 1016 1017 if (rc) 1018 return rc; 1019 1020 q->valid = 1; 1021 1022 return 0; 1023} 1024 1025/* 1026 * hw_queue_fini - destroy queue 1027 * 1028 * @hdev: pointer to hl_device device structure 1029 * @q: pointer to hl_hw_queue queue structure 1030 * 1031 * Free the queue memory 1032 */ 1033static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q) 1034{ 1035 if (!q->valid) 1036 return; 1037 1038 /* 1039 * If we arrived here, there are no jobs waiting on this queue 1040 * so we can safely remove it. 1041 * This is because this function can only called when: 1042 * 1. Either a context is deleted, which only can occur if all its 1043 * jobs were finished 1044 * 2. A context wasn't able to be created due to failure or timeout, 1045 * which means there are no jobs on the queue yet 1046 * 1047 * The only exception are the queues of the kernel context, but 1048 * if they are being destroyed, it means that the entire module is 1049 * being removed. If the module is removed, it means there is no open 1050 * user context. It also means that if a job was submitted by 1051 * the kernel driver (e.g. context creation), the job itself was 1052 * released by the kernel driver when a timeout occurred on its 1053 * Completion. Thus, we don't need to release it again. 1054 */ 1055 1056 if (q->queue_type == QUEUE_TYPE_INT) 1057 return; 1058 1059 kfree(q->shadow_queue); 1060 1061 if (q->queue_type == QUEUE_TYPE_CPU) 1062 hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address); 1063 else 1064 hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address, 1065 q->bus_address); 1066} 1067 1068int hl_hw_queues_create(struct hl_device *hdev) 1069{ 1070 struct asic_fixed_properties *asic = &hdev->asic_prop; 1071 struct hl_hw_queue *q; 1072 int i, rc, q_ready_cnt; 1073 1074 hdev->kernel_queues = kcalloc(asic->max_queues, 1075 sizeof(*hdev->kernel_queues), GFP_KERNEL); 1076 1077 if (!hdev->kernel_queues) { 1078 dev_err(hdev->dev, "Not enough memory for H/W queues\n"); 1079 return -ENOMEM; 1080 } 1081 1082 /* Initialize the H/W queues */ 1083 for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues; 1084 i < asic->max_queues ; i++, q_ready_cnt++, q++) { 1085 1086 q->queue_type = asic->hw_queues_props[i].type; 1087 q->supports_sync_stream = 1088 asic->hw_queues_props[i].supports_sync_stream; 1089 q->collective_mode = asic->hw_queues_props[i].collective_mode; 1090 rc = queue_init(hdev, q, i); 1091 if (rc) { 1092 dev_err(hdev->dev, 1093 "failed to initialize queue %d\n", i); 1094 goto release_queues; 1095 } 1096 } 1097 1098 return 0; 1099 1100release_queues: 1101 for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++) 1102 queue_fini(hdev, q); 1103 1104 kfree(hdev->kernel_queues); 1105 1106 return rc; 1107} 1108 1109void hl_hw_queues_destroy(struct hl_device *hdev) 1110{ 1111 struct hl_hw_queue *q; 1112 u32 max_queues = hdev->asic_prop.max_queues; 1113 int i; 1114 1115 for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) 1116 queue_fini(hdev, q); 1117 1118 kfree(hdev->kernel_queues); 1119} 1120 1121void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset) 1122{ 1123 struct hl_hw_queue *q; 1124 u32 max_queues = hdev->asic_prop.max_queues; 1125 int i; 1126 1127 for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) { 1128 if ((!q->valid) || 1129 ((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU))) 1130 continue; 1131 q->pi = 0; 1132 atomic_set(&q->ci, 0); 1133 1134 if (q->supports_sync_stream) 1135 sync_stream_queue_reset(hdev, q->hw_queue_id); 1136 } 1137} 1138