Lines Matching defs:pfdev
88 static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num)
91 struct panfrost_job_slot *js = pfdev->js;
97 fence->dev = pfdev->ddev;
119 (job->pfdev->features.nr_core_groups == 2))
121 if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))
128 static void panfrost_job_write_affinity(struct panfrost_device *pfdev,
139 affinity = pfdev->features.shader_present;
141 job_write(pfdev, JS_AFFINITY_NEXT_LO(js), lower_32_bits(affinity));
142 job_write(pfdev, JS_AFFINITY_NEXT_HI(js), upper_32_bits(affinity));
150 if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
157 panfrost_dequeue_job(struct panfrost_device *pfdev, int slot)
159 struct panfrost_job *job = pfdev->jobs[slot][0];
162 pfdev->jobs[slot][0] = pfdev->jobs[slot][1];
163 pfdev->jobs[slot][1] = NULL;
169 panfrost_enqueue_job(struct panfrost_device *pfdev, int slot,
175 if (!pfdev->jobs[slot][0]) {
176 pfdev->jobs[slot][0] = job;
180 WARN_ON(pfdev->jobs[slot][1]);
181 pfdev->jobs[slot][1] = job;
183 panfrost_get_job_chain_flag(pfdev->jobs[slot][0]));
189 struct panfrost_device *pfdev = job->pfdev;
195 panfrost_devfreq_record_busy(&pfdev->pfdevfreq);
197 ret = pm_runtime_get_sync(pfdev->dev);
201 if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) {
205 cfg = panfrost_mmu_as_get(pfdev, job->mmu);
207 job_write(pfdev, JS_HEAD_NEXT_LO(js), lower_32_bits(jc_head));
208 job_write(pfdev, JS_HEAD_NEXT_HI(js), upper_32_bits(jc_head));
210 panfrost_job_write_affinity(pfdev, job->requirements, js);
219 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
222 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649))
225 job_write(pfdev, JS_CONFIG_NEXT(js), cfg);
227 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
228 job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);
232 spin_lock(&pfdev->js->job_lock);
233 subslot = panfrost_enqueue_job(pfdev, js, job);
235 if (!atomic_read(&pfdev->reset.pending)) {
236 job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);
237 dev_dbg(pfdev->dev,
241 spin_unlock(&pfdev->js->job_lock);
277 struct panfrost_device *pfdev = job->pfdev;
286 mutex_lock(&pfdev->sched_lock);
294 mutex_unlock(&pfdev->sched_lock);
302 mutex_unlock(&pfdev->sched_lock);
360 struct panfrost_device *pfdev = job->pfdev;
373 fence = panfrost_fence_create(pfdev, slot);
386 void panfrost_job_enable_interrupts(struct panfrost_device *pfdev)
395 job_write(pfdev, JOB_INT_CLEAR, irq_mask);
396 job_write(pfdev, JOB_INT_MASK, irq_mask);
399 static void panfrost_job_handle_err(struct panfrost_device *pfdev,
403 u32 js_status = job_read(pfdev, JS_STATUS(js));
408 dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x",
410 job_read(pfdev, JS_HEAD_LO(js)),
411 job_read(pfdev, JS_TAIL_LO(js)));
413 dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",
415 job_read(pfdev, JS_HEAD_LO(js)),
416 job_read(pfdev, JS_TAIL_LO(js)));
421 job->jc = job_read(pfdev, JS_TAIL_LO(js)) |
422 ((u64)job_read(pfdev, JS_TAIL_HI(js)) << 32);
439 panfrost_mmu_as_put(pfdev, job->mmu);
440 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
445 pm_runtime_put_autosuspend(pfdev->dev);
447 if (panfrost_exception_needs_reset(pfdev, js_status)) {
448 atomic_set(&pfdev->reset.pending, 1);
449 drm_sched_fault(&pfdev->js->queue[js].sched);
453 static void panfrost_job_handle_done(struct panfrost_device *pfdev,
460 panfrost_mmu_as_put(pfdev, job->mmu);
461 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
464 pm_runtime_put_autosuspend(pfdev->dev);
467 static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status)
484 done[j][1] = panfrost_dequeue_job(pfdev, j);
486 done[j][0] = panfrost_dequeue_job(pfdev, j);
494 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);
495 failed[j] = panfrost_dequeue_job(pfdev, j);
511 job_write(pfdev, JOB_INT_CLEAR, status);
513 js_state |= job_read(pfdev, JOB_INT_JS_STATE) & js_state_mask;
515 status = job_read(pfdev, JOB_INT_RAWSTAT);
524 panfrost_job_handle_err(pfdev, failed[j], j);
525 } else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) {
530 * are inactive, but one job remains in pfdev->jobs[j],
536 done[j][0] = panfrost_dequeue_job(pfdev, j);
538 done[j][1] = panfrost_dequeue_job(pfdev, j);
542 panfrost_job_handle_done(pfdev, done[j][i]);
552 if (!failed[j] || !pfdev->jobs[j][0])
555 if (pfdev->jobs[j][0]->jc == 0) {
557 struct panfrost_job *canceled = panfrost_dequeue_job(pfdev, j);
560 panfrost_job_handle_done(pfdev, canceled);
561 } else if (!atomic_read(&pfdev->reset.pending)) {
563 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_START);
568 static void panfrost_job_handle_irqs(struct panfrost_device *pfdev)
570 u32 status = job_read(pfdev, JOB_INT_RAWSTAT);
573 pm_runtime_mark_last_busy(pfdev->dev);
575 spin_lock(&pfdev->js->job_lock);
576 panfrost_job_handle_irq(pfdev, status);
577 spin_unlock(&pfdev->js->job_lock);
578 status = job_read(pfdev, JOB_INT_RAWSTAT);
582 static u32 panfrost_active_slots(struct panfrost_device *pfdev,
590 rawstat = job_read(pfdev, JOB_INT_RAWSTAT);
604 panfrost_reset(struct panfrost_device *pfdev,
612 if (!atomic_read(&pfdev->reset.pending))
629 drm_sched_stop(&pfdev->js->queue[i].sched, bad);
639 job_write(pfdev, JOB_INT_MASK, 0);
640 synchronize_irq(pfdev->js->irq);
644 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);
645 job_write(pfdev, JS_COMMAND(i), JS_COMMAND_SOFT_STOP);
649 ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state,
650 !panfrost_active_slots(pfdev, &js_state_mask, js_state),
654 dev_err(pfdev->dev, "Soft-stop failed\n");
657 panfrost_job_handle_irqs(pfdev);
664 spin_lock(&pfdev->js->job_lock);
666 for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) {
667 pm_runtime_put_noidle(pfdev->dev);
668 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
671 memset(pfdev->jobs, 0, sizeof(pfdev->jobs));
672 spin_unlock(&pfdev->js->job_lock);
675 panfrost_device_reset(pfdev);
680 job_write(pfdev, JOB_INT_MASK, 0);
683 atomic_set(&pfdev->reset.pending, 0);
694 drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);
699 drm_sched_start(&pfdev->js->queue[i].sched, true);
702 job_write(pfdev, JOB_INT_MASK,
713 struct panfrost_device *pfdev = job->pfdev;
732 synchronize_irq(pfdev->js->irq);
735 dev_warn(pfdev->dev, "unexpectedly high interrupt latency\n");
739 dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p",
741 job_read(pfdev, JS_CONFIG(js)),
742 job_read(pfdev, JS_STATUS(js)),
743 job_read(pfdev, JS_HEAD_LO(js)),
744 job_read(pfdev, JS_TAIL_LO(js)),
749 atomic_set(&pfdev->reset.pending, 1);
750 panfrost_reset(pfdev, sched_job);
757 struct panfrost_device *pfdev;
759 pfdev = container_of(work, struct panfrost_device, reset.work);
760 panfrost_reset(pfdev, NULL);
771 struct panfrost_device *pfdev = data;
773 panfrost_job_handle_irqs(pfdev);
774 job_write(pfdev, JOB_INT_MASK,
782 struct panfrost_device *pfdev = data;
783 u32 status = job_read(pfdev, JOB_INT_STAT);
788 job_write(pfdev, JOB_INT_MASK, 0);
792 int panfrost_job_init(struct panfrost_device *pfdev)
802 if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
805 pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);
809 INIT_WORK(&pfdev->reset.work, panfrost_reset_work);
812 js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");
816 ret = devm_request_threaded_irq(pfdev->dev, js->irq,
820 pfdev);
822 dev_err(pfdev->dev, "failed to request job irq");
826 pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0);
827 if (!pfdev->reset.wq)
837 pfdev->reset.wq,
838 NULL, "pan_js", pfdev->dev);
840 dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);
845 panfrost_job_enable_interrupts(pfdev);
853 destroy_workqueue(pfdev->reset.wq);
857 void panfrost_job_fini(struct panfrost_device *pfdev)
859 struct panfrost_job_slot *js = pfdev->js;
862 job_write(pfdev, JOB_INT_MASK, 0);
868 cancel_work_sync(&pfdev->reset.work);
869 destroy_workqueue(pfdev->reset.wq);
874 struct panfrost_device *pfdev = panfrost_priv->pfdev;
875 struct panfrost_job_slot *js = pfdev->js;
892 struct panfrost_device *pfdev = panfrost_priv->pfdev;
899 spin_lock(&pfdev->js->job_lock);
904 for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) {
905 struct panfrost_job *job = pfdev->jobs[i][j];
913 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);
920 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
928 job_write(pfdev, JS_COMMAND(i), cmd);
931 spin_unlock(&pfdev->js->job_lock);
934 int panfrost_job_is_idle(struct panfrost_device *pfdev)
936 struct panfrost_job_slot *js = pfdev->js;