Lines Matching refs:pfdev
29 static int wait_ready(struct panfrost_device *pfdev, u32 as_nr)
36 ret = readl_relaxed_poll_timeout_atomic(pfdev->iomem + AS_STATUS(as_nr),
41 panfrost_device_schedule_reset(pfdev);
42 dev_err(pfdev->dev, "AS_ACTIVE bit stuck\n");
48 static int write_cmd(struct panfrost_device *pfdev, u32 as_nr, u32 cmd)
53 status = wait_ready(pfdev, as_nr);
55 mmu_write(pfdev, AS_COMMAND(as_nr), cmd);
60 static void lock_region(struct panfrost_device *pfdev, u32 as_nr,
90 mmu_write(pfdev, AS_LOCKADDR_LO(as_nr), lower_32_bits(region));
91 mmu_write(pfdev, AS_LOCKADDR_HI(as_nr), upper_32_bits(region));
92 write_cmd(pfdev, as_nr, AS_COMMAND_LOCK);
96 static int mmu_hw_do_operation_locked(struct panfrost_device *pfdev, int as_nr,
103 lock_region(pfdev, as_nr, iova, size);
106 write_cmd(pfdev, as_nr, op);
109 return wait_ready(pfdev, as_nr);
112 static int mmu_hw_do_operation(struct panfrost_device *pfdev,
118 spin_lock(&pfdev->as_lock);
119 ret = mmu_hw_do_operation_locked(pfdev, mmu->as, iova, size, op);
120 spin_unlock(&pfdev->as_lock);
124 static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
131 mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
133 mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), lower_32_bits(transtab));
134 mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), upper_32_bits(transtab));
139 mmu_write(pfdev, AS_MEMATTR_LO(as_nr), lower_32_bits(memattr));
140 mmu_write(pfdev, AS_MEMATTR_HI(as_nr), upper_32_bits(memattr));
142 write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
145 static void panfrost_mmu_disable(struct panfrost_device *pfdev, u32 as_nr)
147 mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
149 mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0);
150 mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0);
152 mmu_write(pfdev, AS_MEMATTR_LO(as_nr), 0);
153 mmu_write(pfdev, AS_MEMATTR_HI(as_nr), 0);
155 write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
158 u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
162 spin_lock(&pfdev->as_lock);
175 list_move(&mmu->list, &pfdev->as_lru_list);
177 if (pfdev->as_faulty_mask & mask) {
182 mmu_write(pfdev, MMU_INT_CLEAR, mask);
183 mmu_write(pfdev, MMU_INT_MASK, ~pfdev->as_faulty_mask);
184 pfdev->as_faulty_mask &= ~mask;
185 panfrost_mmu_enable(pfdev, mmu);
192 as = ffz(pfdev->as_alloc_mask);
193 if (!(BIT(as) & pfdev->features.as_present)) {
196 list_for_each_entry_reverse(lru_mmu, &pfdev->as_lru_list, list) {
200 WARN_ON(&lru_mmu->list == &pfdev->as_lru_list);
211 set_bit(as, &pfdev->as_alloc_mask);
213 list_add(&mmu->list, &pfdev->as_lru_list);
215 dev_dbg(pfdev->dev, "Assigned AS%d to mmu %p, alloc_mask=%lx", as, mmu, pfdev->as_alloc_mask);
217 panfrost_mmu_enable(pfdev, mmu);
220 spin_unlock(&pfdev->as_lock);
224 void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
230 void panfrost_mmu_reset(struct panfrost_device *pfdev)
234 spin_lock(&pfdev->as_lock);
236 pfdev->as_alloc_mask = 0;
237 pfdev->as_faulty_mask = 0;
239 list_for_each_entry_safe(mmu, mmu_tmp, &pfdev->as_lru_list, list) {
245 spin_unlock(&pfdev->as_lock);
247 mmu_write(pfdev, MMU_INT_CLEAR, ~0);
248 mmu_write(pfdev, MMU_INT_MASK, ~0);
272 static void panfrost_mmu_flush_range(struct panfrost_device *pfdev,
279 pm_runtime_get_noresume(pfdev->dev);
282 if (pm_runtime_active(pfdev->dev))
283 mmu_hw_do_operation(pfdev, mmu, iova, size, AS_COMMAND_FLUSH_PT);
285 pm_runtime_put_autosuspend(pfdev->dev);
288 static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu,
300 dev_dbg(pfdev->dev, "map: as=%d, iova=%llx, paddr=%lx, len=%zx", mmu->as, iova, paddr, len);
316 panfrost_mmu_flush_range(pfdev, mmu, start_iova, iova - start_iova);
326 struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
340 mmu_map_sg(pfdev, mapping->mmu, mapping->mmnode.start << PAGE_SHIFT,
351 struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
360 dev_dbg(pfdev->dev, "unmap: as=%d, iova=%llx, len=%zx",
377 panfrost_mmu_flush_range(pfdev, mapping->mmu,
403 addr_to_mapping(struct panfrost_device *pfdev, int as, u64 addr)
410 spin_lock(&pfdev->as_lock);
411 list_for_each_entry(mmu, &pfdev->as_lru_list, list) {
433 spin_unlock(&pfdev->as_lock);
439 static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
451 bomapping = addr_to_mapping(pfdev, as, addr);
457 dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)",
517 ret = dma_map_sgtable(pfdev->dev, sgt, DMA_BIDIRECTIONAL, 0);
521 mmu_map_sg(pfdev, bomapping->mmu, addr,
526 dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr);
550 struct panfrost_device *pfdev = mmu->pfdev;
552 spin_lock(&pfdev->as_lock);
554 pm_runtime_get_noresume(pfdev->dev);
555 if (pm_runtime_active(pfdev->dev))
556 panfrost_mmu_disable(pfdev, mmu->as);
557 pm_runtime_put_autosuspend(pfdev->dev);
559 clear_bit(mmu->as, &pfdev->as_alloc_mask);
560 clear_bit(mmu->as, &pfdev->as_in_use_mask);
563 spin_unlock(&pfdev->as_lock);
608 struct panfrost_mmu *panfrost_mmu_ctx_create(struct panfrost_device *pfdev)
616 mmu->pfdev = pfdev;
628 .ias = FIELD_GET(0xff, pfdev->features.mmu_features),
629 .oas = FIELD_GET(0xff00, pfdev->features.mmu_features),
630 .coherent_walk = pfdev->coherent,
632 .iommu_dev = pfdev->dev,
647 static const char *access_type_name(struct panfrost_device *pfdev,
652 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_AARCH64_MMU))
670 struct panfrost_device *pfdev = data;
672 if (!mmu_read(pfdev, MMU_INT_STAT))
675 mmu_write(pfdev, MMU_INT_MASK, 0);
681 struct panfrost_device *pfdev = data;
682 u32 status = mmu_read(pfdev, MMU_INT_RAWSTAT);
694 fault_status = mmu_read(pfdev, AS_FAULTSTATUS(as));
695 addr = mmu_read(pfdev, AS_FAULTADDRESS_LO(as));
696 addr |= (u64)mmu_read(pfdev, AS_FAULTADDRESS_HI(as)) << 32;
703 mmu_write(pfdev, MMU_INT_CLEAR, mask);
708 ret = panfrost_mmu_map_fault_addr(pfdev, as, addr);
712 dev_err(pfdev->dev,
725 access_type, access_type_name(pfdev, fault_status),
728 spin_lock(&pfdev->as_lock);
732 pfdev->as_faulty_mask |= mask;
735 panfrost_mmu_disable(pfdev, as);
736 spin_unlock(&pfdev->as_lock);
743 status = mmu_read(pfdev, MMU_INT_RAWSTAT) & ~pfdev->as_faulty_mask;
746 spin_lock(&pfdev->as_lock);
747 mmu_write(pfdev, MMU_INT_MASK, ~pfdev->as_faulty_mask);
748 spin_unlock(&pfdev->as_lock);
753 int panfrost_mmu_init(struct panfrost_device *pfdev)
757 irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu");
761 err = devm_request_threaded_irq(pfdev->dev, irq,
765 pfdev);
768 dev_err(pfdev->dev, "failed to request mmu irq");
775 void panfrost_mmu_fini(struct panfrost_device *pfdev)
777 mmu_write(pfdev, MMU_INT_MASK, 0);