1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4#include <linux/interrupt.h>
5#include <linux/iopoll.h>
6#include <linux/device.h>
7
8#include "lima_device.h"
9#include "lima_mmu.h"
10#include "lima_vm.h"
11#include "lima_regs.h"
12
13#define mmu_write(reg, data) writel(data, ip->iomem + reg)
14#define mmu_read(reg) readl(ip->iomem + reg)
15
16#define lima_mmu_send_command(cmd, addr, val, cond)	     \
17({							     \
18	int __ret;					     \
19							     \
20	mmu_write(LIMA_MMU_COMMAND, cmd);		     \
21	__ret = readl_poll_timeout(ip->iomem + (addr), val,  \
22				  cond, 0, 100);	     \
23	if (__ret)					     \
24		dev_err(dev->dev,			     \
25			"mmu command %x timeout\n", cmd);    \
26	__ret;						     \
27})
28
29static irqreturn_t lima_mmu_irq_handler(int irq, void *data)
30{
31	struct lima_ip *ip = data;
32	struct lima_device *dev = ip->dev;
33	u32 status = mmu_read(LIMA_MMU_INT_STATUS);
34	struct lima_sched_pipe *pipe;
35
36	/* for shared irq case */
37	if (!status)
38		return IRQ_NONE;
39
40	if (status & LIMA_MMU_INT_PAGE_FAULT) {
41		u32 fault = mmu_read(LIMA_MMU_PAGE_FAULT_ADDR);
42
43		dev_err(dev->dev, "mmu page fault at 0x%x from bus id %d of type %s on %s\n",
44			fault, LIMA_MMU_STATUS_BUS_ID(status),
45			status & LIMA_MMU_STATUS_PAGE_FAULT_IS_WRITE ? "write" : "read",
46			lima_ip_name(ip));
47	}
48
49	if (status & LIMA_MMU_INT_READ_BUS_ERROR)
50		dev_err(dev->dev, "mmu %s irq bus error\n", lima_ip_name(ip));
51
52	/* mask all interrupts before resume */
53	mmu_write(LIMA_MMU_INT_MASK, 0);
54	mmu_write(LIMA_MMU_INT_CLEAR, status);
55
56	pipe = dev->pipe + (ip->id == lima_ip_gpmmu ? lima_pipe_gp : lima_pipe_pp);
57	lima_sched_pipe_mmu_error(pipe);
58
59	return IRQ_HANDLED;
60}
61
62static int lima_mmu_hw_init(struct lima_ip *ip)
63{
64	struct lima_device *dev = ip->dev;
65	int err;
66	u32 v;
67
68	mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET);
69	err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
70				    LIMA_MMU_DTE_ADDR, v, v == 0);
71	if (err)
72		return err;
73
74	mmu_write(LIMA_MMU_INT_MASK,
75		  LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
76	mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
77	return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
78				     LIMA_MMU_STATUS, v,
79				     v & LIMA_MMU_STATUS_PAGING_ENABLED);
80}
81
82int lima_mmu_resume(struct lima_ip *ip)
83{
84	if (ip->id == lima_ip_ppmmu_bcast)
85		return 0;
86
87	return lima_mmu_hw_init(ip);
88}
89
90void lima_mmu_suspend(struct lima_ip *ip)
91{
92
93}
94
95int lima_mmu_init(struct lima_ip *ip)
96{
97	struct lima_device *dev = ip->dev;
98	int err;
99
100	if (ip->id == lima_ip_ppmmu_bcast)
101		return 0;
102
103	mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
104	if (mmu_read(LIMA_MMU_DTE_ADDR) != 0xCAFEB000) {
105		dev_err(dev->dev, "mmu %s dte write test fail\n", lima_ip_name(ip));
106		return -EIO;
107	}
108
109	err = devm_request_irq(dev->dev, ip->irq, lima_mmu_irq_handler,
110			       IRQF_SHARED, lima_ip_name(ip), ip);
111	if (err) {
112		dev_err(dev->dev, "mmu %s fail to request irq\n", lima_ip_name(ip));
113		return err;
114	}
115
116	return lima_mmu_hw_init(ip);
117}
118
119void lima_mmu_fini(struct lima_ip *ip)
120{
121
122}
123
124void lima_mmu_flush_tlb(struct lima_ip *ip)
125{
126	mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
127}
128
129void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm)
130{
131	struct lima_device *dev = ip->dev;
132	u32 v;
133
134	lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_STALL,
135			      LIMA_MMU_STATUS, v,
136			      v & LIMA_MMU_STATUS_STALL_ACTIVE);
137
138	mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma);
139
140	/* flush the TLB */
141	mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
142
143	lima_mmu_send_command(LIMA_MMU_COMMAND_DISABLE_STALL,
144			      LIMA_MMU_STATUS, v,
145			      !(v & LIMA_MMU_STATUS_STALL_ACTIVE));
146}
147
148void lima_mmu_page_fault_resume(struct lima_ip *ip)
149{
150	struct lima_device *dev = ip->dev;
151	u32 status = mmu_read(LIMA_MMU_STATUS);
152	u32 v;
153
154	if (status & LIMA_MMU_STATUS_PAGE_FAULT_ACTIVE) {
155		dev_info(dev->dev, "mmu resume\n");
156
157		mmu_write(LIMA_MMU_INT_MASK, 0);
158		mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
159		lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
160				      LIMA_MMU_DTE_ADDR, v, v == 0);
161		mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
162		mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
163		lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
164				      LIMA_MMU_STATUS, v,
165				      v & LIMA_MMU_STATUS_PAGING_ENABLED);
166	}
167}
168