1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2019 HiSilicon Limited. */
3#ifndef HISI_ACC_QM_H
4#define HISI_ACC_QM_H
5
6#include <linux/bitfield.h>
7#include <linux/iopoll.h>
8#include <linux/module.h>
9#include <linux/pci.h>
10
11#define QM_QNUM_V1			4096
12#define QM_QNUM_V2			1024
13#define QM_MAX_VFS_NUM_V2		63
14
15/* qm user domain */
16#define QM_ARUSER_M_CFG_1		0x100088
17#define AXUSER_SNOOP_ENABLE		BIT(30)
18#define AXUSER_CMD_TYPE			GENMASK(14, 12)
19#define AXUSER_CMD_SMMU_NORMAL		1
20#define AXUSER_NS			BIT(6)
21#define AXUSER_NO			BIT(5)
22#define AXUSER_FP			BIT(4)
23#define AXUSER_SSV			BIT(0)
24#define AXUSER_BASE			(AXUSER_SNOOP_ENABLE |		\
25					FIELD_PREP(AXUSER_CMD_TYPE,	\
26					AXUSER_CMD_SMMU_NORMAL) |	\
27					AXUSER_NS | AXUSER_NO | AXUSER_FP)
28#define QM_ARUSER_M_CFG_ENABLE		0x100090
29#define ARUSER_M_CFG_ENABLE		0xfffffffe
30#define QM_AWUSER_M_CFG_1		0x100098
31#define QM_AWUSER_M_CFG_ENABLE		0x1000a0
32#define AWUSER_M_CFG_ENABLE		0xfffffffe
33#define QM_WUSER_M_CFG_ENABLE		0x1000a8
34#define WUSER_M_CFG_ENABLE		0xffffffff
35
36/* qm cache */
37#define QM_CACHE_CTL			0x100050
38#define SQC_CACHE_ENABLE		BIT(0)
39#define CQC_CACHE_ENABLE		BIT(1)
40#define SQC_CACHE_WB_ENABLE		BIT(4)
41#define SQC_CACHE_WB_THRD		GENMASK(10, 5)
42#define CQC_CACHE_WB_ENABLE		BIT(11)
43#define CQC_CACHE_WB_THRD		GENMASK(17, 12)
44#define QM_AXI_M_CFG			0x1000ac
45#define AXI_M_CFG			0xffff
46#define QM_AXI_M_CFG_ENABLE		0x1000b0
47#define AM_CFG_SINGLE_PORT_MAX_TRANS	0x300014
48#define AXI_M_CFG_ENABLE		0xffffffff
49#define QM_PEH_AXUSER_CFG		0x1000cc
50#define QM_PEH_AXUSER_CFG_ENABLE	0x1000d0
51#define PEH_AXUSER_CFG			0x401001
52#define PEH_AXUSER_CFG_ENABLE		0xffffffff
53
54#define QM_DFX_MB_CNT_VF		0x104010
55#define QM_DFX_DB_CNT_VF		0x104020
56#define QM_DFX_SQE_CNT_VF_SQN		0x104030
57#define QM_DFX_CQE_CNT_VF_CQN		0x104040
58#define QM_DFX_QN_SHIFT			16
59#define CURRENT_FUN_MASK		GENMASK(5, 0)
60#define CURRENT_Q_MASK			GENMASK(31, 16)
61
62#define QM_AXI_RRESP			BIT(0)
63#define QM_AXI_BRESP			BIT(1)
64#define QM_ECC_MBIT			BIT(2)
65#define QM_ECC_1BIT			BIT(3)
66#define QM_ACC_GET_TASK_TIMEOUT		BIT(4)
67#define QM_ACC_DO_TASK_TIMEOUT		BIT(5)
68#define QM_ACC_WB_NOT_READY_TIMEOUT	BIT(6)
69#define QM_SQ_CQ_VF_INVALID		BIT(7)
70#define QM_CQ_VF_INVALID		BIT(8)
71#define QM_SQ_VF_INVALID		BIT(9)
72#define QM_DB_TIMEOUT			BIT(10)
73#define QM_OF_FIFO_OF			BIT(11)
74#define QM_DB_RANDOM_INVALID		BIT(12)
75
76#define QM_BASE_NFE	(QM_AXI_RRESP | QM_AXI_BRESP | QM_ECC_MBIT | \
77			 QM_ACC_GET_TASK_TIMEOUT | QM_DB_TIMEOUT | \
78			 QM_OF_FIFO_OF | QM_DB_RANDOM_INVALID)
79#define QM_BASE_CE			QM_ECC_1BIT
80
81#define QM_Q_DEPTH			1024
82#define QM_MIN_QNUM                     2
83#define HISI_ACC_SGL_SGE_NR_MAX		255
84
85/* page number for queue file region */
86#define QM_DOORBELL_PAGE_NR		1
87
88enum qm_stop_reason {
89	QM_NORMAL,
90	QM_SOFT_RESET,
91	QM_FLR,
92};
93
94enum qm_state {
95	QM_INIT = 0,
96	QM_START,
97	QM_CLOSE,
98	QM_STOP,
99};
100
101enum qp_state {
102	QP_INIT = 1,
103	QP_START,
104	QP_STOP,
105	QP_CLOSE,
106};
107
108enum qm_hw_ver {
109	QM_HW_UNKNOWN = -1,
110	QM_HW_V1 = 0x20,
111	QM_HW_V2 = 0x21,
112	QM_HW_V3 = 0x30,
113};
114
115enum qm_fun_type {
116	QM_HW_PF,
117	QM_HW_VF,
118};
119
120enum qm_debug_file {
121	CURRENT_Q,
122	CLEAR_ENABLE,
123	DEBUG_FILE_NUM,
124};
125
126struct qm_dfx {
127	atomic64_t err_irq_cnt;
128	atomic64_t aeq_irq_cnt;
129	atomic64_t abnormal_irq_cnt;
130	atomic64_t create_qp_err_cnt;
131	atomic64_t mb_err_cnt;
132};
133
134struct debugfs_file {
135	enum qm_debug_file index;
136	struct mutex lock;
137	struct qm_debug *debug;
138};
139
140struct qm_debug {
141	u32 curr_qm_qp_num;
142	u32 sqe_mask_offset;
143	u32 sqe_mask_len;
144	struct qm_dfx dfx;
145	struct dentry *debug_root;
146	struct dentry *qm_d;
147	struct debugfs_file files[DEBUG_FILE_NUM];
148};
149
150struct qm_dma {
151	void *va;
152	dma_addr_t dma;
153	size_t size;
154};
155
156struct hisi_qm_status {
157	u32 eq_head;
158	bool eqc_phase;
159	u32 aeq_head;
160	bool aeqc_phase;
161	atomic_t flags;
162	int stop_reason;
163};
164
165struct hisi_qm;
166
167struct hisi_qm_err_info {
168	char *acpi_rst;
169	u32 msi_wr_port;
170	u32 ecc_2bits_mask;
171	u32 ce;
172	u32 nfe;
173	u32 fe;
174};
175
176struct hisi_qm_err_status {
177	u32 is_qm_ecc_mbit;
178	u32 is_dev_ecc_mbit;
179};
180
181struct hisi_qm_err_ini {
182	int (*hw_init)(struct hisi_qm *qm);
183	void (*hw_err_enable)(struct hisi_qm *qm);
184	void (*hw_err_disable)(struct hisi_qm *qm);
185	u32 (*get_dev_hw_err_status)(struct hisi_qm *qm);
186	void (*clear_dev_hw_err_status)(struct hisi_qm *qm, u32 err_sts);
187	void (*open_axi_master_ooo)(struct hisi_qm *qm);
188	void (*close_axi_master_ooo)(struct hisi_qm *qm);
189	void (*log_dev_hw_err)(struct hisi_qm *qm, u32 err_sts);
190	struct hisi_qm_err_info err_info;
191};
192
193struct hisi_qm_list {
194	struct mutex lock;
195	struct list_head list;
196	int (*register_to_crypto)(void);
197	void (*unregister_from_crypto)(void);
198};
199
200struct hisi_qm {
201	enum qm_hw_ver ver;
202	enum qm_fun_type fun_type;
203	const char *dev_name;
204	struct pci_dev *pdev;
205	void __iomem *io_base;
206	u32 sqe_size;
207	u32 qp_base;
208	u32 qp_num;
209	u32 qp_in_used;
210	u32 ctrl_qp_num;
211	u32 vfs_num;
212	struct list_head list;
213	struct hisi_qm_list *qm_list;
214
215	struct qm_dma qdma;
216	struct qm_sqc *sqc;
217	struct qm_cqc *cqc;
218	struct qm_eqe *eqe;
219	struct qm_aeqe *aeqe;
220	dma_addr_t sqc_dma;
221	dma_addr_t cqc_dma;
222	dma_addr_t eqe_dma;
223	dma_addr_t aeqe_dma;
224
225	struct hisi_qm_status status;
226	const struct hisi_qm_err_ini *err_ini;
227	struct hisi_qm_err_status err_status;
228	unsigned long reset_flag;
229
230	struct rw_semaphore qps_lock;
231	struct idr qp_idr;
232	struct hisi_qp *qp_array;
233
234	struct mutex mailbox_lock;
235
236	const struct hisi_qm_hw_ops *ops;
237
238	struct qm_debug debug;
239
240	u32 error_mask;
241
242	struct workqueue_struct *wq;
243	struct work_struct work;
244	struct work_struct rst_work;
245
246	const char *algs;
247	bool use_sva;
248	bool is_frozen;
249	resource_size_t phys_base;
250	resource_size_t phys_size;
251	struct uacce_device *uacce;
252};
253
254struct hisi_qp_status {
255	atomic_t used;
256	u16 sq_tail;
257	u16 cq_head;
258	bool cqc_phase;
259	atomic_t flags;
260};
261
262struct hisi_qp_ops {
263	int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm);
264};
265
266struct hisi_qp {
267	u32 qp_id;
268	u8 alg_type;
269	u8 req_type;
270
271	struct qm_dma qdma;
272	void *sqe;
273	struct qm_cqe *cqe;
274	dma_addr_t sqe_dma;
275	dma_addr_t cqe_dma;
276
277	struct hisi_qp_status qp_status;
278	struct hisi_qp_ops *hw_ops;
279	void *qp_ctx;
280	void (*req_cb)(struct hisi_qp *qp, void *data);
281	void (*event_cb)(struct hisi_qp *qp);
282
283	struct hisi_qm *qm;
284	bool is_resetting;
285	u16 pasid;
286	struct uacce_queue *uacce_q;
287};
288
289static inline int q_num_set(const char *val, const struct kernel_param *kp,
290			    unsigned int device)
291{
292	struct pci_dev *pdev;
293	u32 n, q_num;
294	int ret;
295
296	if (!val)
297		return -EINVAL;
298
299	pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, device, NULL);
300	if (!pdev) {
301		q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2);
302		pr_info("No device found currently, suppose queue number is %d\n",
303			q_num);
304	} else {
305		if (pdev->revision == QM_HW_V1)
306			q_num = QM_QNUM_V1;
307		else
308			q_num = QM_QNUM_V2;
309
310		pci_dev_put(pdev);
311	}
312
313	ret = kstrtou32(val, 10, &n);
314	if (ret || n < QM_MIN_QNUM || n > q_num)
315		return -EINVAL;
316
317	return param_set_int(val, kp);
318}
319
320static inline int vfs_num_set(const char *val, const struct kernel_param *kp)
321{
322	u32 n;
323	int ret;
324
325	if (!val)
326		return -EINVAL;
327
328	ret = kstrtou32(val, 10, &n);
329	if (ret < 0)
330		return ret;
331
332	if (n > QM_MAX_VFS_NUM_V2)
333		return -EINVAL;
334
335	return param_set_int(val, kp);
336}
337
338static inline void hisi_qm_init_list(struct hisi_qm_list *qm_list)
339{
340	INIT_LIST_HEAD(&qm_list->list);
341	mutex_init(&qm_list->lock);
342}
343
344int hisi_qm_init(struct hisi_qm *qm);
345void hisi_qm_uninit(struct hisi_qm *qm);
346int hisi_qm_start(struct hisi_qm *qm);
347int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r);
348struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type);
349int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg);
350int hisi_qm_stop_qp(struct hisi_qp *qp);
351void hisi_qm_release_qp(struct hisi_qp *qp);
352int hisi_qp_send(struct hisi_qp *qp, const void *msg);
353int hisi_qm_get_free_qp_num(struct hisi_qm *qm);
354int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number);
355int hisi_qm_debug_init(struct hisi_qm *qm);
356enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev);
357void hisi_qm_debug_regs_clear(struct hisi_qm *qm);
358int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs);
359int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen);
360int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs);
361void hisi_qm_dev_err_init(struct hisi_qm *qm);
362void hisi_qm_dev_err_uninit(struct hisi_qm *qm);
363pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,
364					  pci_channel_state_t state);
365pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev);
366void hisi_qm_reset_prepare(struct pci_dev *pdev);
367void hisi_qm_reset_done(struct pci_dev *pdev);
368
369struct hisi_acc_sgl_pool;
370struct hisi_acc_hw_sgl *hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
371	struct scatterlist *sgl, struct hisi_acc_sgl_pool *pool,
372	u32 index, dma_addr_t *hw_sgl_dma);
373void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
374			   struct hisi_acc_hw_sgl *hw_sgl);
375struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,
376						   u32 count, u32 sge_nr);
377void hisi_acc_free_sgl_pool(struct device *dev,
378			    struct hisi_acc_sgl_pool *pool);
379int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,
380			   u8 alg_type, int node, struct hisi_qp **qps);
381void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num);
382void hisi_qm_dev_shutdown(struct pci_dev *pdev);
383void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list);
384int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list);
385void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list);
386#endif
387