1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Intel IFC VF NIC driver for virtio dataplane offloading
4 *
5 * Copyright (C) 2020 Intel Corporation.
6 *
7 * Author: Zhu Lingshan <lingshan.zhu@intel.com>
8 *
9 */
10
11#include "ifcvf_base.h"
12
13u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector)
14{
15	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
16
17	vp_iowrite16(qid, &cfg->queue_select);
18	vp_iowrite16(vector, &cfg->queue_msix_vector);
19
20	return vp_ioread16(&cfg->queue_msix_vector);
21}
22
23u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector)
24{
25	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
26
27	vp_iowrite16(vector,  &cfg->msix_config);
28
29	return vp_ioread16(&cfg->msix_config);
30}
31
32static void __iomem *get_cap_addr(struct ifcvf_hw *hw,
33				  struct virtio_pci_cap *cap)
34{
35	u32 length, offset;
36	u8 bar;
37
38	length = le32_to_cpu(cap->length);
39	offset = le32_to_cpu(cap->offset);
40	bar = cap->bar;
41
42	if (bar >= IFCVF_PCI_MAX_RESOURCE) {
43		IFCVF_DBG(hw->pdev,
44			  "Invalid bar number %u to get capabilities\n", bar);
45		return NULL;
46	}
47
48	if (offset + length > pci_resource_len(hw->pdev, bar)) {
49		IFCVF_DBG(hw->pdev,
50			  "offset(%u) + len(%u) overflows bar%u's capability\n",
51			  offset, length, bar);
52		return NULL;
53	}
54
55	return hw->base[bar] + offset;
56}
57
58static int ifcvf_read_config_range(struct pci_dev *dev,
59				   uint32_t *val, int size, int where)
60{
61	int ret, i;
62
63	for (i = 0; i < size; i += 4) {
64		ret = pci_read_config_dword(dev, where + i, val + i / 4);
65		if (ret < 0)
66			return ret;
67	}
68
69	return 0;
70}
71
72static u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid)
73{
74	u16 queue_size;
75
76	vp_iowrite16(qid, &hw->common_cfg->queue_select);
77	queue_size = vp_ioread16(&hw->common_cfg->queue_size);
78
79	return queue_size;
80}
81
82/* This function returns the max allowed safe size for
83 * all virtqueues. It is the minimal size that can be
84 * suppprted by all virtqueues.
85 */
86u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw)
87{
88	u16 queue_size, max_size, qid;
89
90	max_size = ifcvf_get_vq_size(hw, 0);
91	for (qid = 1; qid < hw->nr_vring; qid++) {
92		queue_size = ifcvf_get_vq_size(hw, qid);
93		/* 0 means the queue is unavailable */
94		if (!queue_size)
95			continue;
96
97		max_size = min(queue_size, max_size);
98	}
99
100	return max_size;
101}
102
103int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
104{
105	struct virtio_pci_cap cap;
106	u16 notify_off;
107	int ret;
108	u8 pos;
109	u32 i;
110
111	ret = pci_read_config_byte(pdev, PCI_CAPABILITY_LIST, &pos);
112	if (ret < 0) {
113		IFCVF_ERR(pdev, "Failed to read PCI capability list\n");
114		return -EIO;
115	}
116	hw->pdev = pdev;
117
118	while (pos) {
119		ret = ifcvf_read_config_range(pdev, (u32 *)&cap,
120					      sizeof(cap), pos);
121		if (ret < 0) {
122			IFCVF_ERR(pdev,
123				  "Failed to get PCI capability at %x\n", pos);
124			break;
125		}
126
127		if (cap.cap_vndr != PCI_CAP_ID_VNDR)
128			goto next;
129
130		switch (cap.cfg_type) {
131		case VIRTIO_PCI_CAP_COMMON_CFG:
132			hw->common_cfg = get_cap_addr(hw, &cap);
133			IFCVF_DBG(pdev, "hw->common_cfg = %p\n",
134				  hw->common_cfg);
135			break;
136		case VIRTIO_PCI_CAP_NOTIFY_CFG:
137			pci_read_config_dword(pdev, pos + sizeof(cap),
138					      &hw->notify_off_multiplier);
139			hw->notify_bar = cap.bar;
140			hw->notify_base = get_cap_addr(hw, &cap);
141			hw->notify_base_pa = pci_resource_start(pdev, cap.bar) +
142					le32_to_cpu(cap.offset);
143			IFCVF_DBG(pdev, "hw->notify_base = %p\n",
144				  hw->notify_base);
145			break;
146		case VIRTIO_PCI_CAP_ISR_CFG:
147			hw->isr = get_cap_addr(hw, &cap);
148			IFCVF_DBG(pdev, "hw->isr = %p\n", hw->isr);
149			break;
150		case VIRTIO_PCI_CAP_DEVICE_CFG:
151			hw->dev_cfg = get_cap_addr(hw, &cap);
152			hw->cap_dev_config_size = le32_to_cpu(cap.length);
153			IFCVF_DBG(pdev, "hw->dev_cfg = %p\n", hw->dev_cfg);
154			break;
155		}
156
157next:
158		pos = cap.cap_next;
159	}
160
161	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
162	    hw->isr == NULL || hw->dev_cfg == NULL) {
163		IFCVF_ERR(pdev, "Incomplete PCI capabilities\n");
164		return -EIO;
165	}
166
167	hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues);
168	hw->vring = kzalloc(sizeof(struct vring_info) * hw->nr_vring, GFP_KERNEL);
169	if (!hw->vring)
170		return -ENOMEM;
171
172	for (i = 0; i < hw->nr_vring; i++) {
173		vp_iowrite16(i, &hw->common_cfg->queue_select);
174		notify_off = vp_ioread16(&hw->common_cfg->queue_notify_off);
175		hw->vring[i].notify_addr = hw->notify_base +
176			notify_off * hw->notify_off_multiplier;
177		hw->vring[i].notify_pa = hw->notify_base_pa +
178			notify_off * hw->notify_off_multiplier;
179		hw->vring[i].irq = -EINVAL;
180	}
181
182	hw->lm_cfg = hw->base[IFCVF_LM_BAR];
183
184	IFCVF_DBG(pdev,
185		  "PCI capability mapping: common cfg: %p, notify base: %p\n, isr cfg: %p, device cfg: %p, multiplier: %u\n",
186		  hw->common_cfg, hw->notify_base, hw->isr,
187		  hw->dev_cfg, hw->notify_off_multiplier);
188
189	hw->vqs_reused_irq = -EINVAL;
190	hw->config_irq = -EINVAL;
191
192	return 0;
193}
194
195u8 ifcvf_get_status(struct ifcvf_hw *hw)
196{
197	return vp_ioread8(&hw->common_cfg->device_status);
198}
199
200void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)
201{
202	vp_iowrite8(status, &hw->common_cfg->device_status);
203}
204
205void ifcvf_reset(struct ifcvf_hw *hw)
206{
207	ifcvf_set_status(hw, 0);
208	while (ifcvf_get_status(hw))
209		msleep(1);
210}
211
212u64 ifcvf_get_hw_features(struct ifcvf_hw *hw)
213{
214	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
215	u32 features_lo, features_hi;
216	u64 features;
217
218	vp_iowrite32(0, &cfg->device_feature_select);
219	features_lo = vp_ioread32(&cfg->device_feature);
220
221	vp_iowrite32(1, &cfg->device_feature_select);
222	features_hi = vp_ioread32(&cfg->device_feature);
223
224	features = ((u64)features_hi << 32) | features_lo;
225
226	return features;
227}
228
229/* return provisioned vDPA dev features */
230u64 ifcvf_get_dev_features(struct ifcvf_hw *hw)
231{
232	return hw->dev_features;
233}
234
235u64 ifcvf_get_driver_features(struct ifcvf_hw *hw)
236{
237	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
238	u32 features_lo, features_hi;
239	u64 features;
240
241	vp_iowrite32(0, &cfg->device_feature_select);
242	features_lo = vp_ioread32(&cfg->guest_feature);
243
244	vp_iowrite32(1, &cfg->device_feature_select);
245	features_hi = vp_ioread32(&cfg->guest_feature);
246
247	features = ((u64)features_hi << 32) | features_lo;
248
249	return features;
250}
251
252int ifcvf_verify_min_features(struct ifcvf_hw *hw, u64 features)
253{
254	if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) {
255		IFCVF_ERR(hw->pdev, "VIRTIO_F_ACCESS_PLATFORM is not negotiated\n");
256		return -EINVAL;
257	}
258
259	return 0;
260}
261
262u32 ifcvf_get_config_size(struct ifcvf_hw *hw)
263{
264	u32 net_config_size = sizeof(struct virtio_net_config);
265	u32 blk_config_size = sizeof(struct virtio_blk_config);
266	u32 cap_size = hw->cap_dev_config_size;
267	u32 config_size;
268
269	/* If the onboard device config space size is greater than
270	 * the size of struct virtio_net/blk_config, only the spec
271	 * implementing contents size is returned, this is very
272	 * unlikely, defensive programming.
273	 */
274	switch (hw->dev_type) {
275	case VIRTIO_ID_NET:
276		config_size = min(cap_size, net_config_size);
277		break;
278	case VIRTIO_ID_BLOCK:
279		config_size = min(cap_size, blk_config_size);
280		break;
281	default:
282		config_size = 0;
283		IFCVF_ERR(hw->pdev, "VIRTIO ID %u not supported\n", hw->dev_type);
284	}
285
286	return config_size;
287}
288
289void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset,
290			   void *dst, int length)
291{
292	u8 old_gen, new_gen, *p;
293	int i;
294
295	WARN_ON(offset + length > hw->config_size);
296	do {
297		old_gen = vp_ioread8(&hw->common_cfg->config_generation);
298		p = dst;
299		for (i = 0; i < length; i++)
300			*p++ = vp_ioread8(hw->dev_cfg + offset + i);
301
302		new_gen = vp_ioread8(&hw->common_cfg->config_generation);
303	} while (old_gen != new_gen);
304}
305
306void ifcvf_write_dev_config(struct ifcvf_hw *hw, u64 offset,
307			    const void *src, int length)
308{
309	const u8 *p;
310	int i;
311
312	p = src;
313	WARN_ON(offset + length > hw->config_size);
314	for (i = 0; i < length; i++)
315		vp_iowrite8(*p++, hw->dev_cfg + offset + i);
316}
317
318void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features)
319{
320	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
321
322	vp_iowrite32(0, &cfg->guest_feature_select);
323	vp_iowrite32((u32)features, &cfg->guest_feature);
324
325	vp_iowrite32(1, &cfg->guest_feature_select);
326	vp_iowrite32(features >> 32, &cfg->guest_feature);
327}
328
329u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid)
330{
331	struct ifcvf_lm_cfg  __iomem *lm_cfg = hw->lm_cfg;
332	u16 last_avail_idx;
333
334	last_avail_idx = vp_ioread16(&lm_cfg->vq_state_region + qid * 2);
335
336	return last_avail_idx;
337}
338
339int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num)
340{
341	struct ifcvf_lm_cfg  __iomem *lm_cfg = hw->lm_cfg;
342
343	vp_iowrite16(num, &lm_cfg->vq_state_region + qid * 2);
344
345	return 0;
346}
347
348void ifcvf_set_vq_num(struct ifcvf_hw *hw, u16 qid, u32 num)
349{
350	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
351
352	vp_iowrite16(qid, &cfg->queue_select);
353	vp_iowrite16(num, &cfg->queue_size);
354}
355
356int ifcvf_set_vq_address(struct ifcvf_hw *hw, u16 qid, u64 desc_area,
357			 u64 driver_area, u64 device_area)
358{
359	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
360
361	vp_iowrite16(qid, &cfg->queue_select);
362	vp_iowrite64_twopart(desc_area, &cfg->queue_desc_lo,
363			     &cfg->queue_desc_hi);
364	vp_iowrite64_twopart(driver_area, &cfg->queue_avail_lo,
365			     &cfg->queue_avail_hi);
366	vp_iowrite64_twopart(device_area, &cfg->queue_used_lo,
367			     &cfg->queue_used_hi);
368
369	return 0;
370}
371
372bool ifcvf_get_vq_ready(struct ifcvf_hw *hw, u16 qid)
373{
374	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
375	u16 queue_enable;
376
377	vp_iowrite16(qid, &cfg->queue_select);
378	queue_enable = vp_ioread16(&cfg->queue_enable);
379
380	return (bool)queue_enable;
381}
382
383void ifcvf_set_vq_ready(struct ifcvf_hw *hw, u16 qid, bool ready)
384{
385	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
386
387	vp_iowrite16(qid, &cfg->queue_select);
388	vp_iowrite16(ready, &cfg->queue_enable);
389}
390
391static void ifcvf_reset_vring(struct ifcvf_hw *hw)
392{
393	u16 qid;
394
395	for (qid = 0; qid < hw->nr_vring; qid++) {
396		hw->vring[qid].cb.callback = NULL;
397		hw->vring[qid].cb.private = NULL;
398		ifcvf_set_vq_vector(hw, qid, VIRTIO_MSI_NO_VECTOR);
399	}
400}
401
402static void ifcvf_reset_config_handler(struct ifcvf_hw *hw)
403{
404	hw->config_cb.callback = NULL;
405	hw->config_cb.private = NULL;
406	ifcvf_set_config_vector(hw, VIRTIO_MSI_NO_VECTOR);
407}
408
409static void ifcvf_synchronize_irq(struct ifcvf_hw *hw)
410{
411	u32 nvectors = hw->num_msix_vectors;
412	struct pci_dev *pdev = hw->pdev;
413	int i, irq;
414
415	for (i = 0; i < nvectors; i++) {
416		irq = pci_irq_vector(pdev, i);
417		if (irq >= 0)
418			synchronize_irq(irq);
419	}
420}
421
422void ifcvf_stop(struct ifcvf_hw *hw)
423{
424	ifcvf_synchronize_irq(hw);
425	ifcvf_reset_vring(hw);
426	ifcvf_reset_config_handler(hw);
427}
428
429void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
430{
431	vp_iowrite16(qid, hw->vring[qid].notify_addr);
432}
433