1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VIRTIO based driver for vDPA device
4 *
5 * Copyright (c) 2020, Red Hat. All rights reserved.
6 *     Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/device.h>
13#include <linux/kernel.h>
14#include <linux/slab.h>
15#include <linux/uuid.h>
16#include <linux/virtio.h>
17#include <linux/vdpa.h>
18#include <linux/virtio_config.h>
19#include <linux/virtio_ring.h>
20
21#define MOD_VERSION  "0.1"
22#define MOD_AUTHOR   "Jason Wang <jasowang@redhat.com>"
23#define MOD_DESC     "vDPA bus driver for virtio devices"
24#define MOD_LICENSE  "GPL v2"
25
26struct virtio_vdpa_device {
27	struct virtio_device vdev;
28	struct vdpa_device *vdpa;
29	u64 features;
30
31	/* The lock to protect virtqueue list */
32	spinlock_t lock;
33	/* List of virtio_vdpa_vq_info */
34	struct list_head virtqueues;
35};
36
37struct virtio_vdpa_vq_info {
38	/* the actual virtqueue */
39	struct virtqueue *vq;
40
41	/* the list node for the virtqueues list */
42	struct list_head node;
43};
44
45static inline struct virtio_vdpa_device *
46to_virtio_vdpa_device(struct virtio_device *dev)
47{
48	return container_of(dev, struct virtio_vdpa_device, vdev);
49}
50
51static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
52{
53	return to_virtio_vdpa_device(vdev)->vdpa;
54}
55
56static void virtio_vdpa_get(struct virtio_device *vdev, unsigned offset,
57			    void *buf, unsigned len)
58{
59	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
60
61	vdpa_get_config(vdpa, offset, buf, len);
62}
63
64static void virtio_vdpa_set(struct virtio_device *vdev, unsigned offset,
65			    const void *buf, unsigned len)
66{
67	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
68	const struct vdpa_config_ops *ops = vdpa->config;
69
70	ops->set_config(vdpa, offset, buf, len);
71}
72
73static u32 virtio_vdpa_generation(struct virtio_device *vdev)
74{
75	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
76	const struct vdpa_config_ops *ops = vdpa->config;
77
78	if (ops->get_generation)
79		return ops->get_generation(vdpa);
80
81	return 0;
82}
83
84static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
85{
86	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
87	const struct vdpa_config_ops *ops = vdpa->config;
88
89	return ops->get_status(vdpa);
90}
91
92static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
93{
94	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
95	const struct vdpa_config_ops *ops = vdpa->config;
96
97	return ops->set_status(vdpa, status);
98}
99
100static void virtio_vdpa_reset(struct virtio_device *vdev)
101{
102	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
103
104	vdpa_reset(vdpa);
105}
106
107static bool virtio_vdpa_notify(struct virtqueue *vq)
108{
109	struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
110	const struct vdpa_config_ops *ops = vdpa->config;
111
112	ops->kick_vq(vdpa, vq->index);
113
114	return true;
115}
116
117static irqreturn_t virtio_vdpa_config_cb(void *private)
118{
119	struct virtio_vdpa_device *vd_dev = private;
120
121	virtio_config_changed(&vd_dev->vdev);
122
123	return IRQ_HANDLED;
124}
125
126static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
127{
128	struct virtio_vdpa_vq_info *info = private;
129
130	return vring_interrupt(0, info->vq);
131}
132
133static struct virtqueue *
134virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
135		     void (*callback)(struct virtqueue *vq),
136		     const char *name, bool ctx)
137{
138	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
139	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
140	const struct vdpa_config_ops *ops = vdpa->config;
141	struct virtio_vdpa_vq_info *info;
142	struct vdpa_callback cb;
143	struct virtqueue *vq;
144	u64 desc_addr, driver_addr, device_addr;
145	unsigned long flags;
146	u32 align, num;
147	int err;
148
149	if (!name)
150		return NULL;
151
152	if (index >= vdpa->nvqs)
153		return ERR_PTR(-ENOENT);
154
155	/* Queue shouldn't already be set up. */
156	if (ops->get_vq_ready(vdpa, index))
157		return ERR_PTR(-ENOENT);
158
159	/* Allocate and fill out our active queue description */
160	info = kmalloc(sizeof(*info), GFP_KERNEL);
161	if (!info)
162		return ERR_PTR(-ENOMEM);
163
164	num = ops->get_vq_num_max(vdpa);
165	if (num == 0) {
166		err = -ENOENT;
167		goto error_new_virtqueue;
168	}
169
170	/* Create the vring */
171	align = ops->get_vq_align(vdpa);
172	vq = vring_create_virtqueue(index, num, align, vdev,
173				    true, true, ctx,
174				    virtio_vdpa_notify, callback, name);
175	if (!vq) {
176		err = -ENOMEM;
177		goto error_new_virtqueue;
178	}
179
180	/* Setup virtqueue callback */
181	cb.callback = virtio_vdpa_virtqueue_cb;
182	cb.private = info;
183	ops->set_vq_cb(vdpa, index, &cb);
184	ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
185
186	desc_addr = virtqueue_get_desc_addr(vq);
187	driver_addr = virtqueue_get_avail_addr(vq);
188	device_addr = virtqueue_get_used_addr(vq);
189
190	if (ops->set_vq_address(vdpa, index,
191				desc_addr, driver_addr,
192				device_addr)) {
193		err = -EINVAL;
194		goto err_vq;
195	}
196
197	ops->set_vq_ready(vdpa, index, 1);
198
199	vq->priv = info;
200	info->vq = vq;
201
202	spin_lock_irqsave(&vd_dev->lock, flags);
203	list_add(&info->node, &vd_dev->virtqueues);
204	spin_unlock_irqrestore(&vd_dev->lock, flags);
205
206	return vq;
207
208err_vq:
209	vring_del_virtqueue(vq);
210error_new_virtqueue:
211	ops->set_vq_ready(vdpa, index, 0);
212	/* VDPA driver should make sure vq is stopeed here */
213	WARN_ON(ops->get_vq_ready(vdpa, index));
214	kfree(info);
215	return ERR_PTR(err);
216}
217
218static void virtio_vdpa_del_vq(struct virtqueue *vq)
219{
220	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
221	struct vdpa_device *vdpa = vd_dev->vdpa;
222	const struct vdpa_config_ops *ops = vdpa->config;
223	struct virtio_vdpa_vq_info *info = vq->priv;
224	unsigned int index = vq->index;
225	unsigned long flags;
226
227	spin_lock_irqsave(&vd_dev->lock, flags);
228	list_del(&info->node);
229	spin_unlock_irqrestore(&vd_dev->lock, flags);
230
231	/* Select and deactivate the queue */
232	ops->set_vq_ready(vdpa, index, 0);
233	WARN_ON(ops->get_vq_ready(vdpa, index));
234
235	vring_del_virtqueue(vq);
236
237	kfree(info);
238}
239
240static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
241{
242	struct virtqueue *vq, *n;
243
244	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
245		virtio_vdpa_del_vq(vq);
246}
247
248static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned nvqs,
249				struct virtqueue *vqs[],
250				vq_callback_t *callbacks[],
251				const char * const names[],
252				const bool *ctx,
253				struct irq_affinity *desc)
254{
255	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
256	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
257	const struct vdpa_config_ops *ops = vdpa->config;
258	struct vdpa_callback cb;
259	int i, err, queue_idx = 0;
260
261	for (i = 0; i < nvqs; ++i) {
262		if (!names[i]) {
263			vqs[i] = NULL;
264			continue;
265		}
266
267		vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++,
268					      callbacks[i], names[i], ctx ?
269					      ctx[i] : false);
270		if (IS_ERR(vqs[i])) {
271			err = PTR_ERR(vqs[i]);
272			goto err_setup_vq;
273		}
274	}
275
276	cb.callback = virtio_vdpa_config_cb;
277	cb.private = vd_dev;
278	ops->set_config_cb(vdpa, &cb);
279
280	return 0;
281
282err_setup_vq:
283	virtio_vdpa_del_vqs(vdev);
284	return err;
285}
286
287static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
288{
289	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
290	const struct vdpa_config_ops *ops = vdpa->config;
291
292	return ops->get_features(vdpa);
293}
294
295static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
296{
297	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
298
299	/* Give virtio_ring a chance to accept features. */
300	vring_transport_features(vdev);
301
302	return vdpa_set_features(vdpa, vdev->features);
303}
304
305static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
306{
307	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
308	struct vdpa_device *vdpa = vd_dev->vdpa;
309
310	return dev_name(&vdpa->dev);
311}
312
313static const struct virtio_config_ops virtio_vdpa_config_ops = {
314	.get		= virtio_vdpa_get,
315	.set		= virtio_vdpa_set,
316	.generation	= virtio_vdpa_generation,
317	.get_status	= virtio_vdpa_get_status,
318	.set_status	= virtio_vdpa_set_status,
319	.reset		= virtio_vdpa_reset,
320	.find_vqs	= virtio_vdpa_find_vqs,
321	.del_vqs	= virtio_vdpa_del_vqs,
322	.get_features	= virtio_vdpa_get_features,
323	.finalize_features = virtio_vdpa_finalize_features,
324	.bus_name	= virtio_vdpa_bus_name,
325};
326
327static void virtio_vdpa_release_dev(struct device *_d)
328{
329	struct virtio_device *vdev =
330	       container_of(_d, struct virtio_device, dev);
331	struct virtio_vdpa_device *vd_dev =
332	       container_of(vdev, struct virtio_vdpa_device, vdev);
333
334	kfree(vd_dev);
335}
336
337static int virtio_vdpa_probe(struct vdpa_device *vdpa)
338{
339	const struct vdpa_config_ops *ops = vdpa->config;
340	struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
341	int ret = -EINVAL;
342
343	vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);
344	if (!vd_dev)
345		return -ENOMEM;
346
347	vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa);
348	vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
349	vd_dev->vdev.config = &virtio_vdpa_config_ops;
350	vd_dev->vdpa = vdpa;
351	INIT_LIST_HEAD(&vd_dev->virtqueues);
352	spin_lock_init(&vd_dev->lock);
353
354	vd_dev->vdev.id.device = ops->get_device_id(vdpa);
355	if (vd_dev->vdev.id.device == 0)
356		goto err;
357
358	vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
359	ret = register_virtio_device(&vd_dev->vdev);
360	reg_dev = vd_dev;
361	if (ret)
362		goto err;
363
364	vdpa_set_drvdata(vdpa, vd_dev);
365
366	return 0;
367
368err:
369	if (reg_dev)
370		put_device(&vd_dev->vdev.dev);
371	else
372		kfree(vd_dev);
373	return ret;
374}
375
376static void virtio_vdpa_remove(struct vdpa_device *vdpa)
377{
378	struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa);
379
380	unregister_virtio_device(&vd_dev->vdev);
381}
382
383static struct vdpa_driver virtio_vdpa_driver = {
384	.driver = {
385		.name	= "virtio_vdpa",
386	},
387	.probe	= virtio_vdpa_probe,
388	.remove = virtio_vdpa_remove,
389};
390
391module_vdpa_driver(virtio_vdpa_driver);
392
393MODULE_VERSION(MOD_VERSION);
394MODULE_LICENSE(MOD_LICENSE);
395MODULE_AUTHOR(MOD_AUTHOR);
396MODULE_DESCRIPTION(MOD_DESC);
397