1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (c) 2014-2015 Hisilicon Limited.
4 */
5
6#include <linux/dma-mapping.h>
7#include <linux/interrupt.h>
8#include <linux/of.h>
9#include <linux/skbuff.h>
10#include <linux/slab.h>
11#include "hnae.h"
12
13#define cls_to_ae_dev(dev) container_of(dev, struct hnae_ae_dev, cls_dev)
14
15static struct class *hnae_class;
16
17static void
18hnae_list_add(spinlock_t *lock, struct list_head *node, struct list_head *head)
19{
20	unsigned long flags;
21
22	spin_lock_irqsave(lock, flags);
23	list_add_tail_rcu(node, head);
24	spin_unlock_irqrestore(lock, flags);
25}
26
27static void hnae_list_del(spinlock_t *lock, struct list_head *node)
28{
29	unsigned long flags;
30
31	spin_lock_irqsave(lock, flags);
32	list_del_rcu(node);
33	spin_unlock_irqrestore(lock, flags);
34}
35
36static int hnae_alloc_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb)
37{
38	unsigned int order = hnae_page_order(ring);
39	struct page *p = dev_alloc_pages(order);
40
41	if (!p)
42		return -ENOMEM;
43
44	cb->priv = p;
45	cb->page_offset = 0;
46	cb->reuse_flag = 0;
47	cb->buf  = page_address(p);
48	cb->length = hnae_page_size(ring);
49	cb->type = DESC_TYPE_PAGE;
50
51	return 0;
52}
53
54static void hnae_free_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb)
55{
56	if (unlikely(!cb->priv))
57		return;
58
59	if (cb->type == DESC_TYPE_SKB)
60		dev_kfree_skb_any((struct sk_buff *)cb->priv);
61	else if (unlikely(is_rx_ring(ring)))
62		put_page((struct page *)cb->priv);
63
64	cb->priv = NULL;
65}
66
67static int hnae_map_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb)
68{
69	cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
70			       cb->length, ring_to_dma_dir(ring));
71
72	if (dma_mapping_error(ring_to_dev(ring), cb->dma))
73		return -EIO;
74
75	return 0;
76}
77
78static void hnae_unmap_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb)
79{
80	if (cb->type == DESC_TYPE_SKB)
81		dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
82				 ring_to_dma_dir(ring));
83	else if (cb->length)
84		dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
85			       ring_to_dma_dir(ring));
86}
87
88static struct hnae_buf_ops hnae_bops = {
89	.alloc_buffer = hnae_alloc_buffer,
90	.free_buffer = hnae_free_buffer,
91	.map_buffer = hnae_map_buffer,
92	.unmap_buffer = hnae_unmap_buffer,
93};
94
95static int __ae_match(struct device *dev, const void *data)
96{
97	struct hnae_ae_dev *hdev = cls_to_ae_dev(dev);
98
99	if (dev_of_node(hdev->dev))
100		return (data == &hdev->dev->of_node->fwnode);
101	else if (is_acpi_node(hdev->dev->fwnode))
102		return (data == hdev->dev->fwnode);
103
104	dev_err(dev, "__ae_match cannot read cfg data from OF or acpi\n");
105	return 0;
106}
107
108static struct hnae_ae_dev *find_ae(const struct fwnode_handle *fwnode)
109{
110	struct device *dev;
111
112	WARN_ON(!fwnode);
113
114	dev = class_find_device(hnae_class, NULL, fwnode, __ae_match);
115
116	return dev ? cls_to_ae_dev(dev) : NULL;
117}
118
119static void hnae_free_buffers(struct hnae_ring *ring)
120{
121	int i;
122
123	for (i = 0; i < ring->desc_num; i++)
124		hnae_free_buffer_detach(ring, i);
125}
126
127/* Allocate memory for raw pkg, and map with dma */
128static int hnae_alloc_buffers(struct hnae_ring *ring)
129{
130	int i, j, ret;
131
132	for (i = 0; i < ring->desc_num; i++) {
133		ret = hnae_alloc_buffer_attach(ring, i);
134		if (ret)
135			goto out_buffer_fail;
136	}
137
138	return 0;
139
140out_buffer_fail:
141	for (j = i - 1; j >= 0; j--)
142		hnae_free_buffer_detach(ring, j);
143	return ret;
144}
145
146/* free desc along with its attached buffer */
147static void hnae_free_desc(struct hnae_ring *ring)
148{
149	dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,
150			 ring->desc_num * sizeof(ring->desc[0]),
151			 ring_to_dma_dir(ring));
152	ring->desc_dma_addr = 0;
153	kfree(ring->desc);
154	ring->desc = NULL;
155}
156
157/* alloc desc, without buffer attached */
158static int hnae_alloc_desc(struct hnae_ring *ring)
159{
160	int size = ring->desc_num * sizeof(ring->desc[0]);
161
162	ring->desc = kzalloc(size, GFP_KERNEL);
163	if (!ring->desc)
164		return -ENOMEM;
165
166	ring->desc_dma_addr = dma_map_single(ring_to_dev(ring),
167		ring->desc, size, ring_to_dma_dir(ring));
168	if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {
169		ring->desc_dma_addr = 0;
170		kfree(ring->desc);
171		ring->desc = NULL;
172		return -ENOMEM;
173	}
174
175	return 0;
176}
177
178/* fini ring, also free the buffer for the ring */
179static void hnae_fini_ring(struct hnae_ring *ring)
180{
181	if (is_rx_ring(ring))
182		hnae_free_buffers(ring);
183
184	hnae_free_desc(ring);
185	kfree(ring->desc_cb);
186	ring->desc_cb = NULL;
187	ring->next_to_clean = 0;
188	ring->next_to_use = 0;
189}
190
191/* init ring, and with buffer for rx ring */
192static int
193hnae_init_ring(struct hnae_queue *q, struct hnae_ring *ring, int flags)
194{
195	int ret;
196
197	if (ring->desc_num <= 0 || ring->buf_size <= 0)
198		return -EINVAL;
199
200	ring->q = q;
201	ring->flags = flags;
202	ring->coal_param = q->handle->coal_param;
203	assert(!ring->desc && !ring->desc_cb && !ring->desc_dma_addr);
204
205	/* not matter for tx or rx ring, the ntc and ntc start from 0 */
206	assert(ring->next_to_use == 0);
207	assert(ring->next_to_clean == 0);
208
209	ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
210			GFP_KERNEL);
211	if (!ring->desc_cb) {
212		ret = -ENOMEM;
213		goto out;
214	}
215
216	ret = hnae_alloc_desc(ring);
217	if (ret)
218		goto out_with_desc_cb;
219
220	if (is_rx_ring(ring)) {
221		ret = hnae_alloc_buffers(ring);
222		if (ret)
223			goto out_with_desc;
224	}
225
226	return 0;
227
228out_with_desc:
229	hnae_free_desc(ring);
230out_with_desc_cb:
231	kfree(ring->desc_cb);
232	ring->desc_cb = NULL;
233out:
234	return ret;
235}
236
237static int hnae_init_queue(struct hnae_handle *h, struct hnae_queue *q,
238			   struct hnae_ae_dev *dev)
239{
240	int ret;
241
242	q->dev = dev;
243	q->handle = h;
244
245	ret = hnae_init_ring(q, &q->tx_ring, q->tx_ring.flags | RINGF_DIR);
246	if (ret)
247		goto out;
248
249	ret = hnae_init_ring(q, &q->rx_ring, q->rx_ring.flags & ~RINGF_DIR);
250	if (ret)
251		goto out_with_tx_ring;
252
253	if (dev->ops->init_queue)
254		dev->ops->init_queue(q);
255
256	return 0;
257
258out_with_tx_ring:
259	hnae_fini_ring(&q->tx_ring);
260out:
261	return ret;
262}
263
264static void hnae_fini_queue(struct hnae_queue *q)
265{
266	if (q->dev->ops->fini_queue)
267		q->dev->ops->fini_queue(q);
268
269	hnae_fini_ring(&q->tx_ring);
270	hnae_fini_ring(&q->rx_ring);
271}
272
273/*
274 * ae_chain - define ae chain head
275 */
276static RAW_NOTIFIER_HEAD(ae_chain);
277
278int hnae_register_notifier(struct notifier_block *nb)
279{
280	return raw_notifier_chain_register(&ae_chain, nb);
281}
282EXPORT_SYMBOL(hnae_register_notifier);
283
284void hnae_unregister_notifier(struct notifier_block *nb)
285{
286	if (raw_notifier_chain_unregister(&ae_chain, nb))
287		dev_err(NULL, "notifier chain unregister fail\n");
288}
289EXPORT_SYMBOL(hnae_unregister_notifier);
290
291int hnae_reinit_handle(struct hnae_handle *handle)
292{
293	int i, j;
294	int ret;
295
296	for (i = 0; i < handle->q_num; i++) /* free ring*/
297		hnae_fini_queue(handle->qs[i]);
298
299	if (handle->dev->ops->reset)
300		handle->dev->ops->reset(handle);
301
302	for (i = 0; i < handle->q_num; i++) {/* reinit ring*/
303		ret = hnae_init_queue(handle, handle->qs[i], handle->dev);
304		if (ret)
305			goto out_when_init_queue;
306	}
307	return 0;
308out_when_init_queue:
309	for (j = i - 1; j >= 0; j--)
310		hnae_fini_queue(handle->qs[j]);
311	return ret;
312}
313EXPORT_SYMBOL(hnae_reinit_handle);
314
315/* hnae_get_handle - get a handle from the AE
316 * @owner_dev: the dev use this handle
317 * @ae_id: the id of the ae to be used
318 * @ae_opts: the options set for the handle
319 * @bops: the callbacks for buffer management
320 *
321 * return handle ptr or ERR_PTR
322 */
323struct hnae_handle *hnae_get_handle(struct device *owner_dev,
324				    const struct fwnode_handle	*fwnode,
325				    u32 port_id,
326				    struct hnae_buf_ops *bops)
327{
328	struct hnae_ae_dev *dev;
329	struct hnae_handle *handle;
330	int i, j;
331	int ret;
332
333	dev = find_ae(fwnode);
334	if (!dev)
335		return ERR_PTR(-ENODEV);
336
337	handle = dev->ops->get_handle(dev, port_id);
338	if (IS_ERR(handle)) {
339		put_device(&dev->cls_dev);
340		return handle;
341	}
342
343	handle->dev = dev;
344	handle->owner_dev = owner_dev;
345	handle->bops = bops ? bops : &hnae_bops;
346	handle->eport_id = port_id;
347
348	for (i = 0; i < handle->q_num; i++) {
349		ret = hnae_init_queue(handle, handle->qs[i], dev);
350		if (ret)
351			goto out_when_init_queue;
352	}
353
354	__module_get(dev->owner);
355
356	hnae_list_add(&dev->lock, &handle->node, &dev->handle_list);
357
358	return handle;
359
360out_when_init_queue:
361	for (j = i - 1; j >= 0; j--)
362		hnae_fini_queue(handle->qs[j]);
363
364	put_device(&dev->cls_dev);
365
366	return ERR_PTR(-ENOMEM);
367}
368EXPORT_SYMBOL(hnae_get_handle);
369
370void hnae_put_handle(struct hnae_handle *h)
371{
372	struct hnae_ae_dev *dev = h->dev;
373	int i;
374
375	for (i = 0; i < h->q_num; i++)
376		hnae_fini_queue(h->qs[i]);
377
378	if (h->dev->ops->reset)
379		h->dev->ops->reset(h);
380
381	hnae_list_del(&dev->lock, &h->node);
382
383	if (dev->ops->put_handle)
384		dev->ops->put_handle(h);
385
386	module_put(dev->owner);
387
388	put_device(&dev->cls_dev);
389}
390EXPORT_SYMBOL(hnae_put_handle);
391
392static void hnae_release(struct device *dev)
393{
394}
395
396/**
397 * hnae_ae_register - register a AE engine to hnae framework
398 * @hdev: the hnae ae engine device
399 * @owner:  the module who provides this dev
400 * NOTE: the duplicated name will not be checked
401 */
402int hnae_ae_register(struct hnae_ae_dev *hdev, struct module *owner)
403{
404	static atomic_t id = ATOMIC_INIT(-1);
405	int ret;
406
407	if (!hdev->dev)
408		return -ENODEV;
409
410	if (!hdev->ops || !hdev->ops->get_handle ||
411	    !hdev->ops->toggle_ring_irq ||
412	    !hdev->ops->get_status || !hdev->ops->adjust_link)
413		return -EINVAL;
414
415	hdev->owner = owner;
416	hdev->id = (int)atomic_inc_return(&id);
417	hdev->cls_dev.parent = hdev->dev;
418	hdev->cls_dev.class = hnae_class;
419	hdev->cls_dev.release = hnae_release;
420	(void)dev_set_name(&hdev->cls_dev, "hnae%d", hdev->id);
421	ret = device_register(&hdev->cls_dev);
422	if (ret) {
423		put_device(&hdev->cls_dev);
424		return ret;
425	}
426
427	INIT_LIST_HEAD(&hdev->handle_list);
428	spin_lock_init(&hdev->lock);
429
430	ret = raw_notifier_call_chain(&ae_chain, HNAE_AE_REGISTER, NULL);
431	if (ret)
432		dev_dbg(hdev->dev,
433			"has not notifier for AE: %s\n", hdev->name);
434
435	return 0;
436}
437EXPORT_SYMBOL(hnae_ae_register);
438
439/**
440 * hnae_ae_unregister - unregisters a HNAE AE engine
441 * @hdev: the device to unregister
442 */
443void hnae_ae_unregister(struct hnae_ae_dev *hdev)
444{
445	device_unregister(&hdev->cls_dev);
446}
447EXPORT_SYMBOL(hnae_ae_unregister);
448
449static int __init hnae_init(void)
450{
451	hnae_class = class_create("hnae");
452	return PTR_ERR_OR_ZERO(hnae_class);
453}
454
455static void __exit hnae_exit(void)
456{
457	class_destroy(hnae_class);
458}
459
460subsys_initcall(hnae_init);
461module_exit(hnae_exit);
462
463MODULE_AUTHOR("Hisilicon, Inc.");
464MODULE_LICENSE("GPL");
465MODULE_DESCRIPTION("Hisilicon Network Acceleration Engine Framework");
466
467/* vi: set tw=78 noet: */
468