1// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2/*
3 * Copyright(c) 2020 Intel Corporation.
4 *
5 */
6
7/*
8 * This file contains HFI1 support for netdev RX functionality
9 */
10
11#include "sdma.h"
12#include "verbs.h"
13#include "netdev.h"
14#include "hfi.h"
15
16#include <linux/netdevice.h>
17#include <linux/etherdevice.h>
18#include <rdma/ib_verbs.h>
19
20static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_priv *priv,
21				  struct hfi1_ctxtdata *uctxt)
22{
23	unsigned int rcvctrl_ops;
24	struct hfi1_devdata *dd = priv->dd;
25	int ret;
26
27	uctxt->rhf_rcv_function_map = netdev_rhf_rcv_functions;
28	uctxt->do_interrupt = &handle_receive_interrupt_napi_sp;
29
30	/* Now allocate the RcvHdr queue and eager buffers. */
31	ret = hfi1_create_rcvhdrq(dd, uctxt);
32	if (ret)
33		goto done;
34
35	ret = hfi1_setup_eagerbufs(uctxt);
36	if (ret)
37		goto done;
38
39	clear_rcvhdrtail(uctxt);
40
41	rcvctrl_ops = HFI1_RCVCTRL_CTXT_DIS;
42	rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_DIS;
43
44	if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR))
45		rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
46	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL))
47		rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
48	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL))
49		rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
50	if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL))
51		rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;
52
53	hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt);
54done:
55	return ret;
56}
57
58static int hfi1_netdev_allocate_ctxt(struct hfi1_devdata *dd,
59				     struct hfi1_ctxtdata **ctxt)
60{
61	struct hfi1_ctxtdata *uctxt;
62	int ret;
63
64	if (dd->flags & HFI1_FROZEN)
65		return -EIO;
66
67	ret = hfi1_create_ctxtdata(dd->pport, dd->node, &uctxt);
68	if (ret < 0) {
69		dd_dev_err(dd, "Unable to create ctxtdata, failing open\n");
70		return -ENOMEM;
71	}
72
73	uctxt->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
74		HFI1_CAP_KGET(NODROP_RHQ_FULL) |
75		HFI1_CAP_KGET(NODROP_EGR_FULL) |
76		HFI1_CAP_KGET(DMA_RTAIL);
77	/* Netdev contexts are always NO_RDMA_RTAIL */
78	uctxt->fast_handler = handle_receive_interrupt_napi_fp;
79	uctxt->slow_handler = handle_receive_interrupt_napi_sp;
80	hfi1_set_seq_cnt(uctxt, 1);
81	uctxt->is_vnic = true;
82
83	hfi1_stats.sps_ctxts++;
84
85	dd_dev_info(dd, "created netdev context %d\n", uctxt->ctxt);
86	*ctxt = uctxt;
87
88	return 0;
89}
90
91static void hfi1_netdev_deallocate_ctxt(struct hfi1_devdata *dd,
92					struct hfi1_ctxtdata *uctxt)
93{
94	flush_wc();
95
96	/*
97	 * Disable receive context and interrupt available, reset all
98	 * RcvCtxtCtrl bits to default values.
99	 */
100	hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |
101		     HFI1_RCVCTRL_TIDFLOW_DIS |
102		     HFI1_RCVCTRL_INTRAVAIL_DIS |
103		     HFI1_RCVCTRL_ONE_PKT_EGR_DIS |
104		     HFI1_RCVCTRL_NO_RHQ_DROP_DIS |
105		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);
106
107	if (uctxt->msix_intr != CCE_NUM_MSIX_VECTORS)
108		msix_free_irq(dd, uctxt->msix_intr);
109
110	uctxt->msix_intr = CCE_NUM_MSIX_VECTORS;
111	uctxt->event_flags = 0;
112
113	hfi1_clear_tids(uctxt);
114	hfi1_clear_ctxt_pkey(dd, uctxt);
115
116	hfi1_stats.sps_ctxts--;
117
118	hfi1_free_ctxt(uctxt);
119}
120
121static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_priv *priv,
122				  struct hfi1_ctxtdata **ctxt)
123{
124	int rc;
125	struct hfi1_devdata *dd = priv->dd;
126
127	rc = hfi1_netdev_allocate_ctxt(dd, ctxt);
128	if (rc) {
129		dd_dev_err(dd, "netdev ctxt alloc failed %d\n", rc);
130		return rc;
131	}
132
133	rc = hfi1_netdev_setup_ctxt(priv, *ctxt);
134	if (rc) {
135		dd_dev_err(dd, "netdev ctxt setup failed %d\n", rc);
136		hfi1_netdev_deallocate_ctxt(dd, *ctxt);
137		*ctxt = NULL;
138	}
139
140	return rc;
141}
142
143/**
144 * hfi1_num_netdev_contexts - Count of netdev recv contexts to use.
145 * @dd: device on which to allocate netdev contexts
146 * @available_contexts: count of available receive contexts
147 * @cpu_mask: mask of possible cpus to include for contexts
148 *
149 * Return: count of physical cores on a node or the remaining available recv
150 * contexts for netdev recv context usage up to the maximum of
151 * HFI1_MAX_NETDEV_CTXTS.
152 * A value of 0 can be returned when acceleration is explicitly turned off,
153 * a memory allocation error occurs or when there are no available contexts.
154 *
155 */
156u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,
157			     struct cpumask *cpu_mask)
158{
159	cpumask_var_t node_cpu_mask;
160	unsigned int available_cpus;
161
162	if (!HFI1_CAP_IS_KSET(AIP))
163		return 0;
164
165	/* Always give user contexts priority over netdev contexts */
166	if (available_contexts == 0) {
167		dd_dev_info(dd, "No receive contexts available for netdevs.\n");
168		return 0;
169	}
170
171	if (!zalloc_cpumask_var(&node_cpu_mask, GFP_KERNEL)) {
172		dd_dev_err(dd, "Unable to allocate cpu_mask for netdevs.\n");
173		return 0;
174	}
175
176	cpumask_and(node_cpu_mask, cpu_mask, cpumask_of_node(dd->node));
177
178	available_cpus = cpumask_weight(node_cpu_mask);
179
180	free_cpumask_var(node_cpu_mask);
181
182	return min3(available_cpus, available_contexts,
183		    (u32)HFI1_MAX_NETDEV_CTXTS);
184}
185
186static int hfi1_netdev_rxq_init(struct net_device *dev)
187{
188	int i;
189	int rc;
190	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev);
191	struct hfi1_devdata *dd = priv->dd;
192
193	priv->num_rx_q = dd->num_netdev_contexts;
194	priv->rxq = kcalloc_node(priv->num_rx_q, sizeof(struct hfi1_netdev_rxq),
195				 GFP_KERNEL, dd->node);
196
197	if (!priv->rxq) {
198		dd_dev_err(dd, "Unable to allocate netdev queue data\n");
199		return (-ENOMEM);
200	}
201
202	for (i = 0; i < priv->num_rx_q; i++) {
203		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
204
205		rc = hfi1_netdev_allot_ctxt(priv, &rxq->rcd);
206		if (rc)
207			goto bail_context_irq_failure;
208
209		hfi1_rcd_get(rxq->rcd);
210		rxq->priv = priv;
211		rxq->rcd->napi = &rxq->napi;
212		dd_dev_info(dd, "Setting rcv queue %d napi to context %d\n",
213			    i, rxq->rcd->ctxt);
214		/*
215		 * Disable BUSY_POLL on this NAPI as this is not supported
216		 * right now.
217		 */
218		set_bit(NAPI_STATE_NO_BUSY_POLL, &rxq->napi.state);
219		netif_napi_add(dev, &rxq->napi, hfi1_netdev_rx_napi, 64);
220		rc = msix_netdev_request_rcd_irq(rxq->rcd);
221		if (rc)
222			goto bail_context_irq_failure;
223	}
224
225	return 0;
226
227bail_context_irq_failure:
228	dd_dev_err(dd, "Unable to allot receive context\n");
229	for (; i >= 0; i--) {
230		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
231
232		if (rxq->rcd) {
233			hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);
234			hfi1_rcd_put(rxq->rcd);
235			rxq->rcd = NULL;
236		}
237	}
238	kfree(priv->rxq);
239	priv->rxq = NULL;
240
241	return rc;
242}
243
244static void hfi1_netdev_rxq_deinit(struct net_device *dev)
245{
246	int i;
247	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev);
248	struct hfi1_devdata *dd = priv->dd;
249
250	for (i = 0; i < priv->num_rx_q; i++) {
251		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
252
253		netif_napi_del(&rxq->napi);
254		hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);
255		hfi1_rcd_put(rxq->rcd);
256		rxq->rcd = NULL;
257	}
258
259	kfree(priv->rxq);
260	priv->rxq = NULL;
261	priv->num_rx_q = 0;
262}
263
264static void enable_queues(struct hfi1_netdev_priv *priv)
265{
266	int i;
267
268	for (i = 0; i < priv->num_rx_q; i++) {
269		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
270
271		dd_dev_info(priv->dd, "enabling queue %d on context %d\n", i,
272			    rxq->rcd->ctxt);
273		napi_enable(&rxq->napi);
274		hfi1_rcvctrl(priv->dd,
275			     HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB,
276			     rxq->rcd);
277	}
278}
279
280static void disable_queues(struct hfi1_netdev_priv *priv)
281{
282	int i;
283
284	msix_netdev_synchronize_irq(priv->dd);
285
286	for (i = 0; i < priv->num_rx_q; i++) {
287		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
288
289		dd_dev_info(priv->dd, "disabling queue %d on context %d\n", i,
290			    rxq->rcd->ctxt);
291
292		/* wait for napi if it was scheduled */
293		hfi1_rcvctrl(priv->dd,
294			     HFI1_RCVCTRL_CTXT_DIS | HFI1_RCVCTRL_INTRAVAIL_DIS,
295			     rxq->rcd);
296		napi_synchronize(&rxq->napi);
297		napi_disable(&rxq->napi);
298	}
299}
300
301/**
302 * hfi1_netdev_rx_init - Incrememnts netdevs counter. When called first time,
303 * it allocates receive queue data and calls netif_napi_add
304 * for each queue.
305 *
306 * @dd: hfi1 dev data
307 */
308int hfi1_netdev_rx_init(struct hfi1_devdata *dd)
309{
310	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
311	int res;
312
313	if (atomic_fetch_inc(&priv->netdevs))
314		return 0;
315
316	mutex_lock(&hfi1_mutex);
317	init_dummy_netdev(dd->dummy_netdev);
318	res = hfi1_netdev_rxq_init(dd->dummy_netdev);
319	mutex_unlock(&hfi1_mutex);
320	return res;
321}
322
323/**
324 * hfi1_netdev_rx_destroy - Decrements netdevs counter, when it reaches 0
325 * napi is deleted and receive queses memory is freed.
326 *
327 * @dd: hfi1 dev data
328 */
329int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd)
330{
331	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
332
333	/* destroy the RX queues only if it is the last netdev going away */
334	if (atomic_fetch_add_unless(&priv->netdevs, -1, 0) == 1) {
335		mutex_lock(&hfi1_mutex);
336		hfi1_netdev_rxq_deinit(dd->dummy_netdev);
337		mutex_unlock(&hfi1_mutex);
338	}
339
340	return 0;
341}
342
343/**
344 * hfi1_netdev_alloc - Allocates netdev and private data. It is required
345 * because RMT index and MSI-X interrupt can be set only
346 * during driver initialization.
347 *
348 * @dd: hfi1 dev data
349 */
350int hfi1_netdev_alloc(struct hfi1_devdata *dd)
351{
352	struct hfi1_netdev_priv *priv;
353	const int netdev_size = sizeof(*dd->dummy_netdev) +
354		sizeof(struct hfi1_netdev_priv);
355
356	dd_dev_info(dd, "allocating netdev size %d\n", netdev_size);
357	dd->dummy_netdev = kcalloc_node(1, netdev_size, GFP_KERNEL, dd->node);
358
359	if (!dd->dummy_netdev)
360		return -ENOMEM;
361
362	priv = hfi1_netdev_priv(dd->dummy_netdev);
363	priv->dd = dd;
364	xa_init(&priv->dev_tbl);
365	atomic_set(&priv->enabled, 0);
366	atomic_set(&priv->netdevs, 0);
367
368	return 0;
369}
370
371void hfi1_netdev_free(struct hfi1_devdata *dd)
372{
373	if (dd->dummy_netdev) {
374		dd_dev_info(dd, "hfi1 netdev freed\n");
375		kfree(dd->dummy_netdev);
376		dd->dummy_netdev = NULL;
377	}
378}
379
380/**
381 * hfi1_netdev_enable_queues - This is napi enable function.
382 * It enables napi objects associated with queues.
383 * When at least one device has called it it increments atomic counter.
384 * Disable function decrements counter and when it is 0,
385 * calls napi_disable for every queue.
386 *
387 * @dd: hfi1 dev data
388 */
389void hfi1_netdev_enable_queues(struct hfi1_devdata *dd)
390{
391	struct hfi1_netdev_priv *priv;
392
393	if (!dd->dummy_netdev)
394		return;
395
396	priv = hfi1_netdev_priv(dd->dummy_netdev);
397	if (atomic_fetch_inc(&priv->enabled))
398		return;
399
400	mutex_lock(&hfi1_mutex);
401	enable_queues(priv);
402	mutex_unlock(&hfi1_mutex);
403}
404
405void hfi1_netdev_disable_queues(struct hfi1_devdata *dd)
406{
407	struct hfi1_netdev_priv *priv;
408
409	if (!dd->dummy_netdev)
410		return;
411
412	priv = hfi1_netdev_priv(dd->dummy_netdev);
413	if (atomic_dec_if_positive(&priv->enabled))
414		return;
415
416	mutex_lock(&hfi1_mutex);
417	disable_queues(priv);
418	mutex_unlock(&hfi1_mutex);
419}
420
421/**
422 * hfi1_netdev_add_data - Registers data with unique identifier
423 * to be requested later this is needed for VNIC and IPoIB VLANs
424 * implementations.
425 * This call is protected by mutex idr_lock.
426 *
427 * @dd: hfi1 dev data
428 * @id: requested integer id up to INT_MAX
429 * @data: data to be associated with index
430 */
431int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data)
432{
433	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
434
435	return xa_insert(&priv->dev_tbl, id, data, GFP_NOWAIT);
436}
437
438/**
439 * hfi1_netdev_remove_data - Removes data with previously given id.
440 * Returns the reference to removed entry.
441 *
442 * @dd: hfi1 dev data
443 * @id: requested integer id up to INT_MAX
444 */
445void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id)
446{
447	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
448
449	return xa_erase(&priv->dev_tbl, id);
450}
451
452/**
453 * hfi1_netdev_get_data - Gets data with given id
454 *
455 * @dd: hfi1 dev data
456 * @id: requested integer id up to INT_MAX
457 */
458void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id)
459{
460	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
461
462	return xa_load(&priv->dev_tbl, id);
463}
464
465/**
466 * hfi1_netdev_get_first_dat - Gets first entry with greater or equal id.
467 *
468 * @dd: hfi1 dev data
469 * @id: requested integer id up to INT_MAX
470 */
471void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id)
472{
473	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
474	unsigned long index = *start_id;
475	void *ret;
476
477	ret = xa_find(&priv->dev_tbl, &index, UINT_MAX, XA_PRESENT);
478	*start_id = (int)index;
479	return ret;
480}
481