1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7#include "rxe.h"
8#include "rxe_loc.h"
9
10/* info about object pools
11 * note that mr and mw share a single index space
12 * so that one can map an lkey to the correct type of object
13 */
14struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
15	[RXE_TYPE_UC] = {
16		.name		= "rxe-uc",
17		.size		= sizeof(struct rxe_ucontext),
18		.flags          = RXE_POOL_NO_ALLOC,
19	},
20	[RXE_TYPE_PD] = {
21		.name		= "rxe-pd",
22		.size		= sizeof(struct rxe_pd),
23		.flags		= RXE_POOL_NO_ALLOC,
24	},
25	[RXE_TYPE_AH] = {
26		.name		= "rxe-ah",
27		.size		= sizeof(struct rxe_ah),
28		.flags		= RXE_POOL_ATOMIC | RXE_POOL_NO_ALLOC,
29	},
30	[RXE_TYPE_SRQ] = {
31		.name		= "rxe-srq",
32		.size		= sizeof(struct rxe_srq),
33		.flags		= RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
34		.min_index	= RXE_MIN_SRQ_INDEX,
35		.max_index	= RXE_MAX_SRQ_INDEX,
36	},
37	[RXE_TYPE_QP] = {
38		.name		= "rxe-qp",
39		.size		= sizeof(struct rxe_qp),
40		.cleanup	= rxe_qp_cleanup,
41		.flags		= RXE_POOL_INDEX,
42		.min_index	= RXE_MIN_QP_INDEX,
43		.max_index	= RXE_MAX_QP_INDEX,
44	},
45	[RXE_TYPE_CQ] = {
46		.name		= "rxe-cq",
47		.size		= sizeof(struct rxe_cq),
48		.flags          = RXE_POOL_NO_ALLOC,
49		.cleanup	= rxe_cq_cleanup,
50	},
51	[RXE_TYPE_MR] = {
52		.name		= "rxe-mr",
53		.size		= sizeof(struct rxe_mem),
54		.cleanup	= rxe_mem_cleanup,
55		.flags		= RXE_POOL_INDEX,
56		.max_index	= RXE_MAX_MR_INDEX,
57		.min_index	= RXE_MIN_MR_INDEX,
58	},
59	[RXE_TYPE_MW] = {
60		.name		= "rxe-mw",
61		.size		= sizeof(struct rxe_mem),
62		.flags		= RXE_POOL_INDEX,
63		.max_index	= RXE_MAX_MW_INDEX,
64		.min_index	= RXE_MIN_MW_INDEX,
65	},
66	[RXE_TYPE_MC_GRP] = {
67		.name		= "rxe-mc_grp",
68		.size		= sizeof(struct rxe_mc_grp),
69		.cleanup	= rxe_mc_cleanup,
70		.flags		= RXE_POOL_KEY,
71		.key_offset	= offsetof(struct rxe_mc_grp, mgid),
72		.key_size	= sizeof(union ib_gid),
73	},
74	[RXE_TYPE_MC_ELEM] = {
75		.name		= "rxe-mc_elem",
76		.size		= sizeof(struct rxe_mc_elem),
77		.flags		= RXE_POOL_ATOMIC,
78	},
79};
80
81static inline const char *pool_name(struct rxe_pool *pool)
82{
83	return rxe_type_info[pool->type].name;
84}
85
86static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
87{
88	int err = 0;
89	size_t size;
90
91	if ((max - min + 1) < pool->max_elem) {
92		pr_warn("not enough indices for max_elem\n");
93		err = -EINVAL;
94		goto out;
95	}
96
97	pool->max_index = max;
98	pool->min_index = min;
99
100	size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
101	pool->table = kmalloc(size, GFP_KERNEL);
102	if (!pool->table) {
103		err = -ENOMEM;
104		goto out;
105	}
106
107	pool->table_size = size;
108	bitmap_zero(pool->table, max - min + 1);
109
110out:
111	return err;
112}
113
114int rxe_pool_init(
115	struct rxe_dev		*rxe,
116	struct rxe_pool		*pool,
117	enum rxe_elem_type	type,
118	unsigned int		max_elem)
119{
120	int			err = 0;
121	size_t			size = rxe_type_info[type].size;
122
123	memset(pool, 0, sizeof(*pool));
124
125	pool->rxe		= rxe;
126	pool->type		= type;
127	pool->max_elem		= max_elem;
128	pool->elem_size		= ALIGN(size, RXE_POOL_ALIGN);
129	pool->flags		= rxe_type_info[type].flags;
130	pool->tree		= RB_ROOT;
131	pool->cleanup		= rxe_type_info[type].cleanup;
132
133	atomic_set(&pool->num_elem, 0);
134
135	kref_init(&pool->ref_cnt);
136
137	rwlock_init(&pool->pool_lock);
138
139	if (rxe_type_info[type].flags & RXE_POOL_INDEX) {
140		err = rxe_pool_init_index(pool,
141					  rxe_type_info[type].max_index,
142					  rxe_type_info[type].min_index);
143		if (err)
144			goto out;
145	}
146
147	if (rxe_type_info[type].flags & RXE_POOL_KEY) {
148		pool->key_offset = rxe_type_info[type].key_offset;
149		pool->key_size = rxe_type_info[type].key_size;
150	}
151
152	pool->state = RXE_POOL_STATE_VALID;
153
154out:
155	return err;
156}
157
158static void rxe_pool_release(struct kref *kref)
159{
160	struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
161
162	pool->state = RXE_POOL_STATE_INVALID;
163	kfree(pool->table);
164}
165
166static void rxe_pool_put(struct rxe_pool *pool)
167{
168	kref_put(&pool->ref_cnt, rxe_pool_release);
169}
170
171void rxe_pool_cleanup(struct rxe_pool *pool)
172{
173	unsigned long flags;
174
175	write_lock_irqsave(&pool->pool_lock, flags);
176	pool->state = RXE_POOL_STATE_INVALID;
177	if (atomic_read(&pool->num_elem) > 0)
178		pr_warn("%s pool destroyed with unfree'd elem\n",
179			pool_name(pool));
180	write_unlock_irqrestore(&pool->pool_lock, flags);
181
182	rxe_pool_put(pool);
183}
184
185static u32 alloc_index(struct rxe_pool *pool)
186{
187	u32 index;
188	u32 range = pool->max_index - pool->min_index + 1;
189
190	index = find_next_zero_bit(pool->table, range, pool->last);
191	if (index >= range)
192		index = find_first_zero_bit(pool->table, range);
193
194	WARN_ON_ONCE(index >= range);
195	set_bit(index, pool->table);
196	pool->last = index;
197	return index + pool->min_index;
198}
199
200static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
201{
202	struct rb_node **link = &pool->tree.rb_node;
203	struct rb_node *parent = NULL;
204	struct rxe_pool_entry *elem;
205
206	while (*link) {
207		parent = *link;
208		elem = rb_entry(parent, struct rxe_pool_entry, node);
209
210		if (elem->index == new->index) {
211			pr_warn("element already exists!\n");
212			goto out;
213		}
214
215		if (elem->index > new->index)
216			link = &(*link)->rb_left;
217		else
218			link = &(*link)->rb_right;
219	}
220
221	rb_link_node(&new->node, parent, link);
222	rb_insert_color(&new->node, &pool->tree);
223out:
224	return;
225}
226
227static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
228{
229	struct rb_node **link = &pool->tree.rb_node;
230	struct rb_node *parent = NULL;
231	struct rxe_pool_entry *elem;
232	int cmp;
233
234	while (*link) {
235		parent = *link;
236		elem = rb_entry(parent, struct rxe_pool_entry, node);
237
238		cmp = memcmp((u8 *)elem + pool->key_offset,
239			     (u8 *)new + pool->key_offset, pool->key_size);
240
241		if (cmp == 0) {
242			pr_warn("key already exists!\n");
243			goto out;
244		}
245
246		if (cmp > 0)
247			link = &(*link)->rb_left;
248		else
249			link = &(*link)->rb_right;
250	}
251
252	rb_link_node(&new->node, parent, link);
253	rb_insert_color(&new->node, &pool->tree);
254out:
255	return;
256}
257
258void rxe_add_key(void *arg, void *key)
259{
260	struct rxe_pool_entry *elem = arg;
261	struct rxe_pool *pool = elem->pool;
262	unsigned long flags;
263
264	write_lock_irqsave(&pool->pool_lock, flags);
265	memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
266	insert_key(pool, elem);
267	write_unlock_irqrestore(&pool->pool_lock, flags);
268}
269
270void rxe_drop_key(void *arg)
271{
272	struct rxe_pool_entry *elem = arg;
273	struct rxe_pool *pool = elem->pool;
274	unsigned long flags;
275
276	write_lock_irqsave(&pool->pool_lock, flags);
277	rb_erase(&elem->node, &pool->tree);
278	write_unlock_irqrestore(&pool->pool_lock, flags);
279}
280
281void rxe_add_index(void *arg)
282{
283	struct rxe_pool_entry *elem = arg;
284	struct rxe_pool *pool = elem->pool;
285	unsigned long flags;
286
287	write_lock_irqsave(&pool->pool_lock, flags);
288	elem->index = alloc_index(pool);
289	insert_index(pool, elem);
290	write_unlock_irqrestore(&pool->pool_lock, flags);
291}
292
293void rxe_drop_index(void *arg)
294{
295	struct rxe_pool_entry *elem = arg;
296	struct rxe_pool *pool = elem->pool;
297	unsigned long flags;
298
299	write_lock_irqsave(&pool->pool_lock, flags);
300	clear_bit(elem->index - pool->min_index, pool->table);
301	rb_erase(&elem->node, &pool->tree);
302	write_unlock_irqrestore(&pool->pool_lock, flags);
303}
304
305void *rxe_alloc(struct rxe_pool *pool)
306{
307	struct rxe_pool_entry *elem;
308	unsigned long flags;
309
310	might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
311
312	read_lock_irqsave(&pool->pool_lock, flags);
313	if (pool->state != RXE_POOL_STATE_VALID) {
314		read_unlock_irqrestore(&pool->pool_lock, flags);
315		return NULL;
316	}
317	kref_get(&pool->ref_cnt);
318	read_unlock_irqrestore(&pool->pool_lock, flags);
319
320	if (!ib_device_try_get(&pool->rxe->ib_dev))
321		goto out_put_pool;
322
323	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
324		goto out_cnt;
325
326	elem = kzalloc(rxe_type_info[pool->type].size,
327				 (pool->flags & RXE_POOL_ATOMIC) ?
328				 GFP_ATOMIC : GFP_KERNEL);
329	if (!elem)
330		goto out_cnt;
331
332	elem->pool = pool;
333	kref_init(&elem->ref_cnt);
334
335	return elem;
336
337out_cnt:
338	atomic_dec(&pool->num_elem);
339	ib_device_put(&pool->rxe->ib_dev);
340out_put_pool:
341	rxe_pool_put(pool);
342	return NULL;
343}
344
345int rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)
346{
347	unsigned long flags;
348
349	might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
350
351	read_lock_irqsave(&pool->pool_lock, flags);
352	if (pool->state != RXE_POOL_STATE_VALID) {
353		read_unlock_irqrestore(&pool->pool_lock, flags);
354		return -EINVAL;
355	}
356	kref_get(&pool->ref_cnt);
357	read_unlock_irqrestore(&pool->pool_lock, flags);
358
359	if (!ib_device_try_get(&pool->rxe->ib_dev))
360		goto out_put_pool;
361
362	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
363		goto out_cnt;
364
365	elem->pool = pool;
366	kref_init(&elem->ref_cnt);
367
368	return 0;
369
370out_cnt:
371	atomic_dec(&pool->num_elem);
372	ib_device_put(&pool->rxe->ib_dev);
373out_put_pool:
374	rxe_pool_put(pool);
375	return -EINVAL;
376}
377
378void rxe_elem_release(struct kref *kref)
379{
380	struct rxe_pool_entry *elem =
381		container_of(kref, struct rxe_pool_entry, ref_cnt);
382	struct rxe_pool *pool = elem->pool;
383
384	if (pool->cleanup)
385		pool->cleanup(elem);
386
387	if (!(pool->flags & RXE_POOL_NO_ALLOC))
388		kfree(elem);
389	atomic_dec(&pool->num_elem);
390	ib_device_put(&pool->rxe->ib_dev);
391	rxe_pool_put(pool);
392}
393
394void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
395{
396	struct rb_node *node = NULL;
397	struct rxe_pool_entry *elem = NULL;
398	unsigned long flags;
399
400	read_lock_irqsave(&pool->pool_lock, flags);
401
402	if (pool->state != RXE_POOL_STATE_VALID)
403		goto out;
404
405	node = pool->tree.rb_node;
406
407	while (node) {
408		elem = rb_entry(node, struct rxe_pool_entry, node);
409
410		if (elem->index > index)
411			node = node->rb_left;
412		else if (elem->index < index)
413			node = node->rb_right;
414		else {
415			kref_get(&elem->ref_cnt);
416			break;
417		}
418	}
419
420out:
421	read_unlock_irqrestore(&pool->pool_lock, flags);
422	return node ? elem : NULL;
423}
424
425void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
426{
427	struct rb_node *node = NULL;
428	struct rxe_pool_entry *elem = NULL;
429	int cmp;
430	unsigned long flags;
431
432	read_lock_irqsave(&pool->pool_lock, flags);
433
434	if (pool->state != RXE_POOL_STATE_VALID)
435		goto out;
436
437	node = pool->tree.rb_node;
438
439	while (node) {
440		elem = rb_entry(node, struct rxe_pool_entry, node);
441
442		cmp = memcmp((u8 *)elem + pool->key_offset,
443			     key, pool->key_size);
444
445		if (cmp > 0)
446			node = node->rb_left;
447		else if (cmp < 0)
448			node = node->rb_right;
449		else
450			break;
451	}
452
453	if (node)
454		kref_get(&elem->ref_cnt);
455
456out:
457	read_unlock_irqrestore(&pool->pool_lock, flags);
458	return node ? elem : NULL;
459}
460