Lines Matching refs:tbl
38 const struct bucket_table *tbl,
41 return rht_head_hashfn(ht, tbl, he, ht->p);
53 int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash)
57 if (unlikely(tbl->nest))
59 return bit_spin_is_locked(0, (unsigned long *)&tbl->buckets[hash]);
67 const struct bucket_table *tbl)
70 * because it's set at the same time as tbl->nest.
72 return (void *)rcu_dereference_protected(tbl->buckets[0], 1);
94 static void nested_bucket_table_free(const struct bucket_table *tbl)
96 unsigned int size = tbl->size >> tbl->nest;
97 unsigned int len = 1 << tbl->nest;
101 ntbl = nested_table_top(tbl);
109 static void bucket_table_free(const struct bucket_table *tbl)
111 if (tbl->nest)
112 nested_bucket_table_free(tbl);
114 kvfree(tbl);
152 struct bucket_table *tbl;
158 size = sizeof(*tbl) + sizeof(tbl->buckets[0]);
160 tbl = kzalloc(size, gfp);
161 if (!tbl)
164 if (!nested_table_alloc(ht, (union nested_table __rcu **)tbl->buckets,
166 kfree(tbl);
170 tbl->nest = (ilog2(nbuckets) - 1) % shift + 1;
172 return tbl;
179 struct bucket_table *tbl = NULL;
184 tbl = kvzalloc(struct_size(tbl, buckets, nbuckets), gfp);
188 if (tbl == NULL && (gfp & ~__GFP_NOFAIL) != GFP_KERNEL) {
189 tbl = nested_bucket_table_alloc(ht, nbuckets, gfp);
193 if (tbl == NULL)
196 lockdep_init_map(&tbl->dep_map, "rhashtable_bucket", &__key, 0);
198 tbl->size = size;
200 rcu_head_init(&tbl->rcu);
201 INIT_LIST_HEAD(&tbl->walkers);
203 tbl->hash_rnd = get_random_u32();
206 INIT_RHT_NULLS_HEAD(tbl->buckets[i]);
208 return tbl;
212 struct bucket_table *tbl)
217 new_tbl = tbl;
218 tbl = rht_dereference_rcu(tbl->future_tbl, ht);
219 } while (tbl);
228 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
277 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
314 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
332 rcu_assign_pointer(ht->tbl, new_tbl);
336 walker->tbl = NULL;
389 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
410 struct bucket_table *tbl;
416 tbl = rht_dereference(ht->tbl, ht);
417 tbl = rhashtable_last_table(ht, tbl);
419 if (rht_grow_above_75(ht, tbl))
420 err = rhashtable_rehash_alloc(ht, tbl, tbl->size * 2);
421 else if (ht->p.automatic_shrinking && rht_shrink_below_30(ht, tbl))
423 else if (tbl->nest)
424 err = rhashtable_rehash_alloc(ht, tbl, tbl->size);
440 struct bucket_table *tbl)
447 old_tbl = rht_dereference_rcu(ht->tbl, ht);
449 size = tbl->size;
453 if (rht_grow_above_75(ht, tbl))
456 else if (old_tbl != tbl)
465 err = rhashtable_rehash_attach(ht, tbl, new_tbl);
477 if (likely(rcu_access_pointer(tbl->future_tbl)))
489 struct bucket_table *tbl, unsigned int hash,
501 rht_for_each_from(head, rht_ptr(bkt, tbl, hash), tbl, hash) {
521 head = rht_dereference_bucket(head->next, tbl, hash);
540 struct bucket_table *tbl, unsigned int hash, struct rhash_head *obj,
552 new_tbl = rht_dereference_rcu(tbl->future_tbl, ht);
559 if (unlikely(rht_grow_above_max(ht, tbl)))
562 if (unlikely(rht_grow_above_100(ht, tbl)))
565 head = rht_ptr(bkt, tbl, hash);
581 if (rht_grow_above_75(ht, tbl))
591 struct bucket_table *tbl;
596 new_tbl = rcu_dereference(ht->tbl);
599 tbl = new_tbl;
600 hash = rht_head_hashfn(ht, tbl, obj, ht->p);
601 if (rcu_access_pointer(tbl->future_tbl))
603 bkt = rht_bucket_var(tbl, hash);
605 bkt = rht_bucket_insert(ht, tbl, hash);
607 new_tbl = rht_dereference_rcu(tbl->future_tbl, ht);
610 rht_lock(tbl, bkt);
611 data = rhashtable_lookup_one(ht, bkt, tbl,
613 new_tbl = rhashtable_insert_one(ht, bkt, tbl,
618 rht_unlock(tbl, bkt);
623 data = ERR_PTR(rhashtable_insert_rehash(ht, tbl) ?:
674 iter->walker.tbl =
675 rcu_dereference_protected(ht->tbl, lockdep_is_held(&ht->lock));
676 list_add(&iter->walker.list, &iter->walker.tbl->walkers);
690 if (iter->walker.tbl)
723 if (iter->walker.tbl)
729 if (!iter->walker.tbl) {
730 iter->walker.tbl = rht_dereference_rcu(ht->tbl, ht);
743 rht_for_each_rcu(p, iter->walker.tbl, iter->slot) {
758 rht_for_each_rcu(p, iter->walker.tbl, iter->slot) {
789 struct bucket_table *tbl = iter->walker.tbl;
795 if (!tbl)
798 for (; iter->slot < tbl->size; iter->slot++) {
801 rht_for_each_rcu(p, tbl, iter->slot) {
835 iter->walker.tbl = rht_dereference_rcu(tbl->future_tbl, ht);
836 if (iter->walker.tbl) {
934 struct bucket_table *tbl = iter->walker.tbl;
936 if (!tbl)
942 if (rcu_head_after_call_rcu(&tbl->rcu, bucket_table_free_rcu))
944 iter->walker.tbl = NULL;
946 list_add(&iter->walker.list, &tbl->walkers);
1018 struct bucket_table *tbl;
1061 tbl = bucket_table_alloc(ht, size, GFP_KERNEL);
1062 if (unlikely(tbl == NULL)) {
1064 tbl = bucket_table_alloc(ht, size, GFP_KERNEL | __GFP_NOFAIL);
1069 RCU_INIT_POINTER(ht->tbl, tbl);
1134 struct bucket_table *tbl, *next_tbl;
1140 tbl = rht_dereference(ht->tbl, ht);
1143 for (i = 0; i < tbl->size; i++) {
1147 for (pos = rht_ptr_exclusive(rht_bucket(tbl, i)),
1158 next_tbl = rht_dereference(tbl->future_tbl, ht);
1159 bucket_table_free(tbl);
1161 tbl = next_tbl;
1175 const struct bucket_table *tbl, unsigned int hash)
1178 unsigned int index = hash & ((1 << tbl->nest) - 1);
1179 unsigned int size = tbl->size >> tbl->nest;
1183 ntbl = nested_table_top(tbl);
1184 ntbl = rht_dereference_bucket_rcu(ntbl[index].table, tbl, hash);
1185 subhash >>= tbl->nest;
1190 tbl, hash);
1204 const struct bucket_table *tbl, unsigned int hash)
1210 return __rht_bucket_nested(tbl, hash) ?: &rhnull;
1215 struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)
1218 unsigned int index = hash & ((1 << tbl->nest) - 1);
1219 unsigned int size = tbl->size >> tbl->nest;
1222 ntbl = nested_table_top(tbl);
1223 hash >>= tbl->nest;