Lines Matching refs:ref
12 * don't try to detect the ref hitting 0 - which means that get/put can just
23 * the ref hitting 0 on every put - this would require global synchronization
27 * the ref can't hit 0 before the user drops the initial ref, so as long as we
28 * convert to non percpu mode before the initial ref is dropped everything
41 static unsigned long __percpu *percpu_count_ptr(struct percpu_ref *ref)
44 (ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC_DEAD);
49 * @ref: percpu_ref to initialize
54 * Initializes @ref. @ref starts out in percpu mode with a refcount of 1 unless
62 int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release,
70 ref->percpu_count_ptr = (unsigned long)
72 if (!ref->percpu_count_ptr)
75 data = kzalloc(sizeof(*ref->data), gfp);
77 free_percpu((void __percpu *)ref->percpu_count_ptr);
78 ref->percpu_count_ptr = 0;
86 ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC;
93 ref->percpu_count_ptr |= __PERCPU_REF_DEAD;
101 data->ref = ref;
102 ref->data = data;
107 static void __percpu_ref_exit(struct percpu_ref *ref)
109 unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
113 WARN_ON_ONCE(ref->data && ref->data->confirm_switch);
115 ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD;
121 * @ref: percpu_ref to exit
123 * This function exits @ref. The caller is responsible for ensuring that
124 * @ref is no longer in active use. The usual places to invoke this
125 * function from are the @ref->release() callback or in init failure path
129 void percpu_ref_exit(struct percpu_ref *ref)
131 struct percpu_ref_data *data = ref->data;
134 __percpu_ref_exit(ref);
140 ref->percpu_count_ptr |= atomic_long_read(&ref->data->count) <<
142 ref->data = NULL;
153 struct percpu_ref *ref = data->ref;
155 data->confirm_switch(ref);
160 __percpu_ref_exit(ref);
162 /* drop ref from percpu_ref_switch_to_atomic() */
163 percpu_ref_put(ref);
170 struct percpu_ref *ref = data->ref;
171 unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
183 * to &ref->count; since gets could be happening on one cpu while puts
185 * @ref->count to hit 0 before we've got a consistent value - but the
189 * &ref->count; we need the bias value to prevent &ref->count from
196 "percpu ref (%ps) <= 0 (%ld) after switching to atomic",
199 /* @ref is viewed as dead on all CPUs, send out switch confirmation */
203 static void percpu_ref_noop_confirm_switch(struct percpu_ref *ref)
207 static void __percpu_ref_switch_to_atomic(struct percpu_ref *ref,
210 if (ref->percpu_count_ptr & __PERCPU_REF_ATOMIC) {
212 confirm_switch(ref);
217 ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC;
223 ref->data->confirm_switch = confirm_switch ?:
226 percpu_ref_get(ref); /* put after confirmation */
227 call_rcu(&ref->data->rcu, percpu_ref_switch_to_atomic_rcu);
230 static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref)
232 unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
237 if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC))
240 if (WARN_ON_ONCE(!ref->data->allow_reinit))
243 atomic_long_add(PERCPU_COUNT_BIAS, &ref->data->count);
254 smp_store_release(&ref->percpu_count_ptr,
255 ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC);
258 static void __percpu_ref_switch_mode(struct percpu_ref *ref,
261 struct percpu_ref_data *data = ref->data;
273 if (data->force_atomic || (ref->percpu_count_ptr & __PERCPU_REF_DEAD))
274 __percpu_ref_switch_to_atomic(ref, confirm_switch);
276 __percpu_ref_switch_to_percpu(ref);
281 * @ref: percpu_ref to switch to atomic mode
287 * Schedule switching of @ref to atomic mode. All its percpu counts will
292 * operations. Note that @ref will stay in atomic mode across kill/reinit
295 * This function may block if @ref is in the process of switching to atomic
296 * mode. If the caller ensures that @ref is not in the process of
299 void percpu_ref_switch_to_atomic(struct percpu_ref *ref,
306 ref->data->force_atomic = true;
307 __percpu_ref_switch_mode(ref, confirm_switch);
315 * @ref: percpu_ref to switch to atomic mode
317 * Schedule switching the ref to atomic mode, and wait for the
321 void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref)
323 percpu_ref_switch_to_atomic(ref, NULL);
324 wait_event(percpu_ref_switch_waitq, !ref->data->confirm_switch);
330 * @ref: percpu_ref to switch to percpu mode
333 * To re-use an expired ref, use percpu_ref_reinit().
335 * Switch @ref to percpu mode. This function may be invoked concurrently
338 * by PERCPU_REF_INIT_ATOMIC or percpu_ref_switch_to_atomic(). If @ref is
342 * This function may block if @ref is in the process of switching to atomic
343 * mode. If the caller ensures that @ref is not in the process of
346 void percpu_ref_switch_to_percpu(struct percpu_ref *ref)
352 ref->data->force_atomic = false;
353 __percpu_ref_switch_mode(ref, NULL);
360 * percpu_ref_kill_and_confirm - drop the initial ref and schedule confirmation
361 * @ref: percpu_ref to kill
366 * called after @ref is seen as dead from all CPUs at which point all
371 * but it may block if @confirm_kill is specified and @ref is in the
376 void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
383 WARN_ONCE(ref->percpu_count_ptr & __PERCPU_REF_DEAD,
385 ref->data->release);
387 ref->percpu_count_ptr |= __PERCPU_REF_DEAD;
388 __percpu_ref_switch_mode(ref, confirm_kill);
389 percpu_ref_put(ref);
397 * @ref: percpu_ref to test
399 * Returns %true if @ref reached zero.
401 * This function is safe to call as long as @ref is between init and exit.
403 bool percpu_ref_is_zero(struct percpu_ref *ref)
408 if (__ref_is_percpu(ref, &percpu_count))
413 if (ref->data)
414 count = atomic_long_read(&ref->data->count);
416 count = ref->percpu_count_ptr >> __PERCPU_REF_FLAG_BITS;
425 * @ref: perpcu_ref to re-initialize
427 * Re-initialize @ref so that it's in the same state as when it finished
428 * percpu_ref_init() ignoring %PERCPU_REF_INIT_DEAD. @ref must have been
431 * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while
434 void percpu_ref_reinit(struct percpu_ref *ref)
436 WARN_ON_ONCE(!percpu_ref_is_zero(ref));
438 percpu_ref_resurrect(ref);
444 * @ref: perpcu_ref to resurrect
446 * Modify @ref so that it's in the same state as before percpu_ref_kill() was
447 * called. @ref must be dead but must not yet have exited.
449 * If @ref->release() frees @ref then the caller is responsible for
450 * guaranteeing that @ref->release() does not get called while this
453 * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while
456 void percpu_ref_resurrect(struct percpu_ref *ref)
463 WARN_ON_ONCE(!(ref->percpu_count_ptr & __PERCPU_REF_DEAD));
464 WARN_ON_ONCE(__ref_is_percpu(ref, &percpu_count));
466 ref->percpu_count_ptr &= ~__PERCPU_REF_DEAD;
467 percpu_ref_get(ref);
468 __percpu_ref_switch_mode(ref, NULL);