Lines Matching defs:rsclp
82 static void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v)
85 atomic_long_set(&rsclp->len, v);
87 WRITE_ONCE(rsclp->len, v);
98 static void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v)
102 atomic_long_add(v, &rsclp->len);
106 WRITE_ONCE(rsclp->len, rsclp->len + v);
117 void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp)
119 rcu_segcblist_add_len(rsclp, 1);
128 static long rcu_segcblist_xchg_len(struct rcu_segcblist *rsclp, long v)
131 return atomic_long_xchg(&rsclp->len, v);
133 long ret = rsclp->len;
136 WRITE_ONCE(rsclp->len, v);
145 void rcu_segcblist_init(struct rcu_segcblist *rsclp)
149 BUILD_BUG_ON(RCU_NEXT_TAIL + 1 != ARRAY_SIZE(rsclp->gp_seq));
150 BUILD_BUG_ON(ARRAY_SIZE(rsclp->tails) != ARRAY_SIZE(rsclp->gp_seq));
151 rsclp->head = NULL;
153 rsclp->tails[i] = &rsclp->head;
154 rcu_segcblist_set_len(rsclp, 0);
155 rsclp->enabled = 1;
162 void rcu_segcblist_disable(struct rcu_segcblist *rsclp)
164 WARN_ON_ONCE(!rcu_segcblist_empty(rsclp));
165 WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp));
166 rsclp->enabled = 0;
173 void rcu_segcblist_offload(struct rcu_segcblist *rsclp)
175 rsclp->offloaded = 1;
182 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp)
184 return rcu_segcblist_is_enabled(rsclp) &&
185 &rsclp->head != READ_ONCE(rsclp->tails[RCU_DONE_TAIL]);
192 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp)
194 return rcu_segcblist_is_enabled(rsclp) &&
195 !rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL);
202 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp)
204 if (rcu_segcblist_is_enabled(rsclp))
205 return rsclp->head;
216 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp)
218 if (rcu_segcblist_is_enabled(rsclp))
219 return *rsclp->tails[RCU_DONE_TAIL];
227 bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp)
229 if (!rcu_segcblist_pend_cbs(rsclp))
231 *lp = rsclp->gp_seq[RCU_WAIT_TAIL];
244 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
247 rcu_segcblist_inc_len(rsclp);
250 WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp);
251 WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next);
264 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
269 if (rcu_segcblist_n_cbs(rsclp) == 0)
271 rcu_segcblist_inc_len(rsclp);
275 if (rsclp->tails[i] != rsclp->tails[i - 1])
277 WRITE_ONCE(*rsclp->tails[i], rhp);
279 WRITE_ONCE(rsclp->tails[i], &rhp->next);
292 void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
295 rclp->len = rcu_segcblist_xchg_len(rsclp, 0);
303 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
308 if (!rcu_segcblist_ready_cbs(rsclp))
310 *rclp->tail = rsclp->head;
311 WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]);
312 WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
313 rclp->tail = rsclp->tails[RCU_DONE_TAIL];
315 if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL])
316 WRITE_ONCE(rsclp->tails[i], &rsclp->head);
326 void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
331 if (!rcu_segcblist_pend_cbs(rsclp))
333 *rclp->tail = *rsclp->tails[RCU_DONE_TAIL];
334 rclp->tail = rsclp->tails[RCU_NEXT_TAIL];
335 WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
337 WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]);
344 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
347 rcu_segcblist_add_len(rsclp, rclp->len);
355 void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
362 *rclp->tail = rsclp->head;
363 WRITE_ONCE(rsclp->head, rclp->head);
365 if (&rsclp->head == rsclp->tails[i])
366 WRITE_ONCE(rsclp->tails[i], rclp->tail);
377 void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
382 WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);
383 WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);
390 void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)
394 WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp));
395 if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL))
403 if (ULONG_CMP_LT(seq, rsclp->gp_seq[i]))
405 WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]);
414 WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]);
423 if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL])
425 WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]);
426 rsclp->gp_seq[j] = rsclp->gp_seq[i];
445 bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq)
449 WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp));
450 if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL))
462 if (rsclp->tails[i] != rsclp->tails[i - 1] &&
463 ULONG_CMP_LT(rsclp->gp_seq[i], seq))
487 if (rcu_segcblist_restempty(rsclp, i) || ++i >= RCU_NEXT_TAIL)
498 WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]);
499 rsclp->gp_seq[i] = seq;