Lines Matching refs:rtp
15 typedef void (*rcu_tasks_gp_func_t)(struct rcu_tasks *rtp);
20 typedef void (*postgp_func_t)(struct rcu_tasks *rtp);
132 static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate)
134 rtp->gp_state = newstate;
135 rtp->gp_jiffies = jiffies;
140 static const char *tasks_gp_state_getname(struct rcu_tasks *rtp)
142 int i = data_race(rtp->gp_state); // Let KCSAN detect update races
153 struct rcu_tasks *rtp)
160 raw_spin_lock_irqsave(&rtp->cbs_lock, flags);
161 needwake = !rtp->cbs_head;
162 WRITE_ONCE(*rtp->cbs_tail, rhp);
163 rtp->cbs_tail = &rhp->next;
164 raw_spin_unlock_irqrestore(&rtp->cbs_lock, flags);
166 if (needwake && READ_ONCE(rtp->kthread_ptr))
167 wake_up(&rtp->cbs_wq);
171 static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp)
175 "synchronize_%s() called too soon", rtp->name))
179 wait_rcu_gp(rtp->call_func);
188 struct rcu_tasks *rtp = arg;
192 WRITE_ONCE(rtp->kthread_ptr, current); // Let GPs start!
201 set_tasks_gp_state(rtp, RTGS_WAIT_CBS);
204 raw_spin_lock_irqsave(&rtp->cbs_lock, flags);
206 list = rtp->cbs_head;
207 rtp->cbs_head = NULL;
208 rtp->cbs_tail = &rtp->cbs_head;
209 raw_spin_unlock_irqrestore(&rtp->cbs_lock, flags);
213 wait_event_interruptible(rtp->cbs_wq,
214 READ_ONCE(rtp->cbs_head));
215 if (!rtp->cbs_head) {
217 set_tasks_gp_state(rtp, RTGS_WAIT_WAIT_CBS);
224 set_tasks_gp_state(rtp, RTGS_WAIT_GP);
225 rtp->gp_start = jiffies;
226 rtp->gp_func(rtp);
227 rtp->n_gps++;
230 set_tasks_gp_state(rtp, RTGS_INVOKE_CBS);
240 schedule_timeout_idle(rtp->gp_sleep);
245 static void __init rcu_spawn_tasks_kthread_generic(struct rcu_tasks *rtp)
249 t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname);
250 if (WARN_ONCE(IS_ERR(t), "%s: Could not start %s grace-period kthread, OOM is now expected behavior\n", __func__, rtp->name))
281 static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s)
284 rtp->kname,
285 tasks_gp_state_getname(rtp), data_race(rtp->gp_state),
286 jiffies - data_race(rtp->gp_jiffies),
287 data_race(rtp->n_gps),
288 data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis),
289 ".k"[!!data_race(rtp->kthread_ptr)],
290 ".C"[!!data_race(rtp->cbs_head)],
304 static void rcu_tasks_wait_gp(struct rcu_tasks *rtp)
311 set_tasks_gp_state(rtp, RTGS_PRE_WAIT_GP);
312 rtp->pregp_func();
320 set_tasks_gp_state(rtp, RTGS_SCAN_TASKLIST);
323 rtp->pertask_func(t, &holdouts);
326 set_tasks_gp_state(rtp, RTGS_POST_SCAN_TASKLIST);
327 rtp->postscan_func(&holdouts);
337 fract = rtp->init_fract;
350 set_tasks_gp_state(rtp, RTGS_WAIT_SCAN_HOLDOUTS);
362 set_tasks_gp_state(rtp, RTGS_SCAN_HOLDOUTS);
363 rtp->holdouts_func(&holdouts, needreport, &firstreport);
366 set_tasks_gp_state(rtp, RTGS_POST_GP);
367 rtp->postgp_func(rtp);
484 static void rcu_tasks_postgp(struct rcu_tasks *rtp)
650 static void rcu_tasks_rude_wait_gp(struct rcu_tasks *rtp)
652 rtp->n_ipis += cpumask_weight(cpu_online_mask);
1091 static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp)
1114 set_tasks_gp_state(rtp, RTGS_WAIT_READERS);