Lines Matching defs:ssp
27 static int init_srcu_struct_fields(struct srcu_struct *ssp)
29 ssp->srcu_lock_nesting[0] = 0;
30 ssp->srcu_lock_nesting[1] = 0;
31 init_swait_queue_head(&ssp->srcu_wq);
32 ssp->srcu_cb_head = NULL;
33 ssp->srcu_cb_tail = &ssp->srcu_cb_head;
34 ssp->srcu_gp_running = false;
35 ssp->srcu_gp_waiting = false;
36 ssp->srcu_idx = 0;
37 ssp->srcu_idx_max = 0;
38 INIT_WORK(&ssp->srcu_work, srcu_drive_gp);
39 INIT_LIST_HEAD(&ssp->srcu_work.entry);
45 int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
49 debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));
50 lockdep_init_map(&ssp->dep_map, name, key, 0);
51 return init_srcu_struct_fields(ssp);
59 * @ssp: structure to initialize.
65 int init_srcu_struct(struct srcu_struct *ssp)
67 return init_srcu_struct_fields(ssp);
75 * @ssp: structure to clean up.
80 void cleanup_srcu_struct(struct srcu_struct *ssp)
82 WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]);
83 flush_work(&ssp->srcu_work);
84 WARN_ON(ssp->srcu_gp_running);
85 WARN_ON(ssp->srcu_gp_waiting);
86 WARN_ON(ssp->srcu_cb_head);
87 WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);
88 WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max);
89 WARN_ON(ssp->srcu_idx & 0x1);
97 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)
99 int newval = ssp->srcu_lock_nesting[idx] - 1;
101 WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);
102 if (!newval && READ_ONCE(ssp->srcu_gp_waiting))
103 swake_up_one(&ssp->srcu_wq);
117 struct srcu_struct *ssp;
119 ssp = container_of(wp, struct srcu_struct, srcu_work);
120 if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
124 WRITE_ONCE(ssp->srcu_gp_running, true);
126 lh = ssp->srcu_cb_head;
127 ssp->srcu_cb_head = NULL;
128 ssp->srcu_cb_tail = &ssp->srcu_cb_head;
130 idx = (ssp->srcu_idx & 0x2) / 2;
131 WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
132 WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */
133 swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));
134 WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */
135 WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
152 WRITE_ONCE(ssp->srcu_gp_running, false);
153 if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
154 schedule_work(&ssp->srcu_work);
158 static void srcu_gp_start_if_needed(struct srcu_struct *ssp)
162 cookie = get_state_synchronize_srcu(ssp);
163 if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))
165 WRITE_ONCE(ssp->srcu_idx_max, cookie);
166 if (!READ_ONCE(ssp->srcu_gp_running)) {
168 schedule_work(&ssp->srcu_work);
169 else if (list_empty(&ssp->srcu_work.entry))
170 list_add(&ssp->srcu_work.entry, &srcu_boot_list);
178 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
186 *ssp->srcu_cb_tail = rhp;
187 ssp->srcu_cb_tail = &rhp->next;
189 srcu_gp_start_if_needed(ssp);
196 void synchronize_srcu(struct srcu_struct *ssp)
202 call_srcu(ssp, &rs.head, wakeme_after_rcu);
211 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)
216 ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;
229 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)
231 unsigned long ret = get_state_synchronize_srcu(ssp);
233 srcu_gp_start_if_needed(ssp);
241 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)
243 bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie);
263 struct srcu_struct *ssp;
267 ssp = list_first_entry(&srcu_boot_list,
269 list_del_init(&ssp->srcu_work.entry);
270 schedule_work(&ssp->srcu_work);