87 _init_srcu_struct(struct srcu_struct *srcu, const char *name)  in _init_srcu_struct()  argument
92 	srcu->srcu_percpu = percpu_alloc(sizeof(struct srcu_cpu));  in _init_srcu_struct()
93 	mutex_init(&srcu->srcu_lock, MUTEX_DEFAULT, IPL_VM);  in _init_srcu_struct()
94 	cv_init(&srcu->srcu_cv, name);  in _init_srcu_struct()
95 	srcu->srcu_sync = NULL;  in _init_srcu_struct()
96 	srcu->srcu_total = 0;  in _init_srcu_struct()
97 	srcu->srcu_gen = 0;  in _init_srcu_struct()
110 cleanup_srcu_struct(struct srcu_struct *srcu)  in cleanup_srcu_struct()  argument
115 	KASSERTMSG((srcu->srcu_sync == NULL),  in cleanup_srcu_struct()
117 	    __func__, curlwp, srcu->srcu_sync);  in cleanup_srcu_struct()
118 	cv_destroy(&srcu->srcu_cv);  in cleanup_srcu_struct()
119 	mutex_destroy(&srcu->srcu_lock);  in cleanup_srcu_struct()
120 	percpu_free(srcu->srcu_percpu, sizeof(struct srcu_cpu));  in cleanup_srcu_struct()
132 srcu_adjust(struct srcu_struct *srcu, unsigned gen, int delta)  in srcu_adjust()  argument
137 	cpu = percpu_getref(srcu->srcu_percpu);  in srcu_adjust()
139 	percpu_putref(srcu->srcu_percpu);  in srcu_adjust()
152 srcu_read_lock(struct srcu_struct *srcu)  in srcu_read_lock()  argument
161 	gen = srcu->srcu_gen;  in srcu_read_lock()
162 	srcu_adjust(srcu, gen, +1);  in srcu_read_lock()
184 srcu_read_unlock(struct srcu_struct *srcu, int ticket)  in srcu_read_unlock()  argument
202 	if (__predict_true(gen == srcu->srcu_gen)) {  in srcu_read_unlock()
209 		srcu_adjust(srcu, gen, -1);  in srcu_read_unlock()
218 		mutex_spin_enter(&srcu->srcu_lock);  in srcu_read_unlock()
219 		KASSERT(srcu->srcu_sync != NULL);  in srcu_read_unlock()
220 		if (--srcu->srcu_total == 0)  in srcu_read_unlock()
221 			cv_broadcast(&srcu->srcu_cv);  in srcu_read_unlock()
222 		mutex_spin_exit(&srcu->srcu_lock);  in srcu_read_unlock()
238 	struct srcu_struct *srcu = a;  in synchronize_srcu_xc()  local
244 	mutex_spin_enter(&srcu->srcu_lock);  in synchronize_srcu_xc()
246 	gen = srcu->srcu_gen;	/* active generation */  in synchronize_srcu_xc()
250 	cpu = percpu_getref(srcu->srcu_percpu);  in synchronize_srcu_xc()
252 	srcu->srcu_total += local;  in synchronize_srcu_xc()
255 	percpu_putref(srcu->srcu_percpu);  in synchronize_srcu_xc()
257 	mutex_spin_exit(&srcu->srcu_lock);  in synchronize_srcu_xc()
269 synchronize_srcu(struct srcu_struct *srcu)  in synchronize_srcu()  argument
275 	mutex_spin_enter(&srcu->srcu_lock);  in synchronize_srcu()
276 	while (srcu->srcu_sync != NULL)  in synchronize_srcu()
277 		cv_wait(&srcu->srcu_cv, &srcu->srcu_lock);  in synchronize_srcu()
278 	KASSERT(srcu->srcu_total == 0);  in synchronize_srcu()
279 	srcu->srcu_sync = curlwp;  in synchronize_srcu()
280 	srcu->srcu_gen++;  in synchronize_srcu()
281 	mutex_spin_exit(&srcu->srcu_lock);  in synchronize_srcu()
288 	xc_wait(xc_broadcast(0, synchronize_srcu_xc, srcu, NULL));  in synchronize_srcu()
294 	mutex_spin_enter(&srcu->srcu_lock);  in synchronize_srcu()
295 	while (srcu->srcu_total != 0)  in synchronize_srcu()
296 		cv_wait(&srcu->srcu_cv, &srcu->srcu_lock);  in synchronize_srcu()
297 	srcu->srcu_sync = NULL;  in synchronize_srcu()
298 	cv_broadcast(&srcu->srcu_cv);  in synchronize_srcu()
299 	mutex_spin_exit(&srcu->srcu_lock);  in synchronize_srcu()
313 synchronize_srcu_expedited(struct srcu_struct *srcu)  in synchronize_srcu_expedited()  argument
316 	synchronize_srcu(srcu);  in synchronize_srcu_expedited()