221 		struct tasklet_struct *tasklet;  in tasklet_softintr()  local
224 		tasklet = SIMPLEQ_FIRST(&th);  in tasklet_softintr()
227 		KASSERT(atomic_load_relaxed(&tasklet->tl_state) &  in tasklet_softintr()
235 		if (!tasklet_trylock(tasklet)) {  in tasklet_softintr()
241 			tasklet_queue_enqueue(tq, tasklet);  in tasklet_softintr()
250 		if (atomic_load_acquire(&tasklet->tl_disablecount)) {  in tasklet_softintr()
255 			tasklet_unlock(tasklet);  in tasklet_softintr()
256 			tasklet_queue_enqueue(tq, tasklet);  in tasklet_softintr()
261 		KASSERT(atomic_load_relaxed(&tasklet->tl_state) &  in tasklet_softintr()
263 		atomic_and_uint(&tasklet->tl_state, ~TASKLET_SCHEDULED);  in tasklet_softintr()
265 		(*tasklet->func)(tasklet->data);  in tasklet_softintr()
268 		tasklet_unlock(tasklet);  in tasklet_softintr()
280     struct tasklet_struct *tasklet)  in tasklet_queue_schedule()  argument
286 		ostate = atomic_load_relaxed(&tasklet->tl_state);  in tasklet_queue_schedule()
290 	} while (atomic_cas_uint(&tasklet->tl_state, ostate, nstate)  in tasklet_queue_schedule()
297 	tasklet_queue_enqueue(tq, tasklet);  in tasklet_queue_schedule()
307 tasklet_queue_enqueue(struct tasklet_queue *tq, struct tasklet_struct *tasklet)  in tasklet_queue_enqueue()  argument
312 	KASSERT(atomic_load_relaxed(&tasklet->tl_state) & TASKLET_SCHEDULED);  in tasklet_queue_enqueue()
322 	SIMPLEQ_INSERT_TAIL(&tc->tc_head, tasklet, tl_entry);  in tasklet_queue_enqueue()
338 tasklet_init(struct tasklet_struct *tasklet, void (*func)(unsigned long),  in tasklet_init()  argument
342 	atomic_store_relaxed(&tasklet->tl_state, 0);  in tasklet_init()
343 	atomic_store_relaxed(&tasklet->tl_disablecount, 0);  in tasklet_init()
344 	tasklet->func = func;  in tasklet_init()
345 	tasklet->data = data;  in tasklet_init()
355 tasklet_schedule(struct tasklet_struct *tasklet)  in tasklet_schedule()  argument
358 	tasklet_queue_schedule(&tasklet_queue, tasklet);  in tasklet_schedule()
368 tasklet_hi_schedule(struct tasklet_struct *tasklet)  in tasklet_hi_schedule()  argument
371 	tasklet_queue_schedule(&tasklet_hi_queue, tasklet);  in tasklet_hi_schedule()
388 tasklet_disable_nosync(struct tasklet_struct *tasklet)  in tasklet_disable_nosync()  argument
393 	disablecount = atomic_inc_uint_nv(&tasklet->tl_disablecount);  in tasklet_disable_nosync()
419 tasklet_disable(struct tasklet_struct *tasklet)  in tasklet_disable()  argument
423 	tasklet_disable_nosync(tasklet);  in tasklet_disable()
426 	tasklet_unlock_wait(tasklet);  in tasklet_disable()
438 tasklet_enable(struct tasklet_struct *tasklet)  in tasklet_enable()  argument
441 	(void)__tasklet_enable(tasklet);  in tasklet_enable()
452 tasklet_kill(struct tasklet_struct *tasklet)  in tasklet_kill()  argument
459 	while (atomic_load_relaxed(&tasklet->tl_state) & TASKLET_SCHEDULED)  in tasklet_kill()
479 	tasklet_unlock_wait(tasklet);  in tasklet_kill()
489 tasklet_is_locked(const struct tasklet_struct *tasklet)  in tasklet_is_locked()  argument
492 	return atomic_load_relaxed(&tasklet->tl_state) & TASKLET_RUNNING;  in tasklet_is_locked()
504 tasklet_trylock(struct tasklet_struct *tasklet)  in tasklet_trylock()  argument
509 		state = atomic_load_relaxed(&tasklet->tl_state);  in tasklet_trylock()
512 	} while (atomic_cas_uint(&tasklet->tl_state, state,  in tasklet_trylock()
529 tasklet_unlock(struct tasklet_struct *tasklet)  in tasklet_unlock()  argument
532 	KASSERT(atomic_load_relaxed(&tasklet->tl_state) & TASKLET_RUNNING);  in tasklet_unlock()
539 	atomic_and_uint(&tasklet->tl_state, ~TASKLET_RUNNING);  in tasklet_unlock()
550 tasklet_unlock_wait(const struct tasklet_struct *tasklet)  in tasklet_unlock_wait()  argument
554 	while (atomic_load_acquire(&tasklet->tl_state) & TASKLET_RUNNING)  in tasklet_unlock_wait()
577 __tasklet_disable_sync_once(struct tasklet_struct *tasklet)  in __tasklet_disable_sync_once()  argument
582 	disablecount = atomic_inc_uint_nv(&tasklet->tl_disablecount);  in __tasklet_disable_sync_once()
594 		tasklet_unlock_wait(tasklet);  in __tasklet_disable_sync_once()
604 __tasklet_enable_sync_once(struct tasklet_struct *tasklet)  in __tasklet_enable_sync_once()  argument
612 	disablecount = atomic_dec_uint_nv(&tasklet->tl_disablecount);  in __tasklet_enable_sync_once()
620 		tasklet_kill(tasklet);  in __tasklet_enable_sync_once()
631 __tasklet_is_enabled(const struct tasklet_struct *tasklet)  in __tasklet_is_enabled()  argument
635 	disablecount = atomic_load_relaxed(&tasklet->tl_disablecount);  in __tasklet_is_enabled()
648 __tasklet_is_scheduled(const struct tasklet_struct *tasklet)  in __tasklet_is_scheduled()  argument
651 	return atomic_load_relaxed(&tasklet->tl_state) & TASKLET_SCHEDULED;  in __tasklet_is_scheduled()
664 __tasklet_enable(struct tasklet_struct *tasklet)  in __tasklet_enable()  argument
679 	disablecount = atomic_dec_uint_nv(&tasklet->tl_disablecount);  in __tasklet_enable()