Lines Matching refs:work
89 SDT_PROBE_DEFINE2(sdt, linux, work, acquire,
91 SDT_PROBE_DEFINE2(sdt, linux, work, release,
93 SDT_PROBE_DEFINE2(sdt, linux, work, queue,
95 SDT_PROBE_DEFINE2(sdt, linux, work, rcu,
97 SDT_PROBE_DEFINE2(sdt, linux, work, cancel,
99 SDT_PROBE_DEFINE3(sdt, linux, work, schedule,
102 SDT_PROBE_DEFINE2(sdt, linux, work, timer,
104 SDT_PROBE_DEFINE2(sdt, linux, work, wait__start,
106 SDT_PROBE_DEFINE2(sdt, linux, work, wait__done,
108 SDT_PROBE_DEFINE2(sdt, linux, work, run,
110 SDT_PROBE_DEFINE2(sdt, linux, work, done,
112 SDT_PROBE_DEFINE1(sdt, linux, work, batch__start,
114 SDT_PROBE_DEFINE1(sdt, linux, work, batch__done,
116 SDT_PROBE_DEFINE1(sdt, linux, work, flush__self,
118 SDT_PROBE_DEFINE1(sdt, linux, work, flush__start,
120 SDT_PROBE_DEFINE1(sdt, linux, work, flush__done,
329 KASSERT(work_queue(&dw->work) == wq); in destroy_workqueue()
349 SDT_PROBE2(sdt, linux, work, cancel, &dw->work, wq); in destroy_workqueue()
405 struct work_struct marker, *work; in linux_workqueue_thread() local
429 SDT_PROBE1(sdt, linux, work, batch__start, wq); in linux_workqueue_thread()
434 while ((work = TAILQ_FIRST(q[i])) != &marker) { in linux_workqueue_thread()
437 KASSERT(work_queue(work) == wq); in linux_workqueue_thread()
438 KASSERT(work_claimed(work, wq)); in linux_workqueue_thread()
440 container_of(work, struct delayed_work, in linux_workqueue_thread()
441 work)->dw_state == in linux_workqueue_thread()
444 work); in linux_workqueue_thread()
446 TAILQ_REMOVE(q[i], work, work_entry); in linux_workqueue_thread()
448 wq->wq_current_work = work; in linux_workqueue_thread()
449 func = work->func; in linux_workqueue_thread()
450 release_work(work, wq); in linux_workqueue_thread()
454 SDT_PROBE2(sdt, linux, work, run, work, wq); in linux_workqueue_thread()
455 (*func)(work); in linux_workqueue_thread()
456 SDT_PROBE2(sdt, linux, work, done, work, wq); in linux_workqueue_thread()
459 KASSERT(wq->wq_current_work == work); in linux_workqueue_thread()
469 SDT_PROBE1(sdt, linux, work, batch__done, wq); in linux_workqueue_thread()
490 struct workqueue_struct *const wq = work_queue(&dw->work); in linux_workqueue_timeout()
496 SDT_PROBE2(sdt, linux, work, timer, dw, wq); in linux_workqueue_timeout()
499 KASSERT(work_queue(&dw->work) == wq); in linux_workqueue_timeout()
505 TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work, work_entry); in linux_workqueue_timeout()
507 SDT_PROBE2(sdt, linux, work, queue, &dw->work, wq); in linux_workqueue_timeout()
561 INIT_WORK(struct work_struct *work, void (*fn)(struct work_struct *)) in INIT_WORK() argument
564 work->work_owner = 0; in INIT_WORK()
565 work->func = fn; in INIT_WORK()
576 work_claimed(struct work_struct *work, struct workqueue_struct *wq) in work_claimed() argument
579 KASSERT(work_queue(work) == wq); in work_claimed()
582 return atomic_load_relaxed(&work->work_owner) & 1; in work_claimed()
592 work_pending(const struct work_struct *work) in work_pending() argument
595 return atomic_load_relaxed(&work->work_owner) & 1; in work_pending()
605 work_queue(struct work_struct *work) in work_queue() argument
609 (atomic_load_relaxed(&work->work_owner) & ~(uintptr_t)1); in work_queue()
623 acquire_work(struct work_struct *work, struct workqueue_struct *wq) in acquire_work() argument
632 owner0 = atomic_load_relaxed(&work->work_owner); in acquire_work()
638 } while (atomic_cas_uintptr(&work->work_owner, owner0, owner) != in acquire_work()
641 KASSERT(work_queue(work) == wq); in acquire_work()
643 SDT_PROBE2(sdt, linux, work, acquire, work, wq); in acquire_work()
656 release_work(struct work_struct *work, struct workqueue_struct *wq) in release_work() argument
659 KASSERT(work_queue(work) == wq); in release_work()
662 SDT_PROBE2(sdt, linux, work, release, work, wq); in release_work()
670 atomic_store_relaxed(&work->work_owner, in release_work()
671 atomic_load_relaxed(&work->work_owner) & ~(uintptr_t)1); in release_work()
686 schedule_work(struct work_struct *work) in schedule_work() argument
689 return queue_work(system_wq, work); in schedule_work()
704 queue_work(struct workqueue_struct *wq, struct work_struct *work) in queue_work() argument
711 if (__predict_true(acquire_work(work, wq))) { in queue_work()
717 TAILQ_INSERT_TAIL(&wq->wq_queue, work, work_entry); in queue_work()
719 SDT_PROBE2(sdt, linux, work, queue, work, wq); in queue_work()
741 cancel_work(struct work_struct *work) in cancel_work() argument
747 if ((wq = work_queue(work)) == NULL) in cancel_work()
751 if (__predict_false(work_queue(work) != wq)) { in cancel_work()
760 if (work_claimed(work, wq)) { in cancel_work()
765 TAILQ_REMOVE(&wq->wq_queue, work, work_entry); in cancel_work()
766 SDT_PROBE2(sdt, linux, work, cancel, work, wq); in cancel_work()
767 release_work(work, wq); in cancel_work()
790 cancel_work_sync(struct work_struct *work) in cancel_work_sync() argument
796 if ((wq = work_queue(work)) == NULL) in cancel_work_sync()
800 if (__predict_false(work_queue(work) != wq)) { in cancel_work_sync()
809 if (work_claimed(work, wq)) { in cancel_work_sync()
814 TAILQ_REMOVE(&wq->wq_queue, work, work_entry); in cancel_work_sync()
815 SDT_PROBE2(sdt, linux, work, cancel, work, wq); in cancel_work_sync()
816 release_work(work, wq); in cancel_work_sync()
824 if (wq->wq_current_work == work) in cancel_work_sync()
825 wait_for_current_work(work, wq); in cancel_work_sync()
840 wait_for_current_work(struct work_struct *work, struct workqueue_struct *wq) in wait_for_current_work() argument
845 KASSERT(wq->wq_current_work == work); in wait_for_current_work()
848 SDT_PROBE2(sdt, linux, work, wait__start, work, wq); in wait_for_current_work()
852 } while (wq->wq_current_work == work && wq->wq_gen == gen); in wait_for_current_work()
853 SDT_PROBE2(sdt, linux, work, wait__done, work, wq); in wait_for_current_work()
871 INIT_WORK(&dw->work, fn); in INIT_DELAYED_WORK()
912 KASSERT(work_queue(&dw->work) == wq); in dw_callout_init()
931 KASSERT(work_queue(&dw->work) == wq); in dw_callout_destroy()
954 KASSERT(work_queue(&dw->work) == wq); in cancel_delayed_work_done()
958 release_work(&dw->work, wq); in cancel_delayed_work_done()
979 if (__predict_true(acquire_work(&dw->work, wq))) { in queue_delayed_work()
986 TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work, in queue_delayed_work()
989 SDT_PROBE2(sdt, linux, work, queue, &dw->work, wq); in queue_delayed_work()
997 SDT_PROBE3(sdt, linux, work, schedule, dw, wq, ticks); in queue_delayed_work()
1016 SDT_PROBE2(sdt, linux, work, queue, in queue_delayed_work()
1017 &dw->work, wq); in queue_delayed_work()
1021 SDT_PROBE3(sdt, linux, work, schedule, in queue_delayed_work()
1052 if (acquire_work(&dw->work, wq)) { in mod_delayed_work()
1063 TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work, in mod_delayed_work()
1066 SDT_PROBE2(sdt, linux, work, queue, &dw->work, wq); in mod_delayed_work()
1074 SDT_PROBE3(sdt, linux, work, schedule, dw, wq, ticks); in mod_delayed_work()
1084 SDT_PROBE2(sdt, linux, work, cancel, in mod_delayed_work()
1085 &dw->work, wq); in mod_delayed_work()
1086 SDT_PROBE2(sdt, linux, work, queue, in mod_delayed_work()
1087 &dw->work, wq); in mod_delayed_work()
1090 TAILQ_REMOVE(&wq->wq_dqueue, &dw->work, in mod_delayed_work()
1095 SDT_PROBE2(sdt, linux, work, cancel, in mod_delayed_work()
1096 &dw->work, wq); in mod_delayed_work()
1097 SDT_PROBE3(sdt, linux, work, schedule, in mod_delayed_work()
1118 SDT_PROBE2(sdt, linux, work, cancel, in mod_delayed_work()
1119 &dw->work, wq); in mod_delayed_work()
1120 SDT_PROBE2(sdt, linux, work, queue, in mod_delayed_work()
1121 &dw->work, wq); in mod_delayed_work()
1126 SDT_PROBE2(sdt, linux, work, cancel, in mod_delayed_work()
1127 &dw->work, wq); in mod_delayed_work()
1128 SDT_PROBE3(sdt, linux, work, schedule, in mod_delayed_work()
1142 &dw->work, work_entry); in mod_delayed_work()
1144 SDT_PROBE2(sdt, linux, work, cancel, in mod_delayed_work()
1145 &dw->work, wq); in mod_delayed_work()
1146 SDT_PROBE2(sdt, linux, work, queue, in mod_delayed_work()
1147 &dw->work, wq); in mod_delayed_work()
1155 SDT_PROBE2(sdt, linux, work, cancel, in mod_delayed_work()
1156 &dw->work, wq); in mod_delayed_work()
1157 SDT_PROBE3(sdt, linux, work, schedule, in mod_delayed_work()
1178 SDT_PROBE2(sdt, linux, work, cancel, in mod_delayed_work()
1179 &dw->work, wq); in mod_delayed_work()
1180 SDT_PROBE2(sdt, linux, work, queue, in mod_delayed_work()
1181 &dw->work, wq); in mod_delayed_work()
1185 SDT_PROBE2(sdt, linux, work, cancel, in mod_delayed_work()
1186 &dw->work, wq); in mod_delayed_work()
1187 SDT_PROBE3(sdt, linux, work, schedule, in mod_delayed_work()
1206 SDT_PROBE2(sdt, linux, work, queue, in mod_delayed_work()
1207 &dw->work, wq); in mod_delayed_work()
1212 SDT_PROBE3(sdt, linux, work, schedule, in mod_delayed_work()
1243 if ((wq = work_queue(&dw->work)) == NULL) in cancel_delayed_work()
1247 if (__predict_false(work_queue(&dw->work) != wq)) { in cancel_delayed_work()
1256 if (work_claimed(&dw->work, wq)) { in cancel_delayed_work()
1258 TAILQ_REMOVE(&wq->wq_dqueue, &dw->work, in cancel_delayed_work()
1260 SDT_PROBE2(sdt, linux, work, cancel, in cancel_delayed_work()
1261 &dw->work, wq); in cancel_delayed_work()
1262 release_work(&dw->work, wq); in cancel_delayed_work()
1284 SDT_PROBE2(sdt, linux, work, cancel, &dw->work, wq); in cancel_delayed_work()
1296 SDT_PROBE2(sdt, linux, work, cancel, &dw->work, wq); in cancel_delayed_work()
1333 if ((wq = work_queue(&dw->work)) == NULL) in cancel_delayed_work_sync()
1337 if (__predict_false(work_queue(&dw->work) != wq)) { in cancel_delayed_work_sync()
1346 if (work_claimed(&dw->work, wq)) { in cancel_delayed_work_sync()
1348 TAILQ_REMOVE(&wq->wq_dqueue, &dw->work, in cancel_delayed_work_sync()
1350 SDT_PROBE2(sdt, linux, work, cancel, in cancel_delayed_work_sync()
1351 &dw->work, wq); in cancel_delayed_work_sync()
1352 release_work(&dw->work, wq); in cancel_delayed_work_sync()
1360 if (wq->wq_current_work == &dw->work) in cancel_delayed_work_sync()
1361 wait_for_current_work(&dw->work, wq); in cancel_delayed_work_sync()
1379 SDT_PROBE2(sdt, linux, work, cancel, &dw->work, wq); in cancel_delayed_work_sync()
1392 SDT_PROBE2(sdt, linux, work, cancel, &dw->work, wq); in cancel_delayed_work_sync()
1441 flush_work_cb(struct work_struct *work) in flush_work_cb() argument
1443 struct flush_work *fw = container_of(work, struct flush_work, fw_work); in flush_work_cb()
1463 SDT_PROBE1(sdt, linux, work, flush__self, wq); in flush_workqueue()
1472 SDT_PROBE1(sdt, linux, work, flush__start, wq); in flush_workqueue()
1479 SDT_PROBE1(sdt, linux, work, flush__done, wq); in flush_workqueue()
1522 flush_work(struct work_struct *work) in flush_work() argument
1527 if ((wq = work_queue(work)) == NULL) in flush_work()
1548 if ((wq = work_queue(&dw->work)) == NULL) in flush_delayed_work()
1552 if (__predict_false(work_queue(&dw->work) != wq)) { in flush_delayed_work()
1592 TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work, in flush_delayed_work()
1595 SDT_PROBE2(sdt, linux, work, queue, in flush_delayed_work()
1596 &dw->work, wq); in flush_delayed_work()
1625 return work_pending(&dw->work); in delayed_work_pending()
1639 INIT_WORK(&rw->work, fn); in INIT_RCU_WORK()
1646 struct workqueue_struct *wq = work_queue(&rw->work); in queue_rcu_work_cb()
1649 KASSERT(work_pending(&rw->work)); in queue_rcu_work_cb()
1650 KASSERT(work_queue(&rw->work) == wq); in queue_rcu_work_cb()
1652 TAILQ_REMOVE(&wq->wq_rcu, &rw->work, work_entry); in queue_rcu_work_cb()
1653 TAILQ_INSERT_TAIL(&wq->wq_queue, &rw->work, work_entry); in queue_rcu_work_cb()
1655 SDT_PROBE2(sdt, linux, work, queue, &rw->work, wq); in queue_rcu_work_cb()
1669 if (acquire_work(&rw->work, wq)) { in queue_rcu_work()
1671 SDT_PROBE2(sdt, linux, work, rcu, rw, wq); in queue_rcu_work()
1672 TAILQ_INSERT_TAIL(&wq->wq_rcu, &rw->work, work_entry); in queue_rcu_work()