Lines Matching +full:clk +full:- +full:phase +full:-
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
100 0, "Maximum interrupt handler executions without re-entering epoch(9)");
189 ie = ithd->it_event;
190 td = ithd->it_thread;
191 mtx_assert(&ie->ie_lock, MA_OWNED);
194 if (CK_SLIST_EMPTY(&ie->ie_handlers))
197 pri = CK_SLIST_FIRST(&ie->ie_handlers)->ih_pri;
200 strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name));
220 mtx_assert(&ie->ie_lock, MA_OWNED);
221 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
227 CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next) {
228 if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 <
229 sizeof(ie->ie_fullname)) {
230 strcat(ie->ie_fullname, " ");
231 strcat(ie->ie_fullname, ih->ih_name);
235 flags |= ih->ih_flags;
237 ie->ie_hflags = flags;
247 ih = CK_SLIST_FIRST(&ie->ie_handlers);
248 missed = strlen(ie->ie_fullname) + strlen(ih->ih_name) + 2 -
249 sizeof(ie->ie_fullname);
250 strcat(ie->ie_fullname, (missed == 0) ? " " : "-");
251 strcat(ie->ie_fullname, &ih->ih_name[missed]);
254 last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2];
255 while (missed-- > 0) {
256 if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) {
263 strcat(ie->ie_fullname, " +");
266 strcat(ie->ie_fullname, "+");
273 if (ie->ie_thread != NULL)
274 ithread_update(ie->ie_thread);
275 CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname);
291 ie->ie_source = source;
292 ie->ie_pre_ithread = pre_ithread;
293 ie->ie_post_ithread = post_ithread;
294 ie->ie_post_filter = post_filter;
295 ie->ie_assign_cpu = assign_cpu;
296 ie->ie_flags = flags;
297 ie->ie_irq = irq;
298 ie->ie_cpu = NOCPU;
299 CK_SLIST_INIT(&ie->ie_handlers);
300 mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF);
303 vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap);
305 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
311 CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name);
331 if (ie->ie_assign_cpu == NULL)
343 mtx_lock(&ie->ie_lock);
344 if (ie->ie_thread != NULL) {
345 id = ie->ie_thread->it_thread->td_tid;
346 mtx_unlock(&ie->ie_lock);
351 mtx_unlock(&ie->ie_lock);
354 error = ie->ie_assign_cpu(ie->ie_source, cpu);
357 mtx_lock(&ie->ie_lock);
358 if (ie->ie_thread != NULL) {
359 cpu = ie->ie_cpu;
360 id = ie->ie_thread->it_thread->td_tid;
361 mtx_unlock(&ie->ie_lock);
364 mtx_unlock(&ie->ie_lock);
370 mtx_lock(&ie->ie_lock);
371 ie->ie_cpu = cpu;
372 mtx_unlock(&ie->ie_lock);
419 mtx_lock(&ie->ie_lock);
420 if (ie->ie_thread != NULL) {
421 id = ie->ie_thread->it_thread->td_tid;
422 mtx_unlock(&ie->ie_lock);
425 mtx_unlock(&ie->ie_lock);
437 if (ie->ie_irq == irq &&
438 (ie->ie_flags & IE_SOFT) == 0 &&
439 CK_SLIST_FIRST(&ie->ie_handlers) != NULL)
502 mtx_lock(&ie->ie_lock);
503 if (ie->ie_cpu == NOCPU)
506 CPU_SET(ie->ie_cpu, mask);
507 mtx_unlock(&ie->ie_lock);
510 mtx_lock(&ie->ie_lock);
511 if (ie->ie_thread == NULL) {
512 mtx_unlock(&ie->ie_lock);
515 id = ie->ie_thread->it_thread->td_tid;
516 mtx_unlock(&ie->ie_lock);
520 CPU_COPY(&td->td_cpuset->cs_mask, mask);
537 mtx_lock(&ie->ie_lock);
538 if (!CK_SLIST_EMPTY(&ie->ie_handlers)) {
539 mtx_unlock(&ie->ie_lock);
545 if (ie->ie_thread != NULL)
546 ithread_destroy(ie->ie_thread);
547 mtx_unlock(&ie->ie_lock);
548 mtx_destroy(&ie->ie_lock);
571 td->td_pflags |= TDP_ITHREAD;
572 ithd->it_thread = td;
583 td = ithread->it_thread;
584 ie = ithread->it_event;
586 mtx_assert(&ie->ie_lock, MA_OWNED);
588 CTR2(KTR_INTR, "%s: killing %s", __func__, ie->ie_name);
591 ithread->it_flags |= IT_DEAD;
597 while (ie->ie_thread != NULL)
598 msleep(ithread, &ie->ie_lock, 0, "ithd_dth", 0);
621 ih->ih_filter = filter;
622 ih->ih_handler = handler;
623 ih->ih_argument = arg;
624 strlcpy(ih->ih_name, name, sizeof(ih->ih_name));
625 ih->ih_event = ie;
626 ih->ih_pri = pri;
628 ih->ih_flags = IH_EXCLUSIVE;
630 ih->ih_flags |= IH_MPSAFE;
632 ih->ih_flags |= IH_ENTROPY;
634 ih->ih_flags |= IH_NET;
637 mtx_lock(&ie->ie_lock);
638 if (!CK_SLIST_EMPTY(&ie->ie_handlers)) {
640 (CK_SLIST_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
641 mtx_unlock(&ie->ie_lock);
647 ie->ie_flags |= IE_SLEEPABLE;
650 while (ie->ie_thread == NULL && handler != NULL) {
651 if (ie->ie_flags & IE_ADDING_THREAD)
652 msleep(ie, &ie->ie_lock, 0, "ithread", 0);
654 ie->ie_flags |= IE_ADDING_THREAD;
655 mtx_unlock(&ie->ie_lock);
657 mtx_lock(&ie->ie_lock);
658 ie->ie_flags &= ~IE_ADDING_THREAD;
659 ie->ie_thread = it;
660 it->it_event = ie;
667 CK_SLIST_FOREACH_PREVPTR(temp_ih, prevptr, &ie->ie_handlers, ih_next) {
668 if (temp_ih->ih_pri > ih->ih_pri)
675 CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name,
676 ie->ie_name);
677 mtx_unlock(&ie->ie_lock);
696 mtx_lock(&ie->ie_lock);
698 CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next) {
703 mtx_unlock(&ie->ie_lock);
716 start = strchr(ih->ih_name, ':');
718 start = strchr(ih->ih_name, 0);
722 * description + ":". The "- 1" leaves room for the trailing
725 space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1;
727 mtx_unlock(&ie->ie_lock);
735 mtx_unlock(&ie->ie_lock);
752 ie = ih->ih_event;
755 ih->ih_name));
756 return (ie->ie_source);
766 int phase;
768 mtx_assert(&ie->ie_lock, MA_OWNED);
769 phase = ie->ie_phase;
772 * Switch phase to direct future interrupts to the other active counter.
775 KASSERT(ie->ie_active[!phase] == 0, ("idle phase has activity"));
776 atomic_store_rel_int(&ie->ie_phase, !phase);
779 * This code cooperates with wait-free iteration of ie_handlers
781 * Make sure that the removal and the phase update are not reordered
784 * that guarantee as Store->Load sequences can always be reordered.
789 * Now wait on the inactive phase.
790 * The acquire fence is needed so that all post-barrier accesses
793 while (ie->ie_active[phase] > 0)
803 ie = handler->ih_event;
804 mtx_assert(&ie->ie_lock, MA_OWNED);
805 KASSERT((handler->ih_flags & IH_DEAD) == 0,
808 if (ie->ie_thread == NULL) {
812 if ((handler->ih_flags & IH_CHANGED) == 0) {
813 handler->ih_flags |= IH_CHANGED;
816 while ((handler->ih_flags & IH_CHANGED) != 0)
817 msleep(handler, &ie->ie_lock, 0, "ih_barr", 0);
837 if (ie->ie_thread == NULL)
839 ithd = ie->ie_thread;
840 td = ithd->it_thread;
848 ithd->it_flags |= IT_WAIT;
849 while (ithd->it_flags & IT_WAIT) {
869 ie = handler->ih_event;
872 handler->ih_name));
874 mtx_lock(&ie->ie_lock);
875 CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name,
876 ie->ie_name);
877 CK_SLIST_FOREACH_PREVPTR(ih, prevptr, &ie->ie_handlers, ih_next) {
883 "interrupt event \"%s\"", handler->ih_name, ie->ie_name);
886 if (ie->ie_thread == NULL) {
890 * lock-less fashion, so care needs to be taken to keep
902 KASSERT((handler->ih_flags & IH_DEAD) == 0,
904 handler->ih_flags |= IH_DEAD;
906 while (handler->ih_flags & IH_DEAD)
907 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
910 mtx_unlock(&ie->ie_lock);
923 ie = handler->ih_event;
926 handler->ih_name));
927 mtx_lock(&ie->ie_lock);
928 handler->ih_flags |= IH_SUSP;
930 mtx_unlock(&ie->ie_lock);
942 ie = handler->ih_event;
945 handler->ih_name));
951 mtx_lock(&ie->ie_lock);
952 handler->ih_flags &= ~IH_SUSP;
954 mtx_unlock(&ie->ie_lock);
969 if (ie == NULL || CK_SLIST_EMPTY(&ie->ie_handlers) ||
970 ie->ie_thread == NULL)
974 it = ie->ie_thread;
975 td = it->it_thread;
981 if (ie->ie_hflags & IH_ENTROPY) {
987 KASSERT(td->td_proc != NULL, ("ithread %s has no process", ie->ie_name));
998 atomic_store_rel_int(&it->it_need, 1);
1002 it->it_waiting = 0;
1006 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, td->td_proc->p_pid,
1007 td->td_name);
1012 it->it_waiting++;
1014 (it->it_waiting >= intr_hwpmc_waiting_report_threshold))
1018 __func__, td->td_proc->p_pid, td->td_name, it->it_need, TD_GET_STATE(td));
1026 * Allow interrupt event binding for software interrupt handlers -- a no-op,
1054 if (!(ie->ie_flags & IE_SOFT))
1078 struct intr_event *ie = ih->ih_event;
1082 CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name,
1083 ih->ih_need);
1096 ih->ih_need = 1;
1133 CK_SLIST_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) {
1138 if (ih->ih_flags & IH_DEAD) {
1139 mtx_lock(&ie->ie_lock);
1141 CK_SLIST_REMOVE_HEAD(&ie->ie_handlers, ih_next);
1144 ih->ih_flags &= ~IH_DEAD;
1146 mtx_unlock(&ie->ie_lock);
1156 if ((ih->ih_flags & IH_CHANGED) != 0) {
1157 mtx_lock(&ie->ie_lock);
1158 ih->ih_flags &= ~IH_CHANGED;
1160 mtx_unlock(&ie->ie_lock);
1164 if (ih->ih_handler == NULL)
1168 if ((ih->ih_flags & IH_SUSP) != 0)
1180 if ((ie->ie_flags & IE_SOFT) != 0 &&
1181 atomic_cmpset_int(&ih->ih_need, 1, 0) == 0)
1186 __func__, p->p_pid, (void *)ih->ih_handler,
1187 ih->ih_argument, ih->ih_name, ih->ih_flags);
1189 if (!(ih->ih_flags & IH_MPSAFE))
1191 ih->ih_handler(ih->ih_argument);
1192 if (!(ih->ih_flags & IH_MPSAFE))
1202 if (!(ie->ie_flags & (IE_SOFT | IE_SLEEPABLE)))
1205 if (!(ie->ie_flags & (IE_SOFT | IE_SLEEPABLE)))
1219 ie->ie_count >= intr_storm_threshold &&
1220 (ie->ie_flags & IE_SOFT) == 0)) {
1222 if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) {
1225 ie->ie_name);
1229 ie->ie_count++;
1235 if (ie->ie_post_ithread != NULL)
1236 ie->ie_post_ithread(ie->ie_source);
1254 p = td->td_proc;
1256 KASSERT(ithd->it_thread == td,
1258 ie = ithd->it_event;
1259 ie->ie_count = 0;
1269 if (__predict_false((ithd->it_flags & IT_DEAD) != 0)) {
1271 p->p_pid, td->td_name);
1272 mtx_lock(&ie->ie_lock);
1273 ie->ie_thread = NULL;
1275 mtx_unlock(&ie->ie_lock);
1291 (atomic_load_int(&ie->ie_hflags) & IH_NET) != 0;
1296 while (atomic_cmpset_acq_int(&ithd->it_need, 1, 0) != 0) {
1316 if (atomic_load_acq_int(&ithd->it_need) == 0 &&
1317 (ithd->it_flags & (IT_DEAD | IT_WAIT)) == 0) {
1319 ie->ie_count = 0;
1321 } else if ((ithd->it_flags & IT_WAIT) != 0) {
1322 ithd->it_flags &= ~IT_WAIT;
1335 --------------------------------------------------------------------------------
1338 * via curthread->td_intr_frame.
1350 int phase;
1361 if (ie == NULL || CK_SLIST_EMPTY(&ie->ie_handlers))
1367 td->td_intr_nesting_level++;
1372 oldframe = td->td_intr_frame;
1373 td->td_intr_frame = frame;
1375 phase = ie->ie_phase;
1376 atomic_add_int(&ie->ie_active[phase], 1);
1380 * re-ordered before the ie_active store.
1384 CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next) {
1385 if ((ih->ih_flags & IH_SUSP) != 0)
1387 if ((ie->ie_flags & IE_SOFT) != 0 && ih->ih_need == 0)
1389 if (ih->ih_filter == NULL) {
1394 ih->ih_filter, ih->ih_argument, ih->ih_name);
1395 ret = ih->ih_filter(ih->ih_argument);
1403 ih->ih_name));
1419 * functions - a filter wrapper and an ithread wrapper.
1433 atomic_add_rel_int(&ie->ie_active[phase], -1);
1435 td->td_intr_frame = oldframe;
1438 if (ie->ie_pre_ithread != NULL)
1439 ie->ie_pre_ithread(ie->ie_source);
1441 if (ie->ie_post_filter != NULL)
1442 ie->ie_post_filter(ie->ie_source);
1453 td->td_intr_nesting_level--;
1471 db_printf("\t%-10s ", ih->ih_name);
1472 switch (ih->ih_pri) {
1474 db_printf("CLK ");
1480 if (ih->ih_pri >= PI_SOFT)
1483 db_printf("%4u", ih->ih_pri);
1487 if (ih->ih_filter != NULL) {
1489 db_printsym((uintptr_t)ih->ih_filter, DB_STGY_PROC);
1491 if (ih->ih_handler != NULL) {
1492 if (ih->ih_filter != NULL)
1495 db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC);
1497 db_printf("(%p)", ih->ih_argument);
1498 if (ih->ih_need ||
1499 (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD |
1503 if (ih->ih_flags & IH_EXCLUSIVE) {
1509 if (ih->ih_flags & IH_ENTROPY) {
1515 if (ih->ih_flags & IH_DEAD) {
1521 if (ih->ih_flags & IH_MPSAFE) {
1527 if (ih->ih_need) {
1547 db_printf("%s ", ie->ie_fullname);
1548 it = ie->ie_thread;
1550 db_printf("(pid %d)", it->it_thread->td_proc->p_pid);
1553 if ((ie->ie_flags & (IE_SOFT | IE_ADDING_THREAD)) != 0 ||
1554 (it != NULL && it->it_need)) {
1557 if (ie->ie_flags & IE_SOFT) {
1561 if (ie->ie_flags & IE_ADDING_THREAD) {
1567 if (it != NULL && it->it_need) {
1577 CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next)
1592 if (!all && CK_SLIST_EMPTY(&ie->ie_handlers))
1608 if (swi_add(&clk_intr_event, "clk", NULL, NULL, SWI_CLOCK,
1610 panic("died while creating clk swi ithread");
1643 if (req->flags & SCTL_MASK32) {
1644 if (!req->oldptr)