Lines Matching +full:timer +full:- +full:cannot +full:- +full:wake +full:- +full:cpu
1 /*-
2 * Copyright (c) 2016-2018 Netflix, Inc.
34 * The tcp_hpts system is designed to provide a high precision timer
84 * need to wake up the hpts system to get immediate
87 * the current hpts timer run (this is usually set when
88 * a rack timer is up so we know SACK's are happening
93 * knows how to take the input queue of packets from tp->t_inqueue
163 * This gives a range of 10usec - 1024ms to place
166 * when seeing the remainder will re-insert the
174 #define HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED)
175 #define HPTS_LOCK(hpts) mtx_lock(&(hpts)->p_mtx)
176 #define HPTS_TRYLOCK(hpts) mtx_trylock(&(hpts)->p_mtx)
177 #define HPTS_UNLOCK(hpts) mtx_unlock(&(hpts)->p_mtx)
208 uint32_t overidden_sleep; /* what was overrided by min-sleep for logging */
219 uint16_t p_num; /* The hpts number one per cpu */
220 uint16_t p_cpu; /* The hpts CPU */
259 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \
260 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \
261 if ((vvp)->tv_usec < 0) { \
262 (vvp)->tv_sec--; \
263 (vvp)->tv_usec += 1000000; \
271 int cpu[MAXCPU];
334 "Use of irq CPU tunable");
368 if (error == 0 && req->newptr) {
386 if (error == 0 && req->newptr) {
439 * 64 bit - delRate, rttProp, bw_inuse
440 * 16 bit - cwnd_gain
441 * 8 bit - bbr_state, bbr_substate, inhpts;
444 log.u_bbr.flex1 = hpts->p_nxt_slot;
445 log.u_bbr.flex2 = hpts->p_cur_slot;
446 log.u_bbr.flex3 = hpts->p_prev_slot;
448 log.u_bbr.flex5 = hpts->p_curtick;
449 log.u_bbr.flex6 = hpts->p_on_queue_cnt;
450 log.u_bbr.flex7 = hpts->p_cpu;
453 log.u_bbr.applimited = hpts->overidden_sleep;
454 log.u_bbr.delivered = hpts->saved_curtick;
456 log.u_bbr.epoch = hpts->saved_curslot;
457 log.u_bbr.lt_epoch = hpts->saved_prev_slot;
458 log.u_bbr.pkts_out = hpts->p_delayed_by;
459 log.u_bbr.lost = hpts->p_hpts_sleep_time;
460 log.u_bbr.pacing_gain = hpts->p_cpu;
461 log.u_bbr.pkt_epoch = hpts->p_runningslot;
464 &tptosocket(tp)->so_rcv,
465 &tptosocket(tp)->so_snd,
475 if (tcp_hpts_no_wake_over_thresh && (hpts->p_on_queue_cnt >= conn_cnt_thresh)) {
476 hpts->p_direct_wake = 0;
479 if (hpts->p_hpts_wake_scheduled == 0) {
480 hpts->p_hpts_wake_scheduled = 1;
481 swi_sched(hpts->ie_cookie, 0);
491 swi_sched(hpts->ie_cookie, 0);
502 MPASS(hpts->p_cpu == tp->t_hpts_cpu);
503 MPASS(!(inp->inp_flags & INP_DROPPED));
505 hptsh = &hpts->p_hptss[tp->t_hpts_slot];
507 if (tp->t_in_hpts == IHPTS_NONE) {
508 tp->t_in_hpts = IHPTS_ONQUEUE;
510 } else if (tp->t_in_hpts == IHPTS_MOVING) {
511 tp->t_in_hpts = IHPTS_ONQUEUE;
513 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE);
514 tp->t_hpts_gencnt = hptsh->gencnt;
516 TAILQ_INSERT_TAIL(&hptsh->head, tp, t_hpts);
517 hptsh->count++;
518 hpts->p_on_queue_cnt++;
528 hpts = tcp_pace.rp_ent[tp->t_hpts_cpu];
539 tp->t_in_hpts = IHPTS_NONE;
545 * Initialize tcpcb to get ready for use with HPTS. We will know which CPU
547 * a single CPU with newborn connections and use a random one.
549 * can be called once again if stack is switched. In that case we inherit CPU
558 if (__predict_true(tp->t_hpts_cpu == HPTS_CPU_NONE)) {
559 tp->t_hpts_cpu = hpts_random_cpu();
560 MPASS(!(tp->t_flags2 & TF2_HPTS_CPU_SET));
579 if (tp->t_in_hpts == IHPTS_ONQUEUE) {
580 hptsh = &hpts->p_hptss[tp->t_hpts_slot];
581 tp->t_hpts_request = 0;
582 if (__predict_true(tp->t_hpts_gencnt == hptsh->gencnt)) {
583 TAILQ_REMOVE(&hptsh->head, tp, t_hpts);
584 MPASS(hptsh->count > 0);
585 hptsh->count--;
586 MPASS(hpts->p_on_queue_cnt > 0);
587 hpts->p_on_queue_cnt--;
597 TAILQ_FOREACH(tmp, &hptsh->head, t_hpts)
600 tp->t_in_hpts = IHPTS_MOVING;
601 tp->t_hpts_slot = -1;
603 } else if (tp->t_in_hpts == IHPTS_MOVING) {
607 * tcp_hpts_remove() marks as IHPTS_MOVING, slot = -1
612 tp->t_hpts_slot = -1;
649 return (slot_now - prev_slot);
655 return (NUM_OF_HPTSI_SLOTS - 1);
657 return ((NUM_OF_HPTSI_SLOTS - prev_slot) + slot_now);
678 if ((hpts->p_hpts_active == 1) &&
679 (hpts->p_wheel_complete == 0)) {
680 end_slot = hpts->p_runningslot;
683 end_slot = NUM_OF_HPTSI_SLOTS - 1;
685 end_slot--;
697 end_slot = hpts->p_prev_slot;
699 end_slot = NUM_OF_HPTSI_SLOTS - 1;
701 end_slot--;
710 if (hpts->p_prev_slot != wheel_slot)
711 dis_to_travel = hpts_slots_diff(hpts->p_prev_slot, wheel_slot);
722 return (NUM_OF_HPTSI_SLOTS - dis_to_travel);
725 * So how many slots are open between p_runningslot -> p_cur_slot
726 * that is what is currently un-available for insertion. Special
730 if (hpts->p_runningslot == hpts->p_cur_slot)
733 dis_to_travel = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot);
737 if (hpts->p_cur_slot != wheel_slot) {
739 pacer_to_now = hpts_slots_diff(hpts->p_cur_slot, wheel_slot);
749 avail_on_wheel = NUM_OF_HPTSI_SLOTS - dis_to_travel;
764 *target_slot = hpts->p_nxt_slot;
776 return (avail_on_wheel - pacer_to_now);
792 if ((hpts->p_hpts_active) &&
793 (hpts->p_wheel_complete == 0)) {
802 distance = hpts_slots_diff(hpts->p_runningslot, hptsslot);
803 if (hpts->p_runningslot != hpts->p_cur_slot)
804 yet_to_run = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot);
809 hptsslot, distance, yet_to_run, hpts->p_runningslot,
810 hpts->p_cur_slot));
825 MPASS(!(tptoinpcb(tp)->inp_flags & INP_DROPPED));
826 MPASS(!(tp->t_in_hpts == IHPTS_ONQUEUE));
829 * We now return the next-slot the hpts will be on, beyond its
837 diag->p_hpts_active = hpts->p_hpts_active;
838 diag->p_prev_slot = hpts->p_prev_slot;
839 diag->p_runningslot = hpts->p_runningslot;
840 diag->p_nxt_slot = hpts->p_nxt_slot;
841 diag->p_cur_slot = hpts->p_cur_slot;
842 diag->p_curtick = hpts->p_curtick;
843 diag->p_lasttick = hpts->p_lasttick;
844 diag->slot_req = slot;
845 diag->p_on_min_sleep = hpts->p_on_min_sleep;
846 diag->hpts_sleep_time = hpts->p_hpts_sleep_time;
850 tp->t_hpts_request = 0;
851 if ((hpts->p_hpts_active == 0) || (hpts->p_wheel_complete)) {
856 tp->t_hpts_slot = hpts_slot(hpts->p_prev_slot, 1);
857 if ((hpts->p_on_min_sleep == 0) &&
858 (hpts->p_hpts_active == 0))
861 tp->t_hpts_slot = hpts->p_runningslot;
862 if (__predict_true(tp->t_in_hpts != IHPTS_MOVING))
869 hpts->p_direct_wake = 1;
872 slot_on = hpts->p_nxt_slot;
884 diag->wheel_slot = wheel_slot;
885 diag->maxslots = maxslots;
886 diag->wheel_cts = wheel_cts;
898 slot--;
900 tp->t_hpts_slot = last_slot;
901 tp->t_hpts_request = slot;
904 tp->t_hpts_request = 0;
905 tp->t_hpts_slot = hpts_slot(wheel_slot, slot);
908 tp->t_hpts_request = slot - maxslots;
909 tp->t_hpts_slot = last_slot;
912 diag->slot_remaining = tp->t_hpts_request;
913 diag->inp_hptsslot = tp->t_hpts_slot;
916 check_if_slot_would_be_wrong(hpts, tp, tp->t_hpts_slot, line);
918 if (__predict_true(tp->t_in_hpts != IHPTS_MOVING))
920 if ((hpts->p_hpts_active == 0) &&
921 (tp->t_hpts_request == 0) &&
922 (hpts->p_on_min_sleep == 0)) {
926 * it will wake up at and if we need to reschedule
927 * its time-out.
931 /* Now do we need to restart the hpts's timer? */
932 have_slept = hpts_slots_diff(hpts->p_prev_slot, wheel_slot);
933 if (have_slept < hpts->p_hpts_sleep_time)
934 yet_to_sleep = hpts->p_hpts_sleep_time - have_slept;
936 /* We are over-due */
941 diag->have_slept = have_slept;
942 diag->yet_to_sleep = yet_to_sleep;
947 * We need to reschedule the hpts's time-out.
949 hpts->p_hpts_sleep_time = slot;
959 hpts->p_direct_wake = 1;
962 diag->need_new_to = 0;
963 diag->co_ret = 0xffff0000;
974 need_new_to -= HPTS_USEC_IN_SEC;
978 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0,
979 hpts_timeout_swi, hpts, hpts->p_cpu,
982 diag->need_new_to = need_new_to;
983 diag->co_ret = co_ret;
986 slot_on = hpts->p_nxt_slot;
1002 if (tp->t_flags2 & TF2_HPTS_CPU_SET) {
1003 return (tp->t_hpts_cpu);
1006 * If we are using the irq cpu set by LRO or
1010 if (tp->t_lro_cpu == HPTS_CPU_NONE) {
1014 return (tp->t_lro_cpu);
1018 cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype);
1025 * We don't have a flowid -> cpuid mapping, so cheat and just map
1029 if (inp->inp_flowtype == M_HASHTYPE_NONE) {
1040 (inp->inp_numa_domain == M_NODOM)) {
1042 cpuid = inp->inp_flowid % mp_ncpus;
1045 /* Hash into the cpu's that use that domain */
1046 di = &hpts_domains[inp->inp_numa_domain];
1047 cpuid = di->cpu[inp->inp_flowid % di->count];
1059 if ((hpts->p_on_queue_cnt) && (wrap_loop_cnt < 2)) {
1064 for (i = 0, t = hpts_slot(hpts->p_cur_slot, 1); i < NUM_OF_HPTSI_SLOTS; i++) {
1065 if (TAILQ_EMPTY(&hpts->p_hptss[t].head) == 0) {
1070 KASSERT((i != NUM_OF_HPTSI_SLOTS), ("Hpts:%p cnt:%d but none found", hpts, hpts->p_on_queue_cnt));
1071 hpts->p_hpts_sleep_time = min((i + 1), hpts_sleep_max);
1074 hpts->p_hpts_sleep_time = hpts_sleep_max;
1098 hpts->saved_lasttick = hpts->p_lasttick;
1099 hpts->saved_curtick = hpts->p_curtick;
1100 hpts->saved_curslot = hpts->p_cur_slot;
1101 hpts->saved_prev_slot = hpts->p_prev_slot;
1103 hpts->p_lasttick = hpts->p_curtick;
1104 hpts->p_curtick = tcp_gethptstick(&tv);
1105 tcp_pace.cts_last_ran[hpts->p_num] = tcp_tv_to_usectick(&tv);
1106 orig_exit_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick);
1107 if ((hpts->p_on_queue_cnt == 0) ||
1108 (hpts->p_lasttick == hpts->p_curtick)) {
1113 hpts->p_prev_slot = hpts->p_cur_slot;
1114 hpts->p_lasttick = hpts->p_curtick;
1118 hpts->p_wheel_complete = 0;
1120 slots_to_run = hpts_slots_diff(hpts->p_prev_slot, hpts->p_cur_slot);
1121 if (((hpts->p_curtick - hpts->p_lasttick) >
1122 ((NUM_OF_HPTSI_SLOTS-1) * HPTS_TICKS_PER_SLOT)) &&
1123 (hpts->p_on_queue_cnt != 0)) {
1141 hpts->p_nxt_slot = hpts_slot(hpts->p_prev_slot, 1);
1142 hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 2);
1148 hpts->p_cur_slot = hpts->p_prev_slot;
1156 TAILQ_FOREACH(tp, &hpts->p_hptss[hpts->p_nxt_slot].head,
1158 MPASS(tp->t_hpts_slot == hpts->p_nxt_slot);
1159 MPASS(tp->t_hpts_gencnt ==
1160 hpts->p_hptss[hpts->p_nxt_slot].gencnt);
1161 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE);
1169 tp->t_hpts_gencnt =
1170 hpts->p_hptss[hpts->p_runningslot].gencnt;
1171 tp->t_hpts_slot = hpts->p_runningslot;
1173 TAILQ_CONCAT(&hpts->p_hptss[hpts->p_runningslot].head,
1174 &hpts->p_hptss[hpts->p_nxt_slot].head, t_hpts);
1175 hpts->p_hptss[hpts->p_runningslot].count +=
1176 hpts->p_hptss[hpts->p_nxt_slot].count;
1177 hpts->p_hptss[hpts->p_nxt_slot].count = 0;
1178 hpts->p_hptss[hpts->p_nxt_slot].gencnt++;
1179 slots_to_run = NUM_OF_HPTSI_SLOTS - 1;
1186 hpts->p_nxt_slot = hpts->p_prev_slot;
1187 hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 1);
1189 if (hpts->p_on_queue_cnt == 0) {
1202 hpts->p_delayed_by = (slots_to_run - (i + 1)) *
1205 runningslot = hpts->p_runningslot;
1206 hptsh = &hpts->p_hptss[runningslot];
1207 TAILQ_SWAP(&head, &hptsh->head, tcpcb, t_hpts);
1208 hpts->p_on_queue_cnt -= hptsh->count;
1209 hptsh->count = 0;
1210 hptsh->gencnt++;
1233 * re-used and something else (not an
1237 * and we tried to pre-fetch it).
1268 if ((tp->t_flags2 & TF2_HPTS_CPU_SET) == 0) {
1274 if (__predict_false(tp->t_in_hpts == IHPTS_MOVING)) {
1275 if (tp->t_hpts_slot == -1) {
1276 tp->t_in_hpts = IHPTS_NONE;
1288 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE);
1289 MPASS(!(inp->inp_flags & INP_DROPPED));
1290 KASSERT(runningslot == tp->t_hpts_slot,
1291 ("Hpts:%p inp:%p slot mis-aligned %u vs %u",
1292 hpts, inp, runningslot, tp->t_hpts_slot));
1294 if (tp->t_hpts_request) {
1303 remaining_slots = slots_to_run - (i + 1);
1304 if (tp->t_hpts_request > remaining_slots) {
1310 hpts->p_cur_slot, &last_slot);
1311 if (maxslots >= tp->t_hpts_request) {
1314 tp->t_hpts_slot = hpts_slot(
1315 hpts->p_runningslot,
1316 tp->t_hpts_request);
1317 tp->t_hpts_request = 0;
1320 tp->t_hpts_slot = last_slot;
1321 tp->t_hpts_request -=
1329 tp->t_hpts_request = 0;
1337 * the right CPU. This should be a rare
1346 * goes off and sees the mis-match. We
1347 * simply correct it here and the CPU will
1350 * :-)
1354 CURVNET_SET(inp->inp_vnet);
1361 if (tp->t_fb_ptr != NULL) {
1362 kern_prefetch(tp->t_fb_ptr, &did_prefetch);
1382 tp->t_flags2 |= TF2_HPTS_CALLS;
1383 if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) &&
1384 !STAILQ_EMPTY(&tp->t_inqueue))
1385 error = -(*tp->t_fb->tfb_do_queued_segments)(tp,
1396 * slot_pos_of_endpoint <-> orig_exit_slot
1404 hpts->p_runningslot++;
1405 if (hpts->p_runningslot >= NUM_OF_HPTSI_SLOTS) {
1406 hpts->p_runningslot = 0;
1411 hpts->p_delayed_by = 0;
1414 * more ticks (if we did not hit eno-bufs).
1416 hpts->p_prev_slot = hpts->p_cur_slot;
1417 hpts->p_lasttick = hpts->p_curtick;
1437 hpts->p_curtick = tcp_gethptstick(&tv);
1438 hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick);
1441 orig_exit_slot = hpts->p_cur_slot;
1444 (hpts->p_lasttick != hpts->p_curtick)) {
1450 tcp_pace.cts_last_ran[hpts->p_num] = tcp_tv_to_usectick(&tv);
1456 hpts->p_wheel_complete = 1;
1465 KASSERT(((hpts->p_prev_slot == hpts->p_cur_slot) ||
1468 hpts->p_prev_slot, hpts->p_cur_slot));
1469 KASSERT(((hpts->p_lasttick == hpts->p_curtick)
1472 hpts->p_lasttick, hpts->p_curtick));
1473 if (from_callout && (hpts->p_lasttick != hpts->p_curtick)) {
1474 hpts->p_curtick = tcp_gethptstick(&tv);
1476 hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick);
1498 if (tp->t_in_hpts == IHPTS_NONE && !(tp->t_flags2 & TF2_HPTS_CPU_SET)) {
1499 tp->t_hpts_cpu = hpts_cpuid(tp, &failed);
1501 tp->t_flags2 |= TF2_HPTS_CPU_SET;
1519 * this CPU is in.
1523 if (CPU_ISSET(curcpu, &tcp_pace.grps[i]->cg_mask)) {
1524 start = tcp_pace.grps[i]->cg_first;
1525 end = (tcp_pace.grps[i]->cg_last + 1);
1530 oldest_idx = -1;
1533 calc = cts - tcp_pace.cts_last_ran[i];
1556 if (hpts->p_hpts_active) {
1565 if (hpts->p_hpts_active)
1567 hpts->syscall_cnt++;
1569 hpts->p_hpts_active = 1;
1572 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) {
1577 hpts->p_mysleep.tv_usec /= 2;
1578 if (hpts->p_mysleep.tv_usec < dynamic_min_sleep)
1579 hpts->p_mysleep.tv_usec = dynamic_min_sleep;
1583 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT;
1585 if (tv.tv_usec < hpts->p_mysleep.tv_usec) {
1586 hpts->overidden_sleep = tv.tv_usec;
1587 tv.tv_usec = hpts->p_mysleep.tv_usec;
1590 hpts->overidden_sleep = tv.tv_usec;
1594 * In this mode the timer is a backstop to
1601 hpts->sleeping = tv.tv_usec;
1602 callout_reset_sbt_on(&hpts->co, sb, 0,
1603 hpts_timeout_swi, hpts, hpts->p_cpu,
1607 hpts->p_mysleep.tv_usec *= 2;
1608 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep)
1609 hpts->p_mysleep.tv_usec = dynamic_max_sleep;
1611 hpts->p_on_min_sleep = 1;
1613 hpts->p_hpts_active = 0;
1630 if (hpts->p_direct_wake) {
1632 callout_stop(&hpts->co);
1637 if (callout_pending(&hpts->co) ||
1638 !callout_active(&hpts->co)) {
1643 callout_deactivate(&hpts->co);
1644 hpts->p_hpts_wake_scheduled = 0;
1646 if (hpts->p_hpts_active) {
1655 if (hpts->p_direct_wake == 0) {
1657 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) {
1658 hpts->p_mysleep.tv_usec *= 2;
1659 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep)
1660 hpts->p_mysleep.tv_usec = dynamic_max_sleep;
1661 tv.tv_usec = hpts->p_mysleep.tv_usec;
1662 hpts->p_on_min_sleep = 1;
1671 hpts->p_on_min_sleep = 0;
1679 tv.tv_usec = hpts->p_mysleep.tv_usec;
1683 hpts->sleeping = 0;
1684 hpts->p_hpts_active = 1;
1687 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT;
1688 if ((hpts->p_on_queue_cnt > conn_cnt_thresh) && (hpts->hit_callout_thresh == 0)) {
1689 hpts->hit_callout_thresh = 1;
1691 } else if ((hpts->p_on_queue_cnt <= conn_cnt_thresh) && (hpts->hit_callout_thresh == 1)) {
1692 hpts->hit_callout_thresh = 0;
1695 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) {
1696 if(hpts->p_direct_wake == 0) {
1702 hpts->p_mysleep.tv_usec *= 2;
1703 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep)
1704 hpts->p_mysleep.tv_usec = dynamic_max_sleep;
1706 hpts->p_mysleep.tv_usec /= 2;
1707 if (hpts->p_mysleep.tv_usec < dynamic_min_sleep)
1708 hpts->p_mysleep.tv_usec = dynamic_min_sleep;
1711 if (tv.tv_usec < hpts->p_mysleep.tv_usec) {
1712 hpts->overidden_sleep = tv.tv_usec;
1713 tv.tv_usec = hpts->p_mysleep.tv_usec;
1716 hpts->overidden_sleep = tv.tv_usec;
1720 * In this mode the timer is a backstop to
1725 hpts->p_on_min_sleep = 1;
1726 } else if (hpts->p_on_queue_cnt == 0) {
1728 * No one on the wheel, please wake us up
1731 hpts->p_on_min_sleep = 0;
1732 hpts->overidden_sleep = 0;
1741 * where the timer is what runs hpts mainly.
1746 * we cannot be awoken.
1748 hpts->overidden_sleep = tv.tv_usec;
1750 hpts->p_on_min_sleep = 1;
1753 hpts->overidden_sleep = 0;
1754 hpts->p_on_min_sleep = 0;
1758 hpts->p_hpts_active = 0;
1760 hpts->p_direct_wake = 0;
1763 hpts->sleeping = tv.tv_usec;
1764 callout_reset_sbt_on(&hpts->co, sb, 0,
1765 hpts_timeout_swi, hpts, hpts->p_cpu,
1779 if (cg->cg_level == CG_SHARE_L3)
1782 for (i = 0; i < cg->cg_children; i++) {
1783 count_l3 += hpts_count_level(&cg->cg_child[i]);
1794 if (cg->cg_level == CG_SHARE_L3) {
1804 for (i = 0; i < cg->cg_children; i++) {
1805 hpts_gather_grps(grps, at, max, &cg->cg_child[i]);
1861 * All we need is the top level all cpu's are in
1862 * the same cache so when we use grp[0]->cg_mask
1863 * with the cg_first <-> cg_last it will include
1864 * all cpu's in it. The level here is probably
1881 tcp_pace.rp_ent[i]->p_hptss = malloc(asz, M_TCPHPTS, M_WAITOK);
1888 mtx_init(&hpts->p_mtx, "tcp_hpts_lck",
1891 TAILQ_INIT(&hpts->p_hptss[j].head);
1892 hpts->p_hptss[j].count = 0;
1893 hpts->p_hptss[j].gencnt = 0;
1895 sysctl_ctx_init(&hpts->hpts_ctx);
1897 hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx,
1903 SYSCTL_ADD_INT(&hpts->hpts_ctx,
1904 SYSCTL_CHILDREN(hpts->hpts_root),
1906 &hpts->p_on_queue_cnt, 0,
1908 SYSCTL_ADD_U16(&hpts->hpts_ctx,
1909 SYSCTL_CHILDREN(hpts->hpts_root),
1911 &hpts->p_hpts_active, 0,
1913 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1914 SYSCTL_CHILDREN(hpts->hpts_root),
1916 &hpts->p_cur_slot, 0,
1918 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1919 SYSCTL_CHILDREN(hpts->hpts_root),
1921 &hpts->p_runningslot, 0,
1923 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1924 SYSCTL_CHILDREN(hpts->hpts_root),
1926 &hpts->p_curtick, 0,
1928 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1929 SYSCTL_CHILDREN(hpts->hpts_root),
1933 SYSCTL_ADD_LONG(&hpts->hpts_ctx,
1934 SYSCTL_CHILDREN(hpts->hpts_root),
1936 &hpts->p_mysleep.tv_usec,
1938 SYSCTL_ADD_U64(&hpts->hpts_ctx,
1939 SYSCTL_CHILDREN(hpts->hpts_root),
1941 &hpts->sleeping, 0,
1943 SYSCTL_ADD_U64(&hpts->hpts_ctx,
1944 SYSCTL_CHILDREN(hpts->hpts_root),
1946 &hpts->syscall_cnt, 0,
1949 hpts->p_hpts_sleep_time = hpts_sleep_max;
1950 hpts->p_num = i;
1951 hpts->p_curtick = tcp_gethptstick(&tv);
1953 hpts->p_prev_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick);
1954 hpts->p_cpu = 0xffff;
1955 hpts->p_nxt_slot = hpts_slot(hpts->p_cur_slot, 1);
1956 callout_init(&hpts->co, 1);
1967 hpts->p_cpu = i;
1969 error = swi_add(&hpts->ie, "hpts",
1971 SWI_NET, INTR_MPSAFE, &hpts->ie_cookie);
1976 hpts->p_mysleep.tv_sec = 0;
1977 hpts->p_mysleep.tv_usec = tcp_min_hptsi_time;
1979 if (intr_event_bind(hpts->ie, i) == 0)
1982 /* Find the group for this CPU (i) and bind into it */
1984 if (CPU_ISSET(i, &tcp_pace.grps[j]->cg_mask)) {
1985 if (intr_event_bind_ithread_cpuset(hpts->ie,
1986 &tcp_pace.grps[j]->cg_mask) == 0) {
1989 domain = pc->pc_domain;
1991 hpts_domains[domain].cpu[count] = i;
1999 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT;
2000 hpts->sleeping = tv.tv_usec;
2002 callout_reset_sbt_on(&hpts->co, sb, 0,
2003 hpts_timeout_swi, hpts, hpts->p_cpu,
2034 rv = callout_drain(&hpts->co);
2037 rv = swi_remove(hpts->ie_cookie);
2040 rv = sysctl_ctx_free(&hpts->hpts_ctx);
2043 mtx_destroy(&hpts->p_mtx);
2044 free(hpts->p_hptss, M_TCPHPTS);