1 /* $OpenBSD: kern_smr.c,v 1.8 2020/04/03 03:36:56 visa Exp $ */ 2 3 /* 4 * Copyright (c) 2019 Visa Hankala 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 #include <sys/systm.h> 21 #include <sys/kernel.h> 22 #include <sys/kthread.h> 23 #include <sys/mutex.h> 24 #include <sys/percpu.h> 25 #include <sys/proc.h> 26 #include <sys/smr.h> 27 #include <sys/time.h> 28 #include <sys/witness.h> 29 30 #include <machine/cpu.h> 31 32 #define SMR_PAUSE 100 /* pause between rounds in msec */ 33 34 void smr_dispatch(struct schedstate_percpu *); 35 void smr_grace_wait(void); 36 void smr_thread(void *); 37 void smr_wakeup(void *); 38 39 struct mutex smr_lock = MUTEX_INITIALIZER(IPL_HIGH); 40 struct smr_entry_list smr_deferred; 41 struct timeout smr_wakeup_tmo; 42 unsigned int smr_expedite; 43 unsigned int smr_ndeferred; 44 45 #ifdef WITNESS 46 static const char smr_lock_name[] = "smr"; 47 struct lock_object smr_lock_obj = { 48 .lo_name = smr_lock_name, 49 .lo_flags = LO_WITNESS | LO_INITIALIZED | LO_SLEEPABLE | 50 (LO_CLASS_RWLOCK << LO_CLASSSHIFT) 51 }; 52 struct lock_type smr_lock_type = { 53 .lt_name = smr_lock_name 54 }; 55 #endif 56 57 static inline int 58 smr_cpu_is_idle(struct cpu_info *ci) 59 { 60 return ci->ci_curproc == ci->ci_schedstate.spc_idleproc; 61 } 62 63 void 64 smr_startup(void) 65 { 66 SIMPLEQ_INIT(&smr_deferred); 67 WITNESS_INIT(&smr_lock_obj, &smr_lock_type); 68 timeout_set(&smr_wakeup_tmo, smr_wakeup, NULL); 69 } 70 71 void 72 smr_startup_thread(void) 73 { 74 if (kthread_create(smr_thread, NULL, NULL, "smr") != 0) 75 panic("could not create smr thread"); 76 } 77 78 struct timeval smr_logintvl = { 300, 0 }; 79 80 void 81 smr_thread(void *arg) 82 { 83 struct timeval elapsed, end, loglast, start; 84 struct smr_entry_list deferred; 85 struct smr_entry *smr; 86 87 KERNEL_ASSERT_LOCKED(); 88 KERNEL_UNLOCK(); 89 90 memset(&loglast, 0, sizeof(loglast)); 91 SIMPLEQ_INIT(&deferred); 92 93 for (;;) { 94 mtx_enter(&smr_lock); 95 if (smr_ndeferred == 0) { 96 while (smr_ndeferred == 0) 97 msleep_nsec(&smr_ndeferred, &smr_lock, PVM, 98 "bored", INFSLP); 99 } else { 100 if (smr_expedite == 0) 101 msleep_nsec(&smr_ndeferred, &smr_lock, PVM, 102 "pause", MSEC_TO_NSEC(SMR_PAUSE)); 103 } 104 105 SIMPLEQ_CONCAT(&deferred, &smr_deferred); 106 smr_ndeferred = 0; 107 smr_expedite = 0; 108 mtx_leave(&smr_lock); 109 110 getmicrouptime(&start); 111 112 smr_grace_wait(); 113 114 WITNESS_CHECKORDER(&smr_lock_obj, LOP_NEWORDER, NULL); 115 WITNESS_LOCK(&smr_lock_obj, 0); 116 117 while ((smr = SIMPLEQ_FIRST(&deferred)) != NULL) { 118 SIMPLEQ_REMOVE_HEAD(&deferred, smr_list); 119 smr->smr_func(smr->smr_arg); 120 } 121 122 WITNESS_UNLOCK(&smr_lock_obj, 0); 123 124 getmicrouptime(&end); 125 timersub(&end, &start, &elapsed); 126 if (elapsed.tv_sec >= 5 && 127 ratecheck(&loglast, &smr_logintvl)) 128 printf("smr: dispatch took %ld seconds\n", 129 (long)elapsed.tv_sec); 130 } 131 } 132 133 /* 134 * Block until all CPUs have crossed quiescent state. 135 */ 136 void 137 smr_grace_wait(void) 138 { 139 #ifdef MULTIPROCESSOR 140 CPU_INFO_ITERATOR cii; 141 struct cpu_info *ci, *ci_start; 142 143 ci_start = curcpu(); 144 CPU_INFO_FOREACH(cii, ci) { 145 if (ci == ci_start) 146 continue; 147 sched_peg_curproc(ci); 148 } 149 atomic_clearbits_int(&curproc->p_flag, P_CPUPEG); 150 #endif /* MULTIPROCESSOR */ 151 } 152 153 void 154 smr_wakeup(void *arg) 155 { 156 wakeup(&smr_ndeferred); 157 } 158 159 void 160 smr_read_enter(void) 161 { 162 #ifdef DIAGNOSTIC 163 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 164 165 spc->spc_smrdepth++; 166 #endif 167 } 168 169 void 170 smr_read_leave(void) 171 { 172 #ifdef DIAGNOSTIC 173 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 174 175 KASSERT(spc->spc_smrdepth > 0); 176 spc->spc_smrdepth--; 177 #endif 178 } 179 180 /* 181 * Move SMR entries from the local queue to the system-wide queue. 182 */ 183 void 184 smr_dispatch(struct schedstate_percpu *spc) 185 { 186 int expedite = 0, wake = 0; 187 188 mtx_enter(&smr_lock); 189 if (smr_ndeferred == 0) 190 wake = 1; 191 SIMPLEQ_CONCAT(&smr_deferred, &spc->spc_deferred); 192 smr_ndeferred += spc->spc_ndeferred; 193 spc->spc_ndeferred = 0; 194 smr_expedite |= spc->spc_smrexpedite; 195 spc->spc_smrexpedite = 0; 196 expedite = smr_expedite; 197 mtx_leave(&smr_lock); 198 199 if (expedite) 200 wakeup(&smr_ndeferred); 201 else if (wake) 202 timeout_add_msec(&smr_wakeup_tmo, SMR_PAUSE); 203 } 204 205 /* 206 * Signal that the current CPU is in quiescent state. 207 */ 208 void 209 smr_idle(void) 210 { 211 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 212 213 SMR_ASSERT_NONCRITICAL(); 214 215 if (spc->spc_ndeferred > 0) 216 smr_dispatch(spc); 217 } 218 219 void 220 smr_call_impl(struct smr_entry *smr, void (*func)(void *), void *arg, 221 int expedite) 222 { 223 struct cpu_info *ci = curcpu(); 224 struct schedstate_percpu *spc = &ci->ci_schedstate; 225 int s; 226 227 KASSERT(smr->smr_func == NULL); 228 229 smr->smr_func = func; 230 smr->smr_arg = arg; 231 232 s = splhigh(); 233 SIMPLEQ_INSERT_TAIL(&spc->spc_deferred, smr, smr_list); 234 spc->spc_ndeferred++; 235 spc->spc_smrexpedite |= expedite; 236 splx(s); 237 238 /* 239 * If this call was made from an interrupt context that 240 * preempted idle state, dispatch the local queue to the shared 241 * queue immediately. 242 * The entries would linger in the local queue long if the CPU 243 * went to sleep without calling smr_idle(). 244 */ 245 if (smr_cpu_is_idle(ci)) 246 smr_dispatch(spc); 247 } 248 249 void 250 smr_barrier_func(void *arg) 251 { 252 struct cond *c = arg; 253 254 cond_signal(c); 255 } 256 257 void 258 smr_barrier_impl(int expedite) 259 { 260 struct cond c = COND_INITIALIZER(); 261 struct smr_entry smr; 262 263 if (panicstr != NULL || db_active) 264 return; 265 266 WITNESS_CHECKORDER(&smr_lock_obj, LOP_NEWORDER, NULL); 267 268 smr_init(&smr); 269 smr_call_impl(&smr, smr_barrier_func, &c, expedite); 270 cond_wait(&c, "smrbar"); 271 } 272