1 /* $OpenBSD: kern_smr.c,v 1.4 2019/05/17 03:53:08 visa Exp $ */ 2 3 /* 4 * Copyright (c) 2019 Visa Hankala 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 #include <sys/systm.h> 21 #include <sys/kernel.h> 22 #include <sys/kthread.h> 23 #include <sys/mutex.h> 24 #include <sys/percpu.h> 25 #include <sys/proc.h> 26 #include <sys/smr.h> 27 #include <sys/time.h> 28 #include <sys/witness.h> 29 30 #include <machine/cpu.h> 31 32 #define SMR_PAUSE 100 /* pause between rounds in msec */ 33 34 void smr_create_thread(void *); 35 void smr_dispatch(struct schedstate_percpu *); 36 void smr_grace_wait(void); 37 void smr_thread(void *); 38 void smr_wakeup(void *); 39 40 struct mutex smr_lock = MUTEX_INITIALIZER(IPL_HIGH); 41 struct smr_entry_list smr_deferred; 42 struct timeout smr_wakeup_tmo; 43 unsigned int smr_expedite; 44 unsigned int smr_ndeferred; 45 46 #ifdef WITNESS 47 static const char smr_lock_name[] = "smr"; 48 struct lock_object smr_lock_obj = { 49 .lo_name = smr_lock_name, 50 .lo_flags = LO_WITNESS | LO_INITIALIZED | LO_SLEEPABLE | 51 (LO_CLASS_RWLOCK << LO_CLASSSHIFT) 52 }; 53 struct lock_type smr_lock_type = { 54 .lt_name = smr_lock_name 55 }; 56 #endif 57 58 static inline int 59 smr_cpu_is_idle(struct cpu_info *ci) 60 { 61 return ci->ci_curproc == ci->ci_schedstate.spc_idleproc; 62 } 63 64 void 65 smr_startup(void) 66 { 67 SIMPLEQ_INIT(&smr_deferred); 68 WITNESS_INIT(&smr_lock_obj, &smr_lock_type); 69 kthread_create_deferred(smr_create_thread, NULL); 70 } 71 72 void 73 smr_create_thread(void *arg) 74 { 75 timeout_set(&smr_wakeup_tmo, smr_wakeup, NULL); 76 if (kthread_create(smr_thread, NULL, NULL, "smr") != 0) 77 panic("could not create smr thread"); 78 } 79 80 struct timeval smr_logintvl = { 300, 0 }; 81 82 void 83 smr_thread(void *arg) 84 { 85 struct timeval elapsed, end, loglast, start; 86 struct smr_entry_list deferred; 87 struct smr_entry *smr; 88 89 KERNEL_ASSERT_LOCKED(); 90 KERNEL_UNLOCK(); 91 92 memset(&loglast, 0, sizeof(loglast)); 93 SIMPLEQ_INIT(&deferred); 94 95 for (;;) { 96 mtx_enter(&smr_lock); 97 if (smr_ndeferred == 0) { 98 while (smr_ndeferred == 0) 99 msleep(&smr_ndeferred, &smr_lock, PVM, 100 "bored", 0); 101 } else { 102 if (smr_expedite == 0) 103 msleep(&smr_ndeferred, &smr_lock, PVM, 104 "pause", SMR_PAUSE * hz / 1000); 105 } 106 107 SIMPLEQ_CONCAT(&deferred, &smr_deferred); 108 smr_ndeferred = 0; 109 smr_expedite = 0; 110 mtx_leave(&smr_lock); 111 112 getmicrouptime(&start); 113 114 smr_grace_wait(); 115 116 WITNESS_CHECKORDER(&smr_lock_obj, LOP_NEWORDER, NULL); 117 WITNESS_LOCK(&smr_lock_obj, 0); 118 119 while ((smr = SIMPLEQ_FIRST(&deferred)) != NULL) { 120 SIMPLEQ_REMOVE_HEAD(&deferred, smr_list); 121 smr->smr_func(smr->smr_arg); 122 } 123 124 WITNESS_UNLOCK(&smr_lock_obj, 0); 125 126 getmicrouptime(&end); 127 timersub(&end, &start, &elapsed); 128 if (elapsed.tv_sec >= 5 && 129 ratecheck(&loglast, &smr_logintvl)) 130 printf("smr: dispatch took %ld seconds\n", 131 (long)elapsed.tv_sec); 132 } 133 } 134 135 /* 136 * Block until all CPUs have crossed quiescent state. 137 */ 138 void 139 smr_grace_wait(void) 140 { 141 #ifdef MULTIPROCESSOR 142 CPU_INFO_ITERATOR cii; 143 struct cpu_info *ci, *ci_start; 144 145 ci_start = curcpu(); 146 CPU_INFO_FOREACH(cii, ci) { 147 if (ci == ci_start) 148 continue; 149 sched_peg_curproc(ci); 150 } 151 atomic_clearbits_int(&curproc->p_flag, P_CPUPEG); 152 #endif /* MULTIPROCESSOR */ 153 } 154 155 void 156 smr_wakeup(void *arg) 157 { 158 wakeup(&smr_ndeferred); 159 } 160 161 void 162 smr_read_enter(void) 163 { 164 #ifdef DIAGNOSTIC 165 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 166 167 spc->spc_smrdepth++; 168 #endif 169 } 170 171 void 172 smr_read_leave(void) 173 { 174 #ifdef DIAGNOSTIC 175 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 176 177 KASSERT(spc->spc_smrdepth > 0); 178 spc->spc_smrdepth--; 179 #endif 180 } 181 182 /* 183 * Move SMR entries from the local queue to the system-wide queue. 184 */ 185 void 186 smr_dispatch(struct schedstate_percpu *spc) 187 { 188 int expedite = 0, wake = 0; 189 190 mtx_enter(&smr_lock); 191 if (smr_ndeferred == 0) 192 wake = 1; 193 SIMPLEQ_CONCAT(&smr_deferred, &spc->spc_deferred); 194 smr_ndeferred += spc->spc_ndeferred; 195 spc->spc_ndeferred = 0; 196 smr_expedite |= spc->spc_smrexpedite; 197 spc->spc_smrexpedite = 0; 198 expedite = smr_expedite; 199 mtx_leave(&smr_lock); 200 201 if (expedite) 202 wakeup(&smr_ndeferred); 203 else if (wake) 204 timeout_add_msec(&smr_wakeup_tmo, SMR_PAUSE); 205 } 206 207 /* 208 * Signal that the current CPU is in quiescent state. 209 */ 210 void 211 smr_idle(void) 212 { 213 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 214 215 SMR_ASSERT_NONCRITICAL(); 216 217 if (spc->spc_ndeferred > 0) 218 smr_dispatch(spc); 219 } 220 221 #ifdef DIAGNOSTIC 222 223 void 224 smr_assert_critical(void) 225 { 226 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 227 228 if (panicstr == NULL && !db_active) 229 KASSERT(spc->spc_smrdepth > 0); 230 } 231 232 void 233 smr_assert_noncritical(void) 234 { 235 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 236 237 if (panicstr == NULL && !db_active) 238 KASSERT(spc->spc_smrdepth == 0); 239 } 240 241 #endif /* DIAGNOSTIC */ 242 243 void 244 smr_call_impl(struct smr_entry *smr, void (*func)(void *), void *arg, 245 int expedite) 246 { 247 struct cpu_info *ci = curcpu(); 248 struct schedstate_percpu *spc = &ci->ci_schedstate; 249 int s; 250 251 KASSERT(smr->smr_func == NULL); 252 253 smr->smr_func = func; 254 smr->smr_arg = arg; 255 256 s = splhigh(); 257 SIMPLEQ_INSERT_TAIL(&spc->spc_deferred, smr, smr_list); 258 spc->spc_ndeferred++; 259 spc->spc_smrexpedite |= expedite; 260 splx(s); 261 262 /* 263 * If this call was made from an interrupt context that 264 * preempted idle state, dispatch the local queue to the shared 265 * queue immediately. 266 * The entries would linger in the local queue long if the CPU 267 * went to sleep without calling smr_idle(). 268 */ 269 if (smr_cpu_is_idle(ci)) 270 smr_dispatch(spc); 271 } 272 273 void 274 smr_barrier_func(void *arg) 275 { 276 struct cond *c = arg; 277 278 cond_signal(c); 279 } 280 281 void 282 smr_barrier_impl(int expedite) 283 { 284 struct cond c = COND_INITIALIZER(); 285 struct smr_entry smr; 286 287 if (panicstr != NULL || db_active) 288 return; 289 290 WITNESS_CHECKORDER(&smr_lock_obj, LOP_NEWORDER, NULL); 291 292 smr_init(&smr); 293 smr_call_impl(&smr, smr_barrier_func, &c, expedite); 294 cond_wait(&c, "smrbar"); 295 } 296