1 /* $OpenBSD: kern_smr.c,v 1.1 2019/02/26 14:24:21 visa Exp $ */ 2 3 /* 4 * Copyright (c) 2019 Visa Hankala 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 #include <sys/systm.h> 21 #include <sys/kernel.h> 22 #include <sys/kthread.h> 23 #include <sys/mutex.h> 24 #include <sys/percpu.h> 25 #include <sys/proc.h> 26 #include <sys/smr.h> 27 #include <sys/time.h> 28 29 #include <machine/cpu.h> 30 31 #define SMR_ACTIVE 0x01 32 #define SMR_ENTERED 0x02 33 34 #define SMR_PAUSE 100 /* pause between rounds in msec */ 35 36 void smr_create_thread(void *); 37 void smr_dispatch(struct schedstate_percpu *); 38 void smr_grace_wait(void); 39 void smr_thread(void *); 40 void smr_wakeup(void *); 41 42 struct mutex smr_lock = MUTEX_INITIALIZER(IPL_HIGH); 43 struct smr_entry_list smr_deferred; 44 struct timeout smr_wakeup_tmo; 45 unsigned int smr_expedite; 46 unsigned int smr_ndeferred; 47 48 static inline int 49 smr_cpu_is_idle(struct cpu_info *ci) 50 { 51 return ci->ci_curproc == ci->ci_schedstate.spc_idleproc; 52 } 53 54 void 55 smr_startup(void) 56 { 57 SIMPLEQ_INIT(&smr_deferred); 58 kthread_create_deferred(smr_create_thread, NULL); 59 } 60 61 void 62 smr_create_thread(void *arg) 63 { 64 timeout_set(&smr_wakeup_tmo, smr_wakeup, NULL); 65 if (kthread_create(smr_thread, NULL, NULL, "smr") != 0) 66 panic("could not create smr thread"); 67 } 68 69 struct timeval smr_logintvl = { 300, 0 }; 70 71 void 72 smr_thread(void *arg) 73 { 74 struct timeval elapsed, end, loglast, start; 75 struct smr_entry_list deferred; 76 struct smr_entry *smr; 77 78 KERNEL_ASSERT_LOCKED(); 79 KERNEL_UNLOCK(); 80 81 memset(&loglast, 0, sizeof(loglast)); 82 SIMPLEQ_INIT(&deferred); 83 84 for (;;) { 85 mtx_enter(&smr_lock); 86 if (smr_ndeferred == 0) { 87 while (smr_ndeferred == 0) 88 msleep(&smr_ndeferred, &smr_lock, PVM, 89 "bored", 0); 90 } else { 91 if (smr_expedite == 0) 92 msleep(&smr_ndeferred, &smr_lock, PVM, 93 "pause", SMR_PAUSE * hz / 1000); 94 } 95 96 SIMPLEQ_CONCAT(&deferred, &smr_deferred); 97 smr_ndeferred = 0; 98 smr_expedite = 0; 99 mtx_leave(&smr_lock); 100 101 getmicrouptime(&start); 102 103 smr_grace_wait(); 104 105 while ((smr = SIMPLEQ_FIRST(&deferred)) != NULL) { 106 SIMPLEQ_REMOVE_HEAD(&deferred, smr_list); 107 smr->smr_func(smr->smr_arg); 108 } 109 110 getmicrouptime(&end); 111 timersub(&end, &start, &elapsed); 112 if (elapsed.tv_sec >= 5 && 113 ratecheck(&loglast, &smr_logintvl)) 114 printf("smr: dispatch took %ld seconds\n", 115 (long)elapsed.tv_sec); 116 } 117 } 118 119 /* 120 * Block until all CPUs have crossed quiescent state. 121 */ 122 void 123 smr_grace_wait(void) 124 { 125 #ifdef MULTIPROCESSOR 126 CPU_INFO_ITERATOR cii; 127 struct cpu_info *ci, *ci_start; 128 129 ci_start = curcpu(); 130 CPU_INFO_FOREACH(cii, ci) { 131 if (ci == ci_start) 132 continue; 133 if (smr_cpu_is_idle(ci) && ci->ci_schedstate.spc_insmr == 0) 134 continue; 135 sched_peg_curproc(ci); 136 } 137 atomic_clearbits_int(&curproc->p_flag, P_CPUPEG); 138 #endif /* MULTIPROCESSOR */ 139 } 140 141 void 142 smr_wakeup(void *arg) 143 { 144 wakeup(&smr_ndeferred); 145 } 146 147 void 148 smr_read_enter(void) 149 { 150 struct cpu_info *ci = curcpu(); 151 struct schedstate_percpu *spc = &ci->ci_schedstate; 152 153 #ifdef DIAGNOSTIC 154 spc->spc_smrdepth++; 155 #endif 156 157 if (smr_cpu_is_idle(ci) && (spc->spc_insmr & SMR_ENTERED) == 0) { 158 /* 159 * Activate in two steps in order not to miss the memory 160 * barrier with nested interrupts. 161 */ 162 spc->spc_insmr = SMR_ACTIVE; 163 membar_enter(); 164 spc->spc_insmr = SMR_ACTIVE | SMR_ENTERED; 165 } 166 } 167 168 void 169 smr_read_leave(void) 170 { 171 #ifdef DIAGNOSTIC 172 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 173 174 KASSERT(spc->spc_smrdepth > 0); 175 spc->spc_smrdepth--; 176 #endif 177 } 178 179 /* 180 * Move SMR entries from the local queue to the system-wide queue. 181 */ 182 void 183 smr_dispatch(struct schedstate_percpu *spc) 184 { 185 int expedite = 0, wake = 0; 186 187 mtx_enter(&smr_lock); 188 if (smr_ndeferred == 0) 189 wake = 1; 190 SIMPLEQ_CONCAT(&smr_deferred, &spc->spc_deferred); 191 smr_ndeferred += spc->spc_ndeferred; 192 spc->spc_ndeferred = 0; 193 smr_expedite |= spc->spc_smrexpedite; 194 spc->spc_smrexpedite = 0; 195 expedite = smr_expedite; 196 mtx_leave(&smr_lock); 197 198 if (expedite) 199 wakeup(&smr_ndeferred); 200 else if (wake) 201 timeout_add_msec(&smr_wakeup_tmo, SMR_PAUSE); 202 } 203 204 /* 205 * Signal that the current CPU is in quiescent state. 206 */ 207 void 208 smr_idle(void) 209 { 210 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 211 212 SMR_ASSERT_NONCRITICAL(); 213 214 spc->spc_insmr = 0; 215 216 if (spc->spc_ndeferred > 0) 217 smr_dispatch(spc); 218 } 219 220 #ifdef DIAGNOSTIC 221 222 void 223 smr_assert_critical(void) 224 { 225 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 226 227 if (panicstr == NULL && !db_active) 228 KASSERT(spc->spc_smrdepth > 0); 229 } 230 231 void 232 smr_assert_noncritical(void) 233 { 234 struct schedstate_percpu *spc = &curcpu()->ci_schedstate; 235 236 if (panicstr == NULL && !db_active) 237 KASSERT(spc->spc_smrdepth == 0); 238 } 239 240 #endif /* DIAGNOSTIC */ 241 242 void 243 smr_call_impl(struct smr_entry *smr, void (*func)(void *), void *arg, 244 int expedite) 245 { 246 struct cpu_info *ci = curcpu(); 247 struct schedstate_percpu *spc = &ci->ci_schedstate; 248 int s; 249 250 KASSERT(smr->smr_func == NULL); 251 252 smr->smr_func = func; 253 smr->smr_arg = arg; 254 255 s = splhigh(); 256 SIMPLEQ_INSERT_TAIL(&spc->spc_deferred, smr, smr_list); 257 spc->spc_ndeferred++; 258 spc->spc_smrexpedite |= expedite; 259 splx(s); 260 261 /* 262 * If this call was made from an interrupt context that 263 * preempted idle state, dispatch the local queue to the shared 264 * queue immediately. 265 * The entries would linger in the local queue long if the CPU 266 * went to sleep without calling smr_idle(). 267 */ 268 if (smr_cpu_is_idle(ci)) 269 smr_dispatch(spc); 270 } 271 272 void 273 smr_barrier_func(void *arg) 274 { 275 struct cond *c = arg; 276 277 cond_signal(c); 278 } 279 280 void 281 smr_barrier_impl(int expedite) 282 { 283 struct cond c = COND_INITIALIZER(); 284 struct smr_entry smr; 285 286 if (panicstr != NULL || db_active) 287 return; 288 289 assertwaitok(); 290 SMR_ASSERT_NONCRITICAL(); 291 292 smr_init(&smr); 293 smr_call_impl(&smr, smr_barrier_func, &c, expedite); 294 cond_wait(&c, "smrbar"); 295 } 296