1 /* $NetBSD: intr.c,v 1.56 2020/11/01 20:58:38 christos Exp $ */
2
3 /*
4 * Copyright (c) 2008-2010, 2015 Antti Kantee. All Rights Reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
16 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.56 2020/11/01 20:58:38 christos Exp $");
30
31 #include <sys/param.h>
32 #include <sys/atomic.h>
33 #include <sys/cpu.h>
34 #include <sys/kernel.h>
35 #include <sys/kmem.h>
36 #include <sys/kthread.h>
37 #include <sys/malloc.h>
38 #include <sys/intr.h>
39 #include <sys/timetc.h>
40
41 #include <rump-sys/kern.h>
42
43 #include <rump/rumpuser.h>
44
45 /*
46 * Interrupt simulator. It executes hardclock() and softintrs.
47 */
48
49 #define SI_MPSAFE 0x01
50 #define SI_KILLME 0x02
51
52 struct softint_percpu;
53 struct softint {
54 void (*si_func)(void *);
55 void *si_arg;
56 int si_flags;
57 int si_level;
58
59 struct softint_percpu *si_entry; /* [0,ncpu-1] */
60 };
61
62 struct softint_percpu {
63 struct softint *sip_parent;
64 bool sip_onlist;
65 bool sip_onlist_cpu;
66
67 TAILQ_ENTRY(softint_percpu) sip_entries; /* scheduled */
68 TAILQ_ENTRY(softint_percpu) sip_entries_cpu; /* to be scheduled */
69 };
70
71 struct softint_lev {
72 struct rumpuser_cv *si_cv;
73 TAILQ_HEAD(, softint_percpu) si_pending;
74 };
75
76 static TAILQ_HEAD(, softint_percpu) sicpupending \
77 = TAILQ_HEAD_INITIALIZER(sicpupending);
78 static struct rumpuser_mtx *sicpumtx;
79 static struct rumpuser_cv *sicpucv;
80
81 kcondvar_t lbolt; /* Oh Kath Ra */
82
83 static int ncpu_final;
84
noclock(void)85 void noclock(void); void noclock(void) {return;}
86 __strong_alias(sched_schedclock,noclock);
87 __strong_alias(cpu_initclocks,noclock);
88 __strong_alias(addupc_intr,noclock);
89 __strong_alias(sched_tick,noclock);
90 __strong_alias(setstatclockrate,noclock);
91
92 /*
93 * clock "interrupt"
94 */
95 static void
doclock(void * noarg)96 doclock(void *noarg)
97 {
98 struct timespec thetick, curclock;
99 struct clockframe *clkframe;
100 int64_t sec;
101 long nsec;
102 int error;
103 struct cpu_info *ci = curcpu();
104
105 error = rumpuser_clock_gettime(RUMPUSER_CLOCK_ABSMONO, &sec, &nsec);
106 if (error)
107 panic("clock: cannot get monotonic time");
108
109 curclock.tv_sec = sec;
110 curclock.tv_nsec = nsec;
111 thetick.tv_sec = 0;
112 thetick.tv_nsec = 1000000000/hz;
113
114 /* generate dummy clockframe for hardclock to consume */
115 clkframe = rump_cpu_makeclockframe();
116
117 for (;;) {
118 int lbolt_ticks = 0;
119
120 hardclock(clkframe);
121 if (CPU_IS_PRIMARY(ci)) {
122 if (++lbolt_ticks >= hz) {
123 lbolt_ticks = 0;
124 cv_broadcast(&lbolt);
125 }
126 }
127
128 error = rumpuser_clock_sleep(RUMPUSER_CLOCK_ABSMONO,
129 curclock.tv_sec, curclock.tv_nsec);
130 if (error) {
131 panic("rumpuser_clock_sleep failed with error %d",
132 error);
133 }
134 timespecadd(&curclock, &thetick, &curclock);
135 }
136 }
137
138 /*
139 * Soft interrupt execution thread. This thread is pinned to the
140 * same CPU that scheduled the interrupt, so we don't need to do
141 * lock against si_lvl.
142 */
143 static void
sithread(void * arg)144 sithread(void *arg)
145 {
146 struct softint_percpu *sip;
147 struct softint *si;
148 void (*func)(void *) = NULL;
149 void *funarg;
150 bool mpsafe;
151 int mylevel = (uintptr_t)arg;
152 struct softint_lev *si_lvlp, *si_lvl;
153 struct cpu_data *cd = &curcpu()->ci_data;
154
155 si_lvlp = cd->cpu_softcpu;
156 si_lvl = &si_lvlp[mylevel];
157
158 for (;;) {
159 if (!TAILQ_EMPTY(&si_lvl->si_pending)) {
160 sip = TAILQ_FIRST(&si_lvl->si_pending);
161 si = sip->sip_parent;
162
163 func = si->si_func;
164 funarg = si->si_arg;
165 mpsafe = si->si_flags & SI_MPSAFE;
166
167 sip->sip_onlist = false;
168 TAILQ_REMOVE(&si_lvl->si_pending, sip, sip_entries);
169 if (si->si_flags & SI_KILLME) {
170 softint_disestablish(si);
171 continue;
172 }
173 } else {
174 rump_schedlock_cv_wait(si_lvl->si_cv);
175 continue;
176 }
177
178 if (!mpsafe)
179 KERNEL_LOCK(1, curlwp);
180 func(funarg);
181 if (!mpsafe)
182 KERNEL_UNLOCK_ONE(curlwp);
183 }
184
185 panic("sithread unreachable");
186 }
187
188 /*
189 * Helper for softint_schedule_cpu()
190 */
191 static void
sithread_cpu_bouncer(void * arg)192 sithread_cpu_bouncer(void *arg)
193 {
194 struct lwp *me;
195
196 me = curlwp;
197 me->l_pflag |= LP_BOUND;
198
199 rump_unschedule();
200 for (;;) {
201 struct softint_percpu *sip;
202 struct softint *si;
203 struct cpu_info *ci;
204 unsigned int cidx;
205
206 rumpuser_mutex_enter_nowrap(sicpumtx);
207 while (TAILQ_EMPTY(&sicpupending)) {
208 rumpuser_cv_wait_nowrap(sicpucv, sicpumtx);
209 }
210 sip = TAILQ_FIRST(&sicpupending);
211 TAILQ_REMOVE(&sicpupending, sip, sip_entries_cpu);
212 sip->sip_onlist_cpu = false;
213 rumpuser_mutex_exit(sicpumtx);
214
215 /*
216 * ok, now figure out which cpu we need the softint to
217 * be handled on
218 */
219 si = sip->sip_parent;
220 cidx = sip - si->si_entry;
221 ci = cpu_lookup(cidx);
222 me->l_target_cpu = ci;
223
224 /* schedule ourselves there, and then schedule the softint */
225 rump_schedule();
226 KASSERT(curcpu() == ci);
227 softint_schedule(si);
228 rump_unschedule();
229 }
230 panic("sithread_cpu_bouncer unreasonable");
231 }
232
233 static kmutex_t sithr_emtx;
234 static unsigned int sithr_est;
235 static int sithr_canest;
236
237 /*
238 * Create softint handler threads when the softint for each respective
239 * level is established for the first time. Most rump kernels don't
240 * need at least half of the softint levels, so on-demand saves bootstrap
241 * time and memory resources. Note, though, that this routine may be
242 * called before it's possible to call kthread_create(). Creation of
243 * those softints (SOFTINT_CLOCK, as of writing this) will be deferred
244 * to until softint_init() is called for the main CPU.
245 */
246 static void
sithread_establish(int level)247 sithread_establish(int level)
248 {
249 int docreate, rv;
250 int lvlbit = 1<<level;
251 int i;
252
253 KASSERT((level & ~SOFTINT_LVLMASK) == 0);
254 if (__predict_true(sithr_est & lvlbit))
255 return;
256
257 mutex_enter(&sithr_emtx);
258 docreate = (sithr_est & lvlbit) == 0 && sithr_canest;
259 sithr_est |= lvlbit;
260 mutex_exit(&sithr_emtx);
261
262 if (docreate) {
263 for (i = 0; i < ncpu_final; i++) {
264 if ((rv = kthread_create(PRI_NONE,
265 KTHREAD_MPSAFE | KTHREAD_INTR,
266 cpu_lookup(i), sithread, (void *)(uintptr_t)level,
267 NULL, "rsi%d/%d", i, level)) != 0)
268 panic("softint thread create failed: %d", rv);
269 }
270 }
271 }
272
273 void
rump_intr_init(int numcpu)274 rump_intr_init(int numcpu)
275 {
276
277 cv_init(&lbolt, "oh kath ra");
278 mutex_init(&sithr_emtx, MUTEX_DEFAULT, IPL_NONE);
279 ncpu_final = numcpu;
280 }
281
282 void
softint_init(struct cpu_info * ci)283 softint_init(struct cpu_info *ci)
284 {
285 struct cpu_data *cd = &ci->ci_data;
286 struct softint_lev *slev;
287 int rv, i;
288
289 if (!rump_threads)
290 return;
291
292 slev = kmem_alloc(sizeof(struct softint_lev) * SOFTINT_COUNT, KM_SLEEP);
293 for (i = 0; i < SOFTINT_COUNT; i++) {
294 rumpuser_cv_init(&slev[i].si_cv);
295 TAILQ_INIT(&slev[i].si_pending);
296 }
297 cd->cpu_softcpu = slev;
298
299 /* overloaded global init ... */
300 /* XXX: should be done the last time we are called */
301 if (ci->ci_index == 0) {
302 int sithr_swap;
303
304 /* pretend that we have our own for these */
305 stathz = 1;
306 schedhz = 1;
307 profhz = 1;
308
309 initclocks();
310
311 /* create deferred softint threads */
312 mutex_enter(&sithr_emtx);
313 sithr_swap = sithr_est;
314 sithr_est = 0;
315 sithr_canest = 1;
316 mutex_exit(&sithr_emtx);
317 for (i = 0; i < SOFTINT_COUNT; i++) {
318 if (sithr_swap & (1<<i))
319 sithread_establish(i);
320 }
321 }
322
323 /* well, not really a "soft" interrupt ... */
324 if ((rv = kthread_create(PRI_NONE, KTHREAD_MPSAFE,
325 ci, doclock, NULL, NULL, "rumpclk%d", ci->ci_index)) != 0)
326 panic("clock thread creation failed: %d", rv);
327
328 /* not one either, but at least a softint helper */
329 rumpuser_mutex_init(&sicpumtx, RUMPUSER_MTX_SPIN);
330 rumpuser_cv_init(&sicpucv);
331 if ((rv = kthread_create(PRI_NONE, KTHREAD_MPSAFE,
332 NULL, sithread_cpu_bouncer, NULL, NULL, "sipbnc")) != 0)
333 panic("softint cpu bouncer creation failed: %d", rv);
334 }
335
336 void *
softint_establish(u_int flags,void (* func)(void *),void * arg)337 softint_establish(u_int flags, void (*func)(void *), void *arg)
338 {
339 struct softint *si;
340 struct softint_percpu *sip;
341 int level = flags & SOFTINT_LVLMASK;
342 int i;
343
344 si = malloc(sizeof(*si), M_TEMP, M_WAITOK);
345 si->si_func = func;
346 si->si_arg = arg;
347 si->si_flags = flags & SOFTINT_MPSAFE ? SI_MPSAFE : 0;
348 si->si_level = level;
349 KASSERT(si->si_level < SOFTINT_COUNT);
350 si->si_entry = malloc(sizeof(*si->si_entry) * ncpu_final,
351 M_TEMP, M_WAITOK | M_ZERO);
352 for (i = 0; i < ncpu_final; i++) {
353 sip = &si->si_entry[i];
354 sip->sip_parent = si;
355 }
356 sithread_establish(level);
357
358 return si;
359 }
360
361 static struct softint_percpu *
sitosip(struct softint * si,struct cpu_info * ci)362 sitosip(struct softint *si, struct cpu_info *ci)
363 {
364
365 return &si->si_entry[ci->ci_index];
366 }
367
368 /*
369 * Soft interrupts bring two choices. If we are running with thread
370 * support enabled, defer execution, otherwise execute in place.
371 */
372
373 void
softint_schedule(void * arg)374 softint_schedule(void *arg)
375 {
376 struct softint *si = arg;
377 struct cpu_info *ci = curcpu();
378 struct softint_percpu *sip = sitosip(si, ci);
379 struct cpu_data *cd = &ci->ci_data;
380 struct softint_lev *si_lvl = cd->cpu_softcpu;
381
382 if (!rump_threads) {
383 si->si_func(si->si_arg);
384 } else {
385 if (!sip->sip_onlist) {
386 TAILQ_INSERT_TAIL(&si_lvl[si->si_level].si_pending,
387 sip, sip_entries);
388 sip->sip_onlist = true;
389 }
390 }
391 }
392
393 /*
394 * Like softint_schedule(), except schedule softint to be handled on
395 * the core designated by ci_tgt instead of the core the call is made on.
396 *
397 * Unlike softint_schedule(), the performance is not important
398 * (unless ci_tgt == curcpu): high-performance rump kernel I/O stacks
399 * should arrange data to already be on the right core at the driver
400 * layer.
401 */
402 void
softint_schedule_cpu(void * arg,struct cpu_info * ci_tgt)403 softint_schedule_cpu(void *arg, struct cpu_info *ci_tgt)
404 {
405 struct softint *si = arg;
406 struct cpu_info *ci_cur = curcpu();
407 struct softint_percpu *sip;
408
409 KASSERT(rump_threads);
410
411 /* preferred case (which can be optimized some day) */
412 if (ci_cur == ci_tgt) {
413 softint_schedule(si);
414 return;
415 }
416
417 /*
418 * no? then it's softint turtles all the way down
419 */
420
421 sip = sitosip(si, ci_tgt);
422 rumpuser_mutex_enter_nowrap(sicpumtx);
423 if (sip->sip_onlist_cpu) {
424 rumpuser_mutex_exit(sicpumtx);
425 return;
426 }
427 TAILQ_INSERT_TAIL(&sicpupending, sip, sip_entries_cpu);
428 sip->sip_onlist_cpu = true;
429 rumpuser_cv_signal(sicpucv);
430 rumpuser_mutex_exit(sicpumtx);
431 }
432
433 /*
434 * flimsy disestablish: should wait for softints to finish.
435 */
436 void
softint_disestablish(void * cook)437 softint_disestablish(void *cook)
438 {
439 struct softint *si = cook;
440 int i;
441
442 for (i = 0; i < ncpu_final; i++) {
443 struct softint_percpu *sip;
444
445 sip = &si->si_entry[i];
446 if (sip->sip_onlist) {
447 si->si_flags |= SI_KILLME;
448 return;
449 }
450 }
451 free(si->si_entry, M_TEMP);
452 free(si, M_TEMP);
453 }
454
455 void
rump_softint_run(struct cpu_info * ci)456 rump_softint_run(struct cpu_info *ci)
457 {
458 struct cpu_data *cd = &ci->ci_data;
459 struct softint_lev *si_lvl = cd->cpu_softcpu;
460 int i;
461
462 if (!rump_threads)
463 return;
464
465 for (i = 0; i < SOFTINT_COUNT; i++) {
466 if (!TAILQ_EMPTY(&si_lvl[i].si_pending))
467 rump_schedlock_cv_signal(ci, si_lvl[i].si_cv);
468 }
469 }
470
471 bool
cpu_intr_p(void)472 cpu_intr_p(void)
473 {
474
475 return false;
476 }
477