1 /* $NetBSD: subr_interrupt.c,v 1.4 2018/01/28 22:24:58 christos Exp $ */ 2 3 /* 4 * Copyright (c) 2015 Internet Initiative Japan Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: subr_interrupt.c,v 1.4 2018/01/28 22:24:58 christos Exp $"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/errno.h> 36 #include <sys/cpu.h> 37 #include <sys/interrupt.h> 38 #include <sys/intr.h> 39 #include <sys/kcpuset.h> 40 #include <sys/kmem.h> 41 #include <sys/proc.h> 42 #include <sys/xcall.h> 43 #include <sys/sysctl.h> 44 45 #include <sys/conf.h> 46 #include <sys/intrio.h> 47 #include <sys/kauth.h> 48 49 #include <machine/limits.h> 50 51 #ifdef INTR_DEBUG 52 #define DPRINTF(msg) printf msg 53 #else 54 #define DPRINTF(msg) 55 #endif 56 57 static struct intrio_set kintrio_set = { "\0", NULL, 0 }; 58 59 #define UNSET_NOINTR_SHIELD 0 60 #define SET_NOINTR_SHIELD 1 61 62 static void 63 interrupt_shield_xcall(void *arg1, void *arg2) 64 { 65 struct cpu_info *ci; 66 struct schedstate_percpu *spc; 67 int s, shield; 68 69 ci = arg1; 70 shield = (int)(intptr_t)arg2; 71 spc = &ci->ci_schedstate; 72 73 s = splsched(); 74 if (shield == UNSET_NOINTR_SHIELD) 75 spc->spc_flags &= ~SPCF_NOINTR; 76 else if (shield == SET_NOINTR_SHIELD) 77 spc->spc_flags |= SPCF_NOINTR; 78 splx(s); 79 } 80 81 /* 82 * Change SPCF_NOINTR flag of schedstate_percpu->spc_flags. 83 */ 84 static int 85 interrupt_shield(u_int cpu_idx, int shield) 86 { 87 struct cpu_info *ci; 88 struct schedstate_percpu *spc; 89 90 KASSERT(mutex_owned(&cpu_lock)); 91 92 ci = cpu_lookup(cpu_idx); 93 if (ci == NULL) 94 return EINVAL; 95 96 spc = &ci->ci_schedstate; 97 if (shield == UNSET_NOINTR_SHIELD) { 98 if ((spc->spc_flags & SPCF_NOINTR) == 0) 99 return 0; 100 } else if (shield == SET_NOINTR_SHIELD) { 101 if ((spc->spc_flags & SPCF_NOINTR) != 0) 102 return 0; 103 } 104 105 if (ci == curcpu() || !mp_online) { 106 interrupt_shield_xcall(ci, (void *)(intptr_t)shield); 107 } else { 108 uint64_t where; 109 where = xc_unicast(0, interrupt_shield_xcall, ci, 110 (void *)(intptr_t)shield, ci); 111 xc_wait(where); 112 } 113 114 spc->spc_lastmod = time_second; 115 return 0; 116 } 117 118 /* 119 * Move all assigned interrupts from "cpu_idx" to the other cpu as possible. 120 * The destination cpu is the lowest cpuid of available cpus. 121 * If there are no available cpus, give up to move interrupts. 122 */ 123 static int 124 interrupt_avert_intr(u_int cpu_idx) 125 { 126 kcpuset_t *cpuset; 127 struct intrids_handler *ii_handler; 128 intrid_t *ids; 129 int error = 0, i, nids; 130 131 kcpuset_create(&cpuset, true); 132 kcpuset_set(cpuset, cpu_idx); 133 134 ii_handler = interrupt_construct_intrids(cpuset); 135 if (ii_handler == NULL) { 136 error = EINVAL; 137 goto out; 138 } 139 nids = ii_handler->iih_nids; 140 if (nids == 0) { 141 error = 0; 142 goto destruct_out; 143 } 144 145 interrupt_get_available(cpuset); 146 kcpuset_clear(cpuset, cpu_idx); 147 if (kcpuset_iszero(cpuset)) { 148 DPRINTF(("%s: no available cpu\n", __func__)); 149 error = ENOENT; 150 goto destruct_out; 151 } 152 153 ids = ii_handler->iih_intrids; 154 for (i = 0; i < nids; i++) { 155 error = interrupt_distribute_handler(ids[i], cpuset, NULL); 156 if (error) 157 break; 158 } 159 160 destruct_out: 161 interrupt_destruct_intrids(ii_handler); 162 out: 163 kcpuset_destroy(cpuset); 164 return error; 165 } 166 167 /* 168 * Return actual intrio_list_line size. 169 * intrio_list_line size is variable by ncpu. 170 */ 171 static size_t 172 interrupt_intrio_list_line_size(void) 173 { 174 175 return sizeof(struct intrio_list_line) + 176 sizeof(struct intrio_list_line_cpu) * (ncpu - 1); 177 } 178 179 /* 180 * Return the size of interrupts list data on success. 181 * Reterun 0 on failed. 182 */ 183 static int 184 interrupt_intrio_list_size(size_t *ilsize) 185 { 186 struct intrids_handler *ii_handler; 187 188 *ilsize = 0; 189 190 /* buffer header */ 191 *ilsize += sizeof(struct intrio_list); 192 193 /* il_line body */ 194 ii_handler = interrupt_construct_intrids(kcpuset_running); 195 if (ii_handler == NULL) 196 return EOPNOTSUPP; 197 *ilsize += interrupt_intrio_list_line_size() * ii_handler->iih_nids; 198 199 interrupt_destruct_intrids(ii_handler); 200 return 0; 201 } 202 203 /* 204 * Set intrctl list data to "il", and return list structure bytes. 205 * If error occured, return <0. 206 * If "data" == NULL, simply return list structure bytes. 207 */ 208 static int 209 interrupt_intrio_list(struct intrio_list *il, size_t ilsize) 210 { 211 struct intrio_list_line *illine; 212 kcpuset_t *assigned, *avail; 213 struct intrids_handler *ii_handler; 214 intrid_t *ids; 215 u_int cpu_idx; 216 int nids, intr_idx, error, line_size; 217 218 illine = (struct intrio_list_line *) 219 ((char *)il + sizeof(struct intrio_list)); 220 il->il_lineoffset = (off_t)((uintptr_t)illine - (uintptr_t)il); 221 222 kcpuset_create(&avail, true); 223 interrupt_get_available(avail); 224 kcpuset_create(&assigned, true); 225 226 ii_handler = interrupt_construct_intrids(kcpuset_running); 227 if (ii_handler == NULL) { 228 DPRINTF(("%s: interrupt_construct_intrids() failed\n", 229 __func__)); 230 error = EOPNOTSUPP; 231 goto out; 232 } 233 234 line_size = interrupt_intrio_list_line_size(); 235 /* ensure interrupts are not added after interrupt_intrio_list_size() */ 236 nids = ii_handler->iih_nids; 237 ids = ii_handler->iih_intrids; 238 if (ilsize < sizeof(struct intrio_list) + line_size * nids) { 239 DPRINTF(("%s: interrupts are added during execution.\n", 240 __func__)); 241 error = EAGAIN; 242 goto destruct_out; 243 } 244 245 for (intr_idx = 0; intr_idx < nids; intr_idx++) { 246 char devname[INTRDEVNAMEBUF]; 247 248 strncpy(illine->ill_intrid, ids[intr_idx], INTRIDBUF); 249 interrupt_get_devname(ids[intr_idx], devname, sizeof(devname)); 250 strncpy(illine->ill_xname, devname, INTRDEVNAMEBUF); 251 252 interrupt_get_assigned(ids[intr_idx], assigned); 253 for (cpu_idx = 0; cpu_idx < ncpu; cpu_idx++) { 254 struct intrio_list_line_cpu *illcpu = 255 &illine->ill_cpu[cpu_idx]; 256 257 illcpu->illc_assigned = 258 kcpuset_isset(assigned, cpu_idx); 259 illcpu->illc_count = 260 interrupt_get_count(ids[intr_idx], cpu_idx); 261 } 262 263 illine = (struct intrio_list_line *) 264 ((char *)illine + line_size); 265 } 266 267 error = 0; 268 il->il_version = INTRIO_LIST_VERSION; 269 il->il_ncpus = ncpu; 270 il->il_nintrs = nids; 271 il->il_linesize = line_size; 272 il->il_bufsize = ilsize; 273 274 destruct_out: 275 interrupt_destruct_intrids(ii_handler); 276 out: 277 kcpuset_destroy(assigned); 278 kcpuset_destroy(avail); 279 280 return error; 281 } 282 283 /* 284 * "intrctl list" entry 285 */ 286 static int 287 interrupt_intrio_list_sysctl(SYSCTLFN_ARGS) 288 { 289 int error; 290 void *buf; 291 size_t ilsize; 292 293 if (oldlenp == NULL) 294 return EINVAL; 295 296 if ((error = interrupt_intrio_list_size(&ilsize)) != 0) 297 return error; 298 299 /* 300 * If oldp == NULL, the sysctl(8) caller process want to get the size of 301 * intrctl list data only. 302 */ 303 if (oldp == NULL) { 304 *oldlenp = ilsize; 305 return 0; 306 } 307 308 /* 309 * If oldp != NULL, the sysctl(8) caller process want to get both the 310 * size and the contents of intrctl list data. 311 */ 312 if (*oldlenp < ilsize) 313 return ENOMEM; 314 315 buf = kmem_zalloc(ilsize, KM_SLEEP); 316 if ((error = interrupt_intrio_list(buf, ilsize)) != 0) 317 goto out; 318 319 error = copyout(buf, oldp, ilsize); 320 out: 321 kmem_free(buf, ilsize); 322 return error; 323 } 324 325 /* 326 * "intrctl affinity" entry 327 */ 328 static int 329 interrupt_set_affinity_sysctl(SYSCTLFN_ARGS) 330 { 331 struct sysctlnode node; 332 struct intrio_set *iset; 333 cpuset_t *ucpuset; 334 kcpuset_t *kcpuset; 335 int error; 336 337 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_INTR, 338 KAUTH_REQ_SYSTEM_INTR_AFFINITY, NULL, NULL, NULL); 339 if (error) 340 return EPERM; 341 342 node = *rnode; 343 iset = (struct intrio_set *)node.sysctl_data; 344 345 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 346 if (error != 0 || newp == NULL) 347 return error; 348 349 ucpuset = iset->cpuset; 350 kcpuset_create(&kcpuset, true); 351 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); 352 if (error) 353 goto out; 354 if (kcpuset_iszero(kcpuset)) { 355 error = EINVAL; 356 goto out; 357 } 358 359 error = interrupt_distribute_handler(iset->intrid, kcpuset, NULL); 360 361 out: 362 kcpuset_destroy(kcpuset); 363 return error; 364 } 365 366 /* 367 * "intrctl intr" entry 368 */ 369 static int 370 interrupt_intr_sysctl(SYSCTLFN_ARGS) 371 { 372 struct sysctlnode node; 373 struct intrio_set *iset; 374 cpuset_t *ucpuset; 375 kcpuset_t *kcpuset; 376 int error; 377 u_int cpu_idx; 378 379 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU, 380 KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL); 381 if (error) 382 return EPERM; 383 384 node = *rnode; 385 iset = (struct intrio_set *)node.sysctl_data; 386 387 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 388 if (error != 0 || newp == NULL) 389 return error; 390 391 ucpuset = iset->cpuset; 392 kcpuset_create(&kcpuset, true); 393 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); 394 if (error) 395 goto out; 396 if (kcpuset_iszero(kcpuset)) { 397 error = EINVAL; 398 goto out; 399 } 400 401 cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */ 402 403 mutex_enter(&cpu_lock); 404 error = interrupt_shield(cpu_idx, UNSET_NOINTR_SHIELD); 405 mutex_exit(&cpu_lock); 406 407 out: 408 kcpuset_destroy(kcpuset); 409 return error; 410 } 411 412 /* 413 * "intrctl nointr" entry 414 */ 415 static int 416 interrupt_nointr_sysctl(SYSCTLFN_ARGS) 417 { 418 struct sysctlnode node; 419 struct intrio_set *iset; 420 cpuset_t *ucpuset; 421 kcpuset_t *kcpuset; 422 int error; 423 u_int cpu_idx; 424 425 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU, 426 KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL); 427 if (error) 428 return EPERM; 429 430 node = *rnode; 431 iset = (struct intrio_set *)node.sysctl_data; 432 433 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 434 if (error != 0 || newp == NULL) 435 return error; 436 437 ucpuset = iset->cpuset; 438 kcpuset_create(&kcpuset, true); 439 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); 440 if (error) 441 goto out; 442 if (kcpuset_iszero(kcpuset)) { 443 error = EINVAL; 444 goto out; 445 } 446 447 cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */ 448 449 mutex_enter(&cpu_lock); 450 error = interrupt_shield(cpu_idx, SET_NOINTR_SHIELD); 451 mutex_exit(&cpu_lock); 452 if (error) 453 goto out; 454 455 error = interrupt_avert_intr(cpu_idx); 456 457 out: 458 kcpuset_destroy(kcpuset); 459 return error; 460 } 461 462 SYSCTL_SETUP(sysctl_interrupt_setup, "sysctl interrupt setup") 463 { 464 const struct sysctlnode *node = NULL; 465 466 sysctl_createv(clog, 0, NULL, &node, 467 CTLFLAG_PERMANENT, CTLTYPE_NODE, 468 "intr", SYSCTL_DESCR("Interrupt options"), 469 NULL, 0, NULL, 0, 470 CTL_KERN, CTL_CREATE, CTL_EOL); 471 472 sysctl_createv(clog, 0, &node, NULL, 473 CTLFLAG_PERMANENT, CTLTYPE_STRUCT, 474 "list", SYSCTL_DESCR("intrctl list"), 475 interrupt_intrio_list_sysctl, 0, NULL, 476 0, CTL_CREATE, CTL_EOL); 477 478 sysctl_createv(clog, 0, &node, NULL, 479 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, 480 "affinity", SYSCTL_DESCR("set affinity"), 481 interrupt_set_affinity_sysctl, 0, &kintrio_set, 482 sizeof(kintrio_set), CTL_CREATE, CTL_EOL); 483 484 sysctl_createv(clog, 0, &node, NULL, 485 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, 486 "intr", SYSCTL_DESCR("set intr"), 487 interrupt_intr_sysctl, 0, &kintrio_set, 488 sizeof(kintrio_set), CTL_CREATE, CTL_EOL); 489 490 sysctl_createv(clog, 0, &node, NULL, 491 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, 492 "nointr", SYSCTL_DESCR("set nointr"), 493 interrupt_nointr_sysctl, 0, &kintrio_set, 494 sizeof(kintrio_set), CTL_CREATE, CTL_EOL); 495 } 496