1 /* $NetBSD: subr_interrupt.c,v 1.1 2015/08/17 06:16:03 knakahara Exp $ */ 2 3 /* 4 * Copyright (c) 2015 Internet Initiative Japan Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: subr_interrupt.c,v 1.1 2015/08/17 06:16:03 knakahara Exp $"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/errno.h> 36 #include <sys/cpu.h> 37 #include <sys/interrupt.h> 38 #include <sys/intr.h> 39 #include <sys/kcpuset.h> 40 #include <sys/kmem.h> 41 #include <sys/proc.h> 42 #include <sys/xcall.h> 43 #include <sys/sysctl.h> 44 45 #include <sys/conf.h> 46 #include <sys/intrio.h> 47 #include <sys/kauth.h> 48 49 #include <machine/limits.h> 50 51 #ifdef INTR_DEBUG 52 #define DPRINTF(msg) printf msg 53 #else 54 #define DPRINTF(msg) 55 #endif 56 57 static struct intrio_set kintrio_set = { "\0", NULL, 0 }; 58 59 #define UNSET_NOINTR_SHIELD 0 60 #define SET_NOINTR_SHIELD 1 61 62 static void 63 interrupt_shield_xcall(void *arg1, void *arg2) 64 { 65 struct cpu_info *ci; 66 struct schedstate_percpu *spc; 67 int s, shield; 68 69 ci = arg1; 70 shield = (int)(intptr_t)arg2; 71 spc = &ci->ci_schedstate; 72 73 s = splsched(); 74 if (shield == UNSET_NOINTR_SHIELD) 75 spc->spc_flags &= ~SPCF_NOINTR; 76 else if (shield == SET_NOINTR_SHIELD) 77 spc->spc_flags |= SPCF_NOINTR; 78 splx(s); 79 } 80 81 /* 82 * Change SPCF_NOINTR flag of schedstate_percpu->spc_flags. 83 */ 84 static int 85 interrupt_shield(u_int cpu_idx, int shield) 86 { 87 struct cpu_info *ci; 88 struct schedstate_percpu *spc; 89 90 KASSERT(mutex_owned(&cpu_lock)); 91 92 ci = cpu_lookup(cpu_idx); 93 if (ci == NULL) 94 return EINVAL; 95 96 spc = &ci->ci_schedstate; 97 if (shield == UNSET_NOINTR_SHIELD) { 98 if ((spc->spc_flags & SPCF_NOINTR) == 0) 99 return 0; 100 } else if (shield == SET_NOINTR_SHIELD) { 101 if ((spc->spc_flags & SPCF_NOINTR) != 0) 102 return 0; 103 } 104 105 if (ci == curcpu() || !mp_online) { 106 interrupt_shield_xcall(ci, (void *)(intptr_t)shield); 107 } else { 108 uint64_t where; 109 where = xc_unicast(0, interrupt_shield_xcall, ci, 110 (void *)(intptr_t)shield, ci); 111 xc_wait(where); 112 } 113 114 spc->spc_lastmod = time_second; 115 return 0; 116 } 117 118 /* 119 * Move all assigned interrupts from "cpu_idx" to the other cpu as possible. 120 * The destination cpu is the lowest cpuid of available cpus. 121 * If there are no available cpus, give up to move interrupts. 122 */ 123 static int 124 interrupt_avert_intr(u_int cpu_idx) 125 { 126 kcpuset_t *cpuset; 127 struct intrids_handler *ii_handler; 128 intrid_t *ids; 129 int error, i, nids; 130 131 kcpuset_create(&cpuset, true); 132 kcpuset_set(cpuset, cpu_idx); 133 134 ii_handler = interrupt_construct_intrids(cpuset); 135 if (ii_handler == NULL) { 136 error = ENOMEM; 137 goto out; 138 } 139 nids = ii_handler->iih_nids; 140 if (nids == 0) { 141 error = 0; 142 goto destruct_out; 143 } 144 145 interrupt_get_available(cpuset); 146 kcpuset_clear(cpuset, cpu_idx); 147 if (kcpuset_iszero(cpuset)) { 148 DPRINTF(("%s: no available cpu\n", __func__)); 149 error = ENOENT; 150 goto destruct_out; 151 } 152 153 ids = ii_handler->iih_intrids; 154 for (i = 0; i < nids; i++) { 155 error = interrupt_distribute_handler(ids[i], cpuset, NULL); 156 if (error) 157 break; 158 } 159 160 destruct_out: 161 interrupt_destruct_intrids(ii_handler); 162 out: 163 kcpuset_destroy(cpuset); 164 return error; 165 } 166 167 /* 168 * Return actual intrio_list_line size. 169 * intrio_list_line size is variable by ncpu. 170 */ 171 static size_t 172 interrupt_intrio_list_line_size(void) 173 { 174 175 return sizeof(struct intrio_list_line) + 176 sizeof(struct intrio_list_line_cpu) * (ncpu - 1); 177 } 178 179 /* 180 * Return the size of interrupts list data on success. 181 * Reterun 0 on failed. 182 */ 183 static size_t 184 interrupt_intrio_list_size(void) 185 { 186 struct intrids_handler *ii_handler; 187 size_t ilsize; 188 189 ilsize = 0; 190 191 /* buffer header */ 192 ilsize += sizeof(struct intrio_list); 193 194 /* il_line body */ 195 ii_handler = interrupt_construct_intrids(kcpuset_running); 196 if (ii_handler == NULL) 197 return 0; 198 ilsize += interrupt_intrio_list_line_size() * (ii_handler->iih_nids); 199 200 interrupt_destruct_intrids(ii_handler); 201 return ilsize; 202 } 203 204 /* 205 * Set intrctl list data to "il", and return list structure bytes. 206 * If error occured, return <0. 207 * If "data" == NULL, simply return list structure bytes. 208 */ 209 static int 210 interrupt_intrio_list(struct intrio_list *il, int length) 211 { 212 struct intrio_list_line *illine; 213 kcpuset_t *assigned, *avail; 214 struct intrids_handler *ii_handler; 215 intrid_t *ids; 216 size_t ilsize; 217 u_int cpu_idx; 218 int nids, intr_idx, ret, line_size; 219 220 ilsize = interrupt_intrio_list_size(); 221 if (ilsize == 0) 222 return -ENOMEM; 223 224 if (il == NULL) 225 return ilsize; 226 227 if (length < ilsize) 228 return -ENOMEM; 229 230 illine = (struct intrio_list_line *) 231 ((char *)il + sizeof(struct intrio_list)); 232 il->il_lineoffset = (off_t)((uintptr_t)illine - (uintptr_t)il); 233 234 kcpuset_create(&avail, true); 235 interrupt_get_available(avail); 236 kcpuset_create(&assigned, true); 237 238 ii_handler = interrupt_construct_intrids(kcpuset_running); 239 if (ii_handler == NULL) { 240 DPRINTF(("%s: interrupt_construct_intrids() failed\n", 241 __func__)); 242 ret = -ENOMEM; 243 goto out; 244 } 245 246 line_size = interrupt_intrio_list_line_size(); 247 /* ensure interrupts are not added after interrupt_intrio_list_size(). */ 248 nids = ii_handler->iih_nids; 249 ids = ii_handler->iih_intrids; 250 if (ilsize < sizeof(struct intrio_list) + line_size * nids) { 251 DPRINTF(("%s: interrupts are added during execution.\n", 252 __func__)); 253 ret = -ENOMEM; 254 goto destruct_out; 255 } 256 257 for (intr_idx = 0; intr_idx < nids; intr_idx++) { 258 char devname[INTRDEVNAMEBUF]; 259 260 strncpy(illine->ill_intrid, ids[intr_idx], INTRIDBUF); 261 interrupt_get_devname(ids[intr_idx], devname, sizeof(devname)); 262 strncpy(illine->ill_xname, devname, INTRDEVNAMEBUF); 263 264 interrupt_get_assigned(ids[intr_idx], assigned); 265 for (cpu_idx = 0; cpu_idx < ncpu; cpu_idx++) { 266 struct intrio_list_line_cpu *illcpu = 267 &illine->ill_cpu[cpu_idx]; 268 269 illcpu->illc_assigned = 270 kcpuset_isset(assigned, cpu_idx) ? true : false; 271 illcpu->illc_count = 272 interrupt_get_count(ids[intr_idx], cpu_idx); 273 } 274 275 illine = (struct intrio_list_line *) 276 ((char *)illine + line_size); 277 } 278 279 ret = ilsize; 280 il->il_version = INTRIO_LIST_VERSION; 281 il->il_ncpus = ncpu; 282 il->il_nintrs = nids; 283 il->il_linesize = line_size; 284 il->il_bufsize = ilsize; 285 286 destruct_out: 287 interrupt_destruct_intrids(ii_handler); 288 out: 289 kcpuset_destroy(assigned); 290 kcpuset_destroy(avail); 291 292 return ret; 293 } 294 295 /* 296 * "intrctl list" entry 297 */ 298 static int 299 interrupt_intrio_list_sysctl(SYSCTLFN_ARGS) 300 { 301 int ret, error; 302 void *buf; 303 304 if (oldlenp == NULL) 305 return EINVAL; 306 307 /* 308 * If oldp == NULL, the sysctl(8) caller process want to get the size of 309 * intrctl list data only. 310 */ 311 if (oldp == NULL) { 312 ret = interrupt_intrio_list(NULL, 0); 313 if (ret < 0) 314 return -ret; 315 316 *oldlenp = ret; 317 return 0; 318 } 319 320 /* 321 * If oldp != NULL, the sysctl(8) caller process want to get both the size 322 * and the contents of intrctl list data. 323 */ 324 if (*oldlenp == 0) 325 return ENOMEM; 326 327 buf = kmem_zalloc(*oldlenp, KM_SLEEP); 328 if (buf == NULL) 329 return ENOMEM; 330 331 ret = interrupt_intrio_list(buf, *oldlenp); 332 if (ret < 0) { 333 error = -ret; 334 goto out; 335 } 336 error = copyout(buf, oldp, *oldlenp); 337 338 out: 339 kmem_free(buf, *oldlenp); 340 return error; 341 } 342 343 /* 344 * "intrctl affinity" entry 345 */ 346 static int 347 interrupt_set_affinity_sysctl(SYSCTLFN_ARGS) 348 { 349 struct sysctlnode node; 350 struct intrio_set *iset; 351 cpuset_t *ucpuset; 352 kcpuset_t *kcpuset; 353 int error; 354 355 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_INTR, 356 KAUTH_REQ_SYSTEM_INTR_AFFINITY, NULL, NULL, NULL); 357 if (error) 358 return EPERM; 359 360 node = *rnode; 361 iset = (struct intrio_set *)node.sysctl_data; 362 363 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 364 if (error != 0 || newp == NULL) 365 return error; 366 367 ucpuset = iset->cpuset; 368 kcpuset_create(&kcpuset, true); 369 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); 370 if (error) 371 goto out; 372 if (kcpuset_iszero(kcpuset)) { 373 error = EINVAL; 374 goto out; 375 } 376 377 error = interrupt_distribute_handler(iset->intrid, kcpuset, NULL); 378 379 out: 380 kcpuset_destroy(kcpuset); 381 return error; 382 } 383 384 /* 385 * "intrctl intr" entry 386 */ 387 static int 388 interrupt_intr_sysctl(SYSCTLFN_ARGS) 389 { 390 struct sysctlnode node; 391 struct intrio_set *iset; 392 cpuset_t *ucpuset; 393 kcpuset_t *kcpuset; 394 int error; 395 u_int cpu_idx; 396 397 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU, 398 KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL); 399 if (error) 400 return EPERM; 401 402 node = *rnode; 403 iset = (struct intrio_set *)node.sysctl_data; 404 405 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 406 if (error != 0 || newp == NULL) 407 return error; 408 409 ucpuset = iset->cpuset; 410 kcpuset_create(&kcpuset, true); 411 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); 412 if (error) 413 goto out; 414 if (kcpuset_iszero(kcpuset)) { 415 error = EINVAL; 416 goto out; 417 } 418 419 cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */ 420 421 mutex_enter(&cpu_lock); 422 error = interrupt_shield(cpu_idx, UNSET_NOINTR_SHIELD); 423 mutex_exit(&cpu_lock); 424 425 out: 426 kcpuset_destroy(kcpuset); 427 return error; 428 } 429 430 /* 431 * "intrctl nointr" entry 432 */ 433 static int 434 interrupt_nointr_sysctl(SYSCTLFN_ARGS) 435 { 436 struct sysctlnode node; 437 struct intrio_set *iset; 438 cpuset_t *ucpuset; 439 kcpuset_t *kcpuset; 440 int error; 441 u_int cpu_idx; 442 443 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU, 444 KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL); 445 if (error) 446 return EPERM; 447 448 node = *rnode; 449 iset = (struct intrio_set *)node.sysctl_data; 450 451 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 452 if (error != 0 || newp == NULL) 453 return error; 454 455 ucpuset = iset->cpuset; 456 kcpuset_create(&kcpuset, true); 457 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); 458 if (error) 459 goto out; 460 if (kcpuset_iszero(kcpuset)) { 461 error = EINVAL; 462 goto out; 463 } 464 465 cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */ 466 467 mutex_enter(&cpu_lock); 468 error = interrupt_shield(cpu_idx, SET_NOINTR_SHIELD); 469 mutex_exit(&cpu_lock); 470 if (error) 471 goto out; 472 473 error = interrupt_avert_intr(cpu_idx); 474 475 out: 476 kcpuset_destroy(kcpuset); 477 return error; 478 } 479 480 SYSCTL_SETUP(sysctl_interrupt_setup, "sysctl interrupt setup") 481 { 482 const struct sysctlnode *node = NULL; 483 484 sysctl_createv(clog, 0, NULL, &node, 485 CTLFLAG_PERMANENT, CTLTYPE_NODE, 486 "intr", SYSCTL_DESCR("Interrupt options"), 487 NULL, 0, NULL, 0, 488 CTL_KERN, CTL_CREATE, CTL_EOL); 489 490 sysctl_createv(clog, 0, &node, NULL, 491 CTLFLAG_PERMANENT, CTLTYPE_STRUCT, 492 "list", SYSCTL_DESCR("intrctl list"), 493 interrupt_intrio_list_sysctl, 0, NULL, 494 0, CTL_CREATE, CTL_EOL); 495 496 sysctl_createv(clog, 0, &node, NULL, 497 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, 498 "affinity", SYSCTL_DESCR("set affinity"), 499 interrupt_set_affinity_sysctl, 0, &kintrio_set, 500 sizeof(kintrio_set), CTL_CREATE, CTL_EOL); 501 502 sysctl_createv(clog, 0, &node, NULL, 503 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, 504 "intr", SYSCTL_DESCR("set intr"), 505 interrupt_intr_sysctl, 0, &kintrio_set, 506 sizeof(kintrio_set), CTL_CREATE, CTL_EOL); 507 508 sysctl_createv(clog, 0, &node, NULL, 509 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, 510 "nointr", SYSCTL_DESCR("set nointr"), 511 interrupt_nointr_sysctl, 0, &kintrio_set, 512 sizeof(kintrio_set), CTL_CREATE, CTL_EOL); 513 } 514