1 /* $NetBSD: subr_kcpuset.c,v 1.4 2012/01/29 19:08:26 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2011 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Kernel CPU set implementation. 34 * 35 * Interface can be used by kernel subsystems as a unified dynamic CPU 36 * bitset implementation handling many CPUs. Facility also supports early 37 * use by MD code on boot, as it fixups bitsets on further boot. 38 * 39 * TODO: 40 * - Handle "reverse" bitset on fixup/grow. 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: subr_kcpuset.c,v 1.4 2012/01/29 19:08:26 rmind Exp $"); 45 46 #include <sys/param.h> 47 #include <sys/types.h> 48 49 #include <sys/atomic.h> 50 #include <sys/sched.h> 51 #include <sys/kcpuset.h> 52 #include <sys/pool.h> 53 54 /* Number of CPUs to support. */ 55 #define KC_MAXCPUS roundup2(MAXCPUS, 32) 56 57 /* 58 * Structure of dynamic CPU set in the kernel. 59 */ 60 struct kcpuset { 61 uint32_t bits[0]; 62 }; 63 64 typedef struct kcpuset_impl { 65 /* Reference count. */ 66 u_int kc_refcnt; 67 /* Next to free, if non-NULL (used when multiple references). */ 68 struct kcpuset * kc_next; 69 /* Actual variable-sized field of bits. */ 70 struct kcpuset kc_field; 71 } kcpuset_impl_t; 72 73 #define KC_BITS_OFF (offsetof(struct kcpuset_impl, kc_field)) 74 #define KC_GETSTRUCT(b) ((kcpuset_impl_t *)((char *)(b) - KC_BITS_OFF)) 75 76 /* Sizes of a single bitset. */ 77 #define KC_SHIFT 5 78 #define KC_MASK 31 79 80 /* An array of noted early kcpuset creations and data. */ 81 #define KC_SAVE_NITEMS 8 82 83 /* Structures for early boot mechanism (must be statically initialised). */ 84 static kcpuset_t ** kc_noted_early[KC_SAVE_NITEMS]; 85 static uint32_t kc_bits_early[KC_SAVE_NITEMS]; 86 static int kc_last_idx = 0; 87 static bool kc_initialised = false; 88 89 #define KC_BITSIZE_EARLY sizeof(kc_bits_early[0]) 90 #define KC_NFIELDS_EARLY 1 91 92 /* 93 * The size of whole bitset fields and amount of fields. 94 * The whole size must statically initialise for early case. 95 */ 96 static size_t kc_bitsize __read_mostly = KC_BITSIZE_EARLY; 97 static size_t kc_nfields __read_mostly = KC_NFIELDS_EARLY; 98 99 static pool_cache_t kc_cache __read_mostly; 100 101 static kcpuset_t * kcpuset_create_raw(bool); 102 103 /* 104 * kcpuset_sysinit: initialize the subsystem, transfer early boot cases 105 * to dynamically allocated sets. 106 */ 107 void 108 kcpuset_sysinit(void) 109 { 110 kcpuset_t *kc_dynamic[KC_SAVE_NITEMS], *kcp; 111 int i, s; 112 113 /* Set a kcpuset_t sizes. */ 114 kc_nfields = (KC_MAXCPUS >> KC_SHIFT); 115 kc_bitsize = sizeof(uint32_t) * kc_nfields; 116 KASSERT(kc_nfields != 0 && kc_bitsize != 0); 117 118 kc_cache = pool_cache_init(sizeof(kcpuset_impl_t) + kc_bitsize, 119 coherency_unit, 0, 0, "kcpuset", NULL, IPL_NONE, NULL, NULL, NULL); 120 121 /* First, pre-allocate kcpuset entries. */ 122 for (i = 0; i < kc_last_idx; i++) { 123 kcp = kcpuset_create_raw(true); 124 kc_dynamic[i] = kcp; 125 } 126 127 /* 128 * Prepare to convert all early noted kcpuset uses to dynamic sets. 129 * All processors, except the one we are currently running (primary), 130 * must not be spinned yet. Since MD facilities can use kcpuset, 131 * raise the IPL to high. 132 */ 133 KASSERT(mp_online == false); 134 135 s = splhigh(); 136 for (i = 0; i < kc_last_idx; i++) { 137 /* 138 * Transfer the bits from early static storage to the kcpuset. 139 */ 140 KASSERT(kc_bitsize >= KC_BITSIZE_EARLY); 141 memcpy(kc_dynamic[i], &kc_bits_early[i], KC_BITSIZE_EARLY); 142 143 /* 144 * Store the new pointer, pointing to the allocated kcpuset. 145 * Note: we are not in an interrupt context and it is the only 146 * CPU running - thus store is safe (e.g. no need for pointer 147 * variable to be volatile). 148 */ 149 *kc_noted_early[i] = kc_dynamic[i]; 150 } 151 kc_initialised = true; 152 kc_last_idx = 0; 153 splx(s); 154 } 155 156 /* 157 * kcpuset_early_ptr: note an early boot use by saving the pointer and 158 * returning a pointer to a static, temporary bit field. 159 */ 160 static kcpuset_t * 161 kcpuset_early_ptr(kcpuset_t **kcptr) 162 { 163 kcpuset_t *kcp; 164 int s; 165 166 s = splhigh(); 167 if (kc_last_idx < KC_SAVE_NITEMS) { 168 /* 169 * Save the pointer, return pointer to static early field. 170 * Need to zero it out. 171 */ 172 kc_noted_early[kc_last_idx++] = kcptr; 173 kcp = (kcpuset_t *)&kc_bits_early[kc_last_idx]; 174 memset(kcp, 0, KC_BITSIZE_EARLY); 175 KASSERT(kc_bitsize == KC_BITSIZE_EARLY); 176 } else { 177 panic("kcpuset(9): all early-use entries exhausted; " 178 "increase KC_SAVE_NITEMS\n"); 179 } 180 splx(s); 181 182 return kcp; 183 } 184 185 /* 186 * Routines to create or destroy the CPU set. 187 * Early boot case is handled. 188 */ 189 190 static kcpuset_t * 191 kcpuset_create_raw(bool zero) 192 { 193 kcpuset_impl_t *kc; 194 195 kc = pool_cache_get(kc_cache, PR_WAITOK); 196 kc->kc_refcnt = 1; 197 kc->kc_next = NULL; 198 199 if (zero) { 200 memset(&kc->kc_field, 0, kc_bitsize); 201 } 202 203 /* Note: return pointer to the actual field of bits. */ 204 KASSERT((uint8_t *)kc + KC_BITS_OFF == (uint8_t *)&kc->kc_field); 205 return &kc->kc_field; 206 } 207 208 void 209 kcpuset_create(kcpuset_t **retkcp, bool zero) 210 { 211 212 if (__predict_false(!kc_initialised)) { 213 /* Early boot use - special case. */ 214 *retkcp = kcpuset_early_ptr(retkcp); 215 return; 216 } 217 *retkcp = kcpuset_create_raw(zero); 218 } 219 220 void 221 kcpuset_destroy(kcpuset_t *kcp) 222 { 223 kcpuset_impl_t *kc; 224 225 KASSERT(kc_initialised); 226 KASSERT(kcp != NULL); 227 228 do { 229 kc = KC_GETSTRUCT(kcp); 230 kcp = kc->kc_next; 231 pool_cache_put(kc_cache, kc); 232 } while (kcp); 233 } 234 235 /* 236 * Routines to reference/unreference the CPU set. 237 * Note: early boot case is not supported by these routines. 238 */ 239 240 void 241 kcpuset_use(kcpuset_t *kcp) 242 { 243 kcpuset_impl_t *kc = KC_GETSTRUCT(kcp); 244 245 KASSERT(kc_initialised); 246 atomic_inc_uint(&kc->kc_refcnt); 247 } 248 249 void 250 kcpuset_unuse(kcpuset_t *kcp, kcpuset_t **lst) 251 { 252 kcpuset_impl_t *kc = KC_GETSTRUCT(kcp); 253 254 KASSERT(kc_initialised); 255 KASSERT(kc->kc_refcnt > 0); 256 257 if (atomic_dec_uint_nv(&kc->kc_refcnt) != 0) { 258 return; 259 } 260 KASSERT(kc->kc_next == NULL); 261 if (lst == NULL) { 262 kcpuset_destroy(kcp); 263 return; 264 } 265 kc->kc_next = *lst; 266 *lst = kcp; 267 } 268 269 /* 270 * Routines to transfer the CPU set from / to userspace. 271 * Note: early boot case is not supported by these routines. 272 */ 273 274 int 275 kcpuset_copyin(const cpuset_t *ucp, kcpuset_t *kcp, size_t len) 276 { 277 kcpuset_impl_t *kc = KC_GETSTRUCT(kcp); 278 279 KASSERT(kc_initialised); 280 KASSERT(kc->kc_refcnt > 0); 281 KASSERT(kc->kc_next == NULL); 282 (void)kc; 283 284 if (len != kc_bitsize) { /* XXX */ 285 return EINVAL; 286 } 287 return copyin(ucp, kcp, kc_bitsize); 288 } 289 290 int 291 kcpuset_copyout(kcpuset_t *kcp, cpuset_t *ucp, size_t len) 292 { 293 kcpuset_impl_t *kc = KC_GETSTRUCT(kcp); 294 295 KASSERT(kc_initialised); 296 KASSERT(kc->kc_refcnt > 0); 297 KASSERT(kc->kc_next == NULL); 298 (void)kc; 299 300 if (len != kc_bitsize) { /* XXX */ 301 return EINVAL; 302 } 303 return copyout(kcp, ucp, kc_bitsize); 304 } 305 306 /* 307 * Routines to change bit field - zero, fill, copy, set, unset, etc. 308 */ 309 310 void 311 kcpuset_zero(kcpuset_t *kcp) 312 { 313 314 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0); 315 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL); 316 memset(kcp, 0, kc_bitsize); 317 } 318 319 void 320 kcpuset_fill(kcpuset_t *kcp) 321 { 322 323 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0); 324 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL); 325 memset(kcp, ~0, kc_bitsize); 326 } 327 328 void 329 kcpuset_copy(kcpuset_t *dkcp, kcpuset_t *skcp) 330 { 331 332 KASSERT(!kc_initialised || KC_GETSTRUCT(dkcp)->kc_refcnt > 0); 333 KASSERT(!kc_initialised || KC_GETSTRUCT(dkcp)->kc_next == NULL); 334 memcpy(dkcp, skcp, kc_bitsize); 335 } 336 337 void 338 kcpuset_set(kcpuset_t *kcp, cpuid_t i) 339 { 340 const size_t j = i >> KC_SHIFT; 341 342 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL); 343 KASSERT(j < kc_nfields); 344 345 kcp->bits[j] |= 1 << (i & KC_MASK); 346 } 347 348 void 349 kcpuset_clear(kcpuset_t *kcp, cpuid_t i) 350 { 351 const size_t j = i >> KC_SHIFT; 352 353 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL); 354 KASSERT(j < kc_nfields); 355 356 kcp->bits[j] &= ~(1 << (i & KC_MASK)); 357 } 358 359 bool 360 kcpuset_isset(kcpuset_t *kcp, cpuid_t i) 361 { 362 const size_t j = i >> KC_SHIFT; 363 364 KASSERT(kcp != NULL); 365 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0); 366 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL); 367 KASSERT(j < kc_nfields); 368 369 return ((1 << (i & KC_MASK)) & kcp->bits[j]) != 0; 370 } 371 372 bool 373 kcpuset_isotherset(kcpuset_t *kcp, cpuid_t i) 374 { 375 const size_t j2 = i >> KC_SHIFT; 376 const uint32_t mask = ~(1 << (i & KC_MASK)); 377 378 for (size_t j = 0; j < kc_nfields; j++) { 379 const uint32_t bits = kcp->bits[j]; 380 if (bits && (j != j2 || (bits & mask) != 0)) { 381 return true; 382 } 383 } 384 return false; 385 } 386 387 bool 388 kcpuset_iszero(kcpuset_t *kcp) 389 { 390 391 for (size_t j = 0; j < kc_nfields; j++) { 392 if (kcp->bits[j] != 0) { 393 return false; 394 } 395 } 396 return true; 397 } 398 399 bool 400 kcpuset_match(const kcpuset_t *kcp1, const kcpuset_t *kcp2) 401 { 402 403 return memcmp(kcp1, kcp2, kc_bitsize) == 0; 404 } 405 406 void 407 kcpuset_merge(kcpuset_t *kcp1, kcpuset_t *kcp2) 408 { 409 410 for (size_t j = 0; j < kc_nfields; j++) { 411 kcp1->bits[j] |= kcp2->bits[j]; 412 } 413 } 414 415 int 416 kcpuset_countset(kcpuset_t *kcp) 417 { 418 int count = 0; 419 420 for (size_t j = 0; j < kc_nfields; j++) { 421 count += popcount32(kcp->bits[j]); 422 } 423 return count; 424 } 425 426 /* 427 * Routines to set/clear the flags atomically. 428 */ 429 430 void 431 kcpuset_atomic_set(kcpuset_t *kcp, cpuid_t i) 432 { 433 const size_t j = i >> KC_SHIFT; 434 435 KASSERT(j < kc_nfields); 436 atomic_or_32(&kcp->bits[j], 1 << (i & KC_MASK)); 437 } 438 439 void 440 kcpuset_atomic_clear(kcpuset_t *kcp, cpuid_t i) 441 { 442 const size_t j = i >> KC_SHIFT; 443 444 KASSERT(j < kc_nfields); 445 atomic_and_32(&kcp->bits[j], ~(1 << (i & KC_MASK))); 446 } 447