1 /* $NetBSD: subr_kcpuset.c,v 1.3 2011/08/07 21:38:32 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2011 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Kernel CPU set implementation. 34 * 35 * Interface can be used by kernel subsystems as a unified dynamic CPU 36 * bitset implementation handling many CPUs. Facility also supports early 37 * use by MD code on boot, as it fixups bitsets on further boot. 38 * 39 * TODO: 40 * - Handle "reverse" bitset on fixup/grow. 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: subr_kcpuset.c,v 1.3 2011/08/07 21:38:32 rmind Exp $"); 45 46 #include <sys/param.h> 47 #include <sys/types.h> 48 49 #include <sys/atomic.h> 50 #include <sys/sched.h> 51 #include <sys/kcpuset.h> 52 #include <sys/pool.h> 53 54 /* Number of CPUs to support. */ 55 #define KC_MAXCPUS roundup2(MAXCPUS, 32) 56 57 /* 58 * Structure of dynamic CPU set in the kernel. 59 */ 60 struct kcpuset { 61 uint32_t bits[0]; 62 }; 63 64 typedef struct kcpuset_impl { 65 /* Reference count. */ 66 u_int kc_refcnt; 67 /* Next to free, if non-NULL (used when multiple references). */ 68 struct kcpuset * kc_next; 69 /* Actual variable-sized field of bits. */ 70 struct kcpuset kc_field; 71 } kcpuset_impl_t; 72 73 #define KC_BITS_OFF (offsetof(struct kcpuset_impl, kc_field)) 74 #define KC_GETSTRUCT(b) ((kcpuset_impl_t *)((char *)(b) - KC_BITS_OFF)) 75 76 /* Sizes of a single bitset. */ 77 #define KC_SHIFT 5 78 #define KC_MASK 31 79 80 /* An array of noted early kcpuset creations and data. */ 81 #define KC_SAVE_NITEMS 8 82 83 /* Structures for early boot mechanism (must be statically initialised). */ 84 static kcpuset_t ** kc_noted_early[KC_SAVE_NITEMS]; 85 static uint32_t kc_bits_early[KC_SAVE_NITEMS]; 86 static int kc_last_idx = 0; 87 static bool kc_initialised = false; 88 89 #define KC_BITSIZE_EARLY sizeof(kc_bits_early[0]) 90 #define KC_NFIELDS_EARLY (KC_BITSIZE_EARLY >> KC_SHIFT) 91 92 /* 93 * The size of whole bitset fields and amount of fields. 94 * The whole size must statically initialise for early case. 95 */ 96 static size_t kc_bitsize __read_mostly = KC_BITSIZE_EARLY; 97 static size_t kc_nfields __read_mostly = KC_NFIELDS_EARLY; 98 99 static pool_cache_t kc_cache __read_mostly; 100 101 static kcpuset_t * kcpuset_create_raw(bool); 102 103 /* 104 * kcpuset_sysinit: initialize the subsystem, transfer early boot cases 105 * to dynamically allocated sets. 106 */ 107 void 108 kcpuset_sysinit(void) 109 { 110 kcpuset_t *kc_dynamic[KC_SAVE_NITEMS], *kcp; 111 int i, s; 112 113 /* Set a kcpuset_t sizes. */ 114 kc_nfields = (KC_MAXCPUS >> KC_SHIFT); 115 kc_bitsize = sizeof(uint32_t) * kc_nfields; 116 117 kc_cache = pool_cache_init(sizeof(kcpuset_impl_t) + kc_bitsize, 118 coherency_unit, 0, 0, "kcpuset", NULL, IPL_NONE, NULL, NULL, NULL); 119 120 /* First, pre-allocate kcpuset entries. */ 121 for (i = 0; i < kc_last_idx; i++) { 122 kcp = kcpuset_create_raw(true); 123 kc_dynamic[i] = kcp; 124 } 125 126 /* 127 * Prepare to convert all early noted kcpuset uses to dynamic sets. 128 * All processors, except the one we are currently running (primary), 129 * must not be spinned yet. Since MD facilities can use kcpuset, 130 * raise the IPL to high. 131 */ 132 KASSERT(mp_online == false); 133 134 s = splhigh(); 135 for (i = 0; i < kc_last_idx; i++) { 136 /* 137 * Transfer the bits from early static storage to the kcpuset. 138 */ 139 KASSERT(kc_bitsize >= KC_BITSIZE_EARLY); 140 memcpy(kc_dynamic[i], &kc_bits_early[i], KC_BITSIZE_EARLY); 141 142 /* 143 * Store the new pointer, pointing to the allocated kcpuset. 144 * Note: we are not in an interrupt context and it is the only 145 * CPU running - thus store is safe (e.g. no need for pointer 146 * variable to be volatile). 147 */ 148 *kc_noted_early[i] = kc_dynamic[i]; 149 } 150 kc_initialised = true; 151 kc_last_idx = 0; 152 splx(s); 153 } 154 155 /* 156 * kcpuset_early_ptr: note an early boot use by saving the pointer and 157 * returning a pointer to a static, temporary bit field. 158 */ 159 static kcpuset_t * 160 kcpuset_early_ptr(kcpuset_t **kcptr) 161 { 162 kcpuset_t *kcp; 163 int s; 164 165 s = splhigh(); 166 if (kc_last_idx < KC_SAVE_NITEMS) { 167 /* 168 * Save the pointer, return pointer to static early field. 169 * Need to zero it out. 170 */ 171 kc_noted_early[kc_last_idx++] = kcptr; 172 kcp = (kcpuset_t *)&kc_bits_early[kc_last_idx]; 173 memset(kcp, 0, KC_BITSIZE_EARLY); 174 KASSERT(kc_bitsize == KC_BITSIZE_EARLY); 175 } else { 176 panic("kcpuset(9): all early-use entries exhausted; " 177 "increase KC_SAVE_NITEMS\n"); 178 } 179 splx(s); 180 181 return kcp; 182 } 183 184 /* 185 * Routines to create or destroy the CPU set. 186 * Early boot case is handled. 187 */ 188 189 static kcpuset_t * 190 kcpuset_create_raw(bool zero) 191 { 192 kcpuset_impl_t *kc; 193 194 kc = pool_cache_get(kc_cache, PR_WAITOK); 195 kc->kc_refcnt = 1; 196 kc->kc_next = NULL; 197 198 if (zero) { 199 memset(&kc->kc_field, 0, kc_bitsize); 200 } 201 202 /* Note: return pointer to the actual field of bits. */ 203 KASSERT((uint8_t *)kc + KC_BITS_OFF == (uint8_t *)&kc->kc_field); 204 return &kc->kc_field; 205 } 206 207 void 208 kcpuset_create(kcpuset_t **retkcp, bool zero) 209 { 210 211 if (__predict_false(!kc_initialised)) { 212 /* Early boot use - special case. */ 213 *retkcp = kcpuset_early_ptr(retkcp); 214 return; 215 } 216 *retkcp = kcpuset_create_raw(zero); 217 } 218 219 void 220 kcpuset_destroy(kcpuset_t *kcp) 221 { 222 kcpuset_impl_t *kc; 223 224 KASSERT(kc_initialised); 225 KASSERT(kcp != NULL); 226 227 do { 228 kc = KC_GETSTRUCT(kcp); 229 kcp = kc->kc_next; 230 pool_cache_put(kc_cache, kc); 231 } while (kcp); 232 } 233 234 /* 235 * Routines to copy or reference/unreference the CPU set. 236 * Note: early boot case is not supported by these routines. 237 */ 238 239 void 240 kcpuset_copy(kcpuset_t *dkcp, kcpuset_t *skcp) 241 { 242 243 KASSERT(!kc_initialised || KC_GETSTRUCT(dkcp)->kc_refcnt == 1); 244 memcpy(dkcp, skcp, kc_bitsize); 245 } 246 247 void 248 kcpuset_use(kcpuset_t *kcp) 249 { 250 kcpuset_impl_t *kc = KC_GETSTRUCT(kcp); 251 252 KASSERT(kc_initialised); 253 atomic_inc_uint(&kc->kc_refcnt); 254 } 255 256 void 257 kcpuset_unuse(kcpuset_t *kcp, kcpuset_t **lst) 258 { 259 kcpuset_impl_t *kc = KC_GETSTRUCT(kcp); 260 261 KASSERT(kc_initialised); 262 KASSERT(kc->kc_refcnt > 0); 263 264 if (atomic_dec_uint_nv(&kc->kc_refcnt) != 0) { 265 return; 266 } 267 KASSERT(kc->kc_next == NULL); 268 if (lst == NULL) { 269 kcpuset_destroy(kcp); 270 return; 271 } 272 kc->kc_next = *lst; 273 *lst = kcp; 274 } 275 276 /* 277 * Routines to transfer the CPU set from / to userspace. 278 * Note: early boot case is not supported by these routines. 279 */ 280 281 int 282 kcpuset_copyin(const cpuset_t *ucp, kcpuset_t *kcp, size_t len) 283 { 284 kcpuset_impl_t *kc = KC_GETSTRUCT(kcp); 285 286 KASSERT(kc_initialised); 287 KASSERT(kc->kc_refcnt > 0); 288 KASSERT(kc->kc_next == NULL); 289 (void)kc; 290 291 if (len != kc_bitsize) { /* XXX */ 292 return EINVAL; 293 } 294 return copyin(ucp, kcp, kc_bitsize); 295 } 296 297 int 298 kcpuset_copyout(kcpuset_t *kcp, cpuset_t *ucp, size_t len) 299 { 300 kcpuset_impl_t *kc = KC_GETSTRUCT(kcp); 301 302 KASSERT(kc_initialised); 303 KASSERT(kc->kc_refcnt > 0); 304 KASSERT(kc->kc_next == NULL); 305 (void)kc; 306 307 if (len != kc_bitsize) { /* XXX */ 308 return EINVAL; 309 } 310 return copyout(kcp, ucp, kc_bitsize); 311 } 312 313 /* 314 * Routines to change bit field - zero, fill, set, unset, etc. 315 */ 316 317 void 318 kcpuset_zero(kcpuset_t *kcp) 319 { 320 321 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0); 322 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL); 323 memset(kcp, 0, kc_bitsize); 324 } 325 326 void 327 kcpuset_fill(kcpuset_t *kcp) 328 { 329 330 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0); 331 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL); 332 memset(kcp, ~0, kc_bitsize); 333 } 334 335 void 336 kcpuset_set(kcpuset_t *kcp, cpuid_t i) 337 { 338 const size_t j = i >> KC_SHIFT; 339 340 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL); 341 KASSERT(j < kc_nfields); 342 343 kcp->bits[j] |= 1 << (i & KC_MASK); 344 } 345 346 void 347 kcpuset_clear(kcpuset_t *kcp, cpuid_t i) 348 { 349 const size_t j = i >> KC_SHIFT; 350 351 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL); 352 KASSERT(j < kc_nfields); 353 354 kcp->bits[j] &= ~(1 << (i & KC_MASK)); 355 } 356 357 int 358 kcpuset_isset(kcpuset_t *kcp, cpuid_t i) 359 { 360 const size_t j = i >> KC_SHIFT; 361 362 KASSERT(kcp != NULL); 363 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0); 364 KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL); 365 KASSERT(j < kc_nfields); 366 367 return ((1 << (i & KC_MASK)) & kcp->bits[j]) != 0; 368 } 369 370 bool 371 kcpuset_iszero(kcpuset_t *kcp) 372 { 373 374 for (size_t j = 0; j < kc_nfields; j++) { 375 if (kcp->bits[j] != 0) { 376 return false; 377 } 378 } 379 return true; 380 } 381 382 bool 383 kcpuset_match(const kcpuset_t *kcp1, const kcpuset_t *kcp2) 384 { 385 386 return memcmp(kcp1, kcp2, kc_bitsize) == 0; 387 } 388 389 void 390 kcpuset_merge(kcpuset_t *kcp1, kcpuset_t *kcp2) 391 { 392 393 for (size_t j = 0; j < kc_nfields; j++) { 394 kcp1->bits[j] |= kcp2->bits[j]; 395 } 396 } 397 398 /* 399 * Routines to set/clear the flags atomically. 400 */ 401 402 void 403 kcpuset_atomic_set(kcpuset_t *kcp, cpuid_t i) 404 { 405 const size_t j = i >> KC_SHIFT; 406 407 KASSERT(j < kc_nfields); 408 atomic_or_32(&kcp->bits[j], 1 << (i & KC_MASK)); 409 } 410 411 void 412 kcpuset_atomic_clear(kcpuset_t *kcp, cpuid_t i) 413 { 414 const size_t j = i >> KC_SHIFT; 415 416 KASSERT(j < kc_nfields); 417 atomic_and_32(&kcp->bits[j], ~(1 << (i & KC_MASK))); 418 } 419