1 /* 2 * Copyright (c) 2007 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $DragonFly: src/sys/kern/kern_sysref.c,v 1.1 2007/04/29 01:29:33 dillon Exp $ 35 */ 36 /* 37 * System resource control module for all cluster-addressable system resource 38 * structures. 39 * 40 * This module implements the core ref counting, sysid registration, and 41 * objcache-backed allocation mechanism for all major system resource 42 * structures. 43 * 44 * sysid registrations operate via the objcache ctor/dtor mechanism and 45 * sysids will be reused if the resource is not explicitly accessed via 46 * its sysid. This removes all RB tree handling overhead from the critical 47 * path for locally used resources. 48 */ 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/kernel.h> 53 #include <sys/tree.h> 54 #include <sys/syslink.h> 55 #include <sys/spinlock.h> 56 #include <machine/atomic.h> 57 #include <machine/cpufunc.h> 58 59 #include <sys/spinlock2.h> 60 #include <sys/sysref2.h> 61 62 static boolean_t sysref_ctor(void *data, void *private, int ocflags); 63 static void sysref_dtor(void *data, void *private); 64 65 /* 66 * Red-Black tree support 67 */ 68 static int rb_sysref_compare(struct sysref *sr1, struct sysref *sr2); 69 RB_GENERATE2(sysref_rb_tree, sysref, rbnode, rb_sysref_compare, sysid_t, sysid); 70 71 static struct srpercpu { 72 struct sysref_rb_tree rbtree; 73 struct spinlock spin; 74 } sysref_array[MAXCPU]; 75 76 static void 77 sysrefbootinit(void *dummy __unused) 78 { 79 struct srpercpu *sa; 80 int i; 81 82 for (i = 0; i < ncpus; ++i) { 83 sa = &sysref_array[i]; 84 spin_init(&sa->spin); 85 RB_INIT(&sa->rbtree); 86 } 87 } 88 89 SYSINIT(sysref, SI_SUB_OBJCACHE, SI_ORDER_ANY, sysrefbootinit, NULL); 90 91 static 92 int 93 rb_sysref_compare(struct sysref *sr1, struct sysref *sr2) 94 { 95 if (sr1->sysid < sr2->sysid) 96 return(-1); 97 if (sr1->sysid > sr2->sysid) 98 return(1); 99 return(0); 100 } 101 102 /* 103 * Manual initialization of a resource structure's sysref, only used during 104 * booting to set up certain statically declared resources which cannot 105 * be deallocated. 106 */ 107 void 108 sysref_init(struct sysref *sr, struct sysref_class *class) 109 { 110 struct srpercpu *sa; 111 globaldata_t gd; 112 113 gd = mycpu; 114 crit_enter_gd(gd); 115 gd->gd_sysid_alloc += ncpus_fit; /* next unique sysid */ 116 sr->sysid = gd->gd_sysid_alloc; 117 KKASSERT(((int)sr->sysid & ncpus_fit_mask) == gd->gd_cpuid); 118 sr->refcnt = -0x40000000; 119 sr->flags = 0; 120 sr->class = class; 121 122 sa = &sysref_array[gd->gd_cpuid]; 123 spin_lock_wr(&sa->spin); 124 sysref_rb_tree_RB_INSERT(&sa->rbtree, sr); 125 spin_unlock_wr(&sa->spin); 126 crit_exit_gd(gd); 127 } 128 129 /* 130 * Allocate a resource structure of the specified class, initialize a 131 * sysid and add the resource to the RB tree. The caller must complete 132 * initialization of the resource and call sysref_activate() to activate it. 133 */ 134 void * 135 sysref_alloc(struct sysref_class *class) 136 { 137 struct sysref *sr; 138 char *data; 139 int n; 140 141 /* 142 * Create the object cache backing store. 143 */ 144 if (class->oc == NULL) { 145 KKASSERT(class->mtype != NULL); 146 class->oc = objcache_create_mbacked( 147 class->mtype, class->objsize, 148 0, class->mag_capacity, 149 sysref_ctor, sysref_dtor, class); 150 } 151 152 /* 153 * Allocate the resource. 154 */ 155 data = objcache_get(class->oc, M_WAITOK); 156 sr = (struct sysref *)(data + class->offset); 157 158 /* 159 * Refcnt isn't touched while it is zero. The objcache ctor 160 * function has already allocated a sysid and emplaced the 161 * structure in the RB tree. 162 */ 163 KKASSERT(sr->refcnt == 0); 164 sr->refcnt = -0x40000000; 165 166 /* 167 * Clean out the structure unless the caller wants to deal with 168 * it (e.g. like the vmspace code). 169 */ 170 if ((class->flags & SRC_MANAGEDINIT) == 0) { 171 if (class->offset != 0) 172 bzero(data, class->offset); 173 n = class->offset + sizeof(struct sysref); 174 KKASSERT(n <= class->objsize); 175 if (n != class->objsize) 176 bzero(data + n, class->objsize - n); 177 } 178 return(data); 179 } 180 181 /* 182 * Object cache backing store ctor function. 183 * 184 * This allocates the sysid and associates the structure with the 185 * red-black tree, allowing it to be looked up. The actual resource 186 * structure has NOT yet been allocated so it is marked free. 187 * 188 * If the sysid is not used to access the resource, we will just 189 * allow the sysid to be reused when the resource structure is reused, 190 * allowing the RB tree operation to be 'cached'. This results in 191 * virtually no performance penalty for using the sysref facility. 192 */ 193 static 194 boolean_t 195 sysref_ctor(void *data, void *private, int ocflags) 196 { 197 globaldata_t gd; 198 struct srpercpu *sa; 199 struct sysref_class *class = private; 200 struct sysref *sr = (void *)((char *)data + class->offset); 201 202 /* 203 * Resource structures need to be cleared when allocating from 204 * malloc backing store. This is different from the zeroing 205 * that we do in sysref_alloc(). 206 */ 207 bzero(data, class->objsize); 208 209 /* 210 * Resources managed by our objcache do the sysid and RB tree 211 * handling in the objcache ctor/dtor, so we can reuse the 212 * structure without re-treeing it over and over again. 213 */ 214 gd = mycpu; 215 crit_enter_gd(gd); 216 gd->gd_sysid_alloc += ncpus_fit; /* next unique sysid */ 217 sr->sysid = gd->gd_sysid_alloc; 218 KKASSERT(((int)sr->sysid & ncpus_fit_mask) == gd->gd_cpuid); 219 /* sr->refcnt= 0; already zero */ 220 sr->flags = SRF_ALLOCATED; 221 sr->class = class; 222 223 sa = &sysref_array[gd->gd_cpuid]; 224 spin_lock_wr(&sa->spin); 225 sysref_rb_tree_RB_INSERT(&sa->rbtree, sr); 226 spin_unlock_wr(&sa->spin); 227 crit_exit_gd(gd); 228 229 /* 230 * Execute the class's ctor function, if any. NOTE: The class 231 * should not try to zero out the structure, we've already handled 232 * that and preinitialized the sysref. 233 * 234 * XXX ignores return value for now 235 */ 236 if (class->ctor) 237 class->ctor(data, private, ocflags); 238 return TRUE; 239 } 240 241 /* 242 * Object cache destructor, allowing the structure to be returned 243 * to the system memory pool. The resource structure must be 244 * removed from the RB tree. All other references have already 245 * been destroyed and the RB tree will not create any new references 246 * to the structure in its current state. 247 */ 248 static 249 void 250 sysref_dtor(void *data, void *private) 251 { 252 struct srpercpu *sa; 253 struct sysref_class *class = private; 254 struct sysref *sr = (void *)((char *)data + class->offset); 255 256 KKASSERT(sr->refcnt == 0); 257 sa = &sysref_array[(int)sr->sysid & ncpus_fit_mask]; 258 spin_lock_wr(&sa->spin); 259 sysref_rb_tree_RB_REMOVE(&sa->rbtree, sr); 260 spin_unlock_wr(&sa->spin); 261 if (class->dtor) 262 class->dtor(data, private); 263 } 264 265 /* 266 * Activate or reactivate a resource. 0x40000001 is added to the ref count 267 * so -0x40000000 (during initialization) will translate to a ref count of 1. 268 * Any references made during initialization will translate to additional 269 * positive ref counts. 270 */ 271 void 272 sysref_activate(struct sysref *sr) 273 { 274 int count; 275 276 for (;;) { 277 count = sr->refcnt; 278 KKASSERT(count < 0 && count + 0x40000001 > 0); 279 if (atomic_cmpset_int(&sr->refcnt, count, count + 0x40000001)) 280 break; 281 cpu_pause(); 282 } 283 } 284 285 /* 286 * Release a reference under special circumstances. This call is made 287 * from the sysref_put() inline from sys/sysref2.h for any 1->0 transitions, 288 * negative->negative 'termination in progress' transitions, and when the 289 * cmpset instruction fails during a normal transition. 290 * 291 * This function is called from the sysref_put() inline in sys/sysref2.h, 292 * but handles all cases regardless. 293 */ 294 void 295 _sysref_put(struct sysref *sr) 296 { 297 int count; 298 void *data; 299 300 for (;;) { 301 count = sr->refcnt; 302 if (count > 1) { 303 /* 304 * release 1 count, nominal case, active resource 305 * structure, no other action required. 306 */ 307 if (atomic_cmpset_int(&sr->refcnt, count, count - 1)) 308 break; 309 } else if (count == 1) { 310 /* 311 * 1->0 transitions transition to -0x40000000 instead, 312 * placing the resource structure into a termination- 313 * in-progress state. The termination function is 314 * then called. 315 */ 316 if (atomic_cmpset_int(&sr->refcnt, count, -0x40000000)) { 317 data = (char *)sr - sr->class->offset; 318 sr->class->ops.terminate(data); 319 break; 320 } 321 } else if (count > -0x40000000) { 322 /* 323 * release 1 count, nominal case, resource undergoing 324 * termination. The Resource can be ref'd and 325 * deref'd while undergoing termination. 326 */ 327 if (atomic_cmpset_int(&sr->refcnt, count, count - 1)) 328 break; 329 } else { 330 /* 331 * Final release, set refcnt to 0. 332 * Resource must have been allocated. 333 * 334 * If SRF_SYSIDUSED is not set just objcache_put() the 335 * resource, otherwise objcache_dtor() the resource. 336 */ 337 KKASSERT(count == -0x40000000 && 338 (sr->flags & SRF_ALLOCATED)); 339 if (atomic_cmpset_int(&sr->refcnt, count, 0)) { 340 data = (char *)sr - sr->class->offset; 341 if (sr->flags & SRF_SYSIDUSED) 342 objcache_dtor(sr->class->oc, data); 343 else 344 objcache_put(sr->class->oc, data); 345 break; 346 } 347 } 348 /* loop until the cmpset succeeds */ 349 cpu_pause(); 350 } 351 } 352 353