1 /* $NetBSD: npf_tableset.c,v 1.21 2014/02/06 02:51:28 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2009-2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This material is based upon work partially supported by The 8 * NetBSD Foundation under a contract with Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * NPF tableset module. 34 * 35 * Notes 36 * 37 * The tableset is an array of tables. After the creation, the array 38 * is immutable. The caller is responsible to synchronise the access 39 * to the tableset. The table can either be a hash or a tree. Its 40 * entries are protected by a read-write lock. 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: npf_tableset.c,v 1.21 2014/02/06 02:51:28 rmind Exp $"); 45 46 #include <sys/param.h> 47 #include <sys/types.h> 48 49 #include <sys/atomic.h> 50 #include <sys/hash.h> 51 #include <sys/cdbr.h> 52 #include <sys/kmem.h> 53 #include <sys/malloc.h> 54 #include <sys/pool.h> 55 #include <sys/queue.h> 56 #include <sys/rwlock.h> 57 #include <sys/systm.h> 58 #include <sys/types.h> 59 60 #include "npf_impl.h" 61 62 typedef struct npf_tblent { 63 union { 64 LIST_ENTRY(npf_tblent) te_hashent; 65 pt_node_t te_node; 66 } /* C11 */; 67 int te_alen; 68 npf_addr_t te_addr; 69 } npf_tblent_t; 70 71 LIST_HEAD(npf_hashl, npf_tblent); 72 73 struct npf_table { 74 /* 75 * The storage type can be: a) hash b) tree c) cdb. 76 * There are separate trees for IPv4 and IPv6. 77 */ 78 union { 79 struct { 80 struct npf_hashl *t_hashl; 81 u_long t_hashmask; 82 }; 83 struct { 84 pt_tree_t t_tree[2]; 85 }; 86 struct { 87 void * t_blob; 88 size_t t_bsize; 89 struct cdbr * t_cdb; 90 }; 91 } /* C11 */; 92 93 /* 94 * Table ID, type and lock. The ID may change during the 95 * config reload, it is protected by the npf_config_lock. 96 */ 97 int t_type; 98 u_int t_id; 99 krwlock_t t_lock; 100 101 /* The number of items, reference count and table name. */ 102 u_int t_nitems; 103 u_int t_refcnt; 104 char t_name[NPF_TABLE_MAXNAMELEN]; 105 }; 106 107 struct npf_tableset { 108 u_int ts_nitems; 109 npf_table_t * ts_map[]; 110 }; 111 112 #define NPF_TABLESET_SIZE(n) \ 113 (offsetof(npf_tableset_t, ts_map[n]) * sizeof(npf_table_t *)) 114 115 #define NPF_ADDRLEN2TREE(alen) ((alen) >> 4) 116 117 static pool_cache_t tblent_cache __read_mostly; 118 119 /* 120 * npf_table_sysinit: initialise tableset structures. 121 */ 122 void 123 npf_tableset_sysinit(void) 124 { 125 tblent_cache = pool_cache_init(sizeof(npf_tblent_t), coherency_unit, 126 0, 0, "npftblpl", NULL, IPL_NONE, NULL, NULL, NULL); 127 } 128 129 void 130 npf_tableset_sysfini(void) 131 { 132 pool_cache_destroy(tblent_cache); 133 } 134 135 npf_tableset_t * 136 npf_tableset_create(u_int nitems) 137 { 138 npf_tableset_t *ts = kmem_zalloc(NPF_TABLESET_SIZE(nitems), KM_SLEEP); 139 ts->ts_nitems = nitems; 140 return ts; 141 } 142 143 void 144 npf_tableset_destroy(npf_tableset_t *ts) 145 { 146 /* 147 * Destroy all tables (no references should be held, since the 148 * ruleset should be destroyed before). 149 */ 150 for (u_int tid = 0; tid < ts->ts_nitems; tid++) { 151 npf_table_t *t = ts->ts_map[tid]; 152 153 if (t && atomic_dec_uint_nv(&t->t_refcnt) == 0) { 154 npf_table_destroy(t); 155 } 156 } 157 kmem_free(ts, NPF_TABLESET_SIZE(ts->ts_nitems)); 158 } 159 160 /* 161 * npf_tableset_insert: insert the table into the specified tableset. 162 * 163 * => Returns 0 on success. Fails and returns error if ID is already used. 164 */ 165 int 166 npf_tableset_insert(npf_tableset_t *ts, npf_table_t *t) 167 { 168 const u_int tid = t->t_id; 169 int error; 170 171 KASSERT((u_int)tid < ts->ts_nitems); 172 173 if (ts->ts_map[tid] == NULL) { 174 atomic_inc_uint(&t->t_refcnt); 175 ts->ts_map[tid] = t; 176 error = 0; 177 } else { 178 error = EEXIST; 179 } 180 return error; 181 } 182 183 /* 184 * npf_tableset_getbyname: look for a table in the set given the name. 185 */ 186 npf_table_t * 187 npf_tableset_getbyname(npf_tableset_t *ts, const char *name) 188 { 189 npf_table_t *t; 190 191 for (u_int tid = 0; tid < ts->ts_nitems; tid++) { 192 if ((t = ts->ts_map[tid]) == NULL) 193 continue; 194 if (strcmp(name, t->t_name) == 0) 195 return t; 196 } 197 return NULL; 198 } 199 200 npf_table_t * 201 npf_tableset_getbyid(npf_tableset_t *ts, u_int tid) 202 { 203 if (__predict_true(tid < ts->ts_nitems)) { 204 return ts->ts_map[tid]; 205 } 206 return NULL; 207 } 208 209 /* 210 * npf_tableset_reload: iterate all tables and if the new table is of the 211 * same type and has no items, then we preserve the old one and its entries. 212 * 213 * => The caller is responsible for providing synchronisation. 214 */ 215 void 216 npf_tableset_reload(npf_tableset_t *nts, npf_tableset_t *ots) 217 { 218 for (u_int tid = 0; tid < nts->ts_nitems; tid++) { 219 npf_table_t *t, *ot; 220 221 if ((t = nts->ts_map[tid]) == NULL) { 222 continue; 223 } 224 225 /* If our table has entries, just load it. */ 226 if (t->t_nitems) { 227 continue; 228 } 229 230 /* Look for a currently existing table with such name. */ 231 ot = npf_tableset_getbyname(ots, t->t_name); 232 if (ot == NULL) { 233 /* Not found: we have a new table. */ 234 continue; 235 } 236 237 /* Found. Did the type change? */ 238 if (t->t_type != ot->t_type) { 239 /* Yes, load the new. */ 240 continue; 241 } 242 243 /* 244 * Preserve the current table. Acquire a reference since 245 * we are keeping it in the old table set. Update its ID. 246 */ 247 atomic_inc_uint(&ot->t_refcnt); 248 nts->ts_map[tid] = ot; 249 250 KASSERT(npf_config_locked_p()); 251 ot->t_id = tid; 252 253 /* Destroy the new table (we hold the only reference). */ 254 t->t_refcnt--; 255 npf_table_destroy(t); 256 } 257 } 258 259 void 260 npf_tableset_syncdict(const npf_tableset_t *ts, prop_dictionary_t ndict) 261 { 262 prop_array_t tables = prop_array_create(); 263 const npf_table_t *t; 264 265 KASSERT(npf_config_locked_p()); 266 267 for (u_int tid = 0; tid < ts->ts_nitems; tid++) { 268 if ((t = ts->ts_map[tid]) == NULL) { 269 continue; 270 } 271 prop_dictionary_t tdict = prop_dictionary_create(); 272 prop_dictionary_set_cstring(tdict, "name", t->t_name); 273 prop_dictionary_set_uint32(tdict, "type", t->t_type); 274 prop_dictionary_set_uint32(tdict, "id", tid); 275 276 prop_array_add(tables, tdict); 277 prop_object_release(tdict); 278 } 279 prop_dictionary_remove(ndict, "tables"); 280 prop_dictionary_set(ndict, "tables", tables); 281 prop_object_release(tables); 282 } 283 284 /* 285 * Few helper routines. 286 */ 287 288 static npf_tblent_t * 289 table_hash_lookup(const npf_table_t *t, const npf_addr_t *addr, 290 const int alen, struct npf_hashl **rhtbl) 291 { 292 const uint32_t hidx = hash32_buf(addr, alen, HASH32_BUF_INIT); 293 struct npf_hashl *htbl = &t->t_hashl[hidx & t->t_hashmask]; 294 npf_tblent_t *ent; 295 296 /* 297 * Lookup the hash table and check for duplicates. 298 * Note: mask is ignored for the hash storage. 299 */ 300 LIST_FOREACH(ent, htbl, te_hashent) { 301 if (ent->te_alen != alen) { 302 continue; 303 } 304 if (memcmp(&ent->te_addr, addr, alen) == 0) { 305 break; 306 } 307 } 308 *rhtbl = htbl; 309 return ent; 310 } 311 312 static void 313 table_hash_destroy(npf_table_t *t) 314 { 315 for (unsigned n = 0; n <= t->t_hashmask; n++) { 316 npf_tblent_t *ent; 317 318 while ((ent = LIST_FIRST(&t->t_hashl[n])) != NULL) { 319 LIST_REMOVE(ent, te_hashent); 320 pool_cache_put(tblent_cache, ent); 321 } 322 } 323 } 324 325 static void 326 table_tree_destroy(pt_tree_t *tree) 327 { 328 npf_tblent_t *ent; 329 330 while ((ent = ptree_iterate(tree, NULL, PT_ASCENDING)) != NULL) { 331 ptree_remove_node(tree, ent); 332 pool_cache_put(tblent_cache, ent); 333 } 334 } 335 336 /* 337 * npf_table_create: create table with a specified ID. 338 */ 339 npf_table_t * 340 npf_table_create(const char *name, u_int tid, int type, 341 void *blob, size_t size) 342 { 343 npf_table_t *t; 344 345 t = kmem_zalloc(sizeof(npf_table_t), KM_SLEEP); 346 strlcpy(t->t_name, name, NPF_TABLE_MAXNAMELEN); 347 348 switch (type) { 349 case NPF_TABLE_TREE: 350 ptree_init(&t->t_tree[0], &npf_table_ptree_ops, 351 (void *)(sizeof(struct in_addr) / sizeof(uint32_t)), 352 offsetof(npf_tblent_t, te_node), 353 offsetof(npf_tblent_t, te_addr)); 354 ptree_init(&t->t_tree[1], &npf_table_ptree_ops, 355 (void *)(sizeof(struct in6_addr) / sizeof(uint32_t)), 356 offsetof(npf_tblent_t, te_node), 357 offsetof(npf_tblent_t, te_addr)); 358 break; 359 case NPF_TABLE_HASH: 360 t->t_hashl = hashinit(1024, HASH_LIST, true, &t->t_hashmask); 361 if (t->t_hashl == NULL) { 362 kmem_free(t, sizeof(npf_table_t)); 363 return NULL; 364 } 365 break; 366 case NPF_TABLE_CDB: 367 t->t_blob = blob; 368 t->t_bsize = size; 369 t->t_cdb = cdbr_open_mem(blob, size, CDBR_DEFAULT, NULL, NULL); 370 if (t->t_cdb == NULL) { 371 kmem_free(t, sizeof(npf_table_t)); 372 free(blob, M_TEMP); 373 return NULL; 374 } 375 t->t_nitems = cdbr_entries(t->t_cdb); 376 break; 377 default: 378 KASSERT(false); 379 } 380 rw_init(&t->t_lock); 381 t->t_type = type; 382 t->t_id = tid; 383 384 return t; 385 } 386 387 /* 388 * npf_table_destroy: free all table entries and table itself. 389 */ 390 void 391 npf_table_destroy(npf_table_t *t) 392 { 393 KASSERT(t->t_refcnt == 0); 394 395 switch (t->t_type) { 396 case NPF_TABLE_HASH: 397 table_hash_destroy(t); 398 hashdone(t->t_hashl, HASH_LIST, t->t_hashmask); 399 break; 400 case NPF_TABLE_TREE: 401 table_tree_destroy(&t->t_tree[0]); 402 table_tree_destroy(&t->t_tree[1]); 403 break; 404 case NPF_TABLE_CDB: 405 cdbr_close(t->t_cdb); 406 free(t->t_blob, M_TEMP); 407 break; 408 default: 409 KASSERT(false); 410 } 411 rw_destroy(&t->t_lock); 412 kmem_free(t, sizeof(npf_table_t)); 413 } 414 415 /* 416 * npf_table_check: validate the name, ID and type. 417 */ 418 int 419 npf_table_check(npf_tableset_t *ts, const char *name, u_int tid, int type) 420 { 421 if ((u_int)tid >= ts->ts_nitems) { 422 return EINVAL; 423 } 424 if (ts->ts_map[tid] != NULL) { 425 return EEXIST; 426 } 427 switch (type) { 428 case NPF_TABLE_TREE: 429 case NPF_TABLE_HASH: 430 case NPF_TABLE_CDB: 431 break; 432 default: 433 return EINVAL; 434 } 435 if (strlen(name) >= NPF_TABLE_MAXNAMELEN) { 436 return ENAMETOOLONG; 437 } 438 if (npf_tableset_getbyname(ts, name)) { 439 return EEXIST; 440 } 441 return 0; 442 } 443 444 static int 445 table_cidr_check(const u_int aidx, const npf_addr_t *addr, 446 const npf_netmask_t mask) 447 { 448 if (aidx > 1) { 449 return EINVAL; 450 } 451 if (mask > NPF_MAX_NETMASK && mask != NPF_NO_NETMASK) { 452 return EINVAL; 453 } 454 455 /* 456 * For IPv4 (aidx = 0) - 32 and for IPv6 (aidx = 1) - 128. 457 * If it is a host - shall use NPF_NO_NETMASK. 458 */ 459 if (mask >= (aidx ? 128 : 32) && mask != NPF_NO_NETMASK) { 460 return EINVAL; 461 } 462 return 0; 463 } 464 465 /* 466 * npf_table_insert: add an IP CIDR entry into the table. 467 */ 468 int 469 npf_table_insert(npf_table_t *t, const int alen, 470 const npf_addr_t *addr, const npf_netmask_t mask) 471 { 472 const u_int aidx = NPF_ADDRLEN2TREE(alen); 473 npf_tblent_t *ent; 474 int error; 475 476 error = table_cidr_check(aidx, addr, mask); 477 if (error) { 478 return error; 479 } 480 ent = pool_cache_get(tblent_cache, PR_WAITOK); 481 memcpy(&ent->te_addr, addr, alen); 482 ent->te_alen = alen; 483 484 /* 485 * Insert the entry. Return an error on duplicate. 486 */ 487 rw_enter(&t->t_lock, RW_WRITER); 488 switch (t->t_type) { 489 case NPF_TABLE_HASH: { 490 struct npf_hashl *htbl; 491 492 /* 493 * Hash tables by the concept support only IPs. 494 */ 495 if (mask != NPF_NO_NETMASK) { 496 error = EINVAL; 497 break; 498 } 499 if (!table_hash_lookup(t, addr, alen, &htbl)) { 500 LIST_INSERT_HEAD(htbl, ent, te_hashent); 501 t->t_nitems++; 502 } else { 503 error = EEXIST; 504 } 505 break; 506 } 507 case NPF_TABLE_TREE: { 508 pt_tree_t *tree = &t->t_tree[aidx]; 509 bool ok; 510 511 /* 512 * If no mask specified, use maximum mask. 513 */ 514 ok = (mask != NPF_NO_NETMASK) ? 515 ptree_insert_mask_node(tree, ent, mask) : 516 ptree_insert_node(tree, ent); 517 if (ok) { 518 t->t_nitems++; 519 error = 0; 520 } else { 521 error = EEXIST; 522 } 523 break; 524 } 525 case NPF_TABLE_CDB: 526 error = EINVAL; 527 break; 528 default: 529 KASSERT(false); 530 } 531 rw_exit(&t->t_lock); 532 533 if (error) { 534 pool_cache_put(tblent_cache, ent); 535 } 536 return error; 537 } 538 539 /* 540 * npf_table_remove: remove the IP CIDR entry from the table. 541 */ 542 int 543 npf_table_remove(npf_table_t *t, const int alen, 544 const npf_addr_t *addr, const npf_netmask_t mask) 545 { 546 const u_int aidx = NPF_ADDRLEN2TREE(alen); 547 npf_tblent_t *ent = NULL; 548 int error = ENOENT; 549 550 error = table_cidr_check(aidx, addr, mask); 551 if (error) { 552 return error; 553 } 554 555 rw_enter(&t->t_lock, RW_WRITER); 556 switch (t->t_type) { 557 case NPF_TABLE_HASH: { 558 struct npf_hashl *htbl; 559 560 ent = table_hash_lookup(t, addr, alen, &htbl); 561 if (__predict_true(ent != NULL)) { 562 LIST_REMOVE(ent, te_hashent); 563 t->t_nitems--; 564 } 565 break; 566 } 567 case NPF_TABLE_TREE: { 568 pt_tree_t *tree = &t->t_tree[aidx]; 569 570 ent = ptree_find_node(tree, addr); 571 if (__predict_true(ent != NULL)) { 572 ptree_remove_node(tree, ent); 573 t->t_nitems--; 574 } 575 break; 576 } 577 case NPF_TABLE_CDB: 578 error = EINVAL; 579 break; 580 default: 581 KASSERT(false); 582 ent = NULL; 583 } 584 rw_exit(&t->t_lock); 585 586 if (ent) { 587 pool_cache_put(tblent_cache, ent); 588 } 589 return error; 590 } 591 592 /* 593 * npf_table_lookup: find the table according to ID, lookup and match 594 * the contents with the specified IP address. 595 */ 596 int 597 npf_table_lookup(npf_table_t *t, const int alen, const npf_addr_t *addr) 598 { 599 const u_int aidx = NPF_ADDRLEN2TREE(alen); 600 struct npf_hashl *htbl; 601 const void *data; 602 size_t dlen; 603 bool found; 604 605 if (__predict_false(aidx > 1)) { 606 return EINVAL; 607 } 608 609 switch (t->t_type) { 610 case NPF_TABLE_HASH: 611 rw_enter(&t->t_lock, RW_READER); 612 found = table_hash_lookup(t, addr, alen, &htbl) != NULL; 613 rw_exit(&t->t_lock); 614 break; 615 case NPF_TABLE_TREE: 616 rw_enter(&t->t_lock, RW_READER); 617 found = ptree_find_node(&t->t_tree[aidx], addr) != NULL; 618 rw_exit(&t->t_lock); 619 break; 620 case NPF_TABLE_CDB: 621 if (cdbr_find(t->t_cdb, addr, alen, &data, &dlen) == 0) { 622 found = dlen == alen && memcmp(addr, data, dlen) == 0; 623 } else { 624 found = false; 625 } 626 break; 627 default: 628 KASSERT(false); 629 found = false; 630 } 631 632 return found ? 0 : ENOENT; 633 } 634 635 static int 636 table_ent_copyout(const npf_addr_t *addr, const int alen, npf_netmask_t mask, 637 void *ubuf, size_t len, size_t *off) 638 { 639 void *ubufp = (uint8_t *)ubuf + *off; 640 npf_ioctl_ent_t uent; 641 642 if ((*off += sizeof(npf_ioctl_ent_t)) > len) { 643 return ENOMEM; 644 } 645 uent.alen = alen; 646 memcpy(&uent.addr, addr, sizeof(npf_addr_t)); 647 uent.mask = mask; 648 649 return copyout(&uent, ubufp, sizeof(npf_ioctl_ent_t)); 650 } 651 652 static int 653 table_hash_list(const npf_table_t *t, void *ubuf, size_t len) 654 { 655 size_t off = 0; 656 int error = 0; 657 658 for (unsigned n = 0; n <= t->t_hashmask; n++) { 659 npf_tblent_t *ent; 660 661 LIST_FOREACH(ent, &t->t_hashl[n], te_hashent) { 662 error = table_ent_copyout(&ent->te_addr, 663 ent->te_alen, 0, ubuf, len, &off); 664 if (error) 665 break; 666 } 667 } 668 return error; 669 } 670 671 static int 672 table_tree_list(pt_tree_t *tree, npf_netmask_t maxmask, void *ubuf, 673 size_t len, size_t *off) 674 { 675 npf_tblent_t *ent = NULL; 676 int error = 0; 677 678 while ((ent = ptree_iterate(tree, ent, PT_ASCENDING)) != NULL) { 679 pt_bitlen_t blen; 680 681 if (!ptree_mask_node_p(tree, ent, &blen)) { 682 blen = maxmask; 683 } 684 error = table_ent_copyout(&ent->te_addr, ent->te_alen, 685 blen, ubuf, len, off); 686 if (error) 687 break; 688 } 689 return error; 690 } 691 692 static int 693 table_cdb_list(npf_table_t *t, void *ubuf, size_t len) 694 { 695 size_t off = 0, dlen; 696 const void *data; 697 int error = 0; 698 699 for (size_t i = 0; i < t->t_nitems; i++) { 700 if (cdbr_get(t->t_cdb, i, &data, &dlen) != 0) { 701 return EINVAL; 702 } 703 error = table_ent_copyout(data, dlen, 0, ubuf, len, &off); 704 if (error) 705 break; 706 } 707 return error; 708 } 709 710 /* 711 * npf_table_list: copy a list of all table entries into a userspace buffer. 712 */ 713 int 714 npf_table_list(npf_table_t *t, void *ubuf, size_t len) 715 { 716 size_t off = 0; 717 int error = 0; 718 719 rw_enter(&t->t_lock, RW_READER); 720 switch (t->t_type) { 721 case NPF_TABLE_HASH: 722 error = table_hash_list(t, ubuf, len); 723 break; 724 case NPF_TABLE_TREE: 725 error = table_tree_list(&t->t_tree[0], 32, ubuf, len, &off); 726 if (error) 727 break; 728 error = table_tree_list(&t->t_tree[1], 128, ubuf, len, &off); 729 break; 730 case NPF_TABLE_CDB: 731 error = table_cdb_list(t, ubuf, len); 732 break; 733 default: 734 KASSERT(false); 735 } 736 rw_exit(&t->t_lock); 737 738 return error; 739 } 740 741 /* 742 * npf_table_flush: remove all table entries. 743 */ 744 int 745 npf_table_flush(npf_table_t *t) 746 { 747 int error = 0; 748 749 rw_enter(&t->t_lock, RW_WRITER); 750 switch (t->t_type) { 751 case NPF_TABLE_HASH: 752 table_hash_destroy(t); 753 t->t_nitems = 0; 754 break; 755 case NPF_TABLE_TREE: 756 table_tree_destroy(&t->t_tree[0]); 757 table_tree_destroy(&t->t_tree[1]); 758 t->t_nitems = 0; 759 break; 760 case NPF_TABLE_CDB: 761 error = EINVAL; 762 break; 763 default: 764 KASSERT(false); 765 } 766 rw_exit(&t->t_lock); 767 return error; 768 } 769