1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022 NVIDIA Corporation & Affiliates 3 */ 4 5 #include "mlx5dr_internal.h" 6 7 static void mlx5dr_table_init_next_ft_attr(struct mlx5dr_table *tbl, 8 struct mlx5dr_cmd_ft_create_attr *ft_attr) 9 { 10 ft_attr->type = tbl->fw_ft_type; 11 if (tbl->type == MLX5DR_TABLE_TYPE_FDB) 12 ft_attr->level = tbl->ctx->caps->fdb_ft.max_level - 1; 13 else 14 ft_attr->level = tbl->ctx->caps->nic_ft.max_level - 1; 15 ft_attr->rtc_valid = true; 16 } 17 18 /* Call this under ctx->ctrl_lock */ 19 static int 20 mlx5dr_table_up_default_fdb_miss_tbl(struct mlx5dr_table *tbl) 21 { 22 struct mlx5dr_cmd_ft_create_attr ft_attr = {0}; 23 struct mlx5dr_cmd_set_fte_attr fte_attr = {0}; 24 struct mlx5dr_cmd_forward_tbl *default_miss; 25 struct mlx5dr_context *ctx = tbl->ctx; 26 uint8_t tbl_type = tbl->type; 27 28 if (tbl->type != MLX5DR_TABLE_TYPE_FDB) 29 return 0; 30 31 if (ctx->common_res[tbl_type].default_miss) { 32 ctx->common_res[tbl_type].default_miss->refcount++; 33 return 0; 34 } 35 36 ft_attr.type = tbl->fw_ft_type; 37 ft_attr.level = tbl->ctx->caps->fdb_ft.max_level; /* The last level */ 38 ft_attr.rtc_valid = false; 39 40 fte_attr.action_flags = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 41 fte_attr.destination_type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 42 fte_attr.destination_id = ctx->caps->eswitch_manager_vport_number; 43 44 default_miss = mlx5dr_cmd_forward_tbl_create(mlx5dr_context_get_local_ibv(ctx), 45 &ft_attr, &fte_attr); 46 if (!default_miss) { 47 DR_LOG(ERR, "Failed to default miss table type: 0x%x", tbl_type); 48 return rte_errno; 49 } 50 51 ctx->common_res[tbl_type].default_miss = default_miss; 52 ctx->common_res[tbl_type].default_miss->refcount++; 53 return 0; 54 } 55 56 /* Called under pthread_spin_lock(&ctx->ctrl_lock) */ 57 static void mlx5dr_table_down_default_fdb_miss_tbl(struct mlx5dr_table *tbl) 58 { 59 struct mlx5dr_cmd_forward_tbl *default_miss; 60 struct mlx5dr_context *ctx = tbl->ctx; 61 uint8_t tbl_type = tbl->type; 62 63 if (tbl->type != MLX5DR_TABLE_TYPE_FDB) 64 return; 65 66 default_miss = ctx->common_res[tbl_type].default_miss; 67 if (--default_miss->refcount) 68 return; 69 70 mlx5dr_cmd_forward_tbl_destroy(default_miss); 71 ctx->common_res[tbl_type].default_miss = NULL; 72 } 73 74 static int 75 mlx5dr_table_connect_to_default_miss_tbl(struct mlx5dr_table *tbl, 76 struct mlx5dr_devx_obj *ft) 77 { 78 struct mlx5dr_cmd_ft_modify_attr ft_attr = {0}; 79 int ret; 80 81 assert(tbl->type == MLX5DR_TABLE_TYPE_FDB); 82 83 mlx5dr_cmd_set_attr_connect_miss_tbl(tbl->ctx, 84 tbl->fw_ft_type, 85 tbl->type, 86 &ft_attr); 87 88 /* Connect to next */ 89 ret = mlx5dr_cmd_flow_table_modify(ft, &ft_attr); 90 if (ret) { 91 DR_LOG(ERR, "Failed to connect FT to default FDB FT"); 92 return ret; 93 } 94 95 return 0; 96 } 97 98 struct mlx5dr_devx_obj * 99 mlx5dr_table_create_default_ft(struct ibv_context *ibv, 100 struct mlx5dr_table *tbl) 101 { 102 struct mlx5dr_cmd_ft_create_attr ft_attr = {0}; 103 struct mlx5dr_devx_obj *ft_obj; 104 int ret; 105 106 mlx5dr_table_init_next_ft_attr(tbl, &ft_attr); 107 108 ft_obj = mlx5dr_cmd_flow_table_create(ibv, &ft_attr); 109 if (ft_obj && tbl->type == MLX5DR_TABLE_TYPE_FDB) { 110 /* Take/create ref over the default miss */ 111 ret = mlx5dr_table_up_default_fdb_miss_tbl(tbl); 112 if (ret) { 113 DR_LOG(ERR, "Failed to get default fdb miss"); 114 goto free_ft_obj; 115 } 116 ret = mlx5dr_table_connect_to_default_miss_tbl(tbl, ft_obj); 117 if (ret) { 118 DR_LOG(ERR, "Failed connecting to default miss tbl"); 119 goto down_miss_tbl; 120 } 121 } 122 123 return ft_obj; 124 125 down_miss_tbl: 126 mlx5dr_table_down_default_fdb_miss_tbl(tbl); 127 free_ft_obj: 128 mlx5dr_cmd_destroy_obj(ft_obj); 129 return NULL; 130 } 131 132 static int 133 mlx5dr_table_init_check_hws_support(struct mlx5dr_context *ctx, 134 struct mlx5dr_table *tbl) 135 { 136 if (!(ctx->flags & MLX5DR_CONTEXT_FLAG_HWS_SUPPORT)) { 137 DR_LOG(ERR, "HWS not supported, cannot create mlx5dr_table"); 138 rte_errno = EOPNOTSUPP; 139 return rte_errno; 140 } 141 142 if (mlx5dr_context_shared_gvmi_used(ctx) && tbl->type == MLX5DR_TABLE_TYPE_FDB) { 143 DR_LOG(ERR, "FDB with shared port resources is not supported"); 144 rte_errno = EOPNOTSUPP; 145 return rte_errno; 146 } 147 148 return 0; 149 } 150 151 static int 152 mlx5dr_table_shared_gvmi_resource_create(struct mlx5dr_context *ctx, 153 enum mlx5dr_table_type type, 154 struct mlx5dr_context_shared_gvmi_res *gvmi_res) 155 { 156 struct mlx5dr_cmd_ft_create_attr ft_attr = {0}; 157 uint32_t calculated_ft_id; 158 int ret; 159 160 if (!mlx5dr_context_shared_gvmi_used(ctx)) 161 return 0; 162 163 ft_attr.type = mlx5dr_table_get_res_fw_ft_type(type, false); 164 ft_attr.level = ctx->caps->nic_ft.max_level - 1; 165 ft_attr.rtc_valid = true; 166 167 gvmi_res->end_ft = 168 mlx5dr_cmd_flow_table_create(mlx5dr_context_get_local_ibv(ctx), 169 &ft_attr); 170 if (!gvmi_res->end_ft) { 171 DR_LOG(ERR, "Failed to create end-ft"); 172 return rte_errno; 173 } 174 175 calculated_ft_id = 176 mlx5dr_table_get_res_fw_ft_type(type, false) << FT_ID_FT_TYPE_OFFSET; 177 calculated_ft_id |= gvmi_res->end_ft->id; 178 179 /* create alias to that FT */ 180 ret = mlx5dr_matcher_create_aliased_obj(ctx, 181 ctx->local_ibv_ctx, 182 ctx->ibv_ctx, 183 ctx->caps->vhca_id, 184 calculated_ft_id, 185 MLX5_GENERAL_OBJ_TYPE_FT_ALIAS, 186 &gvmi_res->aliased_end_ft); 187 if (ret) { 188 DR_LOG(ERR, "Failed to create alias end-ft"); 189 goto free_end_ft; 190 } 191 192 return 0; 193 194 free_end_ft: 195 mlx5dr_cmd_destroy_obj(gvmi_res->end_ft); 196 197 return rte_errno; 198 } 199 200 static void 201 mlx5dr_table_shared_gvmi_resourse_destroy(struct mlx5dr_context *ctx, 202 struct mlx5dr_context_shared_gvmi_res *gvmi_res) 203 { 204 if (!mlx5dr_context_shared_gvmi_used(ctx)) 205 return; 206 207 if (gvmi_res->aliased_end_ft) { 208 mlx5dr_cmd_destroy_obj(gvmi_res->aliased_end_ft); 209 gvmi_res->aliased_end_ft = NULL; 210 } 211 if (gvmi_res->end_ft) { 212 mlx5dr_cmd_destroy_obj(gvmi_res->end_ft); 213 gvmi_res->end_ft = NULL; 214 } 215 } 216 217 /* called under spinlock ctx->ctrl_lock */ 218 static struct mlx5dr_context_shared_gvmi_res * 219 mlx5dr_table_get_shared_gvmi_res(struct mlx5dr_context *ctx, enum mlx5dr_table_type type) 220 { 221 int ret; 222 223 if (!mlx5dr_context_shared_gvmi_used(ctx)) 224 return NULL; 225 226 if (ctx->gvmi_res[type].aliased_end_ft) { 227 ctx->gvmi_res[type].refcount++; 228 return &ctx->gvmi_res[type]; 229 } 230 231 ret = mlx5dr_table_shared_gvmi_resource_create(ctx, type, &ctx->gvmi_res[type]); 232 if (ret) { 233 DR_LOG(ERR, "Failed to create shared gvmi res for type: %d", type); 234 goto out; 235 } 236 237 ctx->gvmi_res[type].refcount = 1; 238 239 return &ctx->gvmi_res[type]; 240 241 out: 242 return NULL; 243 } 244 245 /* called under spinlock ctx->ctrl_lock */ 246 static void mlx5dr_table_put_shared_gvmi_res(struct mlx5dr_table *tbl) 247 { 248 struct mlx5dr_context *ctx = tbl->ctx; 249 250 if (!mlx5dr_context_shared_gvmi_used(ctx)) 251 return; 252 253 if (--ctx->gvmi_res[tbl->type].refcount) 254 return; 255 256 mlx5dr_table_shared_gvmi_resourse_destroy(ctx, &ctx->gvmi_res[tbl->type]); 257 } 258 259 static void mlx5dr_table_uninit_shared_ctx_res(struct mlx5dr_table *tbl) 260 { 261 struct mlx5dr_context *ctx = tbl->ctx; 262 263 if (!mlx5dr_context_shared_gvmi_used(ctx)) 264 return; 265 266 mlx5dr_cmd_destroy_obj(tbl->local_ft); 267 268 mlx5dr_table_put_shared_gvmi_res(tbl); 269 } 270 271 /* called under spin_lock ctx->ctrl_lock */ 272 static int mlx5dr_table_init_shared_ctx_res(struct mlx5dr_context *ctx, struct mlx5dr_table *tbl) 273 { 274 struct mlx5dr_cmd_ft_modify_attr ft_attr = {0}; 275 int ret; 276 277 if (!mlx5dr_context_shared_gvmi_used(ctx)) 278 return 0; 279 280 /* create local-ft for root access */ 281 tbl->local_ft = 282 mlx5dr_table_create_default_ft(mlx5dr_context_get_local_ibv(ctx), tbl); 283 if (!tbl->local_ft) { 284 DR_LOG(ERR, "Failed to create local-ft"); 285 return rte_errno; 286 } 287 288 if (!mlx5dr_table_get_shared_gvmi_res(tbl->ctx, tbl->type)) { 289 DR_LOG(ERR, "Failed to shared gvmi resources"); 290 goto clean_local_ft; 291 } 292 293 /* On shared gvmi the default behavior is jump to alias end ft */ 294 mlx5dr_cmd_set_attr_connect_miss_tbl(tbl->ctx, 295 tbl->fw_ft_type, 296 tbl->type, 297 &ft_attr); 298 299 ret = mlx5dr_cmd_flow_table_modify(tbl->ft, &ft_attr); 300 if (ret) { 301 DR_LOG(ERR, "Failed to point table to its default miss"); 302 goto clean_shared_res; 303 } 304 305 return 0; 306 307 clean_shared_res: 308 mlx5dr_table_put_shared_gvmi_res(tbl); 309 clean_local_ft: 310 mlx5dr_table_destroy_default_ft(tbl, tbl->local_ft); 311 return rte_errno; 312 } 313 314 void mlx5dr_table_destroy_default_ft(struct mlx5dr_table *tbl, 315 struct mlx5dr_devx_obj *ft_obj) 316 { 317 mlx5dr_cmd_destroy_obj(ft_obj); 318 mlx5dr_table_down_default_fdb_miss_tbl(tbl); 319 } 320 321 static int mlx5dr_table_init(struct mlx5dr_table *tbl) 322 { 323 struct mlx5dr_context *ctx = tbl->ctx; 324 int ret; 325 326 if (mlx5dr_table_is_root(tbl)) 327 return 0; 328 329 ret = mlx5dr_table_init_check_hws_support(ctx, tbl); 330 if (ret) 331 return ret; 332 333 switch (tbl->type) { 334 case MLX5DR_TABLE_TYPE_NIC_RX: 335 tbl->fw_ft_type = FS_FT_NIC_RX; 336 break; 337 case MLX5DR_TABLE_TYPE_NIC_TX: 338 tbl->fw_ft_type = FS_FT_NIC_TX; 339 break; 340 case MLX5DR_TABLE_TYPE_FDB: 341 tbl->fw_ft_type = FS_FT_FDB; 342 break; 343 default: 344 assert(0); 345 break; 346 } 347 348 pthread_spin_lock(&ctx->ctrl_lock); 349 tbl->ft = mlx5dr_table_create_default_ft(tbl->ctx->ibv_ctx, tbl); 350 if (!tbl->ft) { 351 DR_LOG(ERR, "Failed to create flow table devx object"); 352 pthread_spin_unlock(&ctx->ctrl_lock); 353 return rte_errno; 354 } 355 356 ret = mlx5dr_table_init_shared_ctx_res(ctx, tbl); 357 if (ret) 358 goto tbl_destroy; 359 360 ret = mlx5dr_action_get_default_stc(ctx, tbl->type); 361 if (ret) 362 goto free_shared_ctx; 363 364 pthread_spin_unlock(&ctx->ctrl_lock); 365 366 return 0; 367 368 free_shared_ctx: 369 mlx5dr_table_uninit_shared_ctx_res(tbl); 370 tbl_destroy: 371 mlx5dr_table_destroy_default_ft(tbl, tbl->ft); 372 pthread_spin_unlock(&ctx->ctrl_lock); 373 return rte_errno; 374 } 375 376 static void mlx5dr_table_uninit(struct mlx5dr_table *tbl) 377 { 378 if (mlx5dr_table_is_root(tbl)) 379 return; 380 pthread_spin_lock(&tbl->ctx->ctrl_lock); 381 mlx5dr_action_put_default_stc(tbl->ctx, tbl->type); 382 mlx5dr_table_uninit_shared_ctx_res(tbl); 383 mlx5dr_table_destroy_default_ft(tbl, tbl->ft); 384 pthread_spin_unlock(&tbl->ctx->ctrl_lock); 385 } 386 387 struct mlx5dr_table *mlx5dr_table_create(struct mlx5dr_context *ctx, 388 struct mlx5dr_table_attr *attr) 389 { 390 struct mlx5dr_table *tbl; 391 int ret; 392 393 if (attr->type > MLX5DR_TABLE_TYPE_FDB) { 394 DR_LOG(ERR, "Invalid table type %d", attr->type); 395 return NULL; 396 } 397 398 tbl = simple_calloc(1, sizeof(*tbl)); 399 if (!tbl) { 400 rte_errno = ENOMEM; 401 return NULL; 402 } 403 404 tbl->ctx = ctx; 405 tbl->type = attr->type; 406 tbl->level = attr->level; 407 408 ret = mlx5dr_table_init(tbl); 409 if (ret) { 410 DR_LOG(ERR, "Failed to initialise table"); 411 goto free_tbl; 412 } 413 414 pthread_spin_lock(&ctx->ctrl_lock); 415 LIST_INSERT_HEAD(&ctx->head, tbl, next); 416 pthread_spin_unlock(&ctx->ctrl_lock); 417 418 return tbl; 419 420 free_tbl: 421 simple_free(tbl); 422 return NULL; 423 } 424 425 int mlx5dr_table_destroy(struct mlx5dr_table *tbl) 426 { 427 struct mlx5dr_context *ctx = tbl->ctx; 428 pthread_spin_lock(&ctx->ctrl_lock); 429 if (!LIST_EMPTY(&tbl->head)) { 430 DR_LOG(ERR, "Cannot destroy table containing matchers"); 431 rte_errno = EBUSY; 432 goto unlock_err; 433 } 434 435 if (!LIST_EMPTY(&tbl->default_miss.head)) { 436 DR_LOG(ERR, "Cannot destroy table pointed by default miss"); 437 rte_errno = EBUSY; 438 goto unlock_err; 439 } 440 441 LIST_REMOVE(tbl, next); 442 pthread_spin_unlock(&ctx->ctrl_lock); 443 mlx5dr_table_uninit(tbl); 444 simple_free(tbl); 445 446 return 0; 447 448 unlock_err: 449 pthread_spin_unlock(&ctx->ctrl_lock); 450 return -rte_errno; 451 } 452 453 static struct mlx5dr_devx_obj * 454 mlx5dr_table_get_last_ft(struct mlx5dr_table *tbl) 455 { 456 struct mlx5dr_devx_obj *last_ft = tbl->ft; 457 struct mlx5dr_matcher *matcher; 458 459 LIST_FOREACH(matcher, &tbl->head, next) 460 last_ft = matcher->end_ft; 461 462 return last_ft; 463 } 464 465 int mlx5dr_table_ft_set_default_next_ft(struct mlx5dr_table *tbl, 466 struct mlx5dr_devx_obj *ft_obj) 467 { 468 struct mlx5dr_cmd_ft_modify_attr ft_attr = {0}; 469 int ret; 470 471 /* Due to FW limitation, resetting the flow table to default action will 472 * disconnect RTC when ignore_flow_level_rtc_valid is not supported. 473 */ 474 if (!tbl->ctx->caps->nic_ft.ignore_flow_level_rtc_valid) 475 return 0; 476 477 if (tbl->type == MLX5DR_TABLE_TYPE_FDB) 478 return mlx5dr_table_connect_to_default_miss_tbl(tbl, ft_obj); 479 480 ft_attr.type = tbl->fw_ft_type; 481 ft_attr.modify_fs = MLX5_IFC_MODIFY_FLOW_TABLE_MISS_ACTION; 482 ft_attr.table_miss_action = MLX5_IFC_MODIFY_FLOW_TABLE_MISS_ACTION_DEFAULT; 483 484 ret = mlx5dr_cmd_flow_table_modify(ft_obj, &ft_attr); 485 if (ret) { 486 DR_LOG(ERR, "Failed to set FT default miss action"); 487 return ret; 488 } 489 490 return 0; 491 } 492 493 int mlx5dr_table_ft_set_next_rtc(struct mlx5dr_devx_obj *ft, 494 uint32_t fw_ft_type, 495 struct mlx5dr_devx_obj *rtc_0, 496 struct mlx5dr_devx_obj *rtc_1) 497 { 498 struct mlx5dr_cmd_ft_modify_attr ft_attr = {0}; 499 500 ft_attr.modify_fs = MLX5_IFC_MODIFY_FLOW_TABLE_RTC_ID; 501 ft_attr.type = fw_ft_type; 502 ft_attr.rtc_id_0 = rtc_0 ? rtc_0->id : 0; 503 ft_attr.rtc_id_1 = rtc_1 ? rtc_1->id : 0; 504 505 return mlx5dr_cmd_flow_table_modify(ft, &ft_attr); 506 } 507 508 static int mlx5dr_table_ft_set_next_ft(struct mlx5dr_devx_obj *ft, 509 uint32_t fw_ft_type, 510 uint32_t next_ft_id) 511 { 512 struct mlx5dr_cmd_ft_modify_attr ft_attr = {0}; 513 514 ft_attr.modify_fs = MLX5_IFC_MODIFY_FLOW_TABLE_MISS_ACTION; 515 ft_attr.table_miss_action = MLX5_IFC_MODIFY_FLOW_TABLE_MISS_ACTION_GOTO_TBL; 516 ft_attr.type = fw_ft_type; 517 ft_attr.table_miss_id = next_ft_id; 518 519 return mlx5dr_cmd_flow_table_modify(ft, &ft_attr); 520 } 521 522 int mlx5dr_table_update_connected_miss_tables(struct mlx5dr_table *dst_tbl) 523 { 524 struct mlx5dr_table *src_tbl; 525 int ret; 526 527 if (LIST_EMPTY(&dst_tbl->default_miss.head)) 528 return 0; 529 530 LIST_FOREACH(src_tbl, &dst_tbl->default_miss.head, default_miss.next) { 531 ret = mlx5dr_table_connect_to_miss_table(src_tbl, dst_tbl); 532 if (ret) { 533 DR_LOG(ERR, "Failed to update source miss table, unexpected behavior"); 534 return ret; 535 } 536 } 537 538 return 0; 539 } 540 541 int mlx5dr_table_connect_to_miss_table(struct mlx5dr_table *src_tbl, 542 struct mlx5dr_table *dst_tbl) 543 { 544 struct mlx5dr_devx_obj *last_ft; 545 struct mlx5dr_matcher *matcher; 546 int ret; 547 548 last_ft = mlx5dr_table_get_last_ft(src_tbl); 549 550 if (dst_tbl) { 551 if (LIST_EMPTY(&dst_tbl->head)) { 552 /* Connect src_tbl last_ft to dst_tbl start anchor */ 553 ret = mlx5dr_table_ft_set_next_ft(last_ft, 554 src_tbl->fw_ft_type, 555 dst_tbl->ft->id); 556 if (ret) 557 return ret; 558 559 /* Reset last_ft RTC to default RTC */ 560 ret = mlx5dr_table_ft_set_next_rtc(last_ft, 561 src_tbl->fw_ft_type, 562 NULL, NULL); 563 if (ret) 564 return ret; 565 } else { 566 /* Connect src_tbl last_ft to first matcher RTC */ 567 matcher = LIST_FIRST(&dst_tbl->head); 568 ret = mlx5dr_table_ft_set_next_rtc(last_ft, 569 src_tbl->fw_ft_type, 570 matcher->match_ste.rtc_0, 571 matcher->match_ste.rtc_1); 572 if (ret) 573 return ret; 574 575 /* Reset next miss FT to default */ 576 ret = mlx5dr_table_ft_set_default_next_ft(src_tbl, last_ft); 577 if (ret) 578 return ret; 579 } 580 } else { 581 /* Reset next miss FT to default */ 582 ret = mlx5dr_table_ft_set_default_next_ft(src_tbl, last_ft); 583 if (ret) 584 return ret; 585 586 /* Reset last_ft RTC to default RTC */ 587 ret = mlx5dr_table_ft_set_next_rtc(last_ft, 588 src_tbl->fw_ft_type, 589 NULL, NULL); 590 if (ret) 591 return ret; 592 } 593 594 src_tbl->default_miss.miss_tbl = dst_tbl; 595 596 return 0; 597 } 598 599 static int mlx5dr_table_set_default_miss_not_valid(struct mlx5dr_table *tbl, 600 struct mlx5dr_table *miss_tbl) 601 { 602 if (!tbl->ctx->caps->nic_ft.ignore_flow_level_rtc_valid || 603 mlx5dr_context_shared_gvmi_used(tbl->ctx)) { 604 DR_LOG(ERR, "Default miss table is not supported"); 605 rte_errno = EOPNOTSUPP; 606 return -rte_errno; 607 } 608 609 if (mlx5dr_table_is_root(tbl) || 610 (miss_tbl && mlx5dr_table_is_root(miss_tbl)) || 611 (miss_tbl && miss_tbl->type != tbl->type) || 612 (miss_tbl && tbl->default_miss.miss_tbl)) { 613 DR_LOG(ERR, "Invalid arguments"); 614 rte_errno = EINVAL; 615 return -rte_errno; 616 } 617 618 return 0; 619 } 620 621 int mlx5dr_table_set_default_miss(struct mlx5dr_table *tbl, 622 struct mlx5dr_table *miss_tbl) 623 { 624 struct mlx5dr_context *ctx = tbl->ctx; 625 int ret; 626 627 ret = mlx5dr_table_set_default_miss_not_valid(tbl, miss_tbl); 628 if (ret) 629 return ret; 630 631 pthread_spin_lock(&ctx->ctrl_lock); 632 633 ret = mlx5dr_table_connect_to_miss_table(tbl, miss_tbl); 634 if (ret) 635 goto out; 636 637 if (miss_tbl) 638 LIST_INSERT_HEAD(&miss_tbl->default_miss.head, tbl, default_miss.next); 639 else 640 LIST_REMOVE(tbl, default_miss.next); 641 642 pthread_spin_unlock(&ctx->ctrl_lock); 643 return 0; 644 out: 645 pthread_spin_unlock(&ctx->ctrl_lock); 646 return -ret; 647 } 648