1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <rte_bitmap.h> 6 #include <rte_malloc.h> 7 #include "mlx5dr_buddy.h" 8 #include "mlx5dr_internal.h" 9 10 static void mlx5dr_pool_free_one_resource(struct mlx5dr_pool_resource *resource) 11 { 12 mlx5dr_cmd_destroy_obj(resource->devx_obj); 13 14 simple_free(resource); 15 } 16 17 static void mlx5dr_pool_resource_free(struct mlx5dr_pool *pool, 18 int resource_idx) 19 { 20 mlx5dr_pool_free_one_resource(pool->resource[resource_idx]); 21 pool->resource[resource_idx] = NULL; 22 23 if (pool->tbl_type == MLX5DR_TABLE_TYPE_FDB) { 24 mlx5dr_pool_free_one_resource(pool->mirror_resource[resource_idx]); 25 pool->mirror_resource[resource_idx] = NULL; 26 } 27 } 28 29 static struct mlx5dr_pool_resource * 30 mlx5dr_pool_create_one_resource(struct mlx5dr_pool *pool, uint32_t log_range, 31 uint32_t fw_ft_type) 32 { 33 struct mlx5dr_cmd_ste_create_attr ste_attr; 34 struct mlx5dr_cmd_stc_create_attr stc_attr; 35 struct mlx5dr_pool_resource *resource; 36 struct mlx5dr_devx_obj *devx_obj; 37 38 resource = simple_malloc(sizeof(*resource)); 39 if (!resource) { 40 rte_errno = ENOMEM; 41 return NULL; 42 } 43 44 switch (pool->type) { 45 case MLX5DR_POOL_TYPE_STE: 46 ste_attr.log_obj_range = log_range; 47 ste_attr.table_type = fw_ft_type; 48 devx_obj = mlx5dr_cmd_ste_create(pool->ctx->ibv_ctx, &ste_attr); 49 break; 50 case MLX5DR_POOL_TYPE_STC: 51 stc_attr.log_obj_range = log_range; 52 stc_attr.table_type = fw_ft_type; 53 devx_obj = mlx5dr_cmd_stc_create(pool->ctx->ibv_ctx, &stc_attr); 54 break; 55 default: 56 assert(0); 57 break; 58 } 59 60 if (!devx_obj) { 61 DR_LOG(ERR, "Failed to allocate resource objects"); 62 goto free_resource; 63 } 64 65 resource->pool = pool; 66 resource->devx_obj = devx_obj; 67 resource->range = 1 << log_range; 68 resource->base_id = devx_obj->id; 69 70 return resource; 71 72 free_resource: 73 simple_free(resource); 74 return NULL; 75 } 76 77 static int 78 mlx5dr_pool_resource_alloc(struct mlx5dr_pool *pool, uint32_t log_range, int idx) 79 { 80 struct mlx5dr_pool_resource *resource; 81 uint32_t fw_ft_type, opt_log_range; 82 83 fw_ft_type = mlx5dr_table_get_res_fw_ft_type(pool->tbl_type, false); 84 opt_log_range = pool->opt_type == MLX5DR_POOL_OPTIMIZE_ORIG ? 0 : log_range; 85 resource = mlx5dr_pool_create_one_resource(pool, opt_log_range, fw_ft_type); 86 if (!resource) { 87 DR_LOG(ERR, "Failed allocating resource"); 88 return rte_errno; 89 } 90 pool->resource[idx] = resource; 91 92 if (pool->tbl_type == MLX5DR_TABLE_TYPE_FDB) { 93 struct mlx5dr_pool_resource *mir_resource; 94 95 fw_ft_type = mlx5dr_table_get_res_fw_ft_type(pool->tbl_type, true); 96 opt_log_range = pool->opt_type == MLX5DR_POOL_OPTIMIZE_MIRROR ? 0 : log_range; 97 mir_resource = mlx5dr_pool_create_one_resource(pool, opt_log_range, fw_ft_type); 98 if (!mir_resource) { 99 DR_LOG(ERR, "Failed allocating mirrored resource"); 100 mlx5dr_pool_free_one_resource(resource); 101 pool->resource[idx] = NULL; 102 return rte_errno; 103 } 104 pool->mirror_resource[idx] = mir_resource; 105 } 106 107 return 0; 108 } 109 110 static int mlx5dr_pool_bitmap_get_free_slot(struct rte_bitmap *bitmap, uint32_t *iidx) 111 { 112 uint64_t slab = 0; 113 114 __rte_bitmap_scan_init(bitmap); 115 116 if (!rte_bitmap_scan(bitmap, iidx, &slab)) 117 return ENOMEM; 118 119 *iidx += rte_ctz64(slab); 120 121 rte_bitmap_clear(bitmap, *iidx); 122 123 return 0; 124 } 125 126 static struct rte_bitmap *mlx5dr_pool_create_and_init_bitmap(uint32_t log_range) 127 { 128 struct rte_bitmap *cur_bmp; 129 uint32_t bmp_size; 130 void *mem; 131 132 bmp_size = rte_bitmap_get_memory_footprint(1 << log_range); 133 mem = rte_zmalloc("create_stc_bmap", bmp_size, RTE_CACHE_LINE_SIZE); 134 if (!mem) { 135 DR_LOG(ERR, "No mem for bitmap"); 136 rte_errno = ENOMEM; 137 return NULL; 138 } 139 140 cur_bmp = rte_bitmap_init_with_all_set(1 << log_range, mem, bmp_size); 141 if (!cur_bmp) { 142 rte_free(mem); 143 DR_LOG(ERR, "Failed to initialize stc bitmap."); 144 rte_errno = ENOMEM; 145 return NULL; 146 } 147 148 return cur_bmp; 149 } 150 151 static void mlx5dr_pool_buddy_db_put_chunk(struct mlx5dr_pool *pool, 152 struct mlx5dr_pool_chunk *chunk) 153 { 154 struct mlx5dr_buddy_mem *buddy; 155 156 buddy = pool->db.buddy_manager->buddies[chunk->resource_idx]; 157 if (!buddy) { 158 assert(false); 159 DR_LOG(ERR, "No such buddy (%d)", chunk->resource_idx); 160 return; 161 } 162 163 mlx5dr_buddy_free_mem(buddy, chunk->offset, chunk->order); 164 } 165 166 static struct mlx5dr_buddy_mem * 167 mlx5dr_pool_buddy_get_next_buddy(struct mlx5dr_pool *pool, int idx, 168 uint32_t order, bool *is_new_buddy) 169 { 170 static struct mlx5dr_buddy_mem *buddy; 171 uint32_t new_buddy_size; 172 173 buddy = pool->db.buddy_manager->buddies[idx]; 174 if (buddy) 175 return buddy; 176 177 new_buddy_size = RTE_MAX(pool->alloc_log_sz, order); 178 *is_new_buddy = true; 179 buddy = mlx5dr_buddy_create(new_buddy_size); 180 if (!buddy) { 181 DR_LOG(ERR, "Failed to create buddy order: %d index: %d", 182 new_buddy_size, idx); 183 return NULL; 184 } 185 186 if (mlx5dr_pool_resource_alloc(pool, new_buddy_size, idx) != 0) { 187 DR_LOG(ERR, "Failed to create resource type: %d: size %d index: %d", 188 pool->type, new_buddy_size, idx); 189 mlx5dr_buddy_cleanup(buddy); 190 return NULL; 191 } 192 193 pool->db.buddy_manager->buddies[idx] = buddy; 194 195 return buddy; 196 } 197 198 static int mlx5dr_pool_buddy_get_mem_chunk(struct mlx5dr_pool *pool, 199 int order, 200 uint32_t *buddy_idx, 201 int *seg) 202 { 203 struct mlx5dr_buddy_mem *buddy; 204 bool new_mem = false; 205 int err = 0; 206 int i; 207 208 *seg = -1; 209 210 /* Find the next free place from the buddy array */ 211 while (*seg == -1) { 212 for (i = 0; i < MLX5DR_POOL_RESOURCE_ARR_SZ; i++) { 213 buddy = mlx5dr_pool_buddy_get_next_buddy(pool, i, 214 order, 215 &new_mem); 216 if (!buddy) { 217 err = rte_errno; 218 goto out; 219 } 220 221 *seg = mlx5dr_buddy_alloc_mem(buddy, order); 222 if (*seg != -1) 223 goto found; 224 225 if (pool->flags & MLX5DR_POOL_FLAGS_ONE_RESOURCE) { 226 DR_LOG(ERR, "Fail to allocate seg for one resource pool"); 227 err = rte_errno; 228 goto out; 229 } 230 231 if (new_mem) { 232 /* We have new memory pool, should be place for us */ 233 assert(false); 234 DR_LOG(ERR, "No memory for order: %d with buddy no: %d", 235 order, i); 236 rte_errno = ENOMEM; 237 err = ENOMEM; 238 goto out; 239 } 240 } 241 } 242 243 found: 244 *buddy_idx = i; 245 out: 246 return err; 247 } 248 249 static int mlx5dr_pool_buddy_db_get_chunk(struct mlx5dr_pool *pool, 250 struct mlx5dr_pool_chunk *chunk) 251 { 252 int ret = 0; 253 254 /* Go over the buddies and find next free slot */ 255 ret = mlx5dr_pool_buddy_get_mem_chunk(pool, chunk->order, 256 &chunk->resource_idx, 257 &chunk->offset); 258 if (ret) 259 DR_LOG(ERR, "Failed to get free slot for chunk with order: %d", 260 chunk->order); 261 262 return ret; 263 } 264 265 static void mlx5dr_pool_buddy_db_uninit(struct mlx5dr_pool *pool) 266 { 267 struct mlx5dr_buddy_mem *buddy; 268 int i; 269 270 for (i = 0; i < MLX5DR_POOL_RESOURCE_ARR_SZ; i++) { 271 buddy = pool->db.buddy_manager->buddies[i]; 272 if (buddy) { 273 mlx5dr_buddy_cleanup(buddy); 274 simple_free(buddy); 275 pool->db.buddy_manager->buddies[i] = NULL; 276 } 277 } 278 279 simple_free(pool->db.buddy_manager); 280 } 281 282 static int mlx5dr_pool_buddy_db_init(struct mlx5dr_pool *pool, uint32_t log_range) 283 { 284 pool->db.buddy_manager = simple_calloc(1, sizeof(*pool->db.buddy_manager)); 285 if (!pool->db.buddy_manager) { 286 DR_LOG(ERR, "No mem for buddy_manager with log_range: %d", log_range); 287 rte_errno = ENOMEM; 288 return rte_errno; 289 } 290 291 if (pool->flags & MLX5DR_POOL_FLAGS_ALLOC_MEM_ON_CREATE) { 292 bool new_buddy; 293 294 if (!mlx5dr_pool_buddy_get_next_buddy(pool, 0, log_range, &new_buddy)) { 295 DR_LOG(ERR, "Failed allocating memory on create log_sz: %d", log_range); 296 simple_free(pool->db.buddy_manager); 297 return rte_errno; 298 } 299 } 300 301 pool->p_db_uninit = &mlx5dr_pool_buddy_db_uninit; 302 pool->p_get_chunk = &mlx5dr_pool_buddy_db_get_chunk; 303 pool->p_put_chunk = &mlx5dr_pool_buddy_db_put_chunk; 304 305 return 0; 306 } 307 308 static int mlx5dr_pool_create_resource_on_index(struct mlx5dr_pool *pool, 309 uint32_t alloc_size, int idx) 310 { 311 if (mlx5dr_pool_resource_alloc(pool, alloc_size, idx) != 0) { 312 DR_LOG(ERR, "Failed to create resource type: %d: size %d index: %d", 313 pool->type, alloc_size, idx); 314 return rte_errno; 315 } 316 317 return 0; 318 } 319 320 static struct mlx5dr_pool_elements * 321 mlx5dr_pool_element_create_new_elem(struct mlx5dr_pool *pool, uint32_t order, int idx) 322 { 323 struct mlx5dr_pool_elements *elem; 324 uint32_t alloc_size; 325 326 alloc_size = pool->alloc_log_sz; 327 328 elem = simple_calloc(1, sizeof(*elem)); 329 if (!elem) { 330 DR_LOG(ERR, "Failed to create elem order: %d index: %d", 331 order, idx); 332 rte_errno = ENOMEM; 333 return NULL; 334 } 335 /*sharing the same resource, also means that all the elements are with size 1*/ 336 if ((pool->flags & MLX5DR_POOL_FLAGS_FIXED_SIZE_OBJECTS) && 337 !(pool->flags & MLX5DR_POOL_FLAGS_RESOURCE_PER_CHUNK)) { 338 /* Currently all chunks in size 1 */ 339 elem->bitmap = mlx5dr_pool_create_and_init_bitmap(alloc_size - order); 340 if (!elem->bitmap) { 341 DR_LOG(ERR, "Failed to create bitmap type: %d: size %d index: %d", 342 pool->type, alloc_size, idx); 343 goto free_elem; 344 } 345 } 346 347 if (mlx5dr_pool_create_resource_on_index(pool, alloc_size, idx)) { 348 DR_LOG(ERR, "Failed to create resource type: %d: size %d index: %d", 349 pool->type, alloc_size, idx); 350 goto free_db; 351 } 352 353 pool->db.element_manager->elements[idx] = elem; 354 355 return elem; 356 357 free_db: 358 rte_free(elem->bitmap); 359 free_elem: 360 simple_free(elem); 361 return NULL; 362 } 363 364 static int mlx5dr_pool_element_find_seg(struct mlx5dr_pool_elements *elem, int *seg) 365 { 366 if (mlx5dr_pool_bitmap_get_free_slot(elem->bitmap, (uint32_t *)seg)) { 367 elem->is_full = true; 368 return ENOMEM; 369 } 370 return 0; 371 } 372 373 static int 374 mlx5dr_pool_onesize_element_get_mem_chunk(struct mlx5dr_pool *pool, uint32_t order, 375 uint32_t *idx, int *seg) 376 { 377 struct mlx5dr_pool_elements *elem; 378 379 elem = pool->db.element_manager->elements[0]; 380 if (!elem) 381 elem = mlx5dr_pool_element_create_new_elem(pool, order, 0); 382 if (!elem) 383 goto err_no_elem; 384 385 *idx = 0; 386 387 if (mlx5dr_pool_element_find_seg(elem, seg) != 0) { 388 DR_LOG(ERR, "No more resources (last request order: %d)", order); 389 rte_errno = ENOMEM; 390 return ENOMEM; 391 } 392 393 elem->num_of_elements++; 394 return 0; 395 396 err_no_elem: 397 DR_LOG(ERR, "Failed to allocate element for order: %d", order); 398 return ENOMEM; 399 } 400 401 static int 402 mlx5dr_pool_general_element_get_mem_chunk(struct mlx5dr_pool *pool, uint32_t order, 403 uint32_t *idx, int *seg) 404 { 405 int ret; 406 int i; 407 408 for (i = 0; i < MLX5DR_POOL_RESOURCE_ARR_SZ; i++) { 409 if (!pool->resource[i]) { 410 ret = mlx5dr_pool_create_resource_on_index(pool, order, i); 411 if (ret) 412 goto err_no_res; 413 *idx = i; 414 *seg = 0; /* One memory slot in that element */ 415 return 0; 416 } 417 } 418 419 rte_errno = ENOMEM; 420 DR_LOG(ERR, "No more resources (last request order: %d)", order); 421 return ENOMEM; 422 423 err_no_res: 424 DR_LOG(ERR, "Failed to allocate element for order: %d", order); 425 return ENOMEM; 426 } 427 428 static int mlx5dr_pool_general_element_db_get_chunk(struct mlx5dr_pool *pool, 429 struct mlx5dr_pool_chunk *chunk) 430 { 431 int ret; 432 433 /* Go over all memory elements and find/allocate free slot */ 434 ret = mlx5dr_pool_general_element_get_mem_chunk(pool, chunk->order, 435 &chunk->resource_idx, 436 &chunk->offset); 437 if (ret) 438 DR_LOG(ERR, "Failed to get free slot for chunk with order: %d", 439 chunk->order); 440 441 return ret; 442 } 443 444 static void mlx5dr_pool_general_element_db_put_chunk(struct mlx5dr_pool *pool, 445 struct mlx5dr_pool_chunk *chunk) 446 { 447 assert(pool->resource[chunk->resource_idx]); 448 449 if (pool->flags & MLX5DR_POOL_FLAGS_RELEASE_FREE_RESOURCE) 450 mlx5dr_pool_resource_free(pool, chunk->resource_idx); 451 } 452 453 static void mlx5dr_pool_general_element_db_uninit(struct mlx5dr_pool *pool) 454 { 455 (void)pool; 456 } 457 458 /* This memory management works as the following: 459 * - At start doesn't allocate no mem at all. 460 * - When new request for chunk arrived: 461 * allocate resource and give it. 462 * - When free that chunk: 463 * the resource is freed. 464 */ 465 static int mlx5dr_pool_general_element_db_init(struct mlx5dr_pool *pool) 466 { 467 pool->p_db_uninit = &mlx5dr_pool_general_element_db_uninit; 468 pool->p_get_chunk = &mlx5dr_pool_general_element_db_get_chunk; 469 pool->p_put_chunk = &mlx5dr_pool_general_element_db_put_chunk; 470 471 return 0; 472 } 473 474 static void mlx5dr_onesize_element_db_destroy_element(struct mlx5dr_pool *pool, 475 struct mlx5dr_pool_elements *elem, 476 struct mlx5dr_pool_chunk *chunk) 477 { 478 assert(pool->resource[chunk->resource_idx]); 479 480 mlx5dr_pool_resource_free(pool, chunk->resource_idx); 481 482 simple_free(elem); 483 pool->db.element_manager->elements[chunk->resource_idx] = NULL; 484 } 485 486 static void mlx5dr_onesize_element_db_put_chunk(struct mlx5dr_pool *pool, 487 struct mlx5dr_pool_chunk *chunk) 488 { 489 struct mlx5dr_pool_elements *elem; 490 491 assert(chunk->resource_idx == 0); 492 493 elem = pool->db.element_manager->elements[chunk->resource_idx]; 494 if (!elem) { 495 assert(false); 496 DR_LOG(ERR, "No such element (%d)", chunk->resource_idx); 497 return; 498 } 499 500 rte_bitmap_set(elem->bitmap, chunk->offset); 501 elem->is_full = false; 502 elem->num_of_elements--; 503 504 if (pool->flags & MLX5DR_POOL_FLAGS_RELEASE_FREE_RESOURCE && 505 !elem->num_of_elements) 506 mlx5dr_onesize_element_db_destroy_element(pool, elem, chunk); 507 } 508 509 static int mlx5dr_onesize_element_db_get_chunk(struct mlx5dr_pool *pool, 510 struct mlx5dr_pool_chunk *chunk) 511 { 512 int ret = 0; 513 514 /* Go over all memory elements and find/allocate free slot */ 515 ret = mlx5dr_pool_onesize_element_get_mem_chunk(pool, chunk->order, 516 &chunk->resource_idx, 517 &chunk->offset); 518 if (ret) 519 DR_LOG(ERR, "Failed to get free slot for chunk with order: %d", 520 chunk->order); 521 522 return ret; 523 } 524 525 static void mlx5dr_onesize_element_db_uninit(struct mlx5dr_pool *pool) 526 { 527 struct mlx5dr_pool_elements *elem; 528 int i; 529 530 for (i = 0; i < MLX5DR_POOL_RESOURCE_ARR_SZ; i++) { 531 elem = pool->db.element_manager->elements[i]; 532 if (elem) { 533 rte_free(elem->bitmap); 534 simple_free(elem); 535 pool->db.element_manager->elements[i] = NULL; 536 } 537 } 538 simple_free(pool->db.element_manager); 539 } 540 541 /* This memory management works as the following: 542 * - At start doesn't allocate no mem at all. 543 * - When new request for chunk arrived: 544 * aloocate the first and only slot of memory/resource 545 * when it ended return error. 546 */ 547 static int mlx5dr_pool_onesize_element_db_init(struct mlx5dr_pool *pool) 548 { 549 pool->db.element_manager = simple_calloc(1, sizeof(*pool->db.element_manager)); 550 if (!pool->db.element_manager) { 551 DR_LOG(ERR, "No mem for general elemnt_manager"); 552 rte_errno = ENOMEM; 553 return rte_errno; 554 } 555 556 pool->p_db_uninit = &mlx5dr_onesize_element_db_uninit; 557 pool->p_get_chunk = &mlx5dr_onesize_element_db_get_chunk; 558 pool->p_put_chunk = &mlx5dr_onesize_element_db_put_chunk; 559 560 return 0; 561 } 562 563 static int mlx5dr_pool_db_init(struct mlx5dr_pool *pool, 564 enum mlx5dr_db_type db_type) 565 { 566 int ret; 567 568 if (db_type == MLX5DR_POOL_DB_TYPE_GENERAL_SIZE) 569 ret = mlx5dr_pool_general_element_db_init(pool); 570 else if (db_type == MLX5DR_POOL_DB_TYPE_ONE_SIZE_RESOURCE) 571 ret = mlx5dr_pool_onesize_element_db_init(pool); 572 else 573 ret = mlx5dr_pool_buddy_db_init(pool, pool->alloc_log_sz); 574 575 if (ret) { 576 DR_LOG(ERR, "Failed to init general db : %d (ret: %d)", db_type, ret); 577 return ret; 578 } 579 580 return 0; 581 } 582 583 static void mlx5dr_pool_db_unint(struct mlx5dr_pool *pool) 584 { 585 pool->p_db_uninit(pool); 586 } 587 588 int 589 mlx5dr_pool_chunk_alloc(struct mlx5dr_pool *pool, 590 struct mlx5dr_pool_chunk *chunk) 591 { 592 int ret; 593 594 pthread_spin_lock(&pool->lock); 595 ret = pool->p_get_chunk(pool, chunk); 596 pthread_spin_unlock(&pool->lock); 597 598 return ret; 599 } 600 601 void mlx5dr_pool_chunk_free(struct mlx5dr_pool *pool, 602 struct mlx5dr_pool_chunk *chunk) 603 { 604 pthread_spin_lock(&pool->lock); 605 pool->p_put_chunk(pool, chunk); 606 pthread_spin_unlock(&pool->lock); 607 } 608 609 struct mlx5dr_pool * 610 mlx5dr_pool_create(struct mlx5dr_context *ctx, struct mlx5dr_pool_attr *pool_attr) 611 { 612 enum mlx5dr_db_type res_db_type; 613 struct mlx5dr_pool *pool; 614 615 pool = simple_calloc(1, sizeof(*pool)); 616 if (!pool) 617 return NULL; 618 619 pool->ctx = ctx; 620 pool->type = pool_attr->pool_type; 621 pool->alloc_log_sz = pool_attr->alloc_log_sz; 622 pool->flags = pool_attr->flags; 623 pool->tbl_type = pool_attr->table_type; 624 pool->opt_type = pool_attr->opt_type; 625 626 pthread_spin_init(&pool->lock, PTHREAD_PROCESS_PRIVATE); 627 628 /* Support general db */ 629 if (pool->flags == (MLX5DR_POOL_FLAGS_RELEASE_FREE_RESOURCE | 630 MLX5DR_POOL_FLAGS_RESOURCE_PER_CHUNK)) 631 res_db_type = MLX5DR_POOL_DB_TYPE_GENERAL_SIZE; 632 else if (pool->flags == (MLX5DR_POOL_FLAGS_ONE_RESOURCE | 633 MLX5DR_POOL_FLAGS_FIXED_SIZE_OBJECTS)) 634 res_db_type = MLX5DR_POOL_DB_TYPE_ONE_SIZE_RESOURCE; 635 else 636 res_db_type = MLX5DR_POOL_DB_TYPE_BUDDY; 637 638 pool->alloc_log_sz = pool_attr->alloc_log_sz; 639 640 if (mlx5dr_pool_db_init(pool, res_db_type)) 641 goto free_pool; 642 643 return pool; 644 645 free_pool: 646 pthread_spin_destroy(&pool->lock); 647 simple_free(pool); 648 return NULL; 649 } 650 651 int mlx5dr_pool_destroy(struct mlx5dr_pool *pool) 652 { 653 int i; 654 655 for (i = 0; i < MLX5DR_POOL_RESOURCE_ARR_SZ; i++) 656 if (pool->resource[i]) 657 mlx5dr_pool_resource_free(pool, i); 658 659 mlx5dr_pool_db_unint(pool); 660 661 pthread_spin_destroy(&pool->lock); 662 simple_free(pool); 663 return 0; 664 } 665