1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 Mellanox Technologies, Ltd 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_hash_crc.h> 7 8 #include <mlx5_malloc.h> 9 10 #include "mlx5_utils.h" 11 12 struct mlx5_hlist * 13 mlx5_hlist_create(const char *name, uint32_t size) 14 { 15 struct mlx5_hlist *h; 16 uint32_t act_size; 17 uint32_t alloc_size; 18 19 if (!size) 20 return NULL; 21 /* Align to the next power of 2, 32bits integer is enough now. */ 22 if (!rte_is_power_of_2(size)) { 23 act_size = rte_align32pow2(size); 24 DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will " 25 "be aligned to 0x%" PRIX32 ".", size, act_size); 26 } else { 27 act_size = size; 28 } 29 alloc_size = sizeof(struct mlx5_hlist) + 30 sizeof(struct mlx5_hlist_head) * act_size; 31 /* Using zmalloc, then no need to initialize the heads. */ 32 h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE, 33 SOCKET_ID_ANY); 34 if (!h) { 35 DRV_LOG(ERR, "No memory for hash list %s creation", 36 name ? name : "None"); 37 return NULL; 38 } 39 if (name) 40 snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name); 41 h->table_sz = act_size; 42 h->mask = act_size - 1; 43 DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.", 44 h->name, act_size); 45 return h; 46 } 47 48 struct mlx5_hlist_entry * 49 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key) 50 { 51 uint32_t idx; 52 struct mlx5_hlist_head *first; 53 struct mlx5_hlist_entry *node; 54 55 MLX5_ASSERT(h); 56 idx = rte_hash_crc_8byte(key, 0) & h->mask; 57 first = &h->heads[idx]; 58 LIST_FOREACH(node, first, next) { 59 if (node->key == key) 60 return node; 61 } 62 return NULL; 63 } 64 65 int 66 mlx5_hlist_insert(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry) 67 { 68 uint32_t idx; 69 struct mlx5_hlist_head *first; 70 struct mlx5_hlist_entry *node; 71 72 MLX5_ASSERT(h && entry); 73 idx = rte_hash_crc_8byte(entry->key, 0) & h->mask; 74 first = &h->heads[idx]; 75 /* No need to reuse the lookup function. */ 76 LIST_FOREACH(node, first, next) { 77 if (node->key == entry->key) 78 return -EEXIST; 79 } 80 LIST_INSERT_HEAD(first, entry, next); 81 return 0; 82 } 83 84 void 85 mlx5_hlist_remove(struct mlx5_hlist *h __rte_unused, 86 struct mlx5_hlist_entry *entry) 87 { 88 MLX5_ASSERT(entry && entry->next.le_prev); 89 LIST_REMOVE(entry, next); 90 /* Set to NULL to get rid of removing action for more than once. */ 91 entry->next.le_prev = NULL; 92 } 93 94 void 95 mlx5_hlist_destroy(struct mlx5_hlist *h, 96 mlx5_hlist_destroy_callback_fn cb, void *ctx) 97 { 98 uint32_t idx; 99 struct mlx5_hlist_entry *entry; 100 101 MLX5_ASSERT(h); 102 for (idx = 0; idx < h->table_sz; ++idx) { 103 /* no LIST_FOREACH_SAFE, using while instead */ 104 while (!LIST_EMPTY(&h->heads[idx])) { 105 entry = LIST_FIRST(&h->heads[idx]); 106 LIST_REMOVE(entry, next); 107 /* 108 * The owner of whole element which contains data entry 109 * is the user, so it's the user's duty to do the clean 110 * up and the free work because someone may not put the 111 * hlist entry at the beginning(suggested to locate at 112 * the beginning). Or else the default free function 113 * will be used. 114 */ 115 if (cb) 116 cb(entry, ctx); 117 else 118 mlx5_free(entry); 119 } 120 } 121 mlx5_free(h); 122 } 123 124 static inline void 125 mlx5_ipool_lock(struct mlx5_indexed_pool *pool) 126 { 127 if (pool->cfg.need_lock) 128 rte_spinlock_lock(&pool->lock); 129 } 130 131 static inline void 132 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool) 133 { 134 if (pool->cfg.need_lock) 135 rte_spinlock_unlock(&pool->lock); 136 } 137 138 static inline uint32_t 139 mlx5_trunk_idx_get(struct mlx5_indexed_pool *pool, uint32_t entry_idx) 140 { 141 struct mlx5_indexed_pool_config *cfg = &pool->cfg; 142 uint32_t trunk_idx = 0; 143 uint32_t i; 144 145 if (!cfg->grow_trunk) 146 return entry_idx / cfg->trunk_size; 147 if (entry_idx >= pool->grow_tbl[cfg->grow_trunk - 1]) { 148 trunk_idx = (entry_idx - pool->grow_tbl[cfg->grow_trunk - 1]) / 149 (cfg->trunk_size << (cfg->grow_shift * 150 cfg->grow_trunk)) + cfg->grow_trunk; 151 } else { 152 for (i = 0; i < cfg->grow_trunk; i++) { 153 if (entry_idx < pool->grow_tbl[i]) 154 break; 155 } 156 trunk_idx = i; 157 } 158 return trunk_idx; 159 } 160 161 static inline uint32_t 162 mlx5_trunk_size_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx) 163 { 164 struct mlx5_indexed_pool_config *cfg = &pool->cfg; 165 166 return cfg->trunk_size << (cfg->grow_shift * 167 (trunk_idx > cfg->grow_trunk ? cfg->grow_trunk : trunk_idx)); 168 } 169 170 static inline uint32_t 171 mlx5_trunk_idx_offset_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx) 172 { 173 struct mlx5_indexed_pool_config *cfg = &pool->cfg; 174 uint32_t offset = 0; 175 176 if (!trunk_idx) 177 return 0; 178 if (!cfg->grow_trunk) 179 return cfg->trunk_size * trunk_idx; 180 if (trunk_idx < cfg->grow_trunk) 181 offset = pool->grow_tbl[trunk_idx - 1]; 182 else 183 offset = pool->grow_tbl[cfg->grow_trunk - 1] + 184 (cfg->trunk_size << (cfg->grow_shift * 185 cfg->grow_trunk)) * (trunk_idx - cfg->grow_trunk); 186 return offset; 187 } 188 189 struct mlx5_indexed_pool * 190 mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg) 191 { 192 struct mlx5_indexed_pool *pool; 193 uint32_t i; 194 195 if (!cfg || !cfg->size || (!cfg->malloc ^ !cfg->free) || 196 (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) || 197 ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32)))) 198 return NULL; 199 pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool) + cfg->grow_trunk * 200 sizeof(pool->grow_tbl[0]), RTE_CACHE_LINE_SIZE, 201 SOCKET_ID_ANY); 202 if (!pool) 203 return NULL; 204 pool->cfg = *cfg; 205 if (!pool->cfg.trunk_size) 206 pool->cfg.trunk_size = MLX5_IPOOL_DEFAULT_TRUNK_SIZE; 207 if (!cfg->malloc && !cfg->free) { 208 pool->cfg.malloc = mlx5_malloc; 209 pool->cfg.free = mlx5_free; 210 } 211 pool->free_list = TRUNK_INVALID; 212 if (pool->cfg.need_lock) 213 rte_spinlock_init(&pool->lock); 214 /* 215 * Initialize the dynamic grow trunk size lookup table to have a quick 216 * lookup for the trunk entry index offset. 217 */ 218 for (i = 0; i < cfg->grow_trunk; i++) { 219 pool->grow_tbl[i] = cfg->trunk_size << (cfg->grow_shift * i); 220 if (i > 0) 221 pool->grow_tbl[i] += pool->grow_tbl[i - 1]; 222 } 223 return pool; 224 } 225 226 static int 227 mlx5_ipool_grow(struct mlx5_indexed_pool *pool) 228 { 229 struct mlx5_indexed_trunk *trunk; 230 struct mlx5_indexed_trunk **trunk_tmp; 231 struct mlx5_indexed_trunk **p; 232 size_t trunk_size = 0; 233 size_t data_size; 234 size_t bmp_size; 235 uint32_t idx; 236 237 if (pool->n_trunk_valid == TRUNK_MAX_IDX) 238 return -ENOMEM; 239 if (pool->n_trunk_valid == pool->n_trunk) { 240 /* No free trunk flags, expand trunk list. */ 241 int n_grow = pool->n_trunk_valid ? pool->n_trunk : 242 RTE_CACHE_LINE_SIZE / sizeof(void *); 243 244 p = pool->cfg.malloc(0, (pool->n_trunk_valid + n_grow) * 245 sizeof(struct mlx5_indexed_trunk *), 246 RTE_CACHE_LINE_SIZE, rte_socket_id()); 247 if (!p) 248 return -ENOMEM; 249 if (pool->trunks) 250 memcpy(p, pool->trunks, pool->n_trunk_valid * 251 sizeof(struct mlx5_indexed_trunk *)); 252 memset(RTE_PTR_ADD(p, pool->n_trunk_valid * sizeof(void *)), 0, 253 n_grow * sizeof(void *)); 254 trunk_tmp = pool->trunks; 255 pool->trunks = p; 256 if (trunk_tmp) 257 pool->cfg.free(trunk_tmp); 258 pool->n_trunk += n_grow; 259 } 260 if (!pool->cfg.release_mem_en) { 261 idx = pool->n_trunk_valid; 262 } else { 263 /* Find the first available slot in trunk list */ 264 for (idx = 0; idx < pool->n_trunk; idx++) 265 if (pool->trunks[idx] == NULL) 266 break; 267 } 268 trunk_size += sizeof(*trunk); 269 data_size = mlx5_trunk_size_get(pool, idx); 270 bmp_size = rte_bitmap_get_memory_footprint(data_size); 271 /* rte_bitmap requires memory cacheline aligned. */ 272 trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size); 273 trunk_size += bmp_size; 274 trunk = pool->cfg.malloc(0, trunk_size, 275 RTE_CACHE_LINE_SIZE, rte_socket_id()); 276 if (!trunk) 277 return -ENOMEM; 278 pool->trunks[idx] = trunk; 279 trunk->idx = idx; 280 trunk->free = data_size; 281 trunk->prev = TRUNK_INVALID; 282 trunk->next = TRUNK_INVALID; 283 MLX5_ASSERT(pool->free_list == TRUNK_INVALID); 284 pool->free_list = idx; 285 /* Mark all entries as available. */ 286 trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data 287 [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)], 288 bmp_size); 289 MLX5_ASSERT(trunk->bmp); 290 pool->n_trunk_valid++; 291 #ifdef POOL_DEBUG 292 pool->trunk_new++; 293 pool->trunk_avail++; 294 #endif 295 return 0; 296 } 297 298 void * 299 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx) 300 { 301 struct mlx5_indexed_trunk *trunk; 302 uint64_t slab = 0; 303 uint32_t iidx = 0; 304 void *p; 305 306 mlx5_ipool_lock(pool); 307 if (pool->free_list == TRUNK_INVALID) { 308 /* If no available trunks, grow new. */ 309 if (mlx5_ipool_grow(pool)) { 310 mlx5_ipool_unlock(pool); 311 return NULL; 312 } 313 } 314 MLX5_ASSERT(pool->free_list != TRUNK_INVALID); 315 trunk = pool->trunks[pool->free_list]; 316 MLX5_ASSERT(trunk->free); 317 if (!rte_bitmap_scan(trunk->bmp, &iidx, &slab)) { 318 mlx5_ipool_unlock(pool); 319 return NULL; 320 } 321 MLX5_ASSERT(slab); 322 iidx += __builtin_ctzll(slab); 323 MLX5_ASSERT(iidx != UINT32_MAX); 324 MLX5_ASSERT(iidx < mlx5_trunk_size_get(pool, trunk->idx)); 325 rte_bitmap_clear(trunk->bmp, iidx); 326 p = &trunk->data[iidx * pool->cfg.size]; 327 iidx += mlx5_trunk_idx_offset_get(pool, trunk->idx); 328 iidx += 1; /* non-zero index. */ 329 trunk->free--; 330 #ifdef POOL_DEBUG 331 pool->n_entry++; 332 #endif 333 if (!trunk->free) { 334 /* Full trunk will be removed from free list in imalloc. */ 335 MLX5_ASSERT(pool->free_list == trunk->idx); 336 pool->free_list = trunk->next; 337 if (trunk->next != TRUNK_INVALID) 338 pool->trunks[trunk->next]->prev = TRUNK_INVALID; 339 trunk->prev = TRUNK_INVALID; 340 trunk->next = TRUNK_INVALID; 341 #ifdef POOL_DEBUG 342 pool->trunk_empty++; 343 pool->trunk_avail--; 344 #endif 345 } 346 *idx = iidx; 347 mlx5_ipool_unlock(pool); 348 return p; 349 } 350 351 void * 352 mlx5_ipool_zmalloc(struct mlx5_indexed_pool *pool, uint32_t *idx) 353 { 354 void *entry = mlx5_ipool_malloc(pool, idx); 355 356 if (entry) 357 memset(entry, 0, pool->cfg.size); 358 return entry; 359 } 360 361 void 362 mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx) 363 { 364 struct mlx5_indexed_trunk *trunk; 365 uint32_t trunk_idx; 366 uint32_t entry_idx; 367 368 if (!idx) 369 return; 370 idx -= 1; 371 mlx5_ipool_lock(pool); 372 trunk_idx = mlx5_trunk_idx_get(pool, idx); 373 if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) || 374 (pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk)) 375 goto out; 376 trunk = pool->trunks[trunk_idx]; 377 if (!trunk) 378 goto out; 379 entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx); 380 if (trunk_idx != trunk->idx || 381 rte_bitmap_get(trunk->bmp, entry_idx)) 382 goto out; 383 rte_bitmap_set(trunk->bmp, entry_idx); 384 trunk->free++; 385 if (pool->cfg.release_mem_en && trunk->free == mlx5_trunk_size_get 386 (pool, trunk->idx)) { 387 if (pool->free_list == trunk->idx) 388 pool->free_list = trunk->next; 389 if (trunk->next != TRUNK_INVALID) 390 pool->trunks[trunk->next]->prev = trunk->prev; 391 if (trunk->prev != TRUNK_INVALID) 392 pool->trunks[trunk->prev]->next = trunk->next; 393 pool->cfg.free(trunk); 394 pool->trunks[trunk_idx] = NULL; 395 pool->n_trunk_valid--; 396 #ifdef POOL_DEBUG 397 pool->trunk_avail--; 398 pool->trunk_free++; 399 #endif 400 if (pool->n_trunk_valid == 0) { 401 pool->cfg.free(pool->trunks); 402 pool->trunks = NULL; 403 pool->n_trunk = 0; 404 } 405 } else if (trunk->free == 1) { 406 /* Put into free trunk list head. */ 407 MLX5_ASSERT(pool->free_list != trunk->idx); 408 trunk->next = pool->free_list; 409 trunk->prev = TRUNK_INVALID; 410 if (pool->free_list != TRUNK_INVALID) 411 pool->trunks[pool->free_list]->prev = trunk->idx; 412 pool->free_list = trunk->idx; 413 #ifdef POOL_DEBUG 414 pool->trunk_empty--; 415 pool->trunk_avail++; 416 #endif 417 } 418 #ifdef POOL_DEBUG 419 pool->n_entry--; 420 #endif 421 out: 422 mlx5_ipool_unlock(pool); 423 } 424 425 void * 426 mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx) 427 { 428 struct mlx5_indexed_trunk *trunk; 429 void *p = NULL; 430 uint32_t trunk_idx; 431 uint32_t entry_idx; 432 433 if (!idx) 434 return NULL; 435 idx -= 1; 436 mlx5_ipool_lock(pool); 437 trunk_idx = mlx5_trunk_idx_get(pool, idx); 438 if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) || 439 (pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk)) 440 goto out; 441 trunk = pool->trunks[trunk_idx]; 442 if (!trunk) 443 goto out; 444 entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx); 445 if (trunk_idx != trunk->idx || 446 rte_bitmap_get(trunk->bmp, entry_idx)) 447 goto out; 448 p = &trunk->data[entry_idx * pool->cfg.size]; 449 out: 450 mlx5_ipool_unlock(pool); 451 return p; 452 } 453 454 int 455 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool) 456 { 457 struct mlx5_indexed_trunk **trunks; 458 uint32_t i; 459 460 MLX5_ASSERT(pool); 461 mlx5_ipool_lock(pool); 462 trunks = pool->trunks; 463 for (i = 0; i < pool->n_trunk; i++) { 464 if (trunks[i]) 465 pool->cfg.free(trunks[i]); 466 } 467 if (!pool->trunks) 468 pool->cfg.free(pool->trunks); 469 mlx5_ipool_unlock(pool); 470 mlx5_free(pool); 471 return 0; 472 } 473 474 void 475 mlx5_ipool_dump(struct mlx5_indexed_pool *pool) 476 { 477 printf("Pool %s entry size %u, trunks %u, %d entry per trunk, " 478 "total: %d\n", 479 pool->cfg.type, pool->cfg.size, pool->n_trunk_valid, 480 pool->cfg.trunk_size, pool->n_trunk_valid); 481 #ifdef POOL_DEBUG 482 printf("Pool %s entry %u, trunk alloc %u, empty: %u, " 483 "available %u free %u\n", 484 pool->cfg.type, pool->n_entry, pool->trunk_new, 485 pool->trunk_empty, pool->trunk_avail, pool->trunk_free); 486 #endif 487 } 488 489 struct mlx5_l3t_tbl * 490 mlx5_l3t_create(enum mlx5_l3t_type type) 491 { 492 struct mlx5_l3t_tbl *tbl; 493 struct mlx5_indexed_pool_config l3t_ip_cfg = { 494 .trunk_size = 16, 495 .grow_trunk = 6, 496 .grow_shift = 1, 497 .need_lock = 0, 498 .release_mem_en = 1, 499 .malloc = mlx5_malloc, 500 .free = mlx5_free, 501 }; 502 503 if (type >= MLX5_L3T_TYPE_MAX) { 504 rte_errno = EINVAL; 505 return NULL; 506 } 507 tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_l3t_tbl), 1, 508 SOCKET_ID_ANY); 509 if (!tbl) { 510 rte_errno = ENOMEM; 511 return NULL; 512 } 513 tbl->type = type; 514 switch (type) { 515 case MLX5_L3T_TYPE_WORD: 516 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_word) + 517 sizeof(uint16_t) * MLX5_L3T_ET_SIZE; 518 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_w"; 519 break; 520 case MLX5_L3T_TYPE_DWORD: 521 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_dword) + 522 sizeof(uint32_t) * MLX5_L3T_ET_SIZE; 523 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_dw"; 524 break; 525 case MLX5_L3T_TYPE_QWORD: 526 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_qword) + 527 sizeof(uint64_t) * MLX5_L3T_ET_SIZE; 528 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_qw"; 529 break; 530 default: 531 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_ptr) + 532 sizeof(void *) * MLX5_L3T_ET_SIZE; 533 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_tpr"; 534 break; 535 } 536 tbl->eip = mlx5_ipool_create(&l3t_ip_cfg); 537 if (!tbl->eip) { 538 rte_errno = ENOMEM; 539 mlx5_free(tbl); 540 tbl = NULL; 541 } 542 return tbl; 543 } 544 545 void 546 mlx5_l3t_destroy(struct mlx5_l3t_tbl *tbl) 547 { 548 struct mlx5_l3t_level_tbl *g_tbl, *m_tbl; 549 uint32_t i, j; 550 551 if (!tbl) 552 return; 553 g_tbl = tbl->tbl; 554 if (g_tbl) { 555 for (i = 0; i < MLX5_L3T_GT_SIZE; i++) { 556 m_tbl = g_tbl->tbl[i]; 557 if (!m_tbl) 558 continue; 559 for (j = 0; j < MLX5_L3T_MT_SIZE; j++) { 560 if (!m_tbl->tbl[j]) 561 continue; 562 MLX5_ASSERT(!((struct mlx5_l3t_entry_word *) 563 m_tbl->tbl[j])->ref_cnt); 564 mlx5_ipool_free(tbl->eip, 565 ((struct mlx5_l3t_entry_word *) 566 m_tbl->tbl[j])->idx); 567 m_tbl->tbl[j] = 0; 568 if (!(--m_tbl->ref_cnt)) 569 break; 570 } 571 MLX5_ASSERT(!m_tbl->ref_cnt); 572 mlx5_free(g_tbl->tbl[i]); 573 g_tbl->tbl[i] = 0; 574 if (!(--g_tbl->ref_cnt)) 575 break; 576 } 577 MLX5_ASSERT(!g_tbl->ref_cnt); 578 mlx5_free(tbl->tbl); 579 tbl->tbl = 0; 580 } 581 mlx5_ipool_destroy(tbl->eip); 582 mlx5_free(tbl); 583 } 584 585 uint32_t 586 mlx5_l3t_get_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx, 587 union mlx5_l3t_data *data) 588 { 589 struct mlx5_l3t_level_tbl *g_tbl, *m_tbl; 590 void *e_tbl; 591 uint32_t entry_idx; 592 593 g_tbl = tbl->tbl; 594 if (!g_tbl) 595 return -1; 596 m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK]; 597 if (!m_tbl) 598 return -1; 599 e_tbl = m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK]; 600 if (!e_tbl) 601 return -1; 602 entry_idx = idx & MLX5_L3T_ET_MASK; 603 switch (tbl->type) { 604 case MLX5_L3T_TYPE_WORD: 605 data->word = ((struct mlx5_l3t_entry_word *)e_tbl)->entry 606 [entry_idx]; 607 break; 608 case MLX5_L3T_TYPE_DWORD: 609 data->dword = ((struct mlx5_l3t_entry_dword *)e_tbl)->entry 610 [entry_idx]; 611 break; 612 case MLX5_L3T_TYPE_QWORD: 613 data->qword = ((struct mlx5_l3t_entry_qword *)e_tbl)->entry 614 [entry_idx]; 615 break; 616 default: 617 data->ptr = ((struct mlx5_l3t_entry_ptr *)e_tbl)->entry 618 [entry_idx]; 619 break; 620 } 621 return 0; 622 } 623 624 void 625 mlx5_l3t_clear_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx) 626 { 627 struct mlx5_l3t_level_tbl *g_tbl, *m_tbl; 628 struct mlx5_l3t_entry_word *w_e_tbl; 629 struct mlx5_l3t_entry_dword *dw_e_tbl; 630 struct mlx5_l3t_entry_qword *qw_e_tbl; 631 struct mlx5_l3t_entry_ptr *ptr_e_tbl; 632 void *e_tbl; 633 uint32_t entry_idx; 634 uint64_t ref_cnt; 635 636 g_tbl = tbl->tbl; 637 if (!g_tbl) 638 return; 639 m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK]; 640 if (!m_tbl) 641 return; 642 e_tbl = m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK]; 643 if (!e_tbl) 644 return; 645 entry_idx = idx & MLX5_L3T_ET_MASK; 646 switch (tbl->type) { 647 case MLX5_L3T_TYPE_WORD: 648 w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl; 649 w_e_tbl->entry[entry_idx] = 0; 650 ref_cnt = --w_e_tbl->ref_cnt; 651 break; 652 case MLX5_L3T_TYPE_DWORD: 653 dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl; 654 dw_e_tbl->entry[entry_idx] = 0; 655 ref_cnt = --dw_e_tbl->ref_cnt; 656 break; 657 case MLX5_L3T_TYPE_QWORD: 658 qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl; 659 qw_e_tbl->entry[entry_idx] = 0; 660 ref_cnt = --qw_e_tbl->ref_cnt; 661 break; 662 default: 663 ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl; 664 ptr_e_tbl->entry[entry_idx] = NULL; 665 ref_cnt = --ptr_e_tbl->ref_cnt; 666 break; 667 } 668 if (!ref_cnt) { 669 mlx5_ipool_free(tbl->eip, 670 ((struct mlx5_l3t_entry_word *)e_tbl)->idx); 671 m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK] = 672 NULL; 673 if (!(--m_tbl->ref_cnt)) { 674 mlx5_free(m_tbl); 675 g_tbl->tbl 676 [(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK] = NULL; 677 if (!(--g_tbl->ref_cnt)) { 678 mlx5_free(g_tbl); 679 tbl->tbl = 0; 680 } 681 } 682 } 683 } 684 685 uint32_t 686 mlx5_l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx, 687 union mlx5_l3t_data *data) 688 { 689 struct mlx5_l3t_level_tbl *g_tbl, *m_tbl; 690 struct mlx5_l3t_entry_word *w_e_tbl; 691 struct mlx5_l3t_entry_dword *dw_e_tbl; 692 struct mlx5_l3t_entry_qword *qw_e_tbl; 693 struct mlx5_l3t_entry_ptr *ptr_e_tbl; 694 void *e_tbl; 695 uint32_t entry_idx, tbl_idx = 0; 696 697 /* Check the global table, create it if empty. */ 698 g_tbl = tbl->tbl; 699 if (!g_tbl) { 700 g_tbl = mlx5_malloc(MLX5_MEM_ZERO, 701 sizeof(struct mlx5_l3t_level_tbl) + 702 sizeof(void *) * MLX5_L3T_GT_SIZE, 1, 703 SOCKET_ID_ANY); 704 if (!g_tbl) { 705 rte_errno = ENOMEM; 706 return -1; 707 } 708 tbl->tbl = g_tbl; 709 } 710 /* 711 * Check the middle table, create it if empty. Ref_cnt will be 712 * increased if new sub table created. 713 */ 714 m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK]; 715 if (!m_tbl) { 716 m_tbl = mlx5_malloc(MLX5_MEM_ZERO, 717 sizeof(struct mlx5_l3t_level_tbl) + 718 sizeof(void *) * MLX5_L3T_MT_SIZE, 1, 719 SOCKET_ID_ANY); 720 if (!m_tbl) { 721 rte_errno = ENOMEM; 722 return -1; 723 } 724 g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK] = 725 m_tbl; 726 g_tbl->ref_cnt++; 727 } 728 /* 729 * Check the entry table, create it if empty. Ref_cnt will be 730 * increased if new sub entry table created. 731 */ 732 e_tbl = m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK]; 733 if (!e_tbl) { 734 e_tbl = mlx5_ipool_zmalloc(tbl->eip, &tbl_idx); 735 if (!e_tbl) { 736 rte_errno = ENOMEM; 737 return -1; 738 } 739 ((struct mlx5_l3t_entry_word *)e_tbl)->idx = tbl_idx; 740 m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK] = 741 e_tbl; 742 m_tbl->ref_cnt++; 743 } 744 entry_idx = idx & MLX5_L3T_ET_MASK; 745 switch (tbl->type) { 746 case MLX5_L3T_TYPE_WORD: 747 w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl; 748 w_e_tbl->entry[entry_idx] = data->word; 749 w_e_tbl->ref_cnt++; 750 break; 751 case MLX5_L3T_TYPE_DWORD: 752 dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl; 753 dw_e_tbl->entry[entry_idx] = data->dword; 754 dw_e_tbl->ref_cnt++; 755 break; 756 case MLX5_L3T_TYPE_QWORD: 757 qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl; 758 qw_e_tbl->entry[entry_idx] = data->qword; 759 qw_e_tbl->ref_cnt++; 760 break; 761 default: 762 ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl; 763 ptr_e_tbl->entry[entry_idx] = data->ptr; 764 ptr_e_tbl->ref_cnt++; 765 break; 766 } 767 return 0; 768 } 769