1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #ifndef RTE_PMD_MLX5_UTILS_H_ 7 #define RTE_PMD_MLX5_UTILS_H_ 8 9 #include <stddef.h> 10 #include <stdint.h> 11 #include <stdio.h> 12 #include <limits.h> 13 #include <errno.h> 14 15 #include <rte_spinlock.h> 16 #include <rte_rwlock.h> 17 #include <rte_memory.h> 18 #include <rte_bitmap.h> 19 20 #include <mlx5_common.h> 21 22 #include "mlx5_defs.h" 23 24 /* Convert a bit number to the corresponding 64-bit mask */ 25 #define MLX5_BITSHIFT(v) (UINT64_C(1) << (v)) 26 27 /* Save and restore errno around argument evaluation. */ 28 #define ERRNO_SAFE(x) ((errno = (int []){ errno, ((x), 0) }[0])) 29 30 extern int mlx5_logtype; 31 32 /* Generic printf()-like logging macro with automatic line feed. */ 33 #define DRV_LOG(level, ...) \ 34 PMD_DRV_LOG_(level, mlx5_logtype, MLX5_DRIVER_NAME, \ 35 __VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \ 36 PMD_DRV_LOG_CPAREN) 37 38 /* Convenience macros for accessing mbuf fields. */ 39 #define NEXT(m) ((m)->next) 40 #define DATA_LEN(m) ((m)->data_len) 41 #define PKT_LEN(m) ((m)->pkt_len) 42 #define DATA_OFF(m) ((m)->data_off) 43 #define SET_DATA_OFF(m, o) ((m)->data_off = (o)) 44 #define NB_SEGS(m) ((m)->nb_segs) 45 #define PORT(m) ((m)->port) 46 47 /* Transpose flags. Useful to convert IBV to DPDK flags. */ 48 #define TRANSPOSE(val, from, to) \ 49 (((from) >= (to)) ? \ 50 (((val) & (from)) / ((from) / (to))) : \ 51 (((val) & (from)) * ((to) / (from)))) 52 53 /* 54 * For the case which data is linked with sequence increased index, the 55 * array table will be more efficiect than hash table once need to serarch 56 * one data entry in large numbers of entries. Since the traditional hash 57 * tables has fixed table size, when huge numbers of data saved to the hash 58 * table, it also comes lots of hash conflict. 59 * 60 * But simple array table also has fixed size, allocates all the needed 61 * memory at once will waste lots of memory. For the case don't know the 62 * exactly number of entries will be impossible to allocate the array. 63 * 64 * Then the multiple level table helps to balance the two disadvantages. 65 * Allocate a global high level table with sub table entries at first, 66 * the global table contains the sub table entries, and the sub table will 67 * be allocated only once the corresponding index entry need to be saved. 68 * e.g. for up to 32-bits index, three level table with 10-10-12 splitting, 69 * with sequence increased index, the memory grows with every 4K entries. 70 * 71 * The currently implementation introduces 10-10-12 32-bits splitting 72 * Three-Level table to help the cases which have millions of enties to 73 * save. The index entries can be addressed directly by the index, no 74 * search will be needed.q 75 */ 76 77 /* L3 table global table define. */ 78 #define MLX5_L3T_GT_OFFSET 22 79 #define MLX5_L3T_GT_SIZE (1 << 10) 80 #define MLX5_L3T_GT_MASK (MLX5_L3T_GT_SIZE - 1) 81 82 /* L3 table middle table define. */ 83 #define MLX5_L3T_MT_OFFSET 12 84 #define MLX5_L3T_MT_SIZE (1 << 10) 85 #define MLX5_L3T_MT_MASK (MLX5_L3T_MT_SIZE - 1) 86 87 /* L3 table entry table define. */ 88 #define MLX5_L3T_ET_OFFSET 0 89 #define MLX5_L3T_ET_SIZE (1 << 12) 90 #define MLX5_L3T_ET_MASK (MLX5_L3T_ET_SIZE - 1) 91 92 /* L3 table type. */ 93 enum mlx5_l3t_type { 94 MLX5_L3T_TYPE_WORD = 0, 95 MLX5_L3T_TYPE_DWORD, 96 MLX5_L3T_TYPE_QWORD, 97 MLX5_L3T_TYPE_PTR, 98 MLX5_L3T_TYPE_MAX, 99 }; 100 101 struct mlx5_indexed_pool; 102 103 /* Generic data struct. */ 104 union mlx5_l3t_data { 105 uint16_t word; 106 uint32_t dword; 107 uint64_t qword; 108 void *ptr; 109 }; 110 111 /* L3 level table data structure. */ 112 struct mlx5_l3t_level_tbl { 113 uint64_t ref_cnt; /* Table ref_cnt. */ 114 void *tbl[]; /* Table array. */ 115 }; 116 117 /* L3 word entry table data structure. */ 118 struct mlx5_l3t_entry_word { 119 uint32_t idx; /* Table index. */ 120 uint64_t ref_cnt; /* Table ref_cnt. */ 121 struct { 122 uint16_t data; 123 uint32_t ref_cnt; 124 } entry[MLX5_L3T_ET_SIZE]; /* Entry array */ 125 } __rte_packed; 126 127 /* L3 double word entry table data structure. */ 128 struct mlx5_l3t_entry_dword { 129 uint32_t idx; /* Table index. */ 130 uint64_t ref_cnt; /* Table ref_cnt. */ 131 struct { 132 uint32_t data; 133 int32_t ref_cnt; 134 } entry[MLX5_L3T_ET_SIZE]; /* Entry array */ 135 } __rte_packed; 136 137 /* L3 quad word entry table data structure. */ 138 struct mlx5_l3t_entry_qword { 139 uint32_t idx; /* Table index. */ 140 uint64_t ref_cnt; /* Table ref_cnt. */ 141 struct { 142 uint64_t data; 143 uint32_t ref_cnt; 144 } entry[MLX5_L3T_ET_SIZE]; /* Entry array */ 145 } __rte_packed; 146 147 /* L3 pointer entry table data structure. */ 148 struct mlx5_l3t_entry_ptr { 149 uint32_t idx; /* Table index. */ 150 uint64_t ref_cnt; /* Table ref_cnt. */ 151 struct { 152 void *data; 153 uint32_t ref_cnt; 154 } entry[MLX5_L3T_ET_SIZE]; /* Entry array */ 155 } __rte_packed; 156 157 /* L3 table data structure. */ 158 struct mlx5_l3t_tbl { 159 enum mlx5_l3t_type type; /* Table type. */ 160 struct mlx5_indexed_pool *eip; 161 /* Table index pool handles. */ 162 struct mlx5_l3t_level_tbl *tbl; /* Global table index. */ 163 rte_spinlock_t sl; /* The table lock. */ 164 }; 165 166 /** Type of function that is used to handle the data before freeing. */ 167 typedef int32_t (*mlx5_l3t_alloc_callback_fn)(void *ctx, 168 union mlx5_l3t_data *data); 169 170 /* 171 * The indexed memory entry index is made up of trunk index and offset of 172 * the entry in the trunk. Since the entry index is 32 bits, in case user 173 * prefers to have small trunks, user can change the macro below to a big 174 * number which helps the pool contains more trunks with lots of entries 175 * allocated. 176 */ 177 #define TRUNK_IDX_BITS 16 178 #define TRUNK_MAX_IDX ((1 << TRUNK_IDX_BITS) - 1) 179 #define TRUNK_INVALID TRUNK_MAX_IDX 180 #define MLX5_IPOOL_DEFAULT_TRUNK_SIZE (1 << (28 - TRUNK_IDX_BITS)) 181 #ifdef RTE_LIBRTE_MLX5_DEBUG 182 #define POOL_DEBUG 1 183 #endif 184 185 struct mlx5_indexed_pool_config { 186 uint32_t size; /* Pool entry size. */ 187 uint32_t trunk_size:22; 188 /* 189 * Trunk entry number. Must be power of 2. It can be increased 190 * if trunk_grow enable. The trunk entry number increases with 191 * left shift grow_shift. Trunks with index are after grow_trunk 192 * will keep the entry number same with the last grow trunk. 193 */ 194 uint32_t grow_trunk:4; 195 /* 196 * Trunks with entry number increase in the pool. Set it to 0 197 * to make the pool works as trunk entry fixed pool. It works 198 * only if grow_shift is not 0. 199 */ 200 uint32_t grow_shift:4; 201 /* 202 * Trunk entry number increase shift value, stop after grow_trunk. 203 * It works only if grow_trunk is not 0. 204 */ 205 uint32_t need_lock:1; 206 /* Lock is needed for multiple thread usage. */ 207 uint32_t release_mem_en:1; /* Rlease trunk when it is free. */ 208 const char *type; /* Memory allocate type name. */ 209 void *(*malloc)(uint32_t flags, size_t size, unsigned int align, 210 int socket); 211 /* User defined memory allocator. */ 212 void (*free)(void *addr); /* User defined memory release. */ 213 }; 214 215 struct mlx5_indexed_trunk { 216 uint32_t idx; /* Trunk id. */ 217 uint32_t prev; /* Previous free trunk in free list. */ 218 uint32_t next; /* Next free trunk in free list. */ 219 uint32_t free; /* Free entries available */ 220 struct rte_bitmap *bmp; 221 uint8_t data[] __rte_cache_aligned; /* Entry data start. */ 222 }; 223 224 struct mlx5_indexed_pool { 225 struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */ 226 rte_spinlock_t lock; /* Pool lock for multiple thread usage. */ 227 uint32_t n_trunk_valid; /* Trunks allocated. */ 228 uint32_t n_trunk; /* Trunk pointer array size. */ 229 /* Dim of trunk pointer array. */ 230 struct mlx5_indexed_trunk **trunks; 231 uint32_t free_list; /* Index to first free trunk. */ 232 #ifdef POOL_DEBUG 233 uint32_t n_entry; 234 uint32_t trunk_new; 235 uint32_t trunk_avail; 236 uint32_t trunk_empty; 237 uint32_t trunk_free; 238 #endif 239 uint32_t grow_tbl[]; /* Save the index offset for the grow trunks. */ 240 }; 241 242 /** 243 * Return logarithm of the nearest power of two above input value. 244 * 245 * @param v 246 * Input value. 247 * 248 * @return 249 * Logarithm of the nearest power of two above input value. 250 */ 251 static inline unsigned int 252 log2above(unsigned int v) 253 { 254 unsigned int l; 255 unsigned int r; 256 257 for (l = 0, r = 0; (v >> 1); ++l, v >>= 1) 258 r |= (v & 1); 259 return l + r; 260 } 261 262 #define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */ 263 #define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */ 264 265 /** Maximum size of string for naming the hlist table. */ 266 #define MLX5_HLIST_NAMESIZE 32 267 268 struct mlx5_hlist; 269 270 /** 271 * Structure of the entry in the hash list, user should define its own struct 272 * that contains this in order to store the data. The 'key' is 64-bits right 273 * now and its user's responsibility to guarantee there is no collision. 274 */ 275 struct mlx5_hlist_entry { 276 LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */ 277 uint32_t idx; /* Bucket index the entry belongs to. */ 278 uint32_t ref_cnt; /* Reference count. */ 279 }; 280 281 /** Structure for hash head. */ 282 LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry); 283 284 /** 285 * Type of callback function for entry removal. 286 * 287 * @param list 288 * The hash list. 289 * @param entry 290 * The entry in the list. 291 */ 292 typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list, 293 struct mlx5_hlist_entry *entry); 294 295 /** 296 * Type of function for user defined matching. 297 * 298 * @param list 299 * The hash list. 300 * @param entry 301 * The entry in the list. 302 * @param key 303 * The new entry key. 304 * @param ctx 305 * The pointer to new entry context. 306 * 307 * @return 308 * 0 if matching, non-zero number otherwise. 309 */ 310 typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list, 311 struct mlx5_hlist_entry *entry, 312 uint64_t key, void *ctx); 313 314 /** 315 * Type of function for user defined hash list entry creation. 316 * 317 * @param list 318 * The hash list. 319 * @param key 320 * The key of the new entry. 321 * @param ctx 322 * The pointer to new entry context. 323 * 324 * @return 325 * Pointer to allocated entry on success, NULL otherwise. 326 */ 327 typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb) 328 (struct mlx5_hlist *list, 329 uint64_t key, void *ctx); 330 331 /* Hash list bucket head. */ 332 struct mlx5_hlist_bucket { 333 struct mlx5_hlist_head head; /* List head. */ 334 rte_rwlock_t lock; /* Bucket lock. */ 335 uint32_t gen_cnt; /* List modification will update generation count. */ 336 } __rte_cache_aligned; 337 338 /** 339 * Hash list table structure 340 * 341 * Entry in hash list could be reused if entry already exists, reference 342 * count will increase and the existing entry returns. 343 * 344 * When destroy an entry from list, decrease reference count and only 345 * destroy when no further reference. 346 */ 347 struct mlx5_hlist { 348 char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */ 349 /**< number of heads, need to be power of 2. */ 350 uint32_t table_sz; 351 uint32_t entry_sz; /**< Size of entry, used to allocate entry. */ 352 /**< mask to get the index of the list heads. */ 353 uint32_t mask; 354 bool direct_key; /* Use the new entry key directly as hash index. */ 355 bool write_most; /* List mostly used for append new or destroy. */ 356 void *ctx; 357 mlx5_hlist_create_cb cb_create; /**< entry create callback. */ 358 mlx5_hlist_match_cb cb_match; /**< entry match callback. */ 359 mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */ 360 struct mlx5_hlist_bucket buckets[] __rte_cache_aligned; 361 /**< list bucket arrays. */ 362 }; 363 364 /** 365 * Create a hash list table, the user can specify the list heads array size 366 * of the table, now the size should be a power of 2 in order to get better 367 * distribution for the entries. Each entry is a part of the whole data element 368 * and the caller should be responsible for the data element's allocation and 369 * cleanup / free. Key of each entry will be calculated with CRC in order to 370 * generate a little fairer distribution. 371 * 372 * @param name 373 * Name of the hash list(optional). 374 * @param size 375 * Heads array size of the hash list. 376 * @param entry_size 377 * Entry size to allocate if cb_create not specified. 378 * @param flags 379 * The hash list attribute flags. 380 * @param cb_create 381 * Callback function for entry create. 382 * @param cb_match 383 * Callback function for entry match. 384 * @param cb_destroy 385 * Callback function for entry destroy. 386 * @return 387 * Pointer of the hash list table created, NULL on failure. 388 */ 389 struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size, 390 uint32_t entry_size, uint32_t flags, 391 mlx5_hlist_create_cb cb_create, 392 mlx5_hlist_match_cb cb_match, 393 mlx5_hlist_remove_cb cb_destroy); 394 395 /** 396 * Search an entry matching the key. 397 * 398 * Result returned might be destroyed by other thread, must use 399 * this function only in main thread. 400 * 401 * @param h 402 * Pointer to the hast list table. 403 * @param key 404 * Key for the searching entry. 405 * @param ctx 406 * Common context parameter used by entry callback function. 407 * 408 * @return 409 * Pointer of the hlist entry if found, NULL otherwise. 410 */ 411 struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, 412 void *ctx); 413 414 /** 415 * Insert an entry to the hash list table, the entry is only part of whole data 416 * element and a 64B key is used for matching. User should construct the key or 417 * give a calculated hash signature and guarantee there is no collision. 418 * 419 * @param h 420 * Pointer to the hast list table. 421 * @param entry 422 * Entry to be inserted into the hash list table. 423 * @param ctx 424 * Common context parameter used by callback function. 425 * 426 * @return 427 * registered entry on success, NULL otherwise 428 */ 429 struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, 430 void *ctx); 431 432 /** 433 * Remove an entry from the hash list table. User should guarantee the validity 434 * of the entry. 435 * 436 * @param h 437 * Pointer to the hast list table. (not used) 438 * @param entry 439 * Entry to be removed from the hash list table. 440 * @return 441 * 0 on entry removed, 1 on entry still referenced. 442 */ 443 int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry); 444 445 /** 446 * Destroy the hash list table, all the entries already inserted into the lists 447 * will be handled by the callback function provided by the user (including 448 * free if needed) before the table is freed. 449 * 450 * @param h 451 * Pointer to the hast list table. 452 */ 453 void mlx5_hlist_destroy(struct mlx5_hlist *h); 454 455 /************************ cache list *****************************/ 456 457 /** Maximum size of string for naming. */ 458 #define MLX5_NAME_SIZE 32 459 460 struct mlx5_cache_list; 461 462 /** 463 * Structure of the entry in the cache list, user should define its own struct 464 * that contains this in order to store the data. 465 */ 466 struct mlx5_cache_entry { 467 LIST_ENTRY(mlx5_cache_entry) next; /* Entry pointers in the list. */ 468 uint32_t ref_cnt; /* Reference count. */ 469 }; 470 471 /** 472 * Type of callback function for entry removal. 473 * 474 * @param list 475 * The cache list. 476 * @param entry 477 * The entry in the list. 478 */ 479 typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list, 480 struct mlx5_cache_entry *entry); 481 482 /** 483 * Type of function for user defined matching. 484 * 485 * @param list 486 * The cache list. 487 * @param entry 488 * The entry in the list. 489 * @param ctx 490 * The pointer to new entry context. 491 * 492 * @return 493 * 0 if matching, non-zero number otherwise. 494 */ 495 typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list, 496 struct mlx5_cache_entry *entry, void *ctx); 497 498 /** 499 * Type of function for user defined cache list entry creation. 500 * 501 * @param list 502 * The cache list. 503 * @param entry 504 * The new allocated entry, NULL if list entry size unspecified, 505 * New entry has to be allocated in callback and return. 506 * @param ctx 507 * The pointer to new entry context. 508 * 509 * @return 510 * Pointer of entry on success, NULL otherwise. 511 */ 512 typedef struct mlx5_cache_entry *(*mlx5_cache_create_cb) 513 (struct mlx5_cache_list *list, 514 struct mlx5_cache_entry *entry, 515 void *ctx); 516 517 /** 518 * Linked cache list structure. 519 * 520 * Entry in cache list could be reused if entry already exists, 521 * reference count will increase and the existing entry returns. 522 * 523 * When destroy an entry from list, decrease reference count and only 524 * destroy when no further reference. 525 * 526 * Linked list cache is designed for limited number of entries cache, 527 * read mostly, less modification. 528 * 529 * For huge amount of entries cache, please consider hash list cache. 530 * 531 */ 532 struct mlx5_cache_list { 533 char name[MLX5_NAME_SIZE]; /**< Name of the cache list. */ 534 uint32_t entry_sz; /**< Entry size, 0: use create callback. */ 535 rte_rwlock_t lock; /* read/write lock. */ 536 uint32_t gen_cnt; /* List modification will update generation count. */ 537 uint32_t count; /* number of entries in list. */ 538 void *ctx; /* user objects target to callback. */ 539 mlx5_cache_create_cb cb_create; /**< entry create callback. */ 540 mlx5_cache_match_cb cb_match; /**< entry match callback. */ 541 mlx5_cache_remove_cb cb_remove; /**< entry remove callback. */ 542 LIST_HEAD(mlx5_cache_head, mlx5_cache_entry) head; 543 }; 544 545 /** 546 * Initialize a cache list. 547 * 548 * @param list 549 * Pointer to the hast list table. 550 * @param name 551 * Name of the cache list. 552 * @param entry_size 553 * Entry size to allocate, 0 to allocate by creation callback. 554 * @param ctx 555 * Pointer to the list context data. 556 * @param cb_create 557 * Callback function for entry create. 558 * @param cb_match 559 * Callback function for entry match. 560 * @param cb_remove 561 * Callback function for entry remove. 562 * @return 563 * 0 on success, otherwise failure. 564 */ 565 int mlx5_cache_list_init(struct mlx5_cache_list *list, 566 const char *name, uint32_t entry_size, void *ctx, 567 mlx5_cache_create_cb cb_create, 568 mlx5_cache_match_cb cb_match, 569 mlx5_cache_remove_cb cb_remove); 570 571 /** 572 * Search an entry matching the key. 573 * 574 * Result returned might be destroyed by other thread, must use 575 * this function only in main thread. 576 * 577 * @param list 578 * Pointer to the cache list. 579 * @param ctx 580 * Common context parameter used by entry callback function. 581 * 582 * @return 583 * Pointer of the cache entry if found, NULL otherwise. 584 */ 585 struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list, 586 void *ctx); 587 588 /** 589 * Reuse or create an entry to the cache list. 590 * 591 * @param list 592 * Pointer to the hast list table. 593 * @param ctx 594 * Common context parameter used by callback function. 595 * 596 * @return 597 * registered entry on success, NULL otherwise 598 */ 599 struct mlx5_cache_entry *mlx5_cache_register(struct mlx5_cache_list *list, 600 void *ctx); 601 602 /** 603 * Remove an entry from the cache list. 604 * 605 * User should guarantee the validity of the entry. 606 * 607 * @param list 608 * Pointer to the hast list. 609 * @param entry 610 * Entry to be removed from the cache list table. 611 * @return 612 * 0 on entry removed, 1 on entry still referenced. 613 */ 614 int mlx5_cache_unregister(struct mlx5_cache_list *list, 615 struct mlx5_cache_entry *entry); 616 617 /** 618 * Destroy the cache list. 619 * 620 * @param list 621 * Pointer to the cache list. 622 */ 623 void mlx5_cache_list_destroy(struct mlx5_cache_list *list); 624 625 /** 626 * Get entry number from the cache list. 627 * 628 * @param list 629 * Pointer to the hast list. 630 * @return 631 * Cache list entry number. 632 */ 633 uint32_t 634 mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list); 635 636 /********************************* indexed pool *************************/ 637 638 /** 639 * This function allocates non-initialized memory entry from pool. 640 * In NUMA systems, the memory entry allocated resides on the same 641 * NUMA socket as the core that calls this function. 642 * 643 * Memory entry is allocated from memory trunk, no alignment. 644 * 645 * @param pool 646 * Pointer to indexed memory entry pool. 647 * No initialization required. 648 * @param[out] idx 649 * Pointer to memory to save allocated index. 650 * Memory index always positive value. 651 * @return 652 * - Pointer to the allocated memory entry. 653 * - NULL on error. Not enough memory, or invalid arguments. 654 */ 655 void *mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx); 656 657 /** 658 * This function allocates zero initialized memory entry from pool. 659 * In NUMA systems, the memory entry allocated resides on the same 660 * NUMA socket as the core that calls this function. 661 * 662 * Memory entry is allocated from memory trunk, no alignment. 663 * 664 * @param pool 665 * Pointer to indexed memory pool. 666 * No initialization required. 667 * @param[out] idx 668 * Pointer to memory to save allocated index. 669 * Memory index always positive value. 670 * @return 671 * - Pointer to the allocated memory entry . 672 * - NULL on error. Not enough memory, or invalid arguments. 673 */ 674 void *mlx5_ipool_zmalloc(struct mlx5_indexed_pool *pool, uint32_t *idx); 675 676 /** 677 * This function frees indexed memory entry to pool. 678 * Caller has to make sure that the index is allocated from same pool. 679 * 680 * @param pool 681 * Pointer to indexed memory pool. 682 * @param idx 683 * Allocated memory entry index. 684 */ 685 void mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx); 686 687 /** 688 * This function returns pointer of indexed memory entry from index. 689 * Caller has to make sure that the index is valid, and allocated 690 * from same pool. 691 * 692 * @param pool 693 * Pointer to indexed memory pool. 694 * @param idx 695 * Allocated memory index. 696 * @return 697 * - Pointer to indexed memory entry. 698 */ 699 void *mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx); 700 701 /** 702 * This function creates indexed memory pool. 703 * Caller has to configure the configuration accordingly. 704 * 705 * @param pool 706 * Pointer to indexed memory pool. 707 * @param cfg 708 * Allocated memory index. 709 */ 710 struct mlx5_indexed_pool * 711 mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg); 712 713 /** 714 * This function releases all resources of pool. 715 * Caller has to make sure that all indexes and memories allocated 716 * from this pool not referenced anymore. 717 * 718 * @param pool 719 * Pointer to indexed memory pool. 720 * @return 721 * - non-zero value on error. 722 * - 0 on success. 723 */ 724 int mlx5_ipool_destroy(struct mlx5_indexed_pool *pool); 725 726 /** 727 * This function dumps debug info of pool. 728 * 729 * @param pool 730 * Pointer to indexed memory pool. 731 */ 732 void mlx5_ipool_dump(struct mlx5_indexed_pool *pool); 733 734 /** 735 * This function allocates new empty Three-level table. 736 * 737 * @param type 738 * The l3t can set as word, double word, quad word or pointer with index. 739 * 740 * @return 741 * - Pointer to the allocated l3t. 742 * - NULL on error. Not enough memory, or invalid arguments. 743 */ 744 struct mlx5_l3t_tbl *mlx5_l3t_create(enum mlx5_l3t_type type); 745 746 /** 747 * This function destroys Three-level table. 748 * 749 * @param tbl 750 * Pointer to the l3t. 751 */ 752 void mlx5_l3t_destroy(struct mlx5_l3t_tbl *tbl); 753 754 /** 755 * This function gets the index entry from Three-level table. 756 * 757 * @param tbl 758 * Pointer to the l3t. 759 * @param idx 760 * Index to the entry. 761 * @param data 762 * Pointer to the memory which saves the entry data. 763 * When function call returns 0, data contains the entry data get from 764 * l3t. 765 * When function call returns -1, data is not modified. 766 * 767 * @return 768 * 0 if success, -1 on error. 769 */ 770 771 int32_t mlx5_l3t_get_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx, 772 union mlx5_l3t_data *data); 773 774 /** 775 * This function gets the index entry from Three-level table. 776 * 777 * If the index entry is not available, allocate new one by callback 778 * function and fill in the entry. 779 * 780 * @param tbl 781 * Pointer to the l3t. 782 * @param idx 783 * Index to the entry. 784 * @param data 785 * Pointer to the memory which saves the entry data. 786 * When function call returns 0, data contains the entry data get from 787 * l3t. 788 * When function call returns -1, data is not modified. 789 * @param cb 790 * Callback function to allocate new data. 791 * @param ctx 792 * Context for callback function. 793 * 794 * @return 795 * 0 if success, -1 on error. 796 */ 797 798 int32_t mlx5_l3t_prepare_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx, 799 union mlx5_l3t_data *data, 800 mlx5_l3t_alloc_callback_fn cb, void *ctx); 801 802 /** 803 * This function decreases and clear index entry if reference 804 * counter is 0 from Three-level table. 805 * 806 * @param tbl 807 * Pointer to the l3t. 808 * @param idx 809 * Index to the entry. 810 * 811 * @return 812 * The remaining reference count, 0 means entry be cleared, -1 on error. 813 */ 814 int32_t mlx5_l3t_clear_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx); 815 816 /** 817 * This function sets the index entry to Three-level table. 818 * If the entry is already set, the EEXIST errno will be given, and 819 * the set data will be filled to the data. 820 * 821 * @param tbl[in] 822 * Pointer to the l3t. 823 * @param idx[in] 824 * Index to the entry. 825 * @param data[in/out] 826 * Pointer to the memory which contains the entry data save to l3t. 827 * If the entry is already set, the set data will be filled. 828 * 829 * @return 830 * 0 if success, -1 on error. 831 */ 832 int32_t mlx5_l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx, 833 union mlx5_l3t_data *data); 834 835 /* 836 * Macros for linked list based on indexed memory. 837 * Example data structure: 838 * struct Foo { 839 * ILIST_ENTRY(uint16_t) next; 840 * ... 841 * } 842 * 843 */ 844 #define ILIST_ENTRY(type) \ 845 struct { \ 846 type prev; /* Index of previous element. */ \ 847 type next; /* Index of next element. */ \ 848 } 849 850 #define ILIST_INSERT(pool, head, idx, elem, field) \ 851 do { \ 852 typeof(elem) peer; \ 853 MLX5_ASSERT((elem) && (idx)); \ 854 (elem)->field.next = *(head); \ 855 (elem)->field.prev = 0; \ 856 if (*(head)) { \ 857 (peer) = mlx5_ipool_get(pool, *(head)); \ 858 if (peer) \ 859 (peer)->field.prev = (idx); \ 860 } \ 861 *(head) = (idx); \ 862 } while (0) 863 864 #define ILIST_REMOVE(pool, head, idx, elem, field) \ 865 do { \ 866 typeof(elem) peer; \ 867 MLX5_ASSERT(elem); \ 868 MLX5_ASSERT(head); \ 869 if ((elem)->field.prev) { \ 870 (peer) = mlx5_ipool_get \ 871 (pool, (elem)->field.prev); \ 872 if (peer) \ 873 (peer)->field.next = (elem)->field.next;\ 874 } \ 875 if ((elem)->field.next) { \ 876 (peer) = mlx5_ipool_get \ 877 (pool, (elem)->field.next); \ 878 if (peer) \ 879 (peer)->field.prev = (elem)->field.prev;\ 880 } \ 881 if (*(head) == (idx)) \ 882 *(head) = (elem)->field.next; \ 883 } while (0) 884 885 #define ILIST_FOREACH(pool, head, idx, elem, field) \ 886 for ((idx) = (head), (elem) = \ 887 (idx) ? mlx5_ipool_get(pool, (idx)) : NULL; (elem); \ 888 idx = (elem)->field.next, (elem) = \ 889 (idx) ? mlx5_ipool_get(pool, idx) : NULL) 890 891 /* Single index list. */ 892 #define SILIST_ENTRY(type) \ 893 struct { \ 894 type next; /* Index of next element. */ \ 895 } 896 897 #define SILIST_INSERT(head, idx, elem, field) \ 898 do { \ 899 MLX5_ASSERT((elem) && (idx)); \ 900 (elem)->field.next = *(head); \ 901 *(head) = (idx); \ 902 } while (0) 903 904 #define SILIST_FOREACH(pool, head, idx, elem, field) \ 905 for ((idx) = (head), (elem) = \ 906 (idx) ? mlx5_ipool_get(pool, (idx)) : NULL; (elem); \ 907 idx = (elem)->field.next, (elem) = \ 908 (idx) ? mlx5_ipool_get(pool, idx) : NULL) 909 910 #endif /* RTE_PMD_MLX5_UTILS_H_ */ 911