1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #ifndef RTE_PMD_MLX5_UTILS_H_ 7 #define RTE_PMD_MLX5_UTILS_H_ 8 9 #include <stddef.h> 10 #include <stdint.h> 11 #include <stdio.h> 12 #include <limits.h> 13 #include <errno.h> 14 15 #include <rte_spinlock.h> 16 #include <rte_memory.h> 17 #include <rte_bitmap.h> 18 19 #include <mlx5_common.h> 20 21 #include "mlx5_defs.h" 22 23 24 /* 25 * Compilation workaround for PPC64 when AltiVec is fully enabled, e.g. std=c11. 26 * Otherwise there would be a type conflict between stdbool and altivec. 27 */ 28 #if defined(__PPC64__) && !defined(__APPLE_ALTIVEC__) 29 #undef bool 30 /* redefine as in stdbool.h */ 31 #define bool _Bool 32 #endif 33 34 /* Convert a bit number to the corresponding 64-bit mask */ 35 #define MLX5_BITSHIFT(v) (UINT64_C(1) << (v)) 36 37 /* Save and restore errno around argument evaluation. */ 38 #define ERRNO_SAFE(x) ((errno = (int []){ errno, ((x), 0) }[0])) 39 40 extern int mlx5_logtype; 41 42 /* Generic printf()-like logging macro with automatic line feed. */ 43 #define DRV_LOG(level, ...) \ 44 PMD_DRV_LOG_(level, mlx5_logtype, MLX5_DRIVER_NAME, \ 45 __VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \ 46 PMD_DRV_LOG_CPAREN) 47 48 #define INFO(...) DRV_LOG(INFO, __VA_ARGS__) 49 #define WARN(...) DRV_LOG(WARNING, __VA_ARGS__) 50 #define ERROR(...) DRV_LOG(ERR, __VA_ARGS__) 51 52 /* Convenience macros for accessing mbuf fields. */ 53 #define NEXT(m) ((m)->next) 54 #define DATA_LEN(m) ((m)->data_len) 55 #define PKT_LEN(m) ((m)->pkt_len) 56 #define DATA_OFF(m) ((m)->data_off) 57 #define SET_DATA_OFF(m, o) ((m)->data_off = (o)) 58 #define NB_SEGS(m) ((m)->nb_segs) 59 #define PORT(m) ((m)->port) 60 61 /* Transpose flags. Useful to convert IBV to DPDK flags. */ 62 #define TRANSPOSE(val, from, to) \ 63 (((from) >= (to)) ? \ 64 (((val) & (from)) / ((from) / (to))) : \ 65 (((val) & (from)) * ((to) / (from)))) 66 67 /* 68 * The indexed memory entry index is made up of trunk index and offset of 69 * the entry in the trunk. Since the entry index is 32 bits, in case user 70 * prefers to have small trunks, user can change the macro below to a big 71 * number which helps the pool contains more trunks with lots of entries 72 * allocated. 73 */ 74 #define TRUNK_IDX_BITS 16 75 #define TRUNK_MAX_IDX ((1 << TRUNK_IDX_BITS) - 1) 76 #define TRUNK_INVALID TRUNK_MAX_IDX 77 #define MLX5_IPOOL_DEFAULT_TRUNK_SIZE (1 << (28 - TRUNK_IDX_BITS)) 78 #ifdef RTE_LIBRTE_MLX5_DEBUG 79 #define POOL_DEBUG 1 80 #endif 81 82 struct mlx5_indexed_pool_config { 83 uint32_t size; /* Pool entry size. */ 84 uint32_t trunk_size:22; 85 /* 86 * Trunk entry number. Must be power of 2. It can be increased 87 * if trunk_grow enable. The trunk entry number increases with 88 * left shift grow_shift. Trunks with index are after grow_trunk 89 * will keep the entry number same with the last grow trunk. 90 */ 91 uint32_t grow_trunk:4; 92 /* 93 * Trunks with entry number increase in the pool. Set it to 0 94 * to make the pool works as trunk entry fixed pool. It works 95 * only if grow_shift is not 0. 96 */ 97 uint32_t grow_shift:4; 98 /* 99 * Trunk entry number increase shift value, stop after grow_trunk. 100 * It works only if grow_trunk is not 0. 101 */ 102 uint32_t need_lock:1; 103 /* Lock is needed for multiple thread usage. */ 104 uint32_t release_mem_en:1; /* Rlease trunk when it is free. */ 105 const char *type; /* Memory allocate type name. */ 106 void *(*malloc)(const char *type, size_t size, unsigned int align, 107 int socket); 108 /* User defined memory allocator. */ 109 void (*free)(void *addr); /* User defined memory release. */ 110 }; 111 112 struct mlx5_indexed_trunk { 113 uint32_t idx; /* Trunk id. */ 114 uint32_t prev; /* Previous free trunk in free list. */ 115 uint32_t next; /* Next free trunk in free list. */ 116 uint32_t free; /* Free entries available */ 117 struct rte_bitmap *bmp; 118 uint8_t data[] __rte_cache_min_aligned; /* Entry data start. */ 119 }; 120 121 struct mlx5_indexed_pool { 122 struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */ 123 rte_spinlock_t lock; /* Pool lock for multiple thread usage. */ 124 uint32_t n_trunk_valid; /* Trunks allocated. */ 125 uint32_t n_trunk; /* Trunk pointer array size. */ 126 /* Dim of trunk pointer array. */ 127 struct mlx5_indexed_trunk **trunks; 128 uint32_t free_list; /* Index to first free trunk. */ 129 #ifdef POOL_DEBUG 130 uint32_t n_entry; 131 uint32_t trunk_new; 132 uint32_t trunk_avail; 133 uint32_t trunk_empty; 134 uint32_t trunk_free; 135 #endif 136 uint32_t grow_tbl[]; /* Save the index offset for the grow trunks. */ 137 }; 138 139 /** 140 * Return logarithm of the nearest power of two above input value. 141 * 142 * @param v 143 * Input value. 144 * 145 * @return 146 * Logarithm of the nearest power of two above input value. 147 */ 148 static inline unsigned int 149 log2above(unsigned int v) 150 { 151 unsigned int l; 152 unsigned int r; 153 154 for (l = 0, r = 0; (v >> 1); ++l, v >>= 1) 155 r |= (v & 1); 156 return l + r; 157 } 158 159 /** Maximum size of string for naming the hlist table. */ 160 #define MLX5_HLIST_NAMESIZE 32 161 162 /** 163 * Structure of the entry in the hash list, user should define its own struct 164 * that contains this in order to store the data. The 'key' is 64-bits right 165 * now and its user's responsibility to guarantee there is no collision. 166 */ 167 struct mlx5_hlist_entry { 168 LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */ 169 uint64_t key; /* user defined 'key', could be the hash signature. */ 170 }; 171 172 /** Structure for hash head. */ 173 LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry); 174 175 /** Type of function that is used to handle the data before freeing. */ 176 typedef void (*mlx5_hlist_destroy_callback_fn)(void *p, void *ctx); 177 178 /** hash list table structure */ 179 struct mlx5_hlist { 180 char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */ 181 /**< number of heads, need to be power of 2. */ 182 uint32_t table_sz; 183 /**< mask to get the index of the list heads. */ 184 uint32_t mask; 185 struct mlx5_hlist_head heads[]; /**< list head arrays. */ 186 }; 187 188 /** 189 * Create a hash list table, the user can specify the list heads array size 190 * of the table, now the size should be a power of 2 in order to get better 191 * distribution for the entries. Each entry is a part of the whole data element 192 * and the caller should be responsible for the data element's allocation and 193 * cleanup / free. Key of each entry will be calculated with CRC in order to 194 * generate a little fairer distribution. 195 * 196 * @param name 197 * Name of the hash list(optional). 198 * @param size 199 * Heads array size of the hash list. 200 * 201 * @return 202 * Pointer of the hash list table created, NULL on failure. 203 */ 204 struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size); 205 206 /** 207 * Search an entry matching the key. 208 * 209 * @param h 210 * Pointer to the hast list table. 211 * @param key 212 * Key for the searching entry. 213 * 214 * @return 215 * Pointer of the hlist entry if found, NULL otherwise. 216 */ 217 struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key); 218 219 /** 220 * Insert an entry to the hash list table, the entry is only part of whole data 221 * element and a 64B key is used for matching. User should construct the key or 222 * give a calculated hash signature and guarantee there is no collision. 223 * 224 * @param h 225 * Pointer to the hast list table. 226 * @param entry 227 * Entry to be inserted into the hash list table. 228 * 229 * @return 230 * - zero for success. 231 * - -EEXIST if the entry is already inserted. 232 */ 233 int mlx5_hlist_insert(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry); 234 235 /** 236 * Remove an entry from the hash list table. User should guarantee the validity 237 * of the entry. 238 * 239 * @param h 240 * Pointer to the hast list table. (not used) 241 * @param entry 242 * Entry to be removed from the hash list table. 243 */ 244 void mlx5_hlist_remove(struct mlx5_hlist *h __rte_unused, 245 struct mlx5_hlist_entry *entry); 246 247 /** 248 * Destroy the hash list table, all the entries already inserted into the lists 249 * will be handled by the callback function provided by the user (including 250 * free if needed) before the table is freed. 251 * 252 * @param h 253 * Pointer to the hast list table. 254 * @param cb 255 * Callback function for each inserted entry when destroying the hash list. 256 * @param ctx 257 * Common context parameter used by callback function for each entry. 258 */ 259 void mlx5_hlist_destroy(struct mlx5_hlist *h, 260 mlx5_hlist_destroy_callback_fn cb, void *ctx); 261 262 /** 263 * This function allocates non-initialized memory entry from pool. 264 * In NUMA systems, the memory entry allocated resides on the same 265 * NUMA socket as the core that calls this function. 266 * 267 * Memory entry is allocated from memory trunk, no alignment. 268 * 269 * @param pool 270 * Pointer to indexed memory entry pool. 271 * No initialization required. 272 * @param[out] idx 273 * Pointer to memory to save allocated index. 274 * Memory index always positive value. 275 * @return 276 * - Pointer to the allocated memory entry. 277 * - NULL on error. Not enough memory, or invalid arguments. 278 */ 279 void *mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx); 280 281 /** 282 * This function allocates zero initialized memory entry from pool. 283 * In NUMA systems, the memory entry allocated resides on the same 284 * NUMA socket as the core that calls this function. 285 * 286 * Memory entry is allocated from memory trunk, no alignment. 287 * 288 * @param pool 289 * Pointer to indexed memory pool. 290 * No initialization required. 291 * @param[out] idx 292 * Pointer to memory to save allocated index. 293 * Memory index always positive value. 294 * @return 295 * - Pointer to the allocated memory entry . 296 * - NULL on error. Not enough memory, or invalid arguments. 297 */ 298 void *mlx5_ipool_zmalloc(struct mlx5_indexed_pool *pool, uint32_t *idx); 299 300 /** 301 * This function frees indexed memory entry to pool. 302 * Caller has to make sure that the index is allocated from same pool. 303 * 304 * @param pool 305 * Pointer to indexed memory pool. 306 * @param idx 307 * Allocated memory entry index. 308 */ 309 void mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx); 310 311 /** 312 * This function returns pointer of indexed memory entry from index. 313 * Caller has to make sure that the index is valid, and allocated 314 * from same pool. 315 * 316 * @param pool 317 * Pointer to indexed memory pool. 318 * @param idx 319 * Allocated memory index. 320 * @return 321 * - Pointer to indexed memory entry. 322 */ 323 void *mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx); 324 325 /** 326 * This function creates indexed memory pool. 327 * Caller has to configure the configuration accordingly. 328 * 329 * @param pool 330 * Pointer to indexed memory pool. 331 * @param cfg 332 * Allocated memory index. 333 */ 334 struct mlx5_indexed_pool * 335 mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg); 336 337 /** 338 * This function releases all resources of pool. 339 * Caller has to make sure that all indexes and memories allocated 340 * from this pool not referenced anymore. 341 * 342 * @param pool 343 * Pointer to indexed memory pool. 344 * @return 345 * - non-zero value on error. 346 * - 0 on success. 347 */ 348 int mlx5_ipool_destroy(struct mlx5_indexed_pool *pool); 349 350 /** 351 * This function dumps debug info of pool. 352 * 353 * @param pool 354 * Pointer to indexed memory pool. 355 */ 356 void mlx5_ipool_dump(struct mlx5_indexed_pool *pool); 357 358 /* 359 * Macros for linked list based on indexed memory. 360 * Example data structure: 361 * struct Foo { 362 * ILIST_ENTRY(uint16_t) next; 363 * ... 364 * } 365 * 366 */ 367 #define ILIST_ENTRY(type) \ 368 struct { \ 369 type prev; /* Index of previous element. */ \ 370 type next; /* Index of next element. */ \ 371 } 372 373 #define ILIST_INSERT(pool, head, idx, elem, field) \ 374 do { \ 375 typeof(elem) peer; \ 376 MLX5_ASSERT((elem) && (idx)); \ 377 (elem)->field.next = *(head); \ 378 (elem)->field.prev = 0; \ 379 if (*(head)) { \ 380 (peer) = mlx5_ipool_get(pool, *(head)); \ 381 if (peer) \ 382 (peer)->field.prev = (idx); \ 383 } \ 384 *(head) = (idx); \ 385 } while (0) 386 387 #define ILIST_REMOVE(pool, head, idx, elem, field) \ 388 do { \ 389 typeof(elem) peer; \ 390 MLX5_ASSERT(elem); \ 391 MLX5_ASSERT(head); \ 392 if ((elem)->field.prev) { \ 393 (peer) = mlx5_ipool_get \ 394 (pool, (elem)->field.prev); \ 395 if (peer) \ 396 (peer)->field.next = (elem)->field.next;\ 397 } \ 398 if ((elem)->field.next) { \ 399 (peer) = mlx5_ipool_get \ 400 (pool, (elem)->field.next); \ 401 if (peer) \ 402 (peer)->field.prev = (elem)->field.prev;\ 403 } \ 404 if (*(head) == (idx)) \ 405 *(head) = (elem)->field.next; \ 406 } while (0) 407 408 #define ILIST_FOREACH(pool, head, idx, elem, field) \ 409 for ((idx) = (head), (elem) = \ 410 (idx) ? mlx5_ipool_get(pool, (idx)) : NULL; (elem); \ 411 idx = (elem)->field.next, (elem) = \ 412 (idx) ? mlx5_ipool_get(pool, idx) : NULL) 413 414 /* Single index list. */ 415 #define SILIST_ENTRY(type) \ 416 struct { \ 417 type next; /* Index of next element. */ \ 418 } 419 420 #define SILIST_INSERT(head, idx, elem, field) \ 421 do { \ 422 MLX5_ASSERT((elem) && (idx)); \ 423 (elem)->field.next = *(head); \ 424 *(head) = (idx); \ 425 } while (0) 426 427 #define SILIST_FOREACH(pool, head, idx, elem, field) \ 428 for ((idx) = (head), (elem) = \ 429 (idx) ? mlx5_ipool_get(pool, (idx)) : NULL; (elem); \ 430 idx = (elem)->field.next, (elem) = \ 431 (idx) ? mlx5_ipool_get(pool, idx) : NULL) 432 433 #endif /* RTE_PMD_MLX5_UTILS_H_ */ 434