1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 6WIND S.A. 3 */ 4 5 #include <stdalign.h> 6 #include <sys/queue.h> 7 #include <stdint.h> 8 #include <limits.h> 9 10 #include <rte_common.h> 11 #include <rte_eal.h> 12 #include <rte_eal_memconfig.h> 13 #include <rte_tailq.h> 14 #include <rte_errno.h> 15 #include <rte_malloc.h> 16 #include <rte_string_fns.h> 17 #include <rte_bitops.h> 18 #include <rte_mbuf.h> 19 #include <rte_mbuf_dyn.h> 20 21 #include "mbuf_log.h" 22 23 #define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn" 24 25 struct mbuf_dynfield_elt { 26 struct rte_mbuf_dynfield params; 27 size_t offset; 28 }; 29 TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry); 30 31 static struct rte_tailq_elem mbuf_dynfield_tailq = { 32 .name = "RTE_MBUF_DYNFIELD", 33 }; 34 EAL_REGISTER_TAILQ(mbuf_dynfield_tailq); 35 36 struct mbuf_dynflag_elt { 37 struct rte_mbuf_dynflag params; 38 unsigned int bitnum; 39 }; 40 TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry); 41 42 static struct rte_tailq_elem mbuf_dynflag_tailq = { 43 .name = "RTE_MBUF_DYNFLAG", 44 }; 45 EAL_REGISTER_TAILQ(mbuf_dynflag_tailq); 46 47 struct mbuf_dyn_shm { 48 /** 49 * For each mbuf byte, free_space[i] != 0 if space is free. 50 * The value is the size of the biggest aligned element that 51 * can fit in the zone. 52 */ 53 uint8_t free_space[sizeof(struct rte_mbuf)]; 54 /** Bitfield of available flags. */ 55 uint64_t free_flags; 56 }; 57 static struct mbuf_dyn_shm *shm; 58 59 /* Set the value of free_space[] according to the size and alignment of 60 * the free areas. This helps to select the best place when reserving a 61 * dynamic field. Assume tailq is locked. 62 */ 63 static void 64 process_score(void) 65 { 66 size_t off, align, size, i; 67 68 /* first, erase previous info */ 69 for (i = 0; i < sizeof(struct rte_mbuf); i++) { 70 if (shm->free_space[i]) 71 shm->free_space[i] = 1; 72 } 73 74 off = 0; 75 while (off < sizeof(struct rte_mbuf)) { 76 /* get the size of the free zone */ 77 for (size = 0; (off + size) < sizeof(struct rte_mbuf) && 78 shm->free_space[off + size]; size++) 79 ; 80 if (size == 0) { 81 off++; 82 continue; 83 } 84 85 /* get the alignment of biggest object that can fit in 86 * the zone at this offset. 87 */ 88 for (align = 1; 89 (off % (align << 1)) == 0 && (align << 1) <= size; 90 align <<= 1) 91 ; 92 93 /* save it in free_space[] */ 94 for (i = off; i < off + align; i++) 95 shm->free_space[i] = RTE_MAX(align, shm->free_space[i]); 96 97 off += align; 98 } 99 } 100 101 /* Mark the area occupied by a mbuf field as available in the shm. */ 102 #define mark_free(field) \ 103 memset(&shm->free_space[offsetof(struct rte_mbuf, field)], \ 104 1, sizeof(((struct rte_mbuf *)0)->field)) 105 106 /* Allocate and initialize the shared memory. Assume tailq is locked */ 107 static int 108 init_shared_mem(void) 109 { 110 const struct rte_memzone *mz; 111 uint64_t mask; 112 113 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 114 mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME, 115 sizeof(struct mbuf_dyn_shm), 116 SOCKET_ID_ANY, 0, 117 RTE_CACHE_LINE_SIZE); 118 } else { 119 mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME); 120 } 121 if (mz == NULL) { 122 MBUF_LOG(ERR, "Failed to get mbuf dyn shared memory"); 123 return -1; 124 } 125 126 shm = mz->addr; 127 128 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 129 /* init free_space, keep it sync'd with 130 * rte_mbuf_dynfield_copy(). 131 */ 132 memset(shm, 0, sizeof(*shm)); 133 mark_free(dynfield1); 134 #if !RTE_IOVA_IN_MBUF 135 mark_free(dynfield2); 136 #endif 137 138 /* init free_flags */ 139 for (mask = RTE_MBUF_F_FIRST_FREE; mask <= RTE_MBUF_F_LAST_FREE; mask <<= 1) 140 shm->free_flags |= mask; 141 142 process_score(); 143 } 144 145 return 0; 146 } 147 148 /* check if this offset can be used */ 149 static int 150 check_offset(size_t offset, size_t size, size_t align) 151 { 152 size_t i; 153 154 if ((offset & (align - 1)) != 0) 155 return -1; 156 if (offset + size > sizeof(struct rte_mbuf)) 157 return -1; 158 159 for (i = 0; i < size; i++) { 160 if (!shm->free_space[i + offset]) 161 return -1; 162 } 163 164 return 0; 165 } 166 167 /* assume tailq is locked */ 168 static struct mbuf_dynfield_elt * 169 __mbuf_dynfield_lookup(const char *name) 170 { 171 struct mbuf_dynfield_list *mbuf_dynfield_list; 172 struct mbuf_dynfield_elt *mbuf_dynfield; 173 struct rte_tailq_entry *te; 174 175 mbuf_dynfield_list = RTE_TAILQ_CAST( 176 mbuf_dynfield_tailq.head, mbuf_dynfield_list); 177 178 TAILQ_FOREACH(te, mbuf_dynfield_list, next) { 179 mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data; 180 if (strcmp(name, mbuf_dynfield->params.name) == 0) 181 break; 182 } 183 184 if (te == NULL || mbuf_dynfield == NULL) { 185 rte_errno = ENOENT; 186 return NULL; 187 } 188 189 return mbuf_dynfield; 190 } 191 192 int 193 rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params) 194 { 195 struct mbuf_dynfield_elt *mbuf_dynfield; 196 197 rte_mcfg_tailq_read_lock(); 198 if (shm == NULL && init_shared_mem() < 0) 199 mbuf_dynfield = NULL; 200 else 201 mbuf_dynfield = __mbuf_dynfield_lookup(name); 202 rte_mcfg_tailq_read_unlock(); 203 204 if (mbuf_dynfield == NULL) 205 return -1; 206 207 if (params != NULL) 208 memcpy(params, &mbuf_dynfield->params, sizeof(*params)); 209 210 return mbuf_dynfield->offset; 211 } 212 213 static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1, 214 const struct rte_mbuf_dynfield *params2) 215 { 216 if (strcmp(params1->name, params2->name)) 217 return -1; 218 if (params1->size != params2->size) 219 return -1; 220 if (params1->align != params2->align) 221 return -1; 222 if (params1->flags != params2->flags) 223 return -1; 224 return 0; 225 } 226 227 /* assume tailq is locked */ 228 static int 229 __rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params, 230 size_t req) 231 { 232 struct mbuf_dynfield_list *mbuf_dynfield_list; 233 struct mbuf_dynfield_elt *mbuf_dynfield = NULL; 234 struct rte_tailq_entry *te = NULL; 235 unsigned int best_zone = UINT_MAX; 236 size_t i, offset; 237 int ret; 238 239 if (shm == NULL && init_shared_mem() < 0) 240 return -1; 241 242 mbuf_dynfield = __mbuf_dynfield_lookup(params->name); 243 if (mbuf_dynfield != NULL) { 244 if (req != SIZE_MAX && req != mbuf_dynfield->offset) { 245 rte_errno = EEXIST; 246 return -1; 247 } 248 if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) { 249 rte_errno = EEXIST; 250 return -1; 251 } 252 return mbuf_dynfield->offset; 253 } 254 255 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 256 rte_errno = EPERM; 257 return -1; 258 } 259 260 if (req == SIZE_MAX) { 261 /* Find the best place to put this field: we search the 262 * lowest value of shm->free_space[offset]: the zones 263 * containing room for larger fields are kept for later. 264 */ 265 for (offset = 0; 266 offset < sizeof(struct rte_mbuf); 267 offset++) { 268 if (check_offset(offset, params->size, 269 params->align) == 0 && 270 shm->free_space[offset] < best_zone) { 271 best_zone = shm->free_space[offset]; 272 req = offset; 273 } 274 } 275 if (req == SIZE_MAX) { 276 rte_errno = ENOENT; 277 return -1; 278 } 279 } else { 280 if (check_offset(req, params->size, params->align) < 0) { 281 rte_errno = EBUSY; 282 return -1; 283 } 284 } 285 286 offset = req; 287 mbuf_dynfield_list = RTE_TAILQ_CAST( 288 mbuf_dynfield_tailq.head, mbuf_dynfield_list); 289 290 te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0); 291 if (te == NULL) { 292 rte_errno = ENOMEM; 293 return -1; 294 } 295 296 mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0); 297 if (mbuf_dynfield == NULL) { 298 rte_free(te); 299 rte_errno = ENOMEM; 300 return -1; 301 } 302 303 ret = strlcpy(mbuf_dynfield->params.name, params->name, 304 sizeof(mbuf_dynfield->params.name)); 305 if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) { 306 rte_errno = ENAMETOOLONG; 307 rte_free(mbuf_dynfield); 308 rte_free(te); 309 return -1; 310 } 311 memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params)); 312 mbuf_dynfield->offset = offset; 313 te->data = mbuf_dynfield; 314 315 TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next); 316 317 for (i = offset; i < offset + params->size; i++) 318 shm->free_space[i] = 0; 319 process_score(); 320 321 MBUF_LOG(DEBUG, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd", 322 params->name, params->size, params->align, params->flags, 323 offset); 324 325 return offset; 326 } 327 328 int 329 rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params, 330 size_t req) 331 { 332 int ret; 333 334 if (params->size >= sizeof(struct rte_mbuf)) { 335 rte_errno = EINVAL; 336 return -1; 337 } 338 if (!rte_is_power_of_2(params->align)) { 339 rte_errno = EINVAL; 340 return -1; 341 } 342 if (params->flags != 0) { 343 rte_errno = EINVAL; 344 return -1; 345 } 346 347 rte_mcfg_tailq_write_lock(); 348 ret = __rte_mbuf_dynfield_register_offset(params, req); 349 rte_mcfg_tailq_write_unlock(); 350 351 return ret; 352 } 353 354 int 355 rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params) 356 { 357 return rte_mbuf_dynfield_register_offset(params, SIZE_MAX); 358 } 359 360 /* assume tailq is locked */ 361 static struct mbuf_dynflag_elt * 362 __mbuf_dynflag_lookup(const char *name) 363 { 364 struct mbuf_dynflag_list *mbuf_dynflag_list; 365 struct mbuf_dynflag_elt *mbuf_dynflag; 366 struct rte_tailq_entry *te; 367 368 mbuf_dynflag_list = RTE_TAILQ_CAST( 369 mbuf_dynflag_tailq.head, mbuf_dynflag_list); 370 371 TAILQ_FOREACH(te, mbuf_dynflag_list, next) { 372 mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data; 373 if (strncmp(name, mbuf_dynflag->params.name, 374 RTE_MBUF_DYN_NAMESIZE) == 0) 375 break; 376 } 377 378 if (te == NULL) { 379 rte_errno = ENOENT; 380 return NULL; 381 } 382 383 return mbuf_dynflag; 384 } 385 386 int 387 rte_mbuf_dynflag_lookup(const char *name, 388 struct rte_mbuf_dynflag *params) 389 { 390 struct mbuf_dynflag_elt *mbuf_dynflag; 391 392 rte_mcfg_tailq_read_lock(); 393 if (shm == NULL && init_shared_mem() < 0) 394 mbuf_dynflag = NULL; 395 else 396 mbuf_dynflag = __mbuf_dynflag_lookup(name); 397 rte_mcfg_tailq_read_unlock(); 398 399 if (mbuf_dynflag == NULL) 400 return -1; 401 402 if (params != NULL) 403 memcpy(params, &mbuf_dynflag->params, sizeof(*params)); 404 405 return mbuf_dynflag->bitnum; 406 } 407 408 static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1, 409 const struct rte_mbuf_dynflag *params2) 410 { 411 if (strcmp(params1->name, params2->name)) 412 return -1; 413 if (params1->flags != params2->flags) 414 return -1; 415 return 0; 416 } 417 418 /* assume tailq is locked */ 419 static int 420 __rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params, 421 unsigned int req) 422 { 423 struct mbuf_dynflag_list *mbuf_dynflag_list; 424 struct mbuf_dynflag_elt *mbuf_dynflag = NULL; 425 struct rte_tailq_entry *te = NULL; 426 unsigned int bitnum; 427 int ret; 428 429 if (shm == NULL && init_shared_mem() < 0) 430 return -1; 431 432 mbuf_dynflag = __mbuf_dynflag_lookup(params->name); 433 if (mbuf_dynflag != NULL) { 434 if (req != UINT_MAX && req != mbuf_dynflag->bitnum) { 435 rte_errno = EEXIST; 436 return -1; 437 } 438 if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) { 439 rte_errno = EEXIST; 440 return -1; 441 } 442 return mbuf_dynflag->bitnum; 443 } 444 445 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 446 rte_errno = EPERM; 447 return -1; 448 } 449 450 if (req == UINT_MAX) { 451 if (shm->free_flags == 0) { 452 rte_errno = ENOENT; 453 return -1; 454 } 455 bitnum = rte_bsf64(shm->free_flags); 456 } else { 457 if ((shm->free_flags & (1ULL << req)) == 0) { 458 rte_errno = EBUSY; 459 return -1; 460 } 461 bitnum = req; 462 } 463 464 mbuf_dynflag_list = RTE_TAILQ_CAST( 465 mbuf_dynflag_tailq.head, mbuf_dynflag_list); 466 467 te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0); 468 if (te == NULL) { 469 rte_errno = ENOMEM; 470 return -1; 471 } 472 473 mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0); 474 if (mbuf_dynflag == NULL) { 475 rte_free(te); 476 rte_errno = ENOMEM; 477 return -1; 478 } 479 480 ret = strlcpy(mbuf_dynflag->params.name, params->name, 481 sizeof(mbuf_dynflag->params.name)); 482 if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) { 483 rte_free(mbuf_dynflag); 484 rte_free(te); 485 rte_errno = ENAMETOOLONG; 486 return -1; 487 } 488 mbuf_dynflag->bitnum = bitnum; 489 te->data = mbuf_dynflag; 490 491 TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next); 492 493 shm->free_flags &= ~(1ULL << bitnum); 494 495 MBUF_LOG(DEBUG, "Registered dynamic flag %s (fl=0x%x) -> %u", 496 params->name, params->flags, bitnum); 497 498 return bitnum; 499 } 500 501 int 502 rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params, 503 unsigned int req) 504 { 505 int ret; 506 507 if (params->flags != 0) { 508 rte_errno = EINVAL; 509 return -1; 510 } 511 if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT && 512 req != UINT_MAX) { 513 rte_errno = EINVAL; 514 return -1; 515 } 516 517 rte_mcfg_tailq_write_lock(); 518 ret = __rte_mbuf_dynflag_register_bitnum(params, req); 519 rte_mcfg_tailq_write_unlock(); 520 521 return ret; 522 } 523 524 int 525 rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params) 526 { 527 return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX); 528 } 529 530 void rte_mbuf_dyn_dump(FILE *out) 531 { 532 struct mbuf_dynfield_list *mbuf_dynfield_list; 533 struct mbuf_dynfield_elt *dynfield; 534 struct mbuf_dynflag_list *mbuf_dynflag_list; 535 struct mbuf_dynflag_elt *dynflag; 536 struct rte_tailq_entry *te; 537 size_t i; 538 539 rte_mcfg_tailq_write_lock(); 540 if (shm == NULL && init_shared_mem() < 0) { 541 rte_mcfg_tailq_write_unlock(); 542 return; 543 } 544 545 fprintf(out, "Reserved fields:\n"); 546 mbuf_dynfield_list = RTE_TAILQ_CAST( 547 mbuf_dynfield_tailq.head, mbuf_dynfield_list); 548 TAILQ_FOREACH(te, mbuf_dynfield_list, next) { 549 dynfield = (struct mbuf_dynfield_elt *)te->data; 550 fprintf(out, " name=%s offset=%zd size=%zd align=%zd flags=%x\n", 551 dynfield->params.name, dynfield->offset, 552 dynfield->params.size, dynfield->params.align, 553 dynfield->params.flags); 554 } 555 fprintf(out, "Reserved flags:\n"); 556 mbuf_dynflag_list = RTE_TAILQ_CAST( 557 mbuf_dynflag_tailq.head, mbuf_dynflag_list); 558 TAILQ_FOREACH(te, mbuf_dynflag_list, next) { 559 dynflag = (struct mbuf_dynflag_elt *)te->data; 560 fprintf(out, " name=%s bitnum=%u flags=%x\n", 561 dynflag->params.name, dynflag->bitnum, 562 dynflag->params.flags); 563 } 564 fprintf(out, "Free space in mbuf (0 = occupied, value = free zone alignment):\n"); 565 for (i = 0; i < sizeof(struct rte_mbuf); i++) { 566 if ((i % 8) == 0) 567 fprintf(out, " %4.4zx: ", i); 568 fprintf(out, "%2.2x%s", shm->free_space[i], 569 (i % 8 != 7) ? " " : "\n"); 570 } 571 fprintf(out, "Free bit in mbuf->ol_flags (0 = occupied, 1 = free):\n"); 572 for (i = 0; i < sizeof(uint64_t) * CHAR_BIT; i++) { 573 if ((i % 8) == 0) 574 fprintf(out, " %4.4zx: ", i); 575 fprintf(out, "%1.1x%s", (shm->free_flags & (1ULL << i)) ? 1 : 0, 576 (i % 8 != 7) ? " " : "\n"); 577 } 578 579 rte_mcfg_tailq_write_unlock(); 580 } 581 582 static int 583 rte_mbuf_dyn_timestamp_register(int *field_offset, uint64_t *flag, 584 const char *direction, const char *flag_name) 585 { 586 static const struct rte_mbuf_dynfield field_desc = { 587 .name = RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, 588 .size = sizeof(rte_mbuf_timestamp_t), 589 .align = alignof(rte_mbuf_timestamp_t), 590 }; 591 struct rte_mbuf_dynflag flag_desc = {}; 592 int offset; 593 594 offset = rte_mbuf_dynfield_register(&field_desc); 595 if (offset < 0) { 596 MBUF_LOG(ERR, 597 "Failed to register mbuf field for timestamp"); 598 return -1; 599 } 600 if (field_offset != NULL) 601 *field_offset = offset; 602 603 strlcpy(flag_desc.name, flag_name, sizeof(flag_desc.name)); 604 offset = rte_mbuf_dynflag_register(&flag_desc); 605 if (offset < 0) { 606 MBUF_LOG(ERR, 607 "Failed to register mbuf flag for %s timestamp", 608 direction); 609 return -1; 610 } 611 if (flag != NULL) 612 *flag = RTE_BIT64(offset); 613 614 return 0; 615 } 616 617 int 618 rte_mbuf_dyn_rx_timestamp_register(int *field_offset, uint64_t *rx_flag) 619 { 620 return rte_mbuf_dyn_timestamp_register(field_offset, rx_flag, 621 "Rx", RTE_MBUF_DYNFLAG_RX_TIMESTAMP_NAME); 622 } 623 624 int 625 rte_mbuf_dyn_tx_timestamp_register(int *field_offset, uint64_t *tx_flag) 626 { 627 return rte_mbuf_dyn_timestamp_register(field_offset, tx_flag, 628 "Tx", RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME); 629 } 630