1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 6WIND S.A. 3 */ 4 5 #include <sys/queue.h> 6 #include <stdint.h> 7 #include <limits.h> 8 9 #include <rte_common.h> 10 #include <rte_eal.h> 11 #include <rte_eal_memconfig.h> 12 #include <rte_tailq.h> 13 #include <rte_errno.h> 14 #include <rte_malloc.h> 15 #include <rte_string_fns.h> 16 #include <rte_bitops.h> 17 #include <rte_mbuf.h> 18 #include <rte_mbuf_dyn.h> 19 20 #define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn" 21 22 struct mbuf_dynfield_elt { 23 struct rte_mbuf_dynfield params; 24 size_t offset; 25 }; 26 TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry); 27 28 static struct rte_tailq_elem mbuf_dynfield_tailq = { 29 .name = "RTE_MBUF_DYNFIELD", 30 }; 31 EAL_REGISTER_TAILQ(mbuf_dynfield_tailq); 32 33 struct mbuf_dynflag_elt { 34 struct rte_mbuf_dynflag params; 35 unsigned int bitnum; 36 }; 37 TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry); 38 39 static struct rte_tailq_elem mbuf_dynflag_tailq = { 40 .name = "RTE_MBUF_DYNFLAG", 41 }; 42 EAL_REGISTER_TAILQ(mbuf_dynflag_tailq); 43 44 struct mbuf_dyn_shm { 45 /** 46 * For each mbuf byte, free_space[i] != 0 if space is free. 47 * The value is the size of the biggest aligned element that 48 * can fit in the zone. 49 */ 50 uint8_t free_space[sizeof(struct rte_mbuf)]; 51 /** Bitfield of available flags. */ 52 uint64_t free_flags; 53 }; 54 static struct mbuf_dyn_shm *shm; 55 56 /* Set the value of free_space[] according to the size and alignment of 57 * the free areas. This helps to select the best place when reserving a 58 * dynamic field. Assume tailq is locked. 59 */ 60 static void 61 process_score(void) 62 { 63 size_t off, align, size, i; 64 65 /* first, erase previous info */ 66 for (i = 0; i < sizeof(struct rte_mbuf); i++) { 67 if (shm->free_space[i]) 68 shm->free_space[i] = 1; 69 } 70 71 off = 0; 72 while (off < sizeof(struct rte_mbuf)) { 73 /* get the size of the free zone */ 74 for (size = 0; (off + size) < sizeof(struct rte_mbuf) && 75 shm->free_space[off + size]; size++) 76 ; 77 if (size == 0) { 78 off++; 79 continue; 80 } 81 82 /* get the alignment of biggest object that can fit in 83 * the zone at this offset. 84 */ 85 for (align = 1; 86 (off % (align << 1)) == 0 && (align << 1) <= size; 87 align <<= 1) 88 ; 89 90 /* save it in free_space[] */ 91 for (i = off; i < off + align; i++) 92 shm->free_space[i] = RTE_MAX(align, shm->free_space[i]); 93 94 off += align; 95 } 96 } 97 98 /* Mark the area occupied by a mbuf field as available in the shm. */ 99 #define mark_free(field) \ 100 memset(&shm->free_space[offsetof(struct rte_mbuf, field)], \ 101 1, sizeof(((struct rte_mbuf *)0)->field)) 102 103 /* Allocate and initialize the shared memory. Assume tailq is locked */ 104 static int 105 init_shared_mem(void) 106 { 107 const struct rte_memzone *mz; 108 uint64_t mask; 109 110 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 111 mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME, 112 sizeof(struct mbuf_dyn_shm), 113 SOCKET_ID_ANY, 0, 114 RTE_CACHE_LINE_SIZE); 115 } else { 116 mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME); 117 } 118 if (mz == NULL) { 119 RTE_LOG(ERR, MBUF, "Failed to get mbuf dyn shared memory\n"); 120 return -1; 121 } 122 123 shm = mz->addr; 124 125 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 126 /* init free_space, keep it sync'd with 127 * rte_mbuf_dynfield_copy(). 128 */ 129 memset(shm, 0, sizeof(*shm)); 130 mark_free(dynfield1); 131 132 /* init free_flags */ 133 for (mask = RTE_MBUF_F_FIRST_FREE; mask <= RTE_MBUF_F_LAST_FREE; mask <<= 1) 134 shm->free_flags |= mask; 135 136 process_score(); 137 } 138 139 return 0; 140 } 141 142 /* check if this offset can be used */ 143 static int 144 check_offset(size_t offset, size_t size, size_t align) 145 { 146 size_t i; 147 148 if ((offset & (align - 1)) != 0) 149 return -1; 150 if (offset + size > sizeof(struct rte_mbuf)) 151 return -1; 152 153 for (i = 0; i < size; i++) { 154 if (!shm->free_space[i + offset]) 155 return -1; 156 } 157 158 return 0; 159 } 160 161 /* assume tailq is locked */ 162 static struct mbuf_dynfield_elt * 163 __mbuf_dynfield_lookup(const char *name) 164 { 165 struct mbuf_dynfield_list *mbuf_dynfield_list; 166 struct mbuf_dynfield_elt *mbuf_dynfield; 167 struct rte_tailq_entry *te; 168 169 mbuf_dynfield_list = RTE_TAILQ_CAST( 170 mbuf_dynfield_tailq.head, mbuf_dynfield_list); 171 172 TAILQ_FOREACH(te, mbuf_dynfield_list, next) { 173 mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data; 174 if (strcmp(name, mbuf_dynfield->params.name) == 0) 175 break; 176 } 177 178 if (te == NULL || mbuf_dynfield == NULL) { 179 rte_errno = ENOENT; 180 return NULL; 181 } 182 183 return mbuf_dynfield; 184 } 185 186 int 187 rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params) 188 { 189 struct mbuf_dynfield_elt *mbuf_dynfield; 190 191 rte_mcfg_tailq_read_lock(); 192 if (shm == NULL && init_shared_mem() < 0) 193 mbuf_dynfield = NULL; 194 else 195 mbuf_dynfield = __mbuf_dynfield_lookup(name); 196 rte_mcfg_tailq_read_unlock(); 197 198 if (mbuf_dynfield == NULL) 199 return -1; 200 201 if (params != NULL) 202 memcpy(params, &mbuf_dynfield->params, sizeof(*params)); 203 204 return mbuf_dynfield->offset; 205 } 206 207 static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1, 208 const struct rte_mbuf_dynfield *params2) 209 { 210 if (strcmp(params1->name, params2->name)) 211 return -1; 212 if (params1->size != params2->size) 213 return -1; 214 if (params1->align != params2->align) 215 return -1; 216 if (params1->flags != params2->flags) 217 return -1; 218 return 0; 219 } 220 221 /* assume tailq is locked */ 222 static int 223 __rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params, 224 size_t req) 225 { 226 struct mbuf_dynfield_list *mbuf_dynfield_list; 227 struct mbuf_dynfield_elt *mbuf_dynfield = NULL; 228 struct rte_tailq_entry *te = NULL; 229 unsigned int best_zone = UINT_MAX; 230 size_t i, offset; 231 int ret; 232 233 if (shm == NULL && init_shared_mem() < 0) 234 return -1; 235 236 mbuf_dynfield = __mbuf_dynfield_lookup(params->name); 237 if (mbuf_dynfield != NULL) { 238 if (req != SIZE_MAX && req != mbuf_dynfield->offset) { 239 rte_errno = EEXIST; 240 return -1; 241 } 242 if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) { 243 rte_errno = EEXIST; 244 return -1; 245 } 246 return mbuf_dynfield->offset; 247 } 248 249 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 250 rte_errno = EPERM; 251 return -1; 252 } 253 254 if (req == SIZE_MAX) { 255 /* Find the best place to put this field: we search the 256 * lowest value of shm->free_space[offset]: the zones 257 * containing room for larger fields are kept for later. 258 */ 259 for (offset = 0; 260 offset < sizeof(struct rte_mbuf); 261 offset++) { 262 if (check_offset(offset, params->size, 263 params->align) == 0 && 264 shm->free_space[offset] < best_zone) { 265 best_zone = shm->free_space[offset]; 266 req = offset; 267 } 268 } 269 if (req == SIZE_MAX) { 270 rte_errno = ENOENT; 271 return -1; 272 } 273 } else { 274 if (check_offset(req, params->size, params->align) < 0) { 275 rte_errno = EBUSY; 276 return -1; 277 } 278 } 279 280 offset = req; 281 mbuf_dynfield_list = RTE_TAILQ_CAST( 282 mbuf_dynfield_tailq.head, mbuf_dynfield_list); 283 284 te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0); 285 if (te == NULL) { 286 rte_errno = ENOMEM; 287 return -1; 288 } 289 290 mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0); 291 if (mbuf_dynfield == NULL) { 292 rte_free(te); 293 rte_errno = ENOMEM; 294 return -1; 295 } 296 297 ret = strlcpy(mbuf_dynfield->params.name, params->name, 298 sizeof(mbuf_dynfield->params.name)); 299 if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) { 300 rte_errno = ENAMETOOLONG; 301 rte_free(mbuf_dynfield); 302 rte_free(te); 303 return -1; 304 } 305 memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params)); 306 mbuf_dynfield->offset = offset; 307 te->data = mbuf_dynfield; 308 309 TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next); 310 311 for (i = offset; i < offset + params->size; i++) 312 shm->free_space[i] = 0; 313 process_score(); 314 315 RTE_LOG(DEBUG, MBUF, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd\n", 316 params->name, params->size, params->align, params->flags, 317 offset); 318 319 return offset; 320 } 321 322 int 323 rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params, 324 size_t req) 325 { 326 int ret; 327 328 if (params->size >= sizeof(struct rte_mbuf)) { 329 rte_errno = EINVAL; 330 return -1; 331 } 332 if (!rte_is_power_of_2(params->align)) { 333 rte_errno = EINVAL; 334 return -1; 335 } 336 if (params->flags != 0) { 337 rte_errno = EINVAL; 338 return -1; 339 } 340 341 rte_mcfg_tailq_write_lock(); 342 ret = __rte_mbuf_dynfield_register_offset(params, req); 343 rte_mcfg_tailq_write_unlock(); 344 345 return ret; 346 } 347 348 int 349 rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params) 350 { 351 return rte_mbuf_dynfield_register_offset(params, SIZE_MAX); 352 } 353 354 /* assume tailq is locked */ 355 static struct mbuf_dynflag_elt * 356 __mbuf_dynflag_lookup(const char *name) 357 { 358 struct mbuf_dynflag_list *mbuf_dynflag_list; 359 struct mbuf_dynflag_elt *mbuf_dynflag; 360 struct rte_tailq_entry *te; 361 362 mbuf_dynflag_list = RTE_TAILQ_CAST( 363 mbuf_dynflag_tailq.head, mbuf_dynflag_list); 364 365 TAILQ_FOREACH(te, mbuf_dynflag_list, next) { 366 mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data; 367 if (strncmp(name, mbuf_dynflag->params.name, 368 RTE_MBUF_DYN_NAMESIZE) == 0) 369 break; 370 } 371 372 if (te == NULL) { 373 rte_errno = ENOENT; 374 return NULL; 375 } 376 377 return mbuf_dynflag; 378 } 379 380 int 381 rte_mbuf_dynflag_lookup(const char *name, 382 struct rte_mbuf_dynflag *params) 383 { 384 struct mbuf_dynflag_elt *mbuf_dynflag; 385 386 rte_mcfg_tailq_read_lock(); 387 if (shm == NULL && init_shared_mem() < 0) 388 mbuf_dynflag = NULL; 389 else 390 mbuf_dynflag = __mbuf_dynflag_lookup(name); 391 rte_mcfg_tailq_read_unlock(); 392 393 if (mbuf_dynflag == NULL) 394 return -1; 395 396 if (params != NULL) 397 memcpy(params, &mbuf_dynflag->params, sizeof(*params)); 398 399 return mbuf_dynflag->bitnum; 400 } 401 402 static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1, 403 const struct rte_mbuf_dynflag *params2) 404 { 405 if (strcmp(params1->name, params2->name)) 406 return -1; 407 if (params1->flags != params2->flags) 408 return -1; 409 return 0; 410 } 411 412 /* assume tailq is locked */ 413 static int 414 __rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params, 415 unsigned int req) 416 { 417 struct mbuf_dynflag_list *mbuf_dynflag_list; 418 struct mbuf_dynflag_elt *mbuf_dynflag = NULL; 419 struct rte_tailq_entry *te = NULL; 420 unsigned int bitnum; 421 int ret; 422 423 if (shm == NULL && init_shared_mem() < 0) 424 return -1; 425 426 mbuf_dynflag = __mbuf_dynflag_lookup(params->name); 427 if (mbuf_dynflag != NULL) { 428 if (req != UINT_MAX && req != mbuf_dynflag->bitnum) { 429 rte_errno = EEXIST; 430 return -1; 431 } 432 if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) { 433 rte_errno = EEXIST; 434 return -1; 435 } 436 return mbuf_dynflag->bitnum; 437 } 438 439 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 440 rte_errno = EPERM; 441 return -1; 442 } 443 444 if (req == UINT_MAX) { 445 if (shm->free_flags == 0) { 446 rte_errno = ENOENT; 447 return -1; 448 } 449 bitnum = rte_bsf64(shm->free_flags); 450 } else { 451 if ((shm->free_flags & (1ULL << req)) == 0) { 452 rte_errno = EBUSY; 453 return -1; 454 } 455 bitnum = req; 456 } 457 458 mbuf_dynflag_list = RTE_TAILQ_CAST( 459 mbuf_dynflag_tailq.head, mbuf_dynflag_list); 460 461 te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0); 462 if (te == NULL) { 463 rte_errno = ENOMEM; 464 return -1; 465 } 466 467 mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0); 468 if (mbuf_dynflag == NULL) { 469 rte_free(te); 470 rte_errno = ENOMEM; 471 return -1; 472 } 473 474 ret = strlcpy(mbuf_dynflag->params.name, params->name, 475 sizeof(mbuf_dynflag->params.name)); 476 if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) { 477 rte_free(mbuf_dynflag); 478 rte_free(te); 479 rte_errno = ENAMETOOLONG; 480 return -1; 481 } 482 mbuf_dynflag->bitnum = bitnum; 483 te->data = mbuf_dynflag; 484 485 TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next); 486 487 shm->free_flags &= ~(1ULL << bitnum); 488 489 RTE_LOG(DEBUG, MBUF, "Registered dynamic flag %s (fl=0x%x) -> %u\n", 490 params->name, params->flags, bitnum); 491 492 return bitnum; 493 } 494 495 int 496 rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params, 497 unsigned int req) 498 { 499 int ret; 500 501 if (params->flags != 0) { 502 rte_errno = EINVAL; 503 return -1; 504 } 505 if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT && 506 req != UINT_MAX) { 507 rte_errno = EINVAL; 508 return -1; 509 } 510 511 rte_mcfg_tailq_write_lock(); 512 ret = __rte_mbuf_dynflag_register_bitnum(params, req); 513 rte_mcfg_tailq_write_unlock(); 514 515 return ret; 516 } 517 518 int 519 rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params) 520 { 521 return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX); 522 } 523 524 void rte_mbuf_dyn_dump(FILE *out) 525 { 526 struct mbuf_dynfield_list *mbuf_dynfield_list; 527 struct mbuf_dynfield_elt *dynfield; 528 struct mbuf_dynflag_list *mbuf_dynflag_list; 529 struct mbuf_dynflag_elt *dynflag; 530 struct rte_tailq_entry *te; 531 size_t i; 532 533 rte_mcfg_tailq_write_lock(); 534 if (shm == NULL && init_shared_mem() < 0) { 535 rte_mcfg_tailq_write_unlock(); 536 return; 537 } 538 539 fprintf(out, "Reserved fields:\n"); 540 mbuf_dynfield_list = RTE_TAILQ_CAST( 541 mbuf_dynfield_tailq.head, mbuf_dynfield_list); 542 TAILQ_FOREACH(te, mbuf_dynfield_list, next) { 543 dynfield = (struct mbuf_dynfield_elt *)te->data; 544 fprintf(out, " name=%s offset=%zd size=%zd align=%zd flags=%x\n", 545 dynfield->params.name, dynfield->offset, 546 dynfield->params.size, dynfield->params.align, 547 dynfield->params.flags); 548 } 549 fprintf(out, "Reserved flags:\n"); 550 mbuf_dynflag_list = RTE_TAILQ_CAST( 551 mbuf_dynflag_tailq.head, mbuf_dynflag_list); 552 TAILQ_FOREACH(te, mbuf_dynflag_list, next) { 553 dynflag = (struct mbuf_dynflag_elt *)te->data; 554 fprintf(out, " name=%s bitnum=%u flags=%x\n", 555 dynflag->params.name, dynflag->bitnum, 556 dynflag->params.flags); 557 } 558 fprintf(out, "Free space in mbuf (0 = occupied, value = free zone alignment):\n"); 559 for (i = 0; i < sizeof(struct rte_mbuf); i++) { 560 if ((i % 8) == 0) 561 fprintf(out, " %4.4zx: ", i); 562 fprintf(out, "%2.2x%s", shm->free_space[i], 563 (i % 8 != 7) ? " " : "\n"); 564 } 565 fprintf(out, "Free bit in mbuf->ol_flags (0 = occupied, 1 = free):\n"); 566 for (i = 0; i < sizeof(uint64_t) * CHAR_BIT; i++) { 567 if ((i % 8) == 0) 568 fprintf(out, " %4.4zx: ", i); 569 fprintf(out, "%1.1x%s", (shm->free_flags & (1ULL << i)) ? 1 : 0, 570 (i % 8 != 7) ? " " : "\n"); 571 } 572 573 rte_mcfg_tailq_write_unlock(); 574 } 575 576 static int 577 rte_mbuf_dyn_timestamp_register(int *field_offset, uint64_t *flag, 578 const char *direction, const char *flag_name) 579 { 580 static const struct rte_mbuf_dynfield field_desc = { 581 .name = RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, 582 .size = sizeof(rte_mbuf_timestamp_t), 583 .align = __alignof__(rte_mbuf_timestamp_t), 584 }; 585 struct rte_mbuf_dynflag flag_desc = {}; 586 int offset; 587 588 offset = rte_mbuf_dynfield_register(&field_desc); 589 if (offset < 0) { 590 RTE_LOG(ERR, MBUF, 591 "Failed to register mbuf field for timestamp\n"); 592 return -1; 593 } 594 if (field_offset != NULL) 595 *field_offset = offset; 596 597 strlcpy(flag_desc.name, flag_name, sizeof(flag_desc.name)); 598 offset = rte_mbuf_dynflag_register(&flag_desc); 599 if (offset < 0) { 600 RTE_LOG(ERR, MBUF, 601 "Failed to register mbuf flag for %s timestamp\n", 602 direction); 603 return -1; 604 } 605 if (flag != NULL) 606 *flag = RTE_BIT64(offset); 607 608 return 0; 609 } 610 611 int 612 rte_mbuf_dyn_rx_timestamp_register(int *field_offset, uint64_t *rx_flag) 613 { 614 return rte_mbuf_dyn_timestamp_register(field_offset, rx_flag, 615 "Rx", RTE_MBUF_DYNFLAG_RX_TIMESTAMP_NAME); 616 } 617 618 int 619 rte_mbuf_dyn_tx_timestamp_register(int *field_offset, uint64_t *tx_flag) 620 { 621 return rte_mbuf_dyn_timestamp_register(field_offset, tx_flag, 622 "Tx", RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME); 623 } 624