1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation. 3 * Copyright 2014 6WIND S.A. 4 */ 5 6 #include <string.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <stdint.h> 10 #include <stdarg.h> 11 #include <inttypes.h> 12 #include <errno.h> 13 #include <ctype.h> 14 #include <sys/queue.h> 15 16 #include <rte_compat.h> 17 #include <rte_debug.h> 18 #include <rte_common.h> 19 #include <rte_log.h> 20 #include <rte_memory.h> 21 #include <rte_launch.h> 22 #include <rte_eal.h> 23 #include <rte_per_lcore.h> 24 #include <rte_lcore.h> 25 #include <rte_branch_prediction.h> 26 #include <rte_mempool.h> 27 #include <rte_mbuf.h> 28 #include <rte_mbuf_pool_ops.h> 29 #include <rte_string_fns.h> 30 #include <rte_hexdump.h> 31 #include <rte_errno.h> 32 #include <rte_memcpy.h> 33 34 /* 35 * pktmbuf pool constructor, given as a callback function to 36 * rte_mempool_create(), or called directly if using 37 * rte_mempool_create_empty()/rte_mempool_populate() 38 */ 39 void 40 rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg) 41 { 42 struct rte_pktmbuf_pool_private *user_mbp_priv, *mbp_priv; 43 struct rte_pktmbuf_pool_private default_mbp_priv; 44 uint16_t roomsz; 45 46 RTE_ASSERT(mp->private_data_size >= 47 sizeof(struct rte_pktmbuf_pool_private)); 48 RTE_ASSERT(mp->elt_size >= sizeof(struct rte_mbuf)); 49 50 /* if no structure is provided, assume no mbuf private area */ 51 user_mbp_priv = opaque_arg; 52 if (user_mbp_priv == NULL) { 53 memset(&default_mbp_priv, 0, sizeof(default_mbp_priv)); 54 if (mp->elt_size > sizeof(struct rte_mbuf)) 55 roomsz = mp->elt_size - sizeof(struct rte_mbuf); 56 else 57 roomsz = 0; 58 default_mbp_priv.mbuf_data_room_size = roomsz; 59 user_mbp_priv = &default_mbp_priv; 60 } 61 62 RTE_ASSERT(mp->elt_size >= sizeof(struct rte_mbuf) + 63 ((user_mbp_priv->flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) ? 64 sizeof(struct rte_mbuf_ext_shared_info) : 65 user_mbp_priv->mbuf_data_room_size) + 66 user_mbp_priv->mbuf_priv_size); 67 RTE_ASSERT((user_mbp_priv->flags & 68 ~RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) == 0); 69 70 mbp_priv = rte_mempool_get_priv(mp); 71 memcpy(mbp_priv, user_mbp_priv, sizeof(*mbp_priv)); 72 } 73 74 /* 75 * pktmbuf constructor, given as a callback function to 76 * rte_mempool_obj_iter() or rte_mempool_create(). 77 * Set the fields of a packet mbuf to their default values. 78 */ 79 void 80 rte_pktmbuf_init(struct rte_mempool *mp, 81 __rte_unused void *opaque_arg, 82 void *_m, 83 __rte_unused unsigned i) 84 { 85 struct rte_mbuf *m = _m; 86 uint32_t mbuf_size, buf_len, priv_size; 87 88 RTE_ASSERT(mp->private_data_size >= 89 sizeof(struct rte_pktmbuf_pool_private)); 90 91 priv_size = rte_pktmbuf_priv_size(mp); 92 mbuf_size = sizeof(struct rte_mbuf) + priv_size; 93 buf_len = rte_pktmbuf_data_room_size(mp); 94 95 RTE_ASSERT(RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) == priv_size); 96 RTE_ASSERT(mp->elt_size >= mbuf_size); 97 RTE_ASSERT(buf_len <= UINT16_MAX); 98 99 memset(m, 0, mbuf_size); 100 /* start of buffer is after mbuf structure and priv data */ 101 m->priv_size = priv_size; 102 m->buf_addr = (char *)m + mbuf_size; 103 m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; 104 m->buf_len = (uint16_t)buf_len; 105 106 /* keep some headroom between start of buffer and data */ 107 m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->buf_len); 108 109 /* init some constant fields */ 110 m->pool = mp; 111 m->nb_segs = 1; 112 m->port = RTE_MBUF_PORT_INVALID; 113 rte_mbuf_refcnt_set(m, 1); 114 m->next = NULL; 115 } 116 117 /* 118 * @internal The callback routine called when reference counter in shinfo 119 * for mbufs with pinned external buffer reaches zero. It means there is 120 * no more reference to buffer backing mbuf and this one should be freed. 121 * This routine is called for the regular (not with pinned external or 122 * indirect buffer) mbufs on detaching from the mbuf with pinned external 123 * buffer. 124 */ 125 static void 126 rte_pktmbuf_free_pinned_extmem(void *addr, void *opaque) 127 { 128 struct rte_mbuf *m = opaque; 129 130 RTE_SET_USED(addr); 131 RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m)); 132 RTE_ASSERT(RTE_MBUF_HAS_PINNED_EXTBUF(m)); 133 RTE_ASSERT(m->shinfo->fcb_opaque == m); 134 135 rte_mbuf_ext_refcnt_set(m->shinfo, 1); 136 m->ol_flags = RTE_MBUF_F_EXTERNAL; 137 if (m->next != NULL) 138 m->next = NULL; 139 if (m->nb_segs != 1) 140 m->nb_segs = 1; 141 rte_mbuf_raw_free(m); 142 } 143 144 /** The context to initialize the mbufs with pinned external buffers. */ 145 struct rte_pktmbuf_extmem_init_ctx { 146 const struct rte_pktmbuf_extmem *ext_mem; /* descriptor array. */ 147 unsigned int ext_num; /* number of descriptors in array. */ 148 unsigned int ext; /* loop descriptor index. */ 149 size_t off; /* loop buffer offset. */ 150 }; 151 152 /** 153 * @internal Packet mbuf constructor for pools with pinned external memory. 154 * 155 * This function initializes some fields in the mbuf structure that are 156 * not modified by the user once created (origin pool, buffer start 157 * address, and so on). This function is given as a callback function to 158 * rte_mempool_obj_iter() called from rte_mempool_create_extmem(). 159 * 160 * @param mp 161 * The mempool from which mbufs originate. 162 * @param opaque_arg 163 * A pointer to the rte_pktmbuf_extmem_init_ctx - initialization 164 * context structure 165 * @param m 166 * The mbuf to initialize. 167 * @param i 168 * The index of the mbuf in the pool table. 169 */ 170 static void 171 __rte_pktmbuf_init_extmem(struct rte_mempool *mp, 172 void *opaque_arg, 173 void *_m, 174 __rte_unused unsigned int i) 175 { 176 struct rte_mbuf *m = _m; 177 struct rte_pktmbuf_extmem_init_ctx *ctx = opaque_arg; 178 const struct rte_pktmbuf_extmem *ext_mem; 179 uint32_t mbuf_size, buf_len, priv_size; 180 struct rte_mbuf_ext_shared_info *shinfo; 181 182 priv_size = rte_pktmbuf_priv_size(mp); 183 mbuf_size = sizeof(struct rte_mbuf) + priv_size; 184 buf_len = rte_pktmbuf_data_room_size(mp); 185 186 RTE_ASSERT(RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) == priv_size); 187 RTE_ASSERT(mp->elt_size >= mbuf_size); 188 RTE_ASSERT(buf_len <= UINT16_MAX); 189 190 memset(m, 0, mbuf_size); 191 m->priv_size = priv_size; 192 m->buf_len = (uint16_t)buf_len; 193 194 /* set the data buffer pointers to external memory */ 195 ext_mem = ctx->ext_mem + ctx->ext; 196 197 RTE_ASSERT(ctx->ext < ctx->ext_num); 198 RTE_ASSERT(ctx->off + ext_mem->elt_size <= ext_mem->buf_len); 199 200 m->buf_addr = RTE_PTR_ADD(ext_mem->buf_ptr, ctx->off); 201 m->buf_iova = ext_mem->buf_iova == RTE_BAD_IOVA ? 202 RTE_BAD_IOVA : (ext_mem->buf_iova + ctx->off); 203 204 ctx->off += ext_mem->elt_size; 205 if (ctx->off + ext_mem->elt_size > ext_mem->buf_len) { 206 ctx->off = 0; 207 ++ctx->ext; 208 } 209 /* keep some headroom between start of buffer and data */ 210 m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->buf_len); 211 212 /* init some constant fields */ 213 m->pool = mp; 214 m->nb_segs = 1; 215 m->port = RTE_MBUF_PORT_INVALID; 216 m->ol_flags = RTE_MBUF_F_EXTERNAL; 217 rte_mbuf_refcnt_set(m, 1); 218 m->next = NULL; 219 220 /* init external buffer shared info items */ 221 shinfo = RTE_PTR_ADD(m, mbuf_size); 222 m->shinfo = shinfo; 223 shinfo->free_cb = rte_pktmbuf_free_pinned_extmem; 224 shinfo->fcb_opaque = m; 225 rte_mbuf_ext_refcnt_set(shinfo, 1); 226 } 227 228 /* Helper to create a mbuf pool with given mempool ops name*/ 229 struct rte_mempool * 230 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, 231 unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, 232 int socket_id, const char *ops_name) 233 { 234 struct rte_mempool *mp; 235 struct rte_pktmbuf_pool_private mbp_priv; 236 const char *mp_ops_name = ops_name; 237 unsigned elt_size; 238 int ret; 239 240 if (RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) != priv_size) { 241 RTE_LOG(ERR, MBUF, "mbuf priv_size=%u is not aligned\n", 242 priv_size); 243 rte_errno = EINVAL; 244 return NULL; 245 } 246 elt_size = sizeof(struct rte_mbuf) + (unsigned)priv_size + 247 (unsigned)data_room_size; 248 memset(&mbp_priv, 0, sizeof(mbp_priv)); 249 mbp_priv.mbuf_data_room_size = data_room_size; 250 mbp_priv.mbuf_priv_size = priv_size; 251 252 mp = rte_mempool_create_empty(name, n, elt_size, cache_size, 253 sizeof(struct rte_pktmbuf_pool_private), socket_id, 0); 254 if (mp == NULL) 255 return NULL; 256 257 if (mp_ops_name == NULL) 258 mp_ops_name = rte_mbuf_best_mempool_ops(); 259 ret = rte_mempool_set_ops_byname(mp, mp_ops_name, NULL); 260 if (ret != 0) { 261 RTE_LOG(ERR, MBUF, "error setting mempool handler\n"); 262 rte_mempool_free(mp); 263 rte_errno = -ret; 264 return NULL; 265 } 266 rte_pktmbuf_pool_init(mp, &mbp_priv); 267 268 ret = rte_mempool_populate_default(mp); 269 if (ret < 0) { 270 rte_mempool_free(mp); 271 rte_errno = -ret; 272 return NULL; 273 } 274 275 rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL); 276 277 return mp; 278 } 279 280 /* helper to create a mbuf pool */ 281 struct rte_mempool * 282 rte_pktmbuf_pool_create(const char *name, unsigned int n, 283 unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, 284 int socket_id) 285 { 286 return rte_pktmbuf_pool_create_by_ops(name, n, cache_size, priv_size, 287 data_room_size, socket_id, NULL); 288 } 289 290 /* Helper to create a mbuf pool with pinned external data buffers. */ 291 struct rte_mempool * 292 rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, 293 unsigned int cache_size, uint16_t priv_size, 294 uint16_t data_room_size, int socket_id, 295 const struct rte_pktmbuf_extmem *ext_mem, 296 unsigned int ext_num) 297 { 298 struct rte_mempool *mp; 299 struct rte_pktmbuf_pool_private mbp_priv; 300 struct rte_pktmbuf_extmem_init_ctx init_ctx; 301 const char *mp_ops_name; 302 unsigned int elt_size; 303 unsigned int i, n_elts = 0; 304 int ret; 305 306 if (RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) != priv_size) { 307 RTE_LOG(ERR, MBUF, "mbuf priv_size=%u is not aligned\n", 308 priv_size); 309 rte_errno = EINVAL; 310 return NULL; 311 } 312 /* Check the external memory descriptors. */ 313 for (i = 0; i < ext_num; i++) { 314 const struct rte_pktmbuf_extmem *extm = ext_mem + i; 315 316 if (!extm->elt_size || !extm->buf_len || !extm->buf_ptr) { 317 RTE_LOG(ERR, MBUF, "invalid extmem descriptor\n"); 318 rte_errno = EINVAL; 319 return NULL; 320 } 321 if (data_room_size > extm->elt_size) { 322 RTE_LOG(ERR, MBUF, "ext elt_size=%u is too small\n", 323 priv_size); 324 rte_errno = EINVAL; 325 return NULL; 326 } 327 n_elts += extm->buf_len / extm->elt_size; 328 } 329 /* Check whether enough external memory provided. */ 330 if (n_elts < n) { 331 RTE_LOG(ERR, MBUF, "not enough extmem\n"); 332 rte_errno = ENOMEM; 333 return NULL; 334 } 335 elt_size = sizeof(struct rte_mbuf) + 336 (unsigned int)priv_size + 337 sizeof(struct rte_mbuf_ext_shared_info); 338 339 memset(&mbp_priv, 0, sizeof(mbp_priv)); 340 mbp_priv.mbuf_data_room_size = data_room_size; 341 mbp_priv.mbuf_priv_size = priv_size; 342 mbp_priv.flags = RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF; 343 344 mp = rte_mempool_create_empty(name, n, elt_size, cache_size, 345 sizeof(struct rte_pktmbuf_pool_private), socket_id, 0); 346 if (mp == NULL) 347 return NULL; 348 349 mp_ops_name = rte_mbuf_best_mempool_ops(); 350 ret = rte_mempool_set_ops_byname(mp, mp_ops_name, NULL); 351 if (ret != 0) { 352 RTE_LOG(ERR, MBUF, "error setting mempool handler\n"); 353 rte_mempool_free(mp); 354 rte_errno = -ret; 355 return NULL; 356 } 357 rte_pktmbuf_pool_init(mp, &mbp_priv); 358 359 ret = rte_mempool_populate_default(mp); 360 if (ret < 0) { 361 rte_mempool_free(mp); 362 rte_errno = -ret; 363 return NULL; 364 } 365 366 init_ctx = (struct rte_pktmbuf_extmem_init_ctx){ 367 .ext_mem = ext_mem, 368 .ext_num = ext_num, 369 .ext = 0, 370 .off = 0, 371 }; 372 rte_mempool_obj_iter(mp, __rte_pktmbuf_init_extmem, &init_ctx); 373 374 return mp; 375 } 376 377 /* do some sanity checks on a mbuf: panic if it fails */ 378 void 379 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header) 380 { 381 const char *reason; 382 383 if (rte_mbuf_check(m, is_header, &reason)) 384 rte_panic("%s\n", reason); 385 } 386 387 int rte_mbuf_check(const struct rte_mbuf *m, int is_header, 388 const char **reason) 389 { 390 unsigned int nb_segs, pkt_len; 391 392 if (m == NULL) { 393 *reason = "mbuf is NULL"; 394 return -1; 395 } 396 397 /* generic checks */ 398 if (m->pool == NULL) { 399 *reason = "bad mbuf pool"; 400 return -1; 401 } 402 if (m->buf_iova == 0) { 403 *reason = "bad IO addr"; 404 return -1; 405 } 406 if (m->buf_addr == NULL) { 407 *reason = "bad virt addr"; 408 return -1; 409 } 410 411 uint16_t cnt = rte_mbuf_refcnt_read(m); 412 if ((cnt == 0) || (cnt == UINT16_MAX)) { 413 *reason = "bad ref cnt"; 414 return -1; 415 } 416 417 /* nothing to check for sub-segments */ 418 if (is_header == 0) 419 return 0; 420 421 /* data_len is supposed to be not more than pkt_len */ 422 if (m->data_len > m->pkt_len) { 423 *reason = "bad data_len"; 424 return -1; 425 } 426 427 nb_segs = m->nb_segs; 428 pkt_len = m->pkt_len; 429 430 do { 431 if (m->data_off > m->buf_len) { 432 *reason = "data offset too big in mbuf segment"; 433 return -1; 434 } 435 if (m->data_off + m->data_len > m->buf_len) { 436 *reason = "data length too big in mbuf segment"; 437 return -1; 438 } 439 nb_segs -= 1; 440 pkt_len -= m->data_len; 441 } while ((m = m->next) != NULL); 442 443 if (nb_segs) { 444 *reason = "bad nb_segs"; 445 return -1; 446 } 447 if (pkt_len) { 448 *reason = "bad pkt_len"; 449 return -1; 450 } 451 452 return 0; 453 } 454 455 /** 456 * @internal helper function for freeing a bulk of packet mbuf segments 457 * via an array holding the packet mbuf segments from the same mempool 458 * pending to be freed. 459 * 460 * @param m 461 * The packet mbuf segment to be freed. 462 * @param pending 463 * Pointer to the array of packet mbuf segments pending to be freed. 464 * @param nb_pending 465 * Pointer to the number of elements held in the array. 466 * @param pending_sz 467 * Number of elements the array can hold. 468 * Note: The compiler should optimize this parameter away when using a 469 * constant value, such as RTE_PKTMBUF_FREE_PENDING_SZ. 470 */ 471 static void 472 __rte_pktmbuf_free_seg_via_array(struct rte_mbuf *m, 473 struct rte_mbuf ** const pending, unsigned int * const nb_pending, 474 const unsigned int pending_sz) 475 { 476 m = rte_pktmbuf_prefree_seg(m); 477 if (likely(m != NULL)) { 478 if (*nb_pending == pending_sz || 479 (*nb_pending > 0 && m->pool != pending[0]->pool)) { 480 rte_mempool_put_bulk(pending[0]->pool, 481 (void **)pending, *nb_pending); 482 *nb_pending = 0; 483 } 484 485 pending[(*nb_pending)++] = m; 486 } 487 } 488 489 /** 490 * Size of the array holding mbufs from the same mempool pending to be freed 491 * in bulk. 492 */ 493 #define RTE_PKTMBUF_FREE_PENDING_SZ 64 494 495 /* Free a bulk of packet mbufs back into their original mempools. */ 496 void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count) 497 { 498 struct rte_mbuf *m, *m_next, *pending[RTE_PKTMBUF_FREE_PENDING_SZ]; 499 unsigned int idx, nb_pending = 0; 500 501 for (idx = 0; idx < count; idx++) { 502 m = mbufs[idx]; 503 if (unlikely(m == NULL)) 504 continue; 505 506 __rte_mbuf_sanity_check(m, 1); 507 508 do { 509 m_next = m->next; 510 __rte_pktmbuf_free_seg_via_array(m, 511 pending, &nb_pending, 512 RTE_PKTMBUF_FREE_PENDING_SZ); 513 m = m_next; 514 } while (m != NULL); 515 } 516 517 if (nb_pending > 0) 518 rte_mempool_put_bulk(pending[0]->pool, (void **)pending, nb_pending); 519 } 520 521 /* Creates a shallow copy of mbuf */ 522 struct rte_mbuf * 523 rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp) 524 { 525 struct rte_mbuf *mc, *mi, **prev; 526 uint32_t pktlen; 527 uint16_t nseg; 528 529 mc = rte_pktmbuf_alloc(mp); 530 if (unlikely(mc == NULL)) 531 return NULL; 532 533 mi = mc; 534 prev = &mi->next; 535 pktlen = md->pkt_len; 536 nseg = 0; 537 538 do { 539 nseg++; 540 rte_pktmbuf_attach(mi, md); 541 *prev = mi; 542 prev = &mi->next; 543 } while ((md = md->next) != NULL && 544 (mi = rte_pktmbuf_alloc(mp)) != NULL); 545 546 *prev = NULL; 547 mc->nb_segs = nseg; 548 mc->pkt_len = pktlen; 549 550 /* Allocation of new indirect segment failed */ 551 if (unlikely(mi == NULL)) { 552 rte_pktmbuf_free(mc); 553 return NULL; 554 } 555 556 __rte_mbuf_sanity_check(mc, 1); 557 return mc; 558 } 559 560 /* convert multi-segment mbuf to single mbuf */ 561 int 562 __rte_pktmbuf_linearize(struct rte_mbuf *mbuf) 563 { 564 size_t seg_len, copy_len; 565 struct rte_mbuf *m; 566 struct rte_mbuf *m_next; 567 char *buffer; 568 569 /* Extend first segment to the total packet length */ 570 copy_len = rte_pktmbuf_pkt_len(mbuf) - rte_pktmbuf_data_len(mbuf); 571 572 if (unlikely(copy_len > rte_pktmbuf_tailroom(mbuf))) 573 return -1; 574 575 buffer = rte_pktmbuf_mtod_offset(mbuf, char *, mbuf->data_len); 576 mbuf->data_len = (uint16_t)(mbuf->pkt_len); 577 578 /* Append data from next segments to the first one */ 579 m = mbuf->next; 580 while (m != NULL) { 581 m_next = m->next; 582 583 seg_len = rte_pktmbuf_data_len(m); 584 rte_memcpy(buffer, rte_pktmbuf_mtod(m, char *), seg_len); 585 buffer += seg_len; 586 587 rte_pktmbuf_free_seg(m); 588 m = m_next; 589 } 590 591 mbuf->next = NULL; 592 mbuf->nb_segs = 1; 593 594 return 0; 595 } 596 597 /* Create a deep copy of mbuf */ 598 struct rte_mbuf * 599 rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, 600 uint32_t off, uint32_t len) 601 { 602 const struct rte_mbuf *seg = m; 603 struct rte_mbuf *mc, *m_last, **prev; 604 605 /* garbage in check */ 606 __rte_mbuf_sanity_check(m, 1); 607 608 /* check for request to copy at offset past end of mbuf */ 609 if (unlikely(off >= m->pkt_len)) 610 return NULL; 611 612 mc = rte_pktmbuf_alloc(mp); 613 if (unlikely(mc == NULL)) 614 return NULL; 615 616 /* truncate requested length to available data */ 617 if (len > m->pkt_len - off) 618 len = m->pkt_len - off; 619 620 __rte_pktmbuf_copy_hdr(mc, m); 621 622 /* copied mbuf is not indirect or external */ 623 mc->ol_flags = m->ol_flags & ~(RTE_MBUF_F_INDIRECT|RTE_MBUF_F_EXTERNAL); 624 625 prev = &mc->next; 626 m_last = mc; 627 while (len > 0) { 628 uint32_t copy_len; 629 630 /* skip leading mbuf segments */ 631 while (off >= seg->data_len) { 632 off -= seg->data_len; 633 seg = seg->next; 634 } 635 636 /* current buffer is full, chain a new one */ 637 if (rte_pktmbuf_tailroom(m_last) == 0) { 638 m_last = rte_pktmbuf_alloc(mp); 639 if (unlikely(m_last == NULL)) { 640 rte_pktmbuf_free(mc); 641 return NULL; 642 } 643 ++mc->nb_segs; 644 *prev = m_last; 645 prev = &m_last->next; 646 } 647 648 /* 649 * copy the min of data in input segment (seg) 650 * vs space available in output (m_last) 651 */ 652 copy_len = RTE_MIN(seg->data_len - off, len); 653 if (copy_len > rte_pktmbuf_tailroom(m_last)) 654 copy_len = rte_pktmbuf_tailroom(m_last); 655 656 /* append from seg to m_last */ 657 rte_memcpy(rte_pktmbuf_mtod_offset(m_last, char *, 658 m_last->data_len), 659 rte_pktmbuf_mtod_offset(seg, char *, off), 660 copy_len); 661 662 /* update offsets and lengths */ 663 m_last->data_len += copy_len; 664 mc->pkt_len += copy_len; 665 off += copy_len; 666 len -= copy_len; 667 } 668 669 /* garbage out check */ 670 __rte_mbuf_sanity_check(mc, 1); 671 return mc; 672 } 673 674 /* dump a mbuf on console */ 675 void 676 rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len) 677 { 678 unsigned int len; 679 unsigned int nb_segs; 680 681 __rte_mbuf_sanity_check(m, 1); 682 683 fprintf(f, "dump mbuf at %p, iova=%#"PRIx64", buf_len=%u\n", 684 m, m->buf_iova, m->buf_len); 685 fprintf(f, " pkt_len=%u, ol_flags=%#"PRIx64", nb_segs=%u, port=%u", 686 m->pkt_len, m->ol_flags, m->nb_segs, m->port); 687 688 if (m->ol_flags & (RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_TX_VLAN)) 689 fprintf(f, ", vlan_tci=%u", m->vlan_tci); 690 691 fprintf(f, ", ptype=%#"PRIx32"\n", m->packet_type); 692 693 nb_segs = m->nb_segs; 694 695 while (m && nb_segs != 0) { 696 __rte_mbuf_sanity_check(m, 0); 697 698 fprintf(f, " segment at %p, data=%p, len=%u, off=%u, refcnt=%u\n", 699 m, rte_pktmbuf_mtod(m, void *), 700 m->data_len, m->data_off, rte_mbuf_refcnt_read(m)); 701 702 len = dump_len; 703 if (len > m->data_len) 704 len = m->data_len; 705 if (len != 0) 706 rte_hexdump(f, NULL, rte_pktmbuf_mtod(m, void *), len); 707 dump_len -= len; 708 m = m->next; 709 nb_segs --; 710 } 711 } 712 713 /* read len data bytes in a mbuf at specified offset (internal) */ 714 const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, 715 uint32_t len, void *buf) 716 { 717 const struct rte_mbuf *seg = m; 718 uint32_t buf_off = 0, copy_len; 719 720 if (off + len > rte_pktmbuf_pkt_len(m)) 721 return NULL; 722 723 while (off >= rte_pktmbuf_data_len(seg)) { 724 off -= rte_pktmbuf_data_len(seg); 725 seg = seg->next; 726 } 727 728 if (off + len <= rte_pktmbuf_data_len(seg)) 729 return rte_pktmbuf_mtod_offset(seg, char *, off); 730 731 /* rare case: header is split among several segments */ 732 while (len > 0) { 733 copy_len = rte_pktmbuf_data_len(seg) - off; 734 if (copy_len > len) 735 copy_len = len; 736 rte_memcpy((char *)buf + buf_off, 737 rte_pktmbuf_mtod_offset(seg, char *, off), copy_len); 738 off = 0; 739 buf_off += copy_len; 740 len -= copy_len; 741 seg = seg->next; 742 } 743 744 return buf; 745 } 746 747 /* 748 * Get the name of a RX offload flag. Must be kept synchronized with flag 749 * definitions in rte_mbuf.h. 750 */ 751 const char *rte_get_rx_ol_flag_name(uint64_t mask) 752 { 753 switch (mask) { 754 case RTE_MBUF_F_RX_VLAN: return "RTE_MBUF_F_RX_VLAN"; 755 case RTE_MBUF_F_RX_RSS_HASH: return "RTE_MBUF_F_RX_RSS_HASH"; 756 case RTE_MBUF_F_RX_FDIR: return "RTE_MBUF_F_RX_FDIR"; 757 case RTE_MBUF_F_RX_L4_CKSUM_BAD: return "RTE_MBUF_F_RX_L4_CKSUM_BAD"; 758 case RTE_MBUF_F_RX_L4_CKSUM_GOOD: return "RTE_MBUF_F_RX_L4_CKSUM_GOOD"; 759 case RTE_MBUF_F_RX_L4_CKSUM_NONE: return "RTE_MBUF_F_RX_L4_CKSUM_NONE"; 760 case RTE_MBUF_F_RX_IP_CKSUM_BAD: return "RTE_MBUF_F_RX_IP_CKSUM_BAD"; 761 case RTE_MBUF_F_RX_IP_CKSUM_GOOD: return "RTE_MBUF_F_RX_IP_CKSUM_GOOD"; 762 case RTE_MBUF_F_RX_IP_CKSUM_NONE: return "RTE_MBUF_F_RX_IP_CKSUM_NONE"; 763 case RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD: return "RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD"; 764 case RTE_MBUF_F_RX_VLAN_STRIPPED: return "RTE_MBUF_F_RX_VLAN_STRIPPED"; 765 case RTE_MBUF_F_RX_IEEE1588_PTP: return "RTE_MBUF_F_RX_IEEE1588_PTP"; 766 case RTE_MBUF_F_RX_IEEE1588_TMST: return "RTE_MBUF_F_RX_IEEE1588_TMST"; 767 case RTE_MBUF_F_RX_FDIR_ID: return "RTE_MBUF_F_RX_FDIR_ID"; 768 case RTE_MBUF_F_RX_FDIR_FLX: return "RTE_MBUF_F_RX_FDIR_FLX"; 769 case RTE_MBUF_F_RX_QINQ_STRIPPED: return "RTE_MBUF_F_RX_QINQ_STRIPPED"; 770 case RTE_MBUF_F_RX_QINQ: return "RTE_MBUF_F_RX_QINQ"; 771 case RTE_MBUF_F_RX_LRO: return "RTE_MBUF_F_RX_LRO"; 772 case RTE_MBUF_F_RX_SEC_OFFLOAD: return "RTE_MBUF_F_RX_SEC_OFFLOAD"; 773 case RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED: return "RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED"; 774 case RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD: return "RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD"; 775 case RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD: return "RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD"; 776 case RTE_MBUF_F_RX_OUTER_L4_CKSUM_INVALID: 777 return "RTE_MBUF_F_RX_OUTER_L4_CKSUM_INVALID"; 778 779 default: return NULL; 780 } 781 } 782 783 struct flag_mask { 784 uint64_t flag; 785 uint64_t mask; 786 const char *default_name; 787 }; 788 789 /* write the list of rx ol flags in buffer buf */ 790 int 791 rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen) 792 { 793 const struct flag_mask rx_flags[] = { 794 { RTE_MBUF_F_RX_VLAN, RTE_MBUF_F_RX_VLAN, NULL }, 795 { RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, NULL }, 796 { RTE_MBUF_F_RX_FDIR, RTE_MBUF_F_RX_FDIR, NULL }, 797 { RTE_MBUF_F_RX_L4_CKSUM_BAD, RTE_MBUF_F_RX_L4_CKSUM_MASK, NULL }, 798 { RTE_MBUF_F_RX_L4_CKSUM_GOOD, RTE_MBUF_F_RX_L4_CKSUM_MASK, NULL }, 799 { RTE_MBUF_F_RX_L4_CKSUM_NONE, RTE_MBUF_F_RX_L4_CKSUM_MASK, NULL }, 800 { RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN, RTE_MBUF_F_RX_L4_CKSUM_MASK, 801 "RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN" }, 802 { RTE_MBUF_F_RX_IP_CKSUM_BAD, RTE_MBUF_F_RX_IP_CKSUM_MASK, NULL }, 803 { RTE_MBUF_F_RX_IP_CKSUM_GOOD, RTE_MBUF_F_RX_IP_CKSUM_MASK, NULL }, 804 { RTE_MBUF_F_RX_IP_CKSUM_NONE, RTE_MBUF_F_RX_IP_CKSUM_MASK, NULL }, 805 { RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN, RTE_MBUF_F_RX_IP_CKSUM_MASK, 806 "RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN" }, 807 { RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD, RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD, NULL }, 808 { RTE_MBUF_F_RX_VLAN_STRIPPED, RTE_MBUF_F_RX_VLAN_STRIPPED, NULL }, 809 { RTE_MBUF_F_RX_IEEE1588_PTP, RTE_MBUF_F_RX_IEEE1588_PTP, NULL }, 810 { RTE_MBUF_F_RX_IEEE1588_TMST, RTE_MBUF_F_RX_IEEE1588_TMST, NULL }, 811 { RTE_MBUF_F_RX_FDIR_ID, RTE_MBUF_F_RX_FDIR_ID, NULL }, 812 { RTE_MBUF_F_RX_FDIR_FLX, RTE_MBUF_F_RX_FDIR_FLX, NULL }, 813 { RTE_MBUF_F_RX_QINQ_STRIPPED, RTE_MBUF_F_RX_QINQ_STRIPPED, NULL }, 814 { RTE_MBUF_F_RX_LRO, RTE_MBUF_F_RX_LRO, NULL }, 815 { RTE_MBUF_F_RX_SEC_OFFLOAD, RTE_MBUF_F_RX_SEC_OFFLOAD, NULL }, 816 { RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED, RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED, NULL }, 817 { RTE_MBUF_F_RX_QINQ, RTE_MBUF_F_RX_QINQ, NULL }, 818 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, NULL }, 819 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, 820 NULL }, 821 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_INVALID, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, 822 NULL }, 823 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_UNKNOWN, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, 824 "RTE_MBUF_F_RX_OUTER_L4_CKSUM_UNKNOWN" }, 825 }; 826 const char *name; 827 unsigned int i; 828 int ret; 829 830 if (buflen == 0) 831 return -1; 832 833 buf[0] = '\0'; 834 for (i = 0; i < RTE_DIM(rx_flags); i++) { 835 if ((mask & rx_flags[i].mask) != rx_flags[i].flag) 836 continue; 837 name = rte_get_rx_ol_flag_name(rx_flags[i].flag); 838 if (name == NULL) 839 name = rx_flags[i].default_name; 840 ret = snprintf(buf, buflen, "%s ", name); 841 if (ret < 0) 842 return -1; 843 if ((size_t)ret >= buflen) 844 return -1; 845 buf += ret; 846 buflen -= ret; 847 } 848 849 return 0; 850 } 851 852 /* 853 * Get the name of a TX offload flag. Must be kept synchronized with flag 854 * definitions in rte_mbuf.h. 855 */ 856 const char *rte_get_tx_ol_flag_name(uint64_t mask) 857 { 858 switch (mask) { 859 case RTE_MBUF_F_TX_VLAN: return "RTE_MBUF_F_TX_VLAN"; 860 case RTE_MBUF_F_TX_IP_CKSUM: return "RTE_MBUF_F_TX_IP_CKSUM"; 861 case RTE_MBUF_F_TX_TCP_CKSUM: return "RTE_MBUF_F_TX_TCP_CKSUM"; 862 case RTE_MBUF_F_TX_SCTP_CKSUM: return "RTE_MBUF_F_TX_SCTP_CKSUM"; 863 case RTE_MBUF_F_TX_UDP_CKSUM: return "RTE_MBUF_F_TX_UDP_CKSUM"; 864 case RTE_MBUF_F_TX_IEEE1588_TMST: return "RTE_MBUF_F_TX_IEEE1588_TMST"; 865 case RTE_MBUF_F_TX_TCP_SEG: return "RTE_MBUF_F_TX_TCP_SEG"; 866 case RTE_MBUF_F_TX_IPV4: return "RTE_MBUF_F_TX_IPV4"; 867 case RTE_MBUF_F_TX_IPV6: return "RTE_MBUF_F_TX_IPV6"; 868 case RTE_MBUF_F_TX_OUTER_IP_CKSUM: return "RTE_MBUF_F_TX_OUTER_IP_CKSUM"; 869 case RTE_MBUF_F_TX_OUTER_IPV4: return "RTE_MBUF_F_TX_OUTER_IPV4"; 870 case RTE_MBUF_F_TX_OUTER_IPV6: return "RTE_MBUF_F_TX_OUTER_IPV6"; 871 case RTE_MBUF_F_TX_TUNNEL_VXLAN: return "RTE_MBUF_F_TX_TUNNEL_VXLAN"; 872 case RTE_MBUF_F_TX_TUNNEL_GTP: return "RTE_MBUF_F_TX_TUNNEL_GTP"; 873 case RTE_MBUF_F_TX_TUNNEL_GRE: return "RTE_MBUF_F_TX_TUNNEL_GRE"; 874 case RTE_MBUF_F_TX_TUNNEL_IPIP: return "RTE_MBUF_F_TX_TUNNEL_IPIP"; 875 case RTE_MBUF_F_TX_TUNNEL_GENEVE: return "RTE_MBUF_F_TX_TUNNEL_GENEVE"; 876 case RTE_MBUF_F_TX_TUNNEL_MPLSINUDP: return "RTE_MBUF_F_TX_TUNNEL_MPLSINUDP"; 877 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE: return "RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE"; 878 case RTE_MBUF_F_TX_TUNNEL_IP: return "RTE_MBUF_F_TX_TUNNEL_IP"; 879 case RTE_MBUF_F_TX_TUNNEL_UDP: return "RTE_MBUF_F_TX_TUNNEL_UDP"; 880 case RTE_MBUF_F_TX_QINQ: return "RTE_MBUF_F_TX_QINQ"; 881 case RTE_MBUF_F_TX_MACSEC: return "RTE_MBUF_F_TX_MACSEC"; 882 case RTE_MBUF_F_TX_SEC_OFFLOAD: return "RTE_MBUF_F_TX_SEC_OFFLOAD"; 883 case RTE_MBUF_F_TX_UDP_SEG: return "RTE_MBUF_F_TX_UDP_SEG"; 884 case RTE_MBUF_F_TX_OUTER_UDP_CKSUM: return "RTE_MBUF_F_TX_OUTER_UDP_CKSUM"; 885 default: return NULL; 886 } 887 } 888 889 /* write the list of tx ol flags in buffer buf */ 890 int 891 rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen) 892 { 893 const struct flag_mask tx_flags[] = { 894 { RTE_MBUF_F_TX_VLAN, RTE_MBUF_F_TX_VLAN, NULL }, 895 { RTE_MBUF_F_TX_IP_CKSUM, RTE_MBUF_F_TX_IP_CKSUM, NULL }, 896 { RTE_MBUF_F_TX_TCP_CKSUM, RTE_MBUF_F_TX_L4_MASK, NULL }, 897 { RTE_MBUF_F_TX_SCTP_CKSUM, RTE_MBUF_F_TX_L4_MASK, NULL }, 898 { RTE_MBUF_F_TX_UDP_CKSUM, RTE_MBUF_F_TX_L4_MASK, NULL }, 899 { RTE_MBUF_F_TX_L4_NO_CKSUM, RTE_MBUF_F_TX_L4_MASK, "RTE_MBUF_F_TX_L4_NO_CKSUM" }, 900 { RTE_MBUF_F_TX_IEEE1588_TMST, RTE_MBUF_F_TX_IEEE1588_TMST, NULL }, 901 { RTE_MBUF_F_TX_TCP_SEG, RTE_MBUF_F_TX_TCP_SEG, NULL }, 902 { RTE_MBUF_F_TX_IPV4, RTE_MBUF_F_TX_IPV4, NULL }, 903 { RTE_MBUF_F_TX_IPV6, RTE_MBUF_F_TX_IPV6, NULL }, 904 { RTE_MBUF_F_TX_OUTER_IP_CKSUM, RTE_MBUF_F_TX_OUTER_IP_CKSUM, NULL }, 905 { RTE_MBUF_F_TX_OUTER_IPV4, RTE_MBUF_F_TX_OUTER_IPV4, NULL }, 906 { RTE_MBUF_F_TX_OUTER_IPV6, RTE_MBUF_F_TX_OUTER_IPV6, NULL }, 907 { RTE_MBUF_F_TX_TUNNEL_VXLAN, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 908 { RTE_MBUF_F_TX_TUNNEL_GTP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 909 { RTE_MBUF_F_TX_TUNNEL_GRE, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 910 { RTE_MBUF_F_TX_TUNNEL_IPIP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 911 { RTE_MBUF_F_TX_TUNNEL_GENEVE, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 912 { RTE_MBUF_F_TX_TUNNEL_MPLSINUDP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 913 { RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 914 { RTE_MBUF_F_TX_TUNNEL_IP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 915 { RTE_MBUF_F_TX_TUNNEL_UDP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 916 { RTE_MBUF_F_TX_QINQ, RTE_MBUF_F_TX_QINQ, NULL }, 917 { RTE_MBUF_F_TX_MACSEC, RTE_MBUF_F_TX_MACSEC, NULL }, 918 { RTE_MBUF_F_TX_SEC_OFFLOAD, RTE_MBUF_F_TX_SEC_OFFLOAD, NULL }, 919 { RTE_MBUF_F_TX_UDP_SEG, RTE_MBUF_F_TX_UDP_SEG, NULL }, 920 { RTE_MBUF_F_TX_OUTER_UDP_CKSUM, RTE_MBUF_F_TX_OUTER_UDP_CKSUM, NULL }, 921 }; 922 const char *name; 923 unsigned int i; 924 int ret; 925 926 if (buflen == 0) 927 return -1; 928 929 buf[0] = '\0'; 930 for (i = 0; i < RTE_DIM(tx_flags); i++) { 931 if ((mask & tx_flags[i].mask) != tx_flags[i].flag) 932 continue; 933 name = rte_get_tx_ol_flag_name(tx_flags[i].flag); 934 if (name == NULL) 935 name = tx_flags[i].default_name; 936 ret = snprintf(buf, buflen, "%s ", name); 937 if (ret < 0) 938 return -1; 939 if ((size_t)ret >= buflen) 940 return -1; 941 buf += ret; 942 buflen -= ret; 943 } 944 945 return 0; 946 } 947