1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation. 3 * Copyright 2014 6WIND S.A. 4 */ 5 6 #include <string.h> 7 #include <stdio.h> 8 #include <stdint.h> 9 #include <inttypes.h> 10 #include <errno.h> 11 12 #include <rte_debug.h> 13 #include <rte_common.h> 14 #include <rte_log.h> 15 #include <rte_branch_prediction.h> 16 #include <rte_mempool.h> 17 #include <rte_mbuf.h> 18 #include <rte_mbuf_pool_ops.h> 19 #include <rte_hexdump.h> 20 #include <rte_errno.h> 21 #include <rte_memcpy.h> 22 23 #include "mbuf_log.h" 24 25 RTE_LOG_REGISTER_DEFAULT(mbuf_logtype, INFO); 26 27 /* 28 * pktmbuf pool constructor, given as a callback function to 29 * rte_mempool_create(), or called directly if using 30 * rte_mempool_create_empty()/rte_mempool_populate() 31 */ 32 void 33 rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg) 34 { 35 struct rte_pktmbuf_pool_private *user_mbp_priv, *mbp_priv; 36 struct rte_pktmbuf_pool_private default_mbp_priv; 37 uint16_t roomsz; 38 39 RTE_ASSERT(mp->private_data_size >= 40 sizeof(struct rte_pktmbuf_pool_private)); 41 RTE_ASSERT(mp->elt_size >= sizeof(struct rte_mbuf)); 42 43 /* if no structure is provided, assume no mbuf private area */ 44 user_mbp_priv = opaque_arg; 45 if (user_mbp_priv == NULL) { 46 memset(&default_mbp_priv, 0, sizeof(default_mbp_priv)); 47 if (mp->elt_size > sizeof(struct rte_mbuf)) 48 roomsz = mp->elt_size - sizeof(struct rte_mbuf); 49 else 50 roomsz = 0; 51 default_mbp_priv.mbuf_data_room_size = roomsz; 52 user_mbp_priv = &default_mbp_priv; 53 } 54 55 RTE_ASSERT(mp->elt_size >= sizeof(struct rte_mbuf) + 56 ((user_mbp_priv->flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) ? 57 sizeof(struct rte_mbuf_ext_shared_info) : 58 user_mbp_priv->mbuf_data_room_size) + 59 user_mbp_priv->mbuf_priv_size); 60 RTE_ASSERT((user_mbp_priv->flags & 61 ~RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) == 0); 62 63 mbp_priv = rte_mempool_get_priv(mp); 64 memcpy(mbp_priv, user_mbp_priv, sizeof(*mbp_priv)); 65 } 66 67 /* 68 * pktmbuf constructor, given as a callback function to 69 * rte_mempool_obj_iter() or rte_mempool_create(). 70 * Set the fields of a packet mbuf to their default values. 71 */ 72 void 73 rte_pktmbuf_init(struct rte_mempool *mp, 74 __rte_unused void *opaque_arg, 75 void *_m, 76 __rte_unused unsigned i) 77 { 78 struct rte_mbuf *m = _m; 79 uint32_t mbuf_size, buf_len, priv_size; 80 81 RTE_ASSERT(mp->private_data_size >= 82 sizeof(struct rte_pktmbuf_pool_private)); 83 84 priv_size = rte_pktmbuf_priv_size(mp); 85 mbuf_size = sizeof(struct rte_mbuf) + priv_size; 86 buf_len = rte_pktmbuf_data_room_size(mp); 87 88 RTE_ASSERT(RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) == priv_size); 89 RTE_ASSERT(mp->elt_size >= mbuf_size); 90 RTE_ASSERT(buf_len <= UINT16_MAX); 91 92 memset(m, 0, mbuf_size); 93 /* start of buffer is after mbuf structure and priv data */ 94 m->priv_size = priv_size; 95 m->buf_addr = (char *)m + mbuf_size; 96 rte_mbuf_iova_set(m, rte_mempool_virt2iova(m) + mbuf_size); 97 m->buf_len = (uint16_t)buf_len; 98 99 /* keep some headroom between start of buffer and data */ 100 m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->buf_len); 101 102 /* init some constant fields */ 103 m->pool = mp; 104 m->nb_segs = 1; 105 m->port = RTE_MBUF_PORT_INVALID; 106 rte_mbuf_refcnt_set(m, 1); 107 m->next = NULL; 108 } 109 110 /* 111 * @internal The callback routine called when reference counter in shinfo 112 * for mbufs with pinned external buffer reaches zero. It means there is 113 * no more reference to buffer backing mbuf and this one should be freed. 114 * This routine is called for the regular (not with pinned external or 115 * indirect buffer) mbufs on detaching from the mbuf with pinned external 116 * buffer. 117 */ 118 static void 119 rte_pktmbuf_free_pinned_extmem(void *addr, void *opaque) 120 { 121 struct rte_mbuf *m = opaque; 122 123 RTE_SET_USED(addr); 124 RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m)); 125 RTE_ASSERT(RTE_MBUF_HAS_PINNED_EXTBUF(m)); 126 RTE_ASSERT(m->shinfo->fcb_opaque == m); 127 128 rte_mbuf_ext_refcnt_set(m->shinfo, 1); 129 m->ol_flags = RTE_MBUF_F_EXTERNAL; 130 if (m->next != NULL) 131 m->next = NULL; 132 if (m->nb_segs != 1) 133 m->nb_segs = 1; 134 rte_mbuf_raw_free(m); 135 } 136 137 /** The context to initialize the mbufs with pinned external buffers. */ 138 struct rte_pktmbuf_extmem_init_ctx { 139 const struct rte_pktmbuf_extmem *ext_mem; /* descriptor array. */ 140 unsigned int ext_num; /* number of descriptors in array. */ 141 unsigned int ext; /* loop descriptor index. */ 142 size_t off; /* loop buffer offset. */ 143 }; 144 145 /** 146 * @internal Packet mbuf constructor for pools with pinned external memory. 147 * 148 * This function initializes some fields in the mbuf structure that are 149 * not modified by the user once created (origin pool, buffer start 150 * address, and so on). This function is given as a callback function to 151 * rte_mempool_obj_iter() called from rte_mempool_create_extmem(). 152 * 153 * @param mp 154 * The mempool from which mbufs originate. 155 * @param opaque_arg 156 * A pointer to the rte_pktmbuf_extmem_init_ctx - initialization 157 * context structure 158 * @param m 159 * The mbuf to initialize. 160 * @param i 161 * The index of the mbuf in the pool table. 162 */ 163 static void 164 __rte_pktmbuf_init_extmem(struct rte_mempool *mp, 165 void *opaque_arg, 166 void *_m, 167 __rte_unused unsigned int i) 168 { 169 struct rte_mbuf *m = _m; 170 struct rte_pktmbuf_extmem_init_ctx *ctx = opaque_arg; 171 const struct rte_pktmbuf_extmem *ext_mem; 172 uint32_t mbuf_size, buf_len, priv_size; 173 struct rte_mbuf_ext_shared_info *shinfo; 174 175 priv_size = rte_pktmbuf_priv_size(mp); 176 mbuf_size = sizeof(struct rte_mbuf) + priv_size; 177 buf_len = rte_pktmbuf_data_room_size(mp); 178 179 RTE_ASSERT(RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) == priv_size); 180 RTE_ASSERT(mp->elt_size >= mbuf_size); 181 RTE_ASSERT(buf_len <= UINT16_MAX); 182 183 memset(m, 0, mbuf_size); 184 m->priv_size = priv_size; 185 m->buf_len = (uint16_t)buf_len; 186 187 /* set the data buffer pointers to external memory */ 188 ext_mem = ctx->ext_mem + ctx->ext; 189 190 RTE_ASSERT(ctx->ext < ctx->ext_num); 191 RTE_ASSERT(ctx->off + ext_mem->elt_size <= ext_mem->buf_len); 192 193 m->buf_addr = RTE_PTR_ADD(ext_mem->buf_ptr, ctx->off); 194 rte_mbuf_iova_set(m, ext_mem->buf_iova == RTE_BAD_IOVA ? RTE_BAD_IOVA : 195 (ext_mem->buf_iova + ctx->off)); 196 197 ctx->off += ext_mem->elt_size; 198 if (ctx->off + ext_mem->elt_size > ext_mem->buf_len) { 199 ctx->off = 0; 200 ++ctx->ext; 201 } 202 /* keep some headroom between start of buffer and data */ 203 m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->buf_len); 204 205 /* init some constant fields */ 206 m->pool = mp; 207 m->nb_segs = 1; 208 m->port = RTE_MBUF_PORT_INVALID; 209 m->ol_flags = RTE_MBUF_F_EXTERNAL; 210 rte_mbuf_refcnt_set(m, 1); 211 m->next = NULL; 212 213 /* init external buffer shared info items */ 214 shinfo = RTE_PTR_ADD(m, mbuf_size); 215 m->shinfo = shinfo; 216 shinfo->free_cb = rte_pktmbuf_free_pinned_extmem; 217 shinfo->fcb_opaque = m; 218 rte_mbuf_ext_refcnt_set(shinfo, 1); 219 } 220 221 /* Helper to create a mbuf pool with given mempool ops name*/ 222 struct rte_mempool * 223 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, 224 unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, 225 int socket_id, const char *ops_name) 226 { 227 struct rte_mempool *mp; 228 struct rte_pktmbuf_pool_private mbp_priv; 229 const char *mp_ops_name = ops_name; 230 unsigned elt_size; 231 int ret; 232 233 if (RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) != priv_size) { 234 MBUF_LOG(ERR, "mbuf priv_size=%u is not aligned", 235 priv_size); 236 rte_errno = EINVAL; 237 return NULL; 238 } 239 elt_size = sizeof(struct rte_mbuf) + (unsigned)priv_size + 240 (unsigned)data_room_size; 241 memset(&mbp_priv, 0, sizeof(mbp_priv)); 242 mbp_priv.mbuf_data_room_size = data_room_size; 243 mbp_priv.mbuf_priv_size = priv_size; 244 245 mp = rte_mempool_create_empty(name, n, elt_size, cache_size, 246 sizeof(struct rte_pktmbuf_pool_private), socket_id, 0); 247 if (mp == NULL) 248 return NULL; 249 250 if (mp_ops_name == NULL) 251 mp_ops_name = rte_mbuf_best_mempool_ops(); 252 ret = rte_mempool_set_ops_byname(mp, mp_ops_name, NULL); 253 if (ret != 0) { 254 MBUF_LOG(ERR, "error setting mempool handler"); 255 rte_mempool_free(mp); 256 rte_errno = -ret; 257 return NULL; 258 } 259 rte_pktmbuf_pool_init(mp, &mbp_priv); 260 261 ret = rte_mempool_populate_default(mp); 262 if (ret < 0) { 263 rte_mempool_free(mp); 264 rte_errno = -ret; 265 return NULL; 266 } 267 268 rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL); 269 270 return mp; 271 } 272 273 /* helper to create a mbuf pool */ 274 struct rte_mempool * 275 rte_pktmbuf_pool_create(const char *name, unsigned int n, 276 unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, 277 int socket_id) 278 { 279 return rte_pktmbuf_pool_create_by_ops(name, n, cache_size, priv_size, 280 data_room_size, socket_id, NULL); 281 } 282 283 /* Helper to create a mbuf pool with pinned external data buffers. */ 284 struct rte_mempool * 285 rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, 286 unsigned int cache_size, uint16_t priv_size, 287 uint16_t data_room_size, int socket_id, 288 const struct rte_pktmbuf_extmem *ext_mem, 289 unsigned int ext_num) 290 { 291 struct rte_mempool *mp; 292 struct rte_pktmbuf_pool_private mbp_priv; 293 struct rte_pktmbuf_extmem_init_ctx init_ctx; 294 const char *mp_ops_name; 295 unsigned int elt_size; 296 unsigned int i, n_elts = 0; 297 int ret; 298 299 if (RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) != priv_size) { 300 MBUF_LOG(ERR, "mbuf priv_size=%u is not aligned", 301 priv_size); 302 rte_errno = EINVAL; 303 return NULL; 304 } 305 /* Check the external memory descriptors. */ 306 for (i = 0; i < ext_num; i++) { 307 const struct rte_pktmbuf_extmem *extm = ext_mem + i; 308 309 if (!extm->elt_size || !extm->buf_len || !extm->buf_ptr) { 310 MBUF_LOG(ERR, "invalid extmem descriptor"); 311 rte_errno = EINVAL; 312 return NULL; 313 } 314 if (data_room_size > extm->elt_size) { 315 MBUF_LOG(ERR, "ext elt_size=%u is too small", 316 priv_size); 317 rte_errno = EINVAL; 318 return NULL; 319 } 320 n_elts += extm->buf_len / extm->elt_size; 321 } 322 /* Check whether enough external memory provided. */ 323 if (n_elts < n) { 324 MBUF_LOG(ERR, "not enough extmem"); 325 rte_errno = ENOMEM; 326 return NULL; 327 } 328 elt_size = sizeof(struct rte_mbuf) + 329 (unsigned int)priv_size + 330 sizeof(struct rte_mbuf_ext_shared_info); 331 332 memset(&mbp_priv, 0, sizeof(mbp_priv)); 333 mbp_priv.mbuf_data_room_size = data_room_size; 334 mbp_priv.mbuf_priv_size = priv_size; 335 mbp_priv.flags = RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF; 336 337 mp = rte_mempool_create_empty(name, n, elt_size, cache_size, 338 sizeof(struct rte_pktmbuf_pool_private), socket_id, 0); 339 if (mp == NULL) 340 return NULL; 341 342 mp_ops_name = rte_mbuf_best_mempool_ops(); 343 ret = rte_mempool_set_ops_byname(mp, mp_ops_name, NULL); 344 if (ret != 0) { 345 MBUF_LOG(ERR, "error setting mempool handler"); 346 rte_mempool_free(mp); 347 rte_errno = -ret; 348 return NULL; 349 } 350 rte_pktmbuf_pool_init(mp, &mbp_priv); 351 352 ret = rte_mempool_populate_default(mp); 353 if (ret < 0) { 354 rte_mempool_free(mp); 355 rte_errno = -ret; 356 return NULL; 357 } 358 359 init_ctx = (struct rte_pktmbuf_extmem_init_ctx){ 360 .ext_mem = ext_mem, 361 .ext_num = ext_num, 362 .ext = 0, 363 .off = 0, 364 }; 365 rte_mempool_obj_iter(mp, __rte_pktmbuf_init_extmem, &init_ctx); 366 367 return mp; 368 } 369 370 /* do some sanity checks on a mbuf: panic if it fails */ 371 void 372 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header) 373 { 374 const char *reason; 375 376 if (rte_mbuf_check(m, is_header, &reason)) 377 rte_panic("%s\n", reason); 378 } 379 380 int rte_mbuf_check(const struct rte_mbuf *m, int is_header, 381 const char **reason) 382 { 383 unsigned int nb_segs, pkt_len; 384 385 if (m == NULL) { 386 *reason = "mbuf is NULL"; 387 return -1; 388 } 389 390 /* generic checks */ 391 if (m->pool == NULL) { 392 *reason = "bad mbuf pool"; 393 return -1; 394 } 395 if (RTE_IOVA_IN_MBUF && rte_mbuf_iova_get(m) == 0) { 396 *reason = "bad IO addr"; 397 return -1; 398 } 399 if (m->buf_addr == NULL) { 400 *reason = "bad virt addr"; 401 return -1; 402 } 403 404 uint16_t cnt = rte_mbuf_refcnt_read(m); 405 if ((cnt == 0) || (cnt == UINT16_MAX)) { 406 *reason = "bad ref cnt"; 407 return -1; 408 } 409 410 /* nothing to check for sub-segments */ 411 if (is_header == 0) 412 return 0; 413 414 /* data_len is supposed to be not more than pkt_len */ 415 if (m->data_len > m->pkt_len) { 416 *reason = "bad data_len"; 417 return -1; 418 } 419 420 nb_segs = m->nb_segs; 421 pkt_len = m->pkt_len; 422 423 do { 424 if (m->data_off > m->buf_len) { 425 *reason = "data offset too big in mbuf segment"; 426 return -1; 427 } 428 if (m->data_off + m->data_len > m->buf_len) { 429 *reason = "data length too big in mbuf segment"; 430 return -1; 431 } 432 nb_segs -= 1; 433 pkt_len -= m->data_len; 434 } while ((m = m->next) != NULL); 435 436 if (nb_segs) { 437 *reason = "bad nb_segs"; 438 return -1; 439 } 440 if (pkt_len) { 441 *reason = "bad pkt_len"; 442 return -1; 443 } 444 445 return 0; 446 } 447 448 /** 449 * @internal helper function for freeing a bulk of packet mbuf segments 450 * via an array holding the packet mbuf segments from the same mempool 451 * pending to be freed. 452 * 453 * @param m 454 * The packet mbuf segment to be freed. 455 * @param pending 456 * Pointer to the array of packet mbuf segments pending to be freed. 457 * @param nb_pending 458 * Pointer to the number of elements held in the array. 459 * @param pending_sz 460 * Number of elements the array can hold. 461 * Note: The compiler should optimize this parameter away when using a 462 * constant value, such as RTE_PKTMBUF_FREE_PENDING_SZ. 463 */ 464 static void 465 __rte_pktmbuf_free_seg_via_array(struct rte_mbuf *m, 466 struct rte_mbuf ** const pending, unsigned int * const nb_pending, 467 const unsigned int pending_sz) 468 { 469 m = rte_pktmbuf_prefree_seg(m); 470 if (likely(m != NULL)) { 471 if (*nb_pending == pending_sz || 472 (*nb_pending > 0 && m->pool != pending[0]->pool)) { 473 rte_mempool_put_bulk(pending[0]->pool, 474 (void **)pending, *nb_pending); 475 *nb_pending = 0; 476 } 477 478 pending[(*nb_pending)++] = m; 479 } 480 } 481 482 /** 483 * Size of the array holding mbufs from the same mempool pending to be freed 484 * in bulk. 485 */ 486 #define RTE_PKTMBUF_FREE_PENDING_SZ 64 487 488 /* Free a bulk of packet mbufs back into their original mempools. */ 489 void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count) 490 { 491 struct rte_mbuf *m, *m_next, *pending[RTE_PKTMBUF_FREE_PENDING_SZ]; 492 unsigned int idx, nb_pending = 0; 493 494 for (idx = 0; idx < count; idx++) { 495 m = mbufs[idx]; 496 if (unlikely(m == NULL)) 497 continue; 498 499 __rte_mbuf_sanity_check(m, 1); 500 501 do { 502 m_next = m->next; 503 __rte_pktmbuf_free_seg_via_array(m, 504 pending, &nb_pending, 505 RTE_PKTMBUF_FREE_PENDING_SZ); 506 m = m_next; 507 } while (m != NULL); 508 } 509 510 if (nb_pending > 0) 511 rte_mempool_put_bulk(pending[0]->pool, (void **)pending, nb_pending); 512 } 513 514 /* Creates a shallow copy of mbuf */ 515 struct rte_mbuf * 516 rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp) 517 { 518 struct rte_mbuf *mc, *mi, **prev; 519 uint32_t pktlen; 520 uint16_t nseg; 521 522 mc = rte_pktmbuf_alloc(mp); 523 if (unlikely(mc == NULL)) 524 return NULL; 525 526 mi = mc; 527 prev = &mi->next; 528 pktlen = md->pkt_len; 529 nseg = 0; 530 531 do { 532 nseg++; 533 rte_pktmbuf_attach(mi, md); 534 *prev = mi; 535 prev = &mi->next; 536 } while ((md = md->next) != NULL && 537 (mi = rte_pktmbuf_alloc(mp)) != NULL); 538 539 *prev = NULL; 540 mc->nb_segs = nseg; 541 mc->pkt_len = pktlen; 542 543 /* Allocation of new indirect segment failed */ 544 if (unlikely(mi == NULL)) { 545 rte_pktmbuf_free(mc); 546 return NULL; 547 } 548 549 __rte_mbuf_sanity_check(mc, 1); 550 return mc; 551 } 552 553 /* convert multi-segment mbuf to single mbuf */ 554 int 555 __rte_pktmbuf_linearize(struct rte_mbuf *mbuf) 556 { 557 size_t seg_len, copy_len; 558 struct rte_mbuf *m; 559 struct rte_mbuf *m_next; 560 char *buffer; 561 562 /* Extend first segment to the total packet length */ 563 copy_len = rte_pktmbuf_pkt_len(mbuf) - rte_pktmbuf_data_len(mbuf); 564 565 if (unlikely(copy_len > rte_pktmbuf_tailroom(mbuf))) 566 return -1; 567 568 buffer = rte_pktmbuf_mtod_offset(mbuf, char *, mbuf->data_len); 569 mbuf->data_len = (uint16_t)(mbuf->pkt_len); 570 571 /* Append data from next segments to the first one */ 572 m = mbuf->next; 573 while (m != NULL) { 574 m_next = m->next; 575 576 seg_len = rte_pktmbuf_data_len(m); 577 rte_memcpy(buffer, rte_pktmbuf_mtod(m, char *), seg_len); 578 buffer += seg_len; 579 580 rte_pktmbuf_free_seg(m); 581 m = m_next; 582 } 583 584 mbuf->next = NULL; 585 mbuf->nb_segs = 1; 586 587 return 0; 588 } 589 590 /* Create a deep copy of mbuf */ 591 struct rte_mbuf * 592 rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, 593 uint32_t off, uint32_t len) 594 { 595 const struct rte_mbuf *seg = m; 596 struct rte_mbuf *mc, *m_last, **prev; 597 598 /* garbage in check */ 599 __rte_mbuf_sanity_check(m, 1); 600 601 /* check for request to copy at offset past end of mbuf */ 602 if (unlikely(off >= m->pkt_len)) 603 return NULL; 604 605 mc = rte_pktmbuf_alloc(mp); 606 if (unlikely(mc == NULL)) 607 return NULL; 608 609 /* truncate requested length to available data */ 610 if (len > m->pkt_len - off) 611 len = m->pkt_len - off; 612 613 __rte_pktmbuf_copy_hdr(mc, m); 614 615 /* copied mbuf is not indirect or external */ 616 mc->ol_flags = m->ol_flags & ~(RTE_MBUF_F_INDIRECT|RTE_MBUF_F_EXTERNAL); 617 618 prev = &mc->next; 619 m_last = mc; 620 while (len > 0) { 621 uint32_t copy_len; 622 623 /* skip leading mbuf segments */ 624 while (off >= seg->data_len) { 625 off -= seg->data_len; 626 seg = seg->next; 627 } 628 629 /* current buffer is full, chain a new one */ 630 if (rte_pktmbuf_tailroom(m_last) == 0) { 631 m_last = rte_pktmbuf_alloc(mp); 632 if (unlikely(m_last == NULL)) { 633 rte_pktmbuf_free(mc); 634 return NULL; 635 } 636 ++mc->nb_segs; 637 *prev = m_last; 638 prev = &m_last->next; 639 } 640 641 /* 642 * copy the min of data in input segment (seg) 643 * vs space available in output (m_last) 644 */ 645 copy_len = RTE_MIN(seg->data_len - off, len); 646 if (copy_len > rte_pktmbuf_tailroom(m_last)) 647 copy_len = rte_pktmbuf_tailroom(m_last); 648 649 /* append from seg to m_last */ 650 rte_memcpy(rte_pktmbuf_mtod_offset(m_last, char *, 651 m_last->data_len), 652 rte_pktmbuf_mtod_offset(seg, char *, off), 653 copy_len); 654 655 /* update offsets and lengths */ 656 m_last->data_len += copy_len; 657 mc->pkt_len += copy_len; 658 off += copy_len; 659 len -= copy_len; 660 } 661 662 /* garbage out check */ 663 __rte_mbuf_sanity_check(mc, 1); 664 return mc; 665 } 666 667 /* dump a mbuf on console */ 668 void 669 rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len) 670 { 671 unsigned int len; 672 unsigned int nb_segs; 673 674 __rte_mbuf_sanity_check(m, 1); 675 676 fprintf(f, "dump mbuf at %p, iova=%#" PRIx64 ", buf_len=%u\n", m, rte_mbuf_iova_get(m), 677 m->buf_len); 678 fprintf(f, " pkt_len=%u, ol_flags=%#"PRIx64", nb_segs=%u, port=%u", 679 m->pkt_len, m->ol_flags, m->nb_segs, m->port); 680 681 if (m->ol_flags & (RTE_MBUF_F_RX_QINQ | RTE_MBUF_F_TX_QINQ)) 682 fprintf(f, ", vlan_tci_outer=%u", m->vlan_tci_outer); 683 684 if (m->ol_flags & (RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_TX_VLAN)) 685 fprintf(f, ", vlan_tci=%u", m->vlan_tci); 686 687 fprintf(f, ", ptype=%#"PRIx32"\n", m->packet_type); 688 689 nb_segs = m->nb_segs; 690 691 while (m && nb_segs != 0) { 692 __rte_mbuf_sanity_check(m, 0); 693 694 fprintf(f, " segment at %p, data=%p, len=%u, off=%u, refcnt=%u\n", 695 m, rte_pktmbuf_mtod(m, void *), 696 m->data_len, m->data_off, rte_mbuf_refcnt_read(m)); 697 698 len = dump_len; 699 if (len > m->data_len) 700 len = m->data_len; 701 if (len != 0) 702 rte_hexdump(f, NULL, rte_pktmbuf_mtod(m, void *), len); 703 dump_len -= len; 704 m = m->next; 705 nb_segs --; 706 } 707 } 708 709 /* read len data bytes in a mbuf at specified offset (internal) */ 710 const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, 711 uint32_t len, void *buf) 712 { 713 const struct rte_mbuf *seg = m; 714 uint32_t buf_off = 0, copy_len; 715 716 if (off + len > rte_pktmbuf_pkt_len(m)) 717 return NULL; 718 719 while (off >= rte_pktmbuf_data_len(seg)) { 720 off -= rte_pktmbuf_data_len(seg); 721 seg = seg->next; 722 } 723 724 if (off + len <= rte_pktmbuf_data_len(seg)) 725 return rte_pktmbuf_mtod_offset(seg, char *, off); 726 727 /* rare case: header is split among several segments */ 728 while (len > 0) { 729 copy_len = rte_pktmbuf_data_len(seg) - off; 730 if (copy_len > len) 731 copy_len = len; 732 rte_memcpy((char *)buf + buf_off, 733 rte_pktmbuf_mtod_offset(seg, char *, off), copy_len); 734 off = 0; 735 buf_off += copy_len; 736 len -= copy_len; 737 seg = seg->next; 738 } 739 740 return buf; 741 } 742 743 /* 744 * Get the name of a RX offload flag. Must be kept synchronized with flag 745 * definitions in rte_mbuf.h. 746 */ 747 const char *rte_get_rx_ol_flag_name(uint64_t mask) 748 { 749 switch (mask) { 750 case RTE_MBUF_F_RX_VLAN: return "RTE_MBUF_F_RX_VLAN"; 751 case RTE_MBUF_F_RX_RSS_HASH: return "RTE_MBUF_F_RX_RSS_HASH"; 752 case RTE_MBUF_F_RX_FDIR: return "RTE_MBUF_F_RX_FDIR"; 753 case RTE_MBUF_F_RX_L4_CKSUM_BAD: return "RTE_MBUF_F_RX_L4_CKSUM_BAD"; 754 case RTE_MBUF_F_RX_L4_CKSUM_GOOD: return "RTE_MBUF_F_RX_L4_CKSUM_GOOD"; 755 case RTE_MBUF_F_RX_L4_CKSUM_NONE: return "RTE_MBUF_F_RX_L4_CKSUM_NONE"; 756 case RTE_MBUF_F_RX_IP_CKSUM_BAD: return "RTE_MBUF_F_RX_IP_CKSUM_BAD"; 757 case RTE_MBUF_F_RX_IP_CKSUM_GOOD: return "RTE_MBUF_F_RX_IP_CKSUM_GOOD"; 758 case RTE_MBUF_F_RX_IP_CKSUM_NONE: return "RTE_MBUF_F_RX_IP_CKSUM_NONE"; 759 case RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD: return "RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD"; 760 case RTE_MBUF_F_RX_VLAN_STRIPPED: return "RTE_MBUF_F_RX_VLAN_STRIPPED"; 761 case RTE_MBUF_F_RX_IEEE1588_PTP: return "RTE_MBUF_F_RX_IEEE1588_PTP"; 762 case RTE_MBUF_F_RX_IEEE1588_TMST: return "RTE_MBUF_F_RX_IEEE1588_TMST"; 763 case RTE_MBUF_F_RX_FDIR_ID: return "RTE_MBUF_F_RX_FDIR_ID"; 764 case RTE_MBUF_F_RX_FDIR_FLX: return "RTE_MBUF_F_RX_FDIR_FLX"; 765 case RTE_MBUF_F_RX_QINQ_STRIPPED: return "RTE_MBUF_F_RX_QINQ_STRIPPED"; 766 case RTE_MBUF_F_RX_QINQ: return "RTE_MBUF_F_RX_QINQ"; 767 case RTE_MBUF_F_RX_LRO: return "RTE_MBUF_F_RX_LRO"; 768 case RTE_MBUF_F_RX_SEC_OFFLOAD: return "RTE_MBUF_F_RX_SEC_OFFLOAD"; 769 case RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED: return "RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED"; 770 case RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD: return "RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD"; 771 case RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD: return "RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD"; 772 case RTE_MBUF_F_RX_OUTER_L4_CKSUM_INVALID: 773 return "RTE_MBUF_F_RX_OUTER_L4_CKSUM_INVALID"; 774 775 default: return NULL; 776 } 777 } 778 779 struct flag_mask { 780 uint64_t flag; 781 uint64_t mask; 782 const char *default_name; 783 }; 784 785 /* write the list of rx ol flags in buffer buf */ 786 int 787 rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen) 788 { 789 const struct flag_mask rx_flags[] = { 790 { RTE_MBUF_F_RX_VLAN, RTE_MBUF_F_RX_VLAN, NULL }, 791 { RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, NULL }, 792 { RTE_MBUF_F_RX_FDIR, RTE_MBUF_F_RX_FDIR, NULL }, 793 { RTE_MBUF_F_RX_L4_CKSUM_BAD, RTE_MBUF_F_RX_L4_CKSUM_MASK, NULL }, 794 { RTE_MBUF_F_RX_L4_CKSUM_GOOD, RTE_MBUF_F_RX_L4_CKSUM_MASK, NULL }, 795 { RTE_MBUF_F_RX_L4_CKSUM_NONE, RTE_MBUF_F_RX_L4_CKSUM_MASK, NULL }, 796 { RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN, RTE_MBUF_F_RX_L4_CKSUM_MASK, 797 "RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN" }, 798 { RTE_MBUF_F_RX_IP_CKSUM_BAD, RTE_MBUF_F_RX_IP_CKSUM_MASK, NULL }, 799 { RTE_MBUF_F_RX_IP_CKSUM_GOOD, RTE_MBUF_F_RX_IP_CKSUM_MASK, NULL }, 800 { RTE_MBUF_F_RX_IP_CKSUM_NONE, RTE_MBUF_F_RX_IP_CKSUM_MASK, NULL }, 801 { RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN, RTE_MBUF_F_RX_IP_CKSUM_MASK, 802 "RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN" }, 803 { RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD, RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD, NULL }, 804 { RTE_MBUF_F_RX_VLAN_STRIPPED, RTE_MBUF_F_RX_VLAN_STRIPPED, NULL }, 805 { RTE_MBUF_F_RX_IEEE1588_PTP, RTE_MBUF_F_RX_IEEE1588_PTP, NULL }, 806 { RTE_MBUF_F_RX_IEEE1588_TMST, RTE_MBUF_F_RX_IEEE1588_TMST, NULL }, 807 { RTE_MBUF_F_RX_FDIR_ID, RTE_MBUF_F_RX_FDIR_ID, NULL }, 808 { RTE_MBUF_F_RX_FDIR_FLX, RTE_MBUF_F_RX_FDIR_FLX, NULL }, 809 { RTE_MBUF_F_RX_QINQ_STRIPPED, RTE_MBUF_F_RX_QINQ_STRIPPED, NULL }, 810 { RTE_MBUF_F_RX_LRO, RTE_MBUF_F_RX_LRO, NULL }, 811 { RTE_MBUF_F_RX_SEC_OFFLOAD, RTE_MBUF_F_RX_SEC_OFFLOAD, NULL }, 812 { RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED, RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED, NULL }, 813 { RTE_MBUF_F_RX_QINQ, RTE_MBUF_F_RX_QINQ, NULL }, 814 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, NULL }, 815 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, 816 NULL }, 817 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_INVALID, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, 818 NULL }, 819 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_UNKNOWN, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, 820 "RTE_MBUF_F_RX_OUTER_L4_CKSUM_UNKNOWN" }, 821 }; 822 const char *name; 823 unsigned int i; 824 int ret; 825 826 if (buflen == 0) 827 return -1; 828 829 buf[0] = '\0'; 830 for (i = 0; i < RTE_DIM(rx_flags); i++) { 831 if ((mask & rx_flags[i].mask) != rx_flags[i].flag) 832 continue; 833 name = rte_get_rx_ol_flag_name(rx_flags[i].flag); 834 if (name == NULL) 835 name = rx_flags[i].default_name; 836 ret = snprintf(buf, buflen, "%s ", name); 837 if (ret < 0) 838 return -1; 839 if ((size_t)ret >= buflen) 840 return -1; 841 buf += ret; 842 buflen -= ret; 843 } 844 845 return 0; 846 } 847 848 /* 849 * Get the name of a TX offload flag. Must be kept synchronized with flag 850 * definitions in rte_mbuf.h. 851 */ 852 const char *rte_get_tx_ol_flag_name(uint64_t mask) 853 { 854 switch (mask) { 855 case RTE_MBUF_F_TX_VLAN: return "RTE_MBUF_F_TX_VLAN"; 856 case RTE_MBUF_F_TX_IP_CKSUM: return "RTE_MBUF_F_TX_IP_CKSUM"; 857 case RTE_MBUF_F_TX_TCP_CKSUM: return "RTE_MBUF_F_TX_TCP_CKSUM"; 858 case RTE_MBUF_F_TX_SCTP_CKSUM: return "RTE_MBUF_F_TX_SCTP_CKSUM"; 859 case RTE_MBUF_F_TX_UDP_CKSUM: return "RTE_MBUF_F_TX_UDP_CKSUM"; 860 case RTE_MBUF_F_TX_IEEE1588_TMST: return "RTE_MBUF_F_TX_IEEE1588_TMST"; 861 case RTE_MBUF_F_TX_TCP_SEG: return "RTE_MBUF_F_TX_TCP_SEG"; 862 case RTE_MBUF_F_TX_IPV4: return "RTE_MBUF_F_TX_IPV4"; 863 case RTE_MBUF_F_TX_IPV6: return "RTE_MBUF_F_TX_IPV6"; 864 case RTE_MBUF_F_TX_OUTER_IP_CKSUM: return "RTE_MBUF_F_TX_OUTER_IP_CKSUM"; 865 case RTE_MBUF_F_TX_OUTER_IPV4: return "RTE_MBUF_F_TX_OUTER_IPV4"; 866 case RTE_MBUF_F_TX_OUTER_IPV6: return "RTE_MBUF_F_TX_OUTER_IPV6"; 867 case RTE_MBUF_F_TX_TUNNEL_VXLAN: return "RTE_MBUF_F_TX_TUNNEL_VXLAN"; 868 case RTE_MBUF_F_TX_TUNNEL_GTP: return "RTE_MBUF_F_TX_TUNNEL_GTP"; 869 case RTE_MBUF_F_TX_TUNNEL_GRE: return "RTE_MBUF_F_TX_TUNNEL_GRE"; 870 case RTE_MBUF_F_TX_TUNNEL_IPIP: return "RTE_MBUF_F_TX_TUNNEL_IPIP"; 871 case RTE_MBUF_F_TX_TUNNEL_GENEVE: return "RTE_MBUF_F_TX_TUNNEL_GENEVE"; 872 case RTE_MBUF_F_TX_TUNNEL_MPLSINUDP: return "RTE_MBUF_F_TX_TUNNEL_MPLSINUDP"; 873 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE: return "RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE"; 874 case RTE_MBUF_F_TX_TUNNEL_IP: return "RTE_MBUF_F_TX_TUNNEL_IP"; 875 case RTE_MBUF_F_TX_TUNNEL_UDP: return "RTE_MBUF_F_TX_TUNNEL_UDP"; 876 case RTE_MBUF_F_TX_QINQ: return "RTE_MBUF_F_TX_QINQ"; 877 case RTE_MBUF_F_TX_MACSEC: return "RTE_MBUF_F_TX_MACSEC"; 878 case RTE_MBUF_F_TX_SEC_OFFLOAD: return "RTE_MBUF_F_TX_SEC_OFFLOAD"; 879 case RTE_MBUF_F_TX_UDP_SEG: return "RTE_MBUF_F_TX_UDP_SEG"; 880 case RTE_MBUF_F_TX_OUTER_UDP_CKSUM: return "RTE_MBUF_F_TX_OUTER_UDP_CKSUM"; 881 default: return NULL; 882 } 883 } 884 885 /* write the list of tx ol flags in buffer buf */ 886 int 887 rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen) 888 { 889 const struct flag_mask tx_flags[] = { 890 { RTE_MBUF_F_TX_VLAN, RTE_MBUF_F_TX_VLAN, NULL }, 891 { RTE_MBUF_F_TX_IP_CKSUM, RTE_MBUF_F_TX_IP_CKSUM, NULL }, 892 { RTE_MBUF_F_TX_TCP_CKSUM, RTE_MBUF_F_TX_L4_MASK, NULL }, 893 { RTE_MBUF_F_TX_SCTP_CKSUM, RTE_MBUF_F_TX_L4_MASK, NULL }, 894 { RTE_MBUF_F_TX_UDP_CKSUM, RTE_MBUF_F_TX_L4_MASK, NULL }, 895 { RTE_MBUF_F_TX_L4_NO_CKSUM, RTE_MBUF_F_TX_L4_MASK, "RTE_MBUF_F_TX_L4_NO_CKSUM" }, 896 { RTE_MBUF_F_TX_IEEE1588_TMST, RTE_MBUF_F_TX_IEEE1588_TMST, NULL }, 897 { RTE_MBUF_F_TX_TCP_SEG, RTE_MBUF_F_TX_TCP_SEG, NULL }, 898 { RTE_MBUF_F_TX_IPV4, RTE_MBUF_F_TX_IPV4, NULL }, 899 { RTE_MBUF_F_TX_IPV6, RTE_MBUF_F_TX_IPV6, NULL }, 900 { RTE_MBUF_F_TX_OUTER_IP_CKSUM, RTE_MBUF_F_TX_OUTER_IP_CKSUM, NULL }, 901 { RTE_MBUF_F_TX_OUTER_IPV4, RTE_MBUF_F_TX_OUTER_IPV4, NULL }, 902 { RTE_MBUF_F_TX_OUTER_IPV6, RTE_MBUF_F_TX_OUTER_IPV6, NULL }, 903 { RTE_MBUF_F_TX_TUNNEL_VXLAN, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 904 { RTE_MBUF_F_TX_TUNNEL_GTP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 905 { RTE_MBUF_F_TX_TUNNEL_GRE, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 906 { RTE_MBUF_F_TX_TUNNEL_IPIP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 907 { RTE_MBUF_F_TX_TUNNEL_GENEVE, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 908 { RTE_MBUF_F_TX_TUNNEL_MPLSINUDP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 909 { RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 910 { RTE_MBUF_F_TX_TUNNEL_IP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 911 { RTE_MBUF_F_TX_TUNNEL_UDP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 912 { RTE_MBUF_F_TX_QINQ, RTE_MBUF_F_TX_QINQ, NULL }, 913 { RTE_MBUF_F_TX_MACSEC, RTE_MBUF_F_TX_MACSEC, NULL }, 914 { RTE_MBUF_F_TX_SEC_OFFLOAD, RTE_MBUF_F_TX_SEC_OFFLOAD, NULL }, 915 { RTE_MBUF_F_TX_UDP_SEG, RTE_MBUF_F_TX_UDP_SEG, NULL }, 916 { RTE_MBUF_F_TX_OUTER_UDP_CKSUM, RTE_MBUF_F_TX_OUTER_UDP_CKSUM, NULL }, 917 }; 918 const char *name; 919 unsigned int i; 920 int ret; 921 922 if (buflen == 0) 923 return -1; 924 925 buf[0] = '\0'; 926 for (i = 0; i < RTE_DIM(tx_flags); i++) { 927 if ((mask & tx_flags[i].mask) != tx_flags[i].flag) 928 continue; 929 name = rte_get_tx_ol_flag_name(tx_flags[i].flag); 930 if (name == NULL) 931 name = tx_flags[i].default_name; 932 ret = snprintf(buf, buflen, "%s ", name); 933 if (ret < 0) 934 return -1; 935 if ((size_t)ret >= buflen) 936 return -1; 937 buf += ret; 938 buflen -= ret; 939 } 940 941 return 0; 942 } 943