1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation. 3 * Copyright 2014 6WIND S.A. 4 */ 5 6 #include <string.h> 7 #include <stdio.h> 8 #include <stdint.h> 9 #include <inttypes.h> 10 #include <errno.h> 11 12 #include <rte_debug.h> 13 #include <rte_common.h> 14 #include <rte_log.h> 15 #include <rte_branch_prediction.h> 16 #include <rte_mempool.h> 17 #include <rte_mbuf.h> 18 #include <rte_mbuf_pool_ops.h> 19 #include <rte_hexdump.h> 20 #include <rte_errno.h> 21 #include <rte_memcpy.h> 22 23 /* 24 * pktmbuf pool constructor, given as a callback function to 25 * rte_mempool_create(), or called directly if using 26 * rte_mempool_create_empty()/rte_mempool_populate() 27 */ 28 void 29 rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg) 30 { 31 struct rte_pktmbuf_pool_private *user_mbp_priv, *mbp_priv; 32 struct rte_pktmbuf_pool_private default_mbp_priv; 33 uint16_t roomsz; 34 35 RTE_ASSERT(mp->private_data_size >= 36 sizeof(struct rte_pktmbuf_pool_private)); 37 RTE_ASSERT(mp->elt_size >= sizeof(struct rte_mbuf)); 38 39 /* if no structure is provided, assume no mbuf private area */ 40 user_mbp_priv = opaque_arg; 41 if (user_mbp_priv == NULL) { 42 memset(&default_mbp_priv, 0, sizeof(default_mbp_priv)); 43 if (mp->elt_size > sizeof(struct rte_mbuf)) 44 roomsz = mp->elt_size - sizeof(struct rte_mbuf); 45 else 46 roomsz = 0; 47 default_mbp_priv.mbuf_data_room_size = roomsz; 48 user_mbp_priv = &default_mbp_priv; 49 } 50 51 RTE_ASSERT(mp->elt_size >= sizeof(struct rte_mbuf) + 52 ((user_mbp_priv->flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) ? 53 sizeof(struct rte_mbuf_ext_shared_info) : 54 user_mbp_priv->mbuf_data_room_size) + 55 user_mbp_priv->mbuf_priv_size); 56 RTE_ASSERT((user_mbp_priv->flags & 57 ~RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) == 0); 58 59 mbp_priv = rte_mempool_get_priv(mp); 60 memcpy(mbp_priv, user_mbp_priv, sizeof(*mbp_priv)); 61 } 62 63 /* 64 * pktmbuf constructor, given as a callback function to 65 * rte_mempool_obj_iter() or rte_mempool_create(). 66 * Set the fields of a packet mbuf to their default values. 67 */ 68 void 69 rte_pktmbuf_init(struct rte_mempool *mp, 70 __rte_unused void *opaque_arg, 71 void *_m, 72 __rte_unused unsigned i) 73 { 74 struct rte_mbuf *m = _m; 75 uint32_t mbuf_size, buf_len, priv_size; 76 77 RTE_ASSERT(mp->private_data_size >= 78 sizeof(struct rte_pktmbuf_pool_private)); 79 80 priv_size = rte_pktmbuf_priv_size(mp); 81 mbuf_size = sizeof(struct rte_mbuf) + priv_size; 82 buf_len = rte_pktmbuf_data_room_size(mp); 83 84 RTE_ASSERT(RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) == priv_size); 85 RTE_ASSERT(mp->elt_size >= mbuf_size); 86 RTE_ASSERT(buf_len <= UINT16_MAX); 87 88 memset(m, 0, mbuf_size); 89 /* start of buffer is after mbuf structure and priv data */ 90 m->priv_size = priv_size; 91 m->buf_addr = (char *)m + mbuf_size; 92 m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; 93 m->buf_len = (uint16_t)buf_len; 94 95 /* keep some headroom between start of buffer and data */ 96 m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->buf_len); 97 98 /* init some constant fields */ 99 m->pool = mp; 100 m->nb_segs = 1; 101 m->port = RTE_MBUF_PORT_INVALID; 102 rte_mbuf_refcnt_set(m, 1); 103 m->next = NULL; 104 } 105 106 /* 107 * @internal The callback routine called when reference counter in shinfo 108 * for mbufs with pinned external buffer reaches zero. It means there is 109 * no more reference to buffer backing mbuf and this one should be freed. 110 * This routine is called for the regular (not with pinned external or 111 * indirect buffer) mbufs on detaching from the mbuf with pinned external 112 * buffer. 113 */ 114 static void 115 rte_pktmbuf_free_pinned_extmem(void *addr, void *opaque) 116 { 117 struct rte_mbuf *m = opaque; 118 119 RTE_SET_USED(addr); 120 RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m)); 121 RTE_ASSERT(RTE_MBUF_HAS_PINNED_EXTBUF(m)); 122 RTE_ASSERT(m->shinfo->fcb_opaque == m); 123 124 rte_mbuf_ext_refcnt_set(m->shinfo, 1); 125 m->ol_flags = RTE_MBUF_F_EXTERNAL; 126 if (m->next != NULL) 127 m->next = NULL; 128 if (m->nb_segs != 1) 129 m->nb_segs = 1; 130 rte_mbuf_raw_free(m); 131 } 132 133 /** The context to initialize the mbufs with pinned external buffers. */ 134 struct rte_pktmbuf_extmem_init_ctx { 135 const struct rte_pktmbuf_extmem *ext_mem; /* descriptor array. */ 136 unsigned int ext_num; /* number of descriptors in array. */ 137 unsigned int ext; /* loop descriptor index. */ 138 size_t off; /* loop buffer offset. */ 139 }; 140 141 /** 142 * @internal Packet mbuf constructor for pools with pinned external memory. 143 * 144 * This function initializes some fields in the mbuf structure that are 145 * not modified by the user once created (origin pool, buffer start 146 * address, and so on). This function is given as a callback function to 147 * rte_mempool_obj_iter() called from rte_mempool_create_extmem(). 148 * 149 * @param mp 150 * The mempool from which mbufs originate. 151 * @param opaque_arg 152 * A pointer to the rte_pktmbuf_extmem_init_ctx - initialization 153 * context structure 154 * @param m 155 * The mbuf to initialize. 156 * @param i 157 * The index of the mbuf in the pool table. 158 */ 159 static void 160 __rte_pktmbuf_init_extmem(struct rte_mempool *mp, 161 void *opaque_arg, 162 void *_m, 163 __rte_unused unsigned int i) 164 { 165 struct rte_mbuf *m = _m; 166 struct rte_pktmbuf_extmem_init_ctx *ctx = opaque_arg; 167 const struct rte_pktmbuf_extmem *ext_mem; 168 uint32_t mbuf_size, buf_len, priv_size; 169 struct rte_mbuf_ext_shared_info *shinfo; 170 171 priv_size = rte_pktmbuf_priv_size(mp); 172 mbuf_size = sizeof(struct rte_mbuf) + priv_size; 173 buf_len = rte_pktmbuf_data_room_size(mp); 174 175 RTE_ASSERT(RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) == priv_size); 176 RTE_ASSERT(mp->elt_size >= mbuf_size); 177 RTE_ASSERT(buf_len <= UINT16_MAX); 178 179 memset(m, 0, mbuf_size); 180 m->priv_size = priv_size; 181 m->buf_len = (uint16_t)buf_len; 182 183 /* set the data buffer pointers to external memory */ 184 ext_mem = ctx->ext_mem + ctx->ext; 185 186 RTE_ASSERT(ctx->ext < ctx->ext_num); 187 RTE_ASSERT(ctx->off + ext_mem->elt_size <= ext_mem->buf_len); 188 189 m->buf_addr = RTE_PTR_ADD(ext_mem->buf_ptr, ctx->off); 190 m->buf_iova = ext_mem->buf_iova == RTE_BAD_IOVA ? 191 RTE_BAD_IOVA : (ext_mem->buf_iova + ctx->off); 192 193 ctx->off += ext_mem->elt_size; 194 if (ctx->off + ext_mem->elt_size > ext_mem->buf_len) { 195 ctx->off = 0; 196 ++ctx->ext; 197 } 198 /* keep some headroom between start of buffer and data */ 199 m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->buf_len); 200 201 /* init some constant fields */ 202 m->pool = mp; 203 m->nb_segs = 1; 204 m->port = RTE_MBUF_PORT_INVALID; 205 m->ol_flags = RTE_MBUF_F_EXTERNAL; 206 rte_mbuf_refcnt_set(m, 1); 207 m->next = NULL; 208 209 /* init external buffer shared info items */ 210 shinfo = RTE_PTR_ADD(m, mbuf_size); 211 m->shinfo = shinfo; 212 shinfo->free_cb = rte_pktmbuf_free_pinned_extmem; 213 shinfo->fcb_opaque = m; 214 rte_mbuf_ext_refcnt_set(shinfo, 1); 215 } 216 217 /* Helper to create a mbuf pool with given mempool ops name*/ 218 struct rte_mempool * 219 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, 220 unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, 221 int socket_id, const char *ops_name) 222 { 223 struct rte_mempool *mp; 224 struct rte_pktmbuf_pool_private mbp_priv; 225 const char *mp_ops_name = ops_name; 226 unsigned elt_size; 227 int ret; 228 229 if (RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) != priv_size) { 230 RTE_LOG(ERR, MBUF, "mbuf priv_size=%u is not aligned\n", 231 priv_size); 232 rte_errno = EINVAL; 233 return NULL; 234 } 235 elt_size = sizeof(struct rte_mbuf) + (unsigned)priv_size + 236 (unsigned)data_room_size; 237 memset(&mbp_priv, 0, sizeof(mbp_priv)); 238 mbp_priv.mbuf_data_room_size = data_room_size; 239 mbp_priv.mbuf_priv_size = priv_size; 240 241 mp = rte_mempool_create_empty(name, n, elt_size, cache_size, 242 sizeof(struct rte_pktmbuf_pool_private), socket_id, 0); 243 if (mp == NULL) 244 return NULL; 245 246 if (mp_ops_name == NULL) 247 mp_ops_name = rte_mbuf_best_mempool_ops(); 248 ret = rte_mempool_set_ops_byname(mp, mp_ops_name, NULL); 249 if (ret != 0) { 250 RTE_LOG(ERR, MBUF, "error setting mempool handler\n"); 251 rte_mempool_free(mp); 252 rte_errno = -ret; 253 return NULL; 254 } 255 rte_pktmbuf_pool_init(mp, &mbp_priv); 256 257 ret = rte_mempool_populate_default(mp); 258 if (ret < 0) { 259 rte_mempool_free(mp); 260 rte_errno = -ret; 261 return NULL; 262 } 263 264 rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL); 265 266 return mp; 267 } 268 269 /* helper to create a mbuf pool */ 270 struct rte_mempool * 271 rte_pktmbuf_pool_create(const char *name, unsigned int n, 272 unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, 273 int socket_id) 274 { 275 return rte_pktmbuf_pool_create_by_ops(name, n, cache_size, priv_size, 276 data_room_size, socket_id, NULL); 277 } 278 279 /* Helper to create a mbuf pool with pinned external data buffers. */ 280 struct rte_mempool * 281 rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, 282 unsigned int cache_size, uint16_t priv_size, 283 uint16_t data_room_size, int socket_id, 284 const struct rte_pktmbuf_extmem *ext_mem, 285 unsigned int ext_num) 286 { 287 struct rte_mempool *mp; 288 struct rte_pktmbuf_pool_private mbp_priv; 289 struct rte_pktmbuf_extmem_init_ctx init_ctx; 290 const char *mp_ops_name; 291 unsigned int elt_size; 292 unsigned int i, n_elts = 0; 293 int ret; 294 295 if (RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) != priv_size) { 296 RTE_LOG(ERR, MBUF, "mbuf priv_size=%u is not aligned\n", 297 priv_size); 298 rte_errno = EINVAL; 299 return NULL; 300 } 301 /* Check the external memory descriptors. */ 302 for (i = 0; i < ext_num; i++) { 303 const struct rte_pktmbuf_extmem *extm = ext_mem + i; 304 305 if (!extm->elt_size || !extm->buf_len || !extm->buf_ptr) { 306 RTE_LOG(ERR, MBUF, "invalid extmem descriptor\n"); 307 rte_errno = EINVAL; 308 return NULL; 309 } 310 if (data_room_size > extm->elt_size) { 311 RTE_LOG(ERR, MBUF, "ext elt_size=%u is too small\n", 312 priv_size); 313 rte_errno = EINVAL; 314 return NULL; 315 } 316 n_elts += extm->buf_len / extm->elt_size; 317 } 318 /* Check whether enough external memory provided. */ 319 if (n_elts < n) { 320 RTE_LOG(ERR, MBUF, "not enough extmem\n"); 321 rte_errno = ENOMEM; 322 return NULL; 323 } 324 elt_size = sizeof(struct rte_mbuf) + 325 (unsigned int)priv_size + 326 sizeof(struct rte_mbuf_ext_shared_info); 327 328 memset(&mbp_priv, 0, sizeof(mbp_priv)); 329 mbp_priv.mbuf_data_room_size = data_room_size; 330 mbp_priv.mbuf_priv_size = priv_size; 331 mbp_priv.flags = RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF; 332 333 mp = rte_mempool_create_empty(name, n, elt_size, cache_size, 334 sizeof(struct rte_pktmbuf_pool_private), socket_id, 0); 335 if (mp == NULL) 336 return NULL; 337 338 mp_ops_name = rte_mbuf_best_mempool_ops(); 339 ret = rte_mempool_set_ops_byname(mp, mp_ops_name, NULL); 340 if (ret != 0) { 341 RTE_LOG(ERR, MBUF, "error setting mempool handler\n"); 342 rte_mempool_free(mp); 343 rte_errno = -ret; 344 return NULL; 345 } 346 rte_pktmbuf_pool_init(mp, &mbp_priv); 347 348 ret = rte_mempool_populate_default(mp); 349 if (ret < 0) { 350 rte_mempool_free(mp); 351 rte_errno = -ret; 352 return NULL; 353 } 354 355 init_ctx = (struct rte_pktmbuf_extmem_init_ctx){ 356 .ext_mem = ext_mem, 357 .ext_num = ext_num, 358 .ext = 0, 359 .off = 0, 360 }; 361 rte_mempool_obj_iter(mp, __rte_pktmbuf_init_extmem, &init_ctx); 362 363 return mp; 364 } 365 366 /* do some sanity checks on a mbuf: panic if it fails */ 367 void 368 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header) 369 { 370 const char *reason; 371 372 if (rte_mbuf_check(m, is_header, &reason)) 373 rte_panic("%s\n", reason); 374 } 375 376 int rte_mbuf_check(const struct rte_mbuf *m, int is_header, 377 const char **reason) 378 { 379 unsigned int nb_segs, pkt_len; 380 381 if (m == NULL) { 382 *reason = "mbuf is NULL"; 383 return -1; 384 } 385 386 /* generic checks */ 387 if (m->pool == NULL) { 388 *reason = "bad mbuf pool"; 389 return -1; 390 } 391 if (m->buf_iova == 0) { 392 *reason = "bad IO addr"; 393 return -1; 394 } 395 if (m->buf_addr == NULL) { 396 *reason = "bad virt addr"; 397 return -1; 398 } 399 400 uint16_t cnt = rte_mbuf_refcnt_read(m); 401 if ((cnt == 0) || (cnt == UINT16_MAX)) { 402 *reason = "bad ref cnt"; 403 return -1; 404 } 405 406 /* nothing to check for sub-segments */ 407 if (is_header == 0) 408 return 0; 409 410 /* data_len is supposed to be not more than pkt_len */ 411 if (m->data_len > m->pkt_len) { 412 *reason = "bad data_len"; 413 return -1; 414 } 415 416 nb_segs = m->nb_segs; 417 pkt_len = m->pkt_len; 418 419 do { 420 if (m->data_off > m->buf_len) { 421 *reason = "data offset too big in mbuf segment"; 422 return -1; 423 } 424 if (m->data_off + m->data_len > m->buf_len) { 425 *reason = "data length too big in mbuf segment"; 426 return -1; 427 } 428 nb_segs -= 1; 429 pkt_len -= m->data_len; 430 } while ((m = m->next) != NULL); 431 432 if (nb_segs) { 433 *reason = "bad nb_segs"; 434 return -1; 435 } 436 if (pkt_len) { 437 *reason = "bad pkt_len"; 438 return -1; 439 } 440 441 return 0; 442 } 443 444 /** 445 * @internal helper function for freeing a bulk of packet mbuf segments 446 * via an array holding the packet mbuf segments from the same mempool 447 * pending to be freed. 448 * 449 * @param m 450 * The packet mbuf segment to be freed. 451 * @param pending 452 * Pointer to the array of packet mbuf segments pending to be freed. 453 * @param nb_pending 454 * Pointer to the number of elements held in the array. 455 * @param pending_sz 456 * Number of elements the array can hold. 457 * Note: The compiler should optimize this parameter away when using a 458 * constant value, such as RTE_PKTMBUF_FREE_PENDING_SZ. 459 */ 460 static void 461 __rte_pktmbuf_free_seg_via_array(struct rte_mbuf *m, 462 struct rte_mbuf ** const pending, unsigned int * const nb_pending, 463 const unsigned int pending_sz) 464 { 465 m = rte_pktmbuf_prefree_seg(m); 466 if (likely(m != NULL)) { 467 if (*nb_pending == pending_sz || 468 (*nb_pending > 0 && m->pool != pending[0]->pool)) { 469 rte_mempool_put_bulk(pending[0]->pool, 470 (void **)pending, *nb_pending); 471 *nb_pending = 0; 472 } 473 474 pending[(*nb_pending)++] = m; 475 } 476 } 477 478 /** 479 * Size of the array holding mbufs from the same mempool pending to be freed 480 * in bulk. 481 */ 482 #define RTE_PKTMBUF_FREE_PENDING_SZ 64 483 484 /* Free a bulk of packet mbufs back into their original mempools. */ 485 void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count) 486 { 487 struct rte_mbuf *m, *m_next, *pending[RTE_PKTMBUF_FREE_PENDING_SZ]; 488 unsigned int idx, nb_pending = 0; 489 490 for (idx = 0; idx < count; idx++) { 491 m = mbufs[idx]; 492 if (unlikely(m == NULL)) 493 continue; 494 495 __rte_mbuf_sanity_check(m, 1); 496 497 do { 498 m_next = m->next; 499 __rte_pktmbuf_free_seg_via_array(m, 500 pending, &nb_pending, 501 RTE_PKTMBUF_FREE_PENDING_SZ); 502 m = m_next; 503 } while (m != NULL); 504 } 505 506 if (nb_pending > 0) 507 rte_mempool_put_bulk(pending[0]->pool, (void **)pending, nb_pending); 508 } 509 510 /* Creates a shallow copy of mbuf */ 511 struct rte_mbuf * 512 rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp) 513 { 514 struct rte_mbuf *mc, *mi, **prev; 515 uint32_t pktlen; 516 uint16_t nseg; 517 518 mc = rte_pktmbuf_alloc(mp); 519 if (unlikely(mc == NULL)) 520 return NULL; 521 522 mi = mc; 523 prev = &mi->next; 524 pktlen = md->pkt_len; 525 nseg = 0; 526 527 do { 528 nseg++; 529 rte_pktmbuf_attach(mi, md); 530 *prev = mi; 531 prev = &mi->next; 532 } while ((md = md->next) != NULL && 533 (mi = rte_pktmbuf_alloc(mp)) != NULL); 534 535 *prev = NULL; 536 mc->nb_segs = nseg; 537 mc->pkt_len = pktlen; 538 539 /* Allocation of new indirect segment failed */ 540 if (unlikely(mi == NULL)) { 541 rte_pktmbuf_free(mc); 542 return NULL; 543 } 544 545 __rte_mbuf_sanity_check(mc, 1); 546 return mc; 547 } 548 549 /* convert multi-segment mbuf to single mbuf */ 550 int 551 __rte_pktmbuf_linearize(struct rte_mbuf *mbuf) 552 { 553 size_t seg_len, copy_len; 554 struct rte_mbuf *m; 555 struct rte_mbuf *m_next; 556 char *buffer; 557 558 /* Extend first segment to the total packet length */ 559 copy_len = rte_pktmbuf_pkt_len(mbuf) - rte_pktmbuf_data_len(mbuf); 560 561 if (unlikely(copy_len > rte_pktmbuf_tailroom(mbuf))) 562 return -1; 563 564 buffer = rte_pktmbuf_mtod_offset(mbuf, char *, mbuf->data_len); 565 mbuf->data_len = (uint16_t)(mbuf->pkt_len); 566 567 /* Append data from next segments to the first one */ 568 m = mbuf->next; 569 while (m != NULL) { 570 m_next = m->next; 571 572 seg_len = rte_pktmbuf_data_len(m); 573 rte_memcpy(buffer, rte_pktmbuf_mtod(m, char *), seg_len); 574 buffer += seg_len; 575 576 rte_pktmbuf_free_seg(m); 577 m = m_next; 578 } 579 580 mbuf->next = NULL; 581 mbuf->nb_segs = 1; 582 583 return 0; 584 } 585 586 /* Create a deep copy of mbuf */ 587 struct rte_mbuf * 588 rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp, 589 uint32_t off, uint32_t len) 590 { 591 const struct rte_mbuf *seg = m; 592 struct rte_mbuf *mc, *m_last, **prev; 593 594 /* garbage in check */ 595 __rte_mbuf_sanity_check(m, 1); 596 597 /* check for request to copy at offset past end of mbuf */ 598 if (unlikely(off >= m->pkt_len)) 599 return NULL; 600 601 mc = rte_pktmbuf_alloc(mp); 602 if (unlikely(mc == NULL)) 603 return NULL; 604 605 /* truncate requested length to available data */ 606 if (len > m->pkt_len - off) 607 len = m->pkt_len - off; 608 609 __rte_pktmbuf_copy_hdr(mc, m); 610 611 /* copied mbuf is not indirect or external */ 612 mc->ol_flags = m->ol_flags & ~(RTE_MBUF_F_INDIRECT|RTE_MBUF_F_EXTERNAL); 613 614 prev = &mc->next; 615 m_last = mc; 616 while (len > 0) { 617 uint32_t copy_len; 618 619 /* skip leading mbuf segments */ 620 while (off >= seg->data_len) { 621 off -= seg->data_len; 622 seg = seg->next; 623 } 624 625 /* current buffer is full, chain a new one */ 626 if (rte_pktmbuf_tailroom(m_last) == 0) { 627 m_last = rte_pktmbuf_alloc(mp); 628 if (unlikely(m_last == NULL)) { 629 rte_pktmbuf_free(mc); 630 return NULL; 631 } 632 ++mc->nb_segs; 633 *prev = m_last; 634 prev = &m_last->next; 635 } 636 637 /* 638 * copy the min of data in input segment (seg) 639 * vs space available in output (m_last) 640 */ 641 copy_len = RTE_MIN(seg->data_len - off, len); 642 if (copy_len > rte_pktmbuf_tailroom(m_last)) 643 copy_len = rte_pktmbuf_tailroom(m_last); 644 645 /* append from seg to m_last */ 646 rte_memcpy(rte_pktmbuf_mtod_offset(m_last, char *, 647 m_last->data_len), 648 rte_pktmbuf_mtod_offset(seg, char *, off), 649 copy_len); 650 651 /* update offsets and lengths */ 652 m_last->data_len += copy_len; 653 mc->pkt_len += copy_len; 654 off += copy_len; 655 len -= copy_len; 656 } 657 658 /* garbage out check */ 659 __rte_mbuf_sanity_check(mc, 1); 660 return mc; 661 } 662 663 /* dump a mbuf on console */ 664 void 665 rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len) 666 { 667 unsigned int len; 668 unsigned int nb_segs; 669 670 __rte_mbuf_sanity_check(m, 1); 671 672 fprintf(f, "dump mbuf at %p, iova=%#"PRIx64", buf_len=%u\n", 673 m, m->buf_iova, m->buf_len); 674 fprintf(f, " pkt_len=%u, ol_flags=%#"PRIx64", nb_segs=%u, port=%u", 675 m->pkt_len, m->ol_flags, m->nb_segs, m->port); 676 677 if (m->ol_flags & (RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_TX_VLAN)) 678 fprintf(f, ", vlan_tci=%u", m->vlan_tci); 679 680 fprintf(f, ", ptype=%#"PRIx32"\n", m->packet_type); 681 682 nb_segs = m->nb_segs; 683 684 while (m && nb_segs != 0) { 685 __rte_mbuf_sanity_check(m, 0); 686 687 fprintf(f, " segment at %p, data=%p, len=%u, off=%u, refcnt=%u\n", 688 m, rte_pktmbuf_mtod(m, void *), 689 m->data_len, m->data_off, rte_mbuf_refcnt_read(m)); 690 691 len = dump_len; 692 if (len > m->data_len) 693 len = m->data_len; 694 if (len != 0) 695 rte_hexdump(f, NULL, rte_pktmbuf_mtod(m, void *), len); 696 dump_len -= len; 697 m = m->next; 698 nb_segs --; 699 } 700 } 701 702 /* read len data bytes in a mbuf at specified offset (internal) */ 703 const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, 704 uint32_t len, void *buf) 705 { 706 const struct rte_mbuf *seg = m; 707 uint32_t buf_off = 0, copy_len; 708 709 if (off + len > rte_pktmbuf_pkt_len(m)) 710 return NULL; 711 712 while (off >= rte_pktmbuf_data_len(seg)) { 713 off -= rte_pktmbuf_data_len(seg); 714 seg = seg->next; 715 } 716 717 if (off + len <= rte_pktmbuf_data_len(seg)) 718 return rte_pktmbuf_mtod_offset(seg, char *, off); 719 720 /* rare case: header is split among several segments */ 721 while (len > 0) { 722 copy_len = rte_pktmbuf_data_len(seg) - off; 723 if (copy_len > len) 724 copy_len = len; 725 rte_memcpy((char *)buf + buf_off, 726 rte_pktmbuf_mtod_offset(seg, char *, off), copy_len); 727 off = 0; 728 buf_off += copy_len; 729 len -= copy_len; 730 seg = seg->next; 731 } 732 733 return buf; 734 } 735 736 /* 737 * Get the name of a RX offload flag. Must be kept synchronized with flag 738 * definitions in rte_mbuf.h. 739 */ 740 const char *rte_get_rx_ol_flag_name(uint64_t mask) 741 { 742 switch (mask) { 743 case RTE_MBUF_F_RX_VLAN: return "RTE_MBUF_F_RX_VLAN"; 744 case RTE_MBUF_F_RX_RSS_HASH: return "RTE_MBUF_F_RX_RSS_HASH"; 745 case RTE_MBUF_F_RX_FDIR: return "RTE_MBUF_F_RX_FDIR"; 746 case RTE_MBUF_F_RX_L4_CKSUM_BAD: return "RTE_MBUF_F_RX_L4_CKSUM_BAD"; 747 case RTE_MBUF_F_RX_L4_CKSUM_GOOD: return "RTE_MBUF_F_RX_L4_CKSUM_GOOD"; 748 case RTE_MBUF_F_RX_L4_CKSUM_NONE: return "RTE_MBUF_F_RX_L4_CKSUM_NONE"; 749 case RTE_MBUF_F_RX_IP_CKSUM_BAD: return "RTE_MBUF_F_RX_IP_CKSUM_BAD"; 750 case RTE_MBUF_F_RX_IP_CKSUM_GOOD: return "RTE_MBUF_F_RX_IP_CKSUM_GOOD"; 751 case RTE_MBUF_F_RX_IP_CKSUM_NONE: return "RTE_MBUF_F_RX_IP_CKSUM_NONE"; 752 case RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD: return "RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD"; 753 case RTE_MBUF_F_RX_VLAN_STRIPPED: return "RTE_MBUF_F_RX_VLAN_STRIPPED"; 754 case RTE_MBUF_F_RX_IEEE1588_PTP: return "RTE_MBUF_F_RX_IEEE1588_PTP"; 755 case RTE_MBUF_F_RX_IEEE1588_TMST: return "RTE_MBUF_F_RX_IEEE1588_TMST"; 756 case RTE_MBUF_F_RX_FDIR_ID: return "RTE_MBUF_F_RX_FDIR_ID"; 757 case RTE_MBUF_F_RX_FDIR_FLX: return "RTE_MBUF_F_RX_FDIR_FLX"; 758 case RTE_MBUF_F_RX_QINQ_STRIPPED: return "RTE_MBUF_F_RX_QINQ_STRIPPED"; 759 case RTE_MBUF_F_RX_QINQ: return "RTE_MBUF_F_RX_QINQ"; 760 case RTE_MBUF_F_RX_LRO: return "RTE_MBUF_F_RX_LRO"; 761 case RTE_MBUF_F_RX_SEC_OFFLOAD: return "RTE_MBUF_F_RX_SEC_OFFLOAD"; 762 case RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED: return "RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED"; 763 case RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD: return "RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD"; 764 case RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD: return "RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD"; 765 case RTE_MBUF_F_RX_OUTER_L4_CKSUM_INVALID: 766 return "RTE_MBUF_F_RX_OUTER_L4_CKSUM_INVALID"; 767 768 default: return NULL; 769 } 770 } 771 772 struct flag_mask { 773 uint64_t flag; 774 uint64_t mask; 775 const char *default_name; 776 }; 777 778 /* write the list of rx ol flags in buffer buf */ 779 int 780 rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen) 781 { 782 const struct flag_mask rx_flags[] = { 783 { RTE_MBUF_F_RX_VLAN, RTE_MBUF_F_RX_VLAN, NULL }, 784 { RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, NULL }, 785 { RTE_MBUF_F_RX_FDIR, RTE_MBUF_F_RX_FDIR, NULL }, 786 { RTE_MBUF_F_RX_L4_CKSUM_BAD, RTE_MBUF_F_RX_L4_CKSUM_MASK, NULL }, 787 { RTE_MBUF_F_RX_L4_CKSUM_GOOD, RTE_MBUF_F_RX_L4_CKSUM_MASK, NULL }, 788 { RTE_MBUF_F_RX_L4_CKSUM_NONE, RTE_MBUF_F_RX_L4_CKSUM_MASK, NULL }, 789 { RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN, RTE_MBUF_F_RX_L4_CKSUM_MASK, 790 "RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN" }, 791 { RTE_MBUF_F_RX_IP_CKSUM_BAD, RTE_MBUF_F_RX_IP_CKSUM_MASK, NULL }, 792 { RTE_MBUF_F_RX_IP_CKSUM_GOOD, RTE_MBUF_F_RX_IP_CKSUM_MASK, NULL }, 793 { RTE_MBUF_F_RX_IP_CKSUM_NONE, RTE_MBUF_F_RX_IP_CKSUM_MASK, NULL }, 794 { RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN, RTE_MBUF_F_RX_IP_CKSUM_MASK, 795 "RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN" }, 796 { RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD, RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD, NULL }, 797 { RTE_MBUF_F_RX_VLAN_STRIPPED, RTE_MBUF_F_RX_VLAN_STRIPPED, NULL }, 798 { RTE_MBUF_F_RX_IEEE1588_PTP, RTE_MBUF_F_RX_IEEE1588_PTP, NULL }, 799 { RTE_MBUF_F_RX_IEEE1588_TMST, RTE_MBUF_F_RX_IEEE1588_TMST, NULL }, 800 { RTE_MBUF_F_RX_FDIR_ID, RTE_MBUF_F_RX_FDIR_ID, NULL }, 801 { RTE_MBUF_F_RX_FDIR_FLX, RTE_MBUF_F_RX_FDIR_FLX, NULL }, 802 { RTE_MBUF_F_RX_QINQ_STRIPPED, RTE_MBUF_F_RX_QINQ_STRIPPED, NULL }, 803 { RTE_MBUF_F_RX_LRO, RTE_MBUF_F_RX_LRO, NULL }, 804 { RTE_MBUF_F_RX_SEC_OFFLOAD, RTE_MBUF_F_RX_SEC_OFFLOAD, NULL }, 805 { RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED, RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED, NULL }, 806 { RTE_MBUF_F_RX_QINQ, RTE_MBUF_F_RX_QINQ, NULL }, 807 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, NULL }, 808 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, 809 NULL }, 810 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_INVALID, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, 811 NULL }, 812 { RTE_MBUF_F_RX_OUTER_L4_CKSUM_UNKNOWN, RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK, 813 "RTE_MBUF_F_RX_OUTER_L4_CKSUM_UNKNOWN" }, 814 }; 815 const char *name; 816 unsigned int i; 817 int ret; 818 819 if (buflen == 0) 820 return -1; 821 822 buf[0] = '\0'; 823 for (i = 0; i < RTE_DIM(rx_flags); i++) { 824 if ((mask & rx_flags[i].mask) != rx_flags[i].flag) 825 continue; 826 name = rte_get_rx_ol_flag_name(rx_flags[i].flag); 827 if (name == NULL) 828 name = rx_flags[i].default_name; 829 ret = snprintf(buf, buflen, "%s ", name); 830 if (ret < 0) 831 return -1; 832 if ((size_t)ret >= buflen) 833 return -1; 834 buf += ret; 835 buflen -= ret; 836 } 837 838 return 0; 839 } 840 841 /* 842 * Get the name of a TX offload flag. Must be kept synchronized with flag 843 * definitions in rte_mbuf.h. 844 */ 845 const char *rte_get_tx_ol_flag_name(uint64_t mask) 846 { 847 switch (mask) { 848 case RTE_MBUF_F_TX_VLAN: return "RTE_MBUF_F_TX_VLAN"; 849 case RTE_MBUF_F_TX_IP_CKSUM: return "RTE_MBUF_F_TX_IP_CKSUM"; 850 case RTE_MBUF_F_TX_TCP_CKSUM: return "RTE_MBUF_F_TX_TCP_CKSUM"; 851 case RTE_MBUF_F_TX_SCTP_CKSUM: return "RTE_MBUF_F_TX_SCTP_CKSUM"; 852 case RTE_MBUF_F_TX_UDP_CKSUM: return "RTE_MBUF_F_TX_UDP_CKSUM"; 853 case RTE_MBUF_F_TX_IEEE1588_TMST: return "RTE_MBUF_F_TX_IEEE1588_TMST"; 854 case RTE_MBUF_F_TX_TCP_SEG: return "RTE_MBUF_F_TX_TCP_SEG"; 855 case RTE_MBUF_F_TX_IPV4: return "RTE_MBUF_F_TX_IPV4"; 856 case RTE_MBUF_F_TX_IPV6: return "RTE_MBUF_F_TX_IPV6"; 857 case RTE_MBUF_F_TX_OUTER_IP_CKSUM: return "RTE_MBUF_F_TX_OUTER_IP_CKSUM"; 858 case RTE_MBUF_F_TX_OUTER_IPV4: return "RTE_MBUF_F_TX_OUTER_IPV4"; 859 case RTE_MBUF_F_TX_OUTER_IPV6: return "RTE_MBUF_F_TX_OUTER_IPV6"; 860 case RTE_MBUF_F_TX_TUNNEL_VXLAN: return "RTE_MBUF_F_TX_TUNNEL_VXLAN"; 861 case RTE_MBUF_F_TX_TUNNEL_GTP: return "RTE_MBUF_F_TX_TUNNEL_GTP"; 862 case RTE_MBUF_F_TX_TUNNEL_GRE: return "RTE_MBUF_F_TX_TUNNEL_GRE"; 863 case RTE_MBUF_F_TX_TUNNEL_IPIP: return "RTE_MBUF_F_TX_TUNNEL_IPIP"; 864 case RTE_MBUF_F_TX_TUNNEL_GENEVE: return "RTE_MBUF_F_TX_TUNNEL_GENEVE"; 865 case RTE_MBUF_F_TX_TUNNEL_MPLSINUDP: return "RTE_MBUF_F_TX_TUNNEL_MPLSINUDP"; 866 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE: return "RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE"; 867 case RTE_MBUF_F_TX_TUNNEL_IP: return "RTE_MBUF_F_TX_TUNNEL_IP"; 868 case RTE_MBUF_F_TX_TUNNEL_UDP: return "RTE_MBUF_F_TX_TUNNEL_UDP"; 869 case RTE_MBUF_F_TX_QINQ: return "RTE_MBUF_F_TX_QINQ"; 870 case RTE_MBUF_F_TX_MACSEC: return "RTE_MBUF_F_TX_MACSEC"; 871 case RTE_MBUF_F_TX_SEC_OFFLOAD: return "RTE_MBUF_F_TX_SEC_OFFLOAD"; 872 case RTE_MBUF_F_TX_UDP_SEG: return "RTE_MBUF_F_TX_UDP_SEG"; 873 case RTE_MBUF_F_TX_OUTER_UDP_CKSUM: return "RTE_MBUF_F_TX_OUTER_UDP_CKSUM"; 874 default: return NULL; 875 } 876 } 877 878 /* write the list of tx ol flags in buffer buf */ 879 int 880 rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen) 881 { 882 const struct flag_mask tx_flags[] = { 883 { RTE_MBUF_F_TX_VLAN, RTE_MBUF_F_TX_VLAN, NULL }, 884 { RTE_MBUF_F_TX_IP_CKSUM, RTE_MBUF_F_TX_IP_CKSUM, NULL }, 885 { RTE_MBUF_F_TX_TCP_CKSUM, RTE_MBUF_F_TX_L4_MASK, NULL }, 886 { RTE_MBUF_F_TX_SCTP_CKSUM, RTE_MBUF_F_TX_L4_MASK, NULL }, 887 { RTE_MBUF_F_TX_UDP_CKSUM, RTE_MBUF_F_TX_L4_MASK, NULL }, 888 { RTE_MBUF_F_TX_L4_NO_CKSUM, RTE_MBUF_F_TX_L4_MASK, "RTE_MBUF_F_TX_L4_NO_CKSUM" }, 889 { RTE_MBUF_F_TX_IEEE1588_TMST, RTE_MBUF_F_TX_IEEE1588_TMST, NULL }, 890 { RTE_MBUF_F_TX_TCP_SEG, RTE_MBUF_F_TX_TCP_SEG, NULL }, 891 { RTE_MBUF_F_TX_IPV4, RTE_MBUF_F_TX_IPV4, NULL }, 892 { RTE_MBUF_F_TX_IPV6, RTE_MBUF_F_TX_IPV6, NULL }, 893 { RTE_MBUF_F_TX_OUTER_IP_CKSUM, RTE_MBUF_F_TX_OUTER_IP_CKSUM, NULL }, 894 { RTE_MBUF_F_TX_OUTER_IPV4, RTE_MBUF_F_TX_OUTER_IPV4, NULL }, 895 { RTE_MBUF_F_TX_OUTER_IPV6, RTE_MBUF_F_TX_OUTER_IPV6, NULL }, 896 { RTE_MBUF_F_TX_TUNNEL_VXLAN, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 897 { RTE_MBUF_F_TX_TUNNEL_GTP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 898 { RTE_MBUF_F_TX_TUNNEL_GRE, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 899 { RTE_MBUF_F_TX_TUNNEL_IPIP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 900 { RTE_MBUF_F_TX_TUNNEL_GENEVE, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 901 { RTE_MBUF_F_TX_TUNNEL_MPLSINUDP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 902 { RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 903 { RTE_MBUF_F_TX_TUNNEL_IP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 904 { RTE_MBUF_F_TX_TUNNEL_UDP, RTE_MBUF_F_TX_TUNNEL_MASK, NULL }, 905 { RTE_MBUF_F_TX_QINQ, RTE_MBUF_F_TX_QINQ, NULL }, 906 { RTE_MBUF_F_TX_MACSEC, RTE_MBUF_F_TX_MACSEC, NULL }, 907 { RTE_MBUF_F_TX_SEC_OFFLOAD, RTE_MBUF_F_TX_SEC_OFFLOAD, NULL }, 908 { RTE_MBUF_F_TX_UDP_SEG, RTE_MBUF_F_TX_UDP_SEG, NULL }, 909 { RTE_MBUF_F_TX_OUTER_UDP_CKSUM, RTE_MBUF_F_TX_OUTER_UDP_CKSUM, NULL }, 910 }; 911 const char *name; 912 unsigned int i; 913 int ret; 914 915 if (buflen == 0) 916 return -1; 917 918 buf[0] = '\0'; 919 for (i = 0; i < RTE_DIM(tx_flags); i++) { 920 if ((mask & tx_flags[i].mask) != tx_flags[i].flag) 921 continue; 922 name = rte_get_tx_ol_flag_name(tx_flags[i].flag); 923 if (name == NULL) 924 name = tx_flags[i].default_name; 925 ret = snprintf(buf, buflen, "%s ", name); 926 if (ret < 0) 927 return -1; 928 if ((size_t)ret >= buflen) 929 return -1; 930 buf += ret; 931 buflen -= ret; 932 } 933 934 return 0; 935 } 936