1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <string.h> 6 #include <stdarg.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <stdint.h> 10 #include <inttypes.h> 11 #include <errno.h> 12 #include <sys/queue.h> 13 14 #include <rte_common.h> 15 #include <rte_debug.h> 16 #include <rte_log.h> 17 #include <rte_memory.h> 18 #include <rte_memcpy.h> 19 #include <rte_launch.h> 20 #include <rte_eal.h> 21 #include <rte_per_lcore.h> 22 #include <rte_lcore.h> 23 #include <rte_atomic.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_ring.h> 26 #include <rte_mempool.h> 27 #include <rte_mbuf.h> 28 #include <rte_random.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc.h> 31 32 #include "test.h" 33 34 #define MBUF_DATA_SIZE 2048 35 #define NB_MBUF 128 36 #define MBUF_TEST_DATA_LEN 1464 37 #define MBUF_TEST_DATA_LEN2 50 38 #define MBUF_TEST_HDR1_LEN 20 39 #define MBUF_TEST_HDR2_LEN 30 40 #define MBUF_TEST_ALL_HDRS_LEN (MBUF_TEST_HDR1_LEN+MBUF_TEST_HDR2_LEN) 41 42 /* size of private data for mbuf in pktmbuf_pool2 */ 43 #define MBUF2_PRIV_SIZE 128 44 45 #define REFCNT_MAX_ITER 64 46 #define REFCNT_MAX_TIMEOUT 10 47 #define REFCNT_MAX_REF (RTE_MAX_LCORE) 48 #define REFCNT_MBUF_NUM 64 49 #define REFCNT_RING_SIZE (REFCNT_MBUF_NUM * REFCNT_MAX_REF) 50 51 #define MAGIC_DATA 0x42424242 52 53 #define MAKE_STRING(x) # x 54 55 #ifdef RTE_MBUF_REFCNT_ATOMIC 56 57 static volatile uint32_t refcnt_stop_slaves; 58 static unsigned refcnt_lcore[RTE_MAX_LCORE]; 59 60 #endif 61 62 /* 63 * MBUF 64 * ==== 65 * 66 * #. Allocate a mbuf pool. 67 * 68 * - The pool contains NB_MBUF elements, where each mbuf is MBUF_SIZE 69 * bytes long. 70 * 71 * #. Test multiple allocations of mbufs from this pool. 72 * 73 * - Allocate NB_MBUF and store pointers in a table. 74 * - If an allocation fails, return an error. 75 * - Free all these mbufs. 76 * - Repeat the same test to check that mbufs were freed correctly. 77 * 78 * #. Test data manipulation in pktmbuf. 79 * 80 * - Alloc an mbuf. 81 * - Append data using rte_pktmbuf_append(). 82 * - Test for error in rte_pktmbuf_append() when len is too large. 83 * - Trim data at the end of mbuf using rte_pktmbuf_trim(). 84 * - Test for error in rte_pktmbuf_trim() when len is too large. 85 * - Prepend a header using rte_pktmbuf_prepend(). 86 * - Test for error in rte_pktmbuf_prepend() when len is too large. 87 * - Remove data at the beginning of mbuf using rte_pktmbuf_adj(). 88 * - Test for error in rte_pktmbuf_adj() when len is too large. 89 * - Check that appended data is not corrupt. 90 * - Free the mbuf. 91 * - Between all these tests, check data_len and pkt_len, and 92 * that the mbuf is contiguous. 93 * - Repeat the test to check that allocation operations 94 * reinitialize the mbuf correctly. 95 * 96 * #. Test packet cloning 97 * - Clone a mbuf and verify the data 98 * - Clone the cloned mbuf and verify the data 99 * - Attach a mbuf to another that does not have the same priv_size. 100 */ 101 102 #define GOTO_FAIL(str, ...) do { \ 103 printf("mbuf test FAILED (l.%d): <" str ">\n", \ 104 __LINE__, ##__VA_ARGS__); \ 105 goto fail; \ 106 } while(0) 107 108 /* 109 * test data manipulation in mbuf with non-ascii data 110 */ 111 static int 112 test_pktmbuf_with_non_ascii_data(struct rte_mempool *pktmbuf_pool) 113 { 114 struct rte_mbuf *m = NULL; 115 char *data; 116 117 m = rte_pktmbuf_alloc(pktmbuf_pool); 118 if (m == NULL) 119 GOTO_FAIL("Cannot allocate mbuf"); 120 if (rte_pktmbuf_pkt_len(m) != 0) 121 GOTO_FAIL("Bad length"); 122 123 data = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN); 124 if (data == NULL) 125 GOTO_FAIL("Cannot append data"); 126 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) 127 GOTO_FAIL("Bad pkt length"); 128 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) 129 GOTO_FAIL("Bad data length"); 130 memset(data, 0xff, rte_pktmbuf_pkt_len(m)); 131 if (!rte_pktmbuf_is_contiguous(m)) 132 GOTO_FAIL("Buffer should be continuous"); 133 rte_pktmbuf_dump(stdout, m, MBUF_TEST_DATA_LEN); 134 135 rte_pktmbuf_free(m); 136 137 return 0; 138 139 fail: 140 if(m) { 141 rte_pktmbuf_free(m); 142 } 143 return -1; 144 } 145 146 /* 147 * test data manipulation in mbuf 148 */ 149 static int 150 test_one_pktmbuf(struct rte_mempool *pktmbuf_pool) 151 { 152 struct rte_mbuf *m = NULL; 153 char *data, *data2, *hdr; 154 unsigned i; 155 156 printf("Test pktmbuf API\n"); 157 158 /* alloc a mbuf */ 159 160 m = rte_pktmbuf_alloc(pktmbuf_pool); 161 if (m == NULL) 162 GOTO_FAIL("Cannot allocate mbuf"); 163 if (rte_pktmbuf_pkt_len(m) != 0) 164 GOTO_FAIL("Bad length"); 165 166 rte_pktmbuf_dump(stdout, m, 0); 167 168 /* append data */ 169 170 data = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN); 171 if (data == NULL) 172 GOTO_FAIL("Cannot append data"); 173 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) 174 GOTO_FAIL("Bad pkt length"); 175 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) 176 GOTO_FAIL("Bad data length"); 177 memset(data, 0x66, rte_pktmbuf_pkt_len(m)); 178 if (!rte_pktmbuf_is_contiguous(m)) 179 GOTO_FAIL("Buffer should be continuous"); 180 rte_pktmbuf_dump(stdout, m, MBUF_TEST_DATA_LEN); 181 rte_pktmbuf_dump(stdout, m, 2*MBUF_TEST_DATA_LEN); 182 183 /* this append should fail */ 184 185 data2 = rte_pktmbuf_append(m, (uint16_t)(rte_pktmbuf_tailroom(m) + 1)); 186 if (data2 != NULL) 187 GOTO_FAIL("Append should not succeed"); 188 189 /* append some more data */ 190 191 data2 = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN2); 192 if (data2 == NULL) 193 GOTO_FAIL("Cannot append data"); 194 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_DATA_LEN2) 195 GOTO_FAIL("Bad pkt length"); 196 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_DATA_LEN2) 197 GOTO_FAIL("Bad data length"); 198 if (!rte_pktmbuf_is_contiguous(m)) 199 GOTO_FAIL("Buffer should be continuous"); 200 201 /* trim data at the end of mbuf */ 202 203 if (rte_pktmbuf_trim(m, MBUF_TEST_DATA_LEN2) < 0) 204 GOTO_FAIL("Cannot trim data"); 205 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) 206 GOTO_FAIL("Bad pkt length"); 207 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) 208 GOTO_FAIL("Bad data length"); 209 if (!rte_pktmbuf_is_contiguous(m)) 210 GOTO_FAIL("Buffer should be continuous"); 211 212 /* this trim should fail */ 213 214 if (rte_pktmbuf_trim(m, (uint16_t)(rte_pktmbuf_data_len(m) + 1)) == 0) 215 GOTO_FAIL("trim should not succeed"); 216 217 /* prepend one header */ 218 219 hdr = rte_pktmbuf_prepend(m, MBUF_TEST_HDR1_LEN); 220 if (hdr == NULL) 221 GOTO_FAIL("Cannot prepend"); 222 if (data - hdr != MBUF_TEST_HDR1_LEN) 223 GOTO_FAIL("Prepend failed"); 224 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_HDR1_LEN) 225 GOTO_FAIL("Bad pkt length"); 226 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_HDR1_LEN) 227 GOTO_FAIL("Bad data length"); 228 if (!rte_pktmbuf_is_contiguous(m)) 229 GOTO_FAIL("Buffer should be continuous"); 230 memset(hdr, 0x55, MBUF_TEST_HDR1_LEN); 231 232 /* prepend another header */ 233 234 hdr = rte_pktmbuf_prepend(m, MBUF_TEST_HDR2_LEN); 235 if (hdr == NULL) 236 GOTO_FAIL("Cannot prepend"); 237 if (data - hdr != MBUF_TEST_ALL_HDRS_LEN) 238 GOTO_FAIL("Prepend failed"); 239 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_ALL_HDRS_LEN) 240 GOTO_FAIL("Bad pkt length"); 241 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_ALL_HDRS_LEN) 242 GOTO_FAIL("Bad data length"); 243 if (!rte_pktmbuf_is_contiguous(m)) 244 GOTO_FAIL("Buffer should be continuous"); 245 memset(hdr, 0x55, MBUF_TEST_HDR2_LEN); 246 247 rte_mbuf_sanity_check(m, 1); 248 rte_mbuf_sanity_check(m, 0); 249 rte_pktmbuf_dump(stdout, m, 0); 250 251 /* this prepend should fail */ 252 253 hdr = rte_pktmbuf_prepend(m, (uint16_t)(rte_pktmbuf_headroom(m) + 1)); 254 if (hdr != NULL) 255 GOTO_FAIL("prepend should not succeed"); 256 257 /* remove data at beginning of mbuf (adj) */ 258 259 if (data != rte_pktmbuf_adj(m, MBUF_TEST_ALL_HDRS_LEN)) 260 GOTO_FAIL("rte_pktmbuf_adj failed"); 261 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) 262 GOTO_FAIL("Bad pkt length"); 263 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) 264 GOTO_FAIL("Bad data length"); 265 if (!rte_pktmbuf_is_contiguous(m)) 266 GOTO_FAIL("Buffer should be continuous"); 267 268 /* this adj should fail */ 269 270 if (rte_pktmbuf_adj(m, (uint16_t)(rte_pktmbuf_data_len(m) + 1)) != NULL) 271 GOTO_FAIL("rte_pktmbuf_adj should not succeed"); 272 273 /* check data */ 274 275 if (!rte_pktmbuf_is_contiguous(m)) 276 GOTO_FAIL("Buffer should be continuous"); 277 278 for (i=0; i<MBUF_TEST_DATA_LEN; i++) { 279 if (data[i] != 0x66) 280 GOTO_FAIL("Data corrupted at offset %u", i); 281 } 282 283 /* free mbuf */ 284 285 rte_pktmbuf_free(m); 286 m = NULL; 287 return 0; 288 289 fail: 290 if (m) 291 rte_pktmbuf_free(m); 292 return -1; 293 } 294 295 static int 296 testclone_testupdate_testdetach(struct rte_mempool *pktmbuf_pool) 297 { 298 struct rte_mbuf *m = NULL; 299 struct rte_mbuf *clone = NULL; 300 struct rte_mbuf *clone2 = NULL; 301 unaligned_uint32_t *data; 302 303 /* alloc a mbuf */ 304 m = rte_pktmbuf_alloc(pktmbuf_pool); 305 if (m == NULL) 306 GOTO_FAIL("ooops not allocating mbuf"); 307 308 if (rte_pktmbuf_pkt_len(m) != 0) 309 GOTO_FAIL("Bad length"); 310 311 rte_pktmbuf_append(m, sizeof(uint32_t)); 312 data = rte_pktmbuf_mtod(m, unaligned_uint32_t *); 313 *data = MAGIC_DATA; 314 315 /* clone the allocated mbuf */ 316 clone = rte_pktmbuf_clone(m, pktmbuf_pool); 317 if (clone == NULL) 318 GOTO_FAIL("cannot clone data\n"); 319 320 data = rte_pktmbuf_mtod(clone, unaligned_uint32_t *); 321 if (*data != MAGIC_DATA) 322 GOTO_FAIL("invalid data in clone\n"); 323 324 if (rte_mbuf_refcnt_read(m) != 2) 325 GOTO_FAIL("invalid refcnt in m\n"); 326 327 /* free the clone */ 328 rte_pktmbuf_free(clone); 329 clone = NULL; 330 331 /* same test with a chained mbuf */ 332 m->next = rte_pktmbuf_alloc(pktmbuf_pool); 333 if (m->next == NULL) 334 GOTO_FAIL("Next Pkt Null\n"); 335 336 rte_pktmbuf_append(m->next, sizeof(uint32_t)); 337 data = rte_pktmbuf_mtod(m->next, unaligned_uint32_t *); 338 *data = MAGIC_DATA; 339 340 clone = rte_pktmbuf_clone(m, pktmbuf_pool); 341 if (clone == NULL) 342 GOTO_FAIL("cannot clone data\n"); 343 344 data = rte_pktmbuf_mtod(clone, unaligned_uint32_t *); 345 if (*data != MAGIC_DATA) 346 GOTO_FAIL("invalid data in clone\n"); 347 348 data = rte_pktmbuf_mtod(clone->next, unaligned_uint32_t *); 349 if (*data != MAGIC_DATA) 350 GOTO_FAIL("invalid data in clone->next\n"); 351 352 if (rte_mbuf_refcnt_read(m) != 2) 353 GOTO_FAIL("invalid refcnt in m\n"); 354 355 if (rte_mbuf_refcnt_read(m->next) != 2) 356 GOTO_FAIL("invalid refcnt in m->next\n"); 357 358 /* try to clone the clone */ 359 360 clone2 = rte_pktmbuf_clone(clone, pktmbuf_pool); 361 if (clone2 == NULL) 362 GOTO_FAIL("cannot clone the clone\n"); 363 364 data = rte_pktmbuf_mtod(clone2, unaligned_uint32_t *); 365 if (*data != MAGIC_DATA) 366 GOTO_FAIL("invalid data in clone2\n"); 367 368 data = rte_pktmbuf_mtod(clone2->next, unaligned_uint32_t *); 369 if (*data != MAGIC_DATA) 370 GOTO_FAIL("invalid data in clone2->next\n"); 371 372 if (rte_mbuf_refcnt_read(m) != 3) 373 GOTO_FAIL("invalid refcnt in m\n"); 374 375 if (rte_mbuf_refcnt_read(m->next) != 3) 376 GOTO_FAIL("invalid refcnt in m->next\n"); 377 378 /* free mbuf */ 379 rte_pktmbuf_free(m); 380 rte_pktmbuf_free(clone); 381 rte_pktmbuf_free(clone2); 382 383 m = NULL; 384 clone = NULL; 385 clone2 = NULL; 386 printf("%s ok\n", __func__); 387 return 0; 388 389 fail: 390 if (m) 391 rte_pktmbuf_free(m); 392 if (clone) 393 rte_pktmbuf_free(clone); 394 if (clone2) 395 rte_pktmbuf_free(clone2); 396 return -1; 397 } 398 399 static int 400 test_attach_from_different_pool(struct rte_mempool *pktmbuf_pool, 401 struct rte_mempool *pktmbuf_pool2) 402 { 403 struct rte_mbuf *m = NULL; 404 struct rte_mbuf *clone = NULL; 405 struct rte_mbuf *clone2 = NULL; 406 char *data, *c_data, *c_data2; 407 408 /* alloc a mbuf */ 409 m = rte_pktmbuf_alloc(pktmbuf_pool); 410 if (m == NULL) 411 GOTO_FAIL("cannot allocate mbuf"); 412 413 if (rte_pktmbuf_pkt_len(m) != 0) 414 GOTO_FAIL("Bad length"); 415 416 data = rte_pktmbuf_mtod(m, char *); 417 418 /* allocate a new mbuf from the second pool, and attach it to the first 419 * mbuf */ 420 clone = rte_pktmbuf_alloc(pktmbuf_pool2); 421 if (clone == NULL) 422 GOTO_FAIL("cannot allocate mbuf from second pool\n"); 423 424 /* check data room size and priv size, and erase priv */ 425 if (rte_pktmbuf_data_room_size(clone->pool) != 0) 426 GOTO_FAIL("data room size should be 0\n"); 427 if (rte_pktmbuf_priv_size(clone->pool) != MBUF2_PRIV_SIZE) 428 GOTO_FAIL("data room size should be %d\n", MBUF2_PRIV_SIZE); 429 memset(clone + 1, 0, MBUF2_PRIV_SIZE); 430 431 /* save data pointer to compare it after detach() */ 432 c_data = rte_pktmbuf_mtod(clone, char *); 433 if (c_data != (char *)clone + sizeof(*clone) + MBUF2_PRIV_SIZE) 434 GOTO_FAIL("bad data pointer in clone"); 435 if (rte_pktmbuf_headroom(clone) != 0) 436 GOTO_FAIL("bad headroom in clone"); 437 438 rte_pktmbuf_attach(clone, m); 439 440 if (rte_pktmbuf_mtod(clone, char *) != data) 441 GOTO_FAIL("clone was not attached properly\n"); 442 if (rte_pktmbuf_headroom(clone) != RTE_PKTMBUF_HEADROOM) 443 GOTO_FAIL("bad headroom in clone after attach"); 444 if (rte_mbuf_refcnt_read(m) != 2) 445 GOTO_FAIL("invalid refcnt in m\n"); 446 447 /* allocate a new mbuf from the second pool, and attach it to the first 448 * cloned mbuf */ 449 clone2 = rte_pktmbuf_alloc(pktmbuf_pool2); 450 if (clone2 == NULL) 451 GOTO_FAIL("cannot allocate clone2 from second pool\n"); 452 453 /* check data room size and priv size, and erase priv */ 454 if (rte_pktmbuf_data_room_size(clone2->pool) != 0) 455 GOTO_FAIL("data room size should be 0\n"); 456 if (rte_pktmbuf_priv_size(clone2->pool) != MBUF2_PRIV_SIZE) 457 GOTO_FAIL("data room size should be %d\n", MBUF2_PRIV_SIZE); 458 memset(clone2 + 1, 0, MBUF2_PRIV_SIZE); 459 460 /* save data pointer to compare it after detach() */ 461 c_data2 = rte_pktmbuf_mtod(clone2, char *); 462 if (c_data2 != (char *)clone2 + sizeof(*clone2) + MBUF2_PRIV_SIZE) 463 GOTO_FAIL("bad data pointer in clone2"); 464 if (rte_pktmbuf_headroom(clone2) != 0) 465 GOTO_FAIL("bad headroom in clone2"); 466 467 rte_pktmbuf_attach(clone2, clone); 468 469 if (rte_pktmbuf_mtod(clone2, char *) != data) 470 GOTO_FAIL("clone2 was not attached properly\n"); 471 if (rte_pktmbuf_headroom(clone2) != RTE_PKTMBUF_HEADROOM) 472 GOTO_FAIL("bad headroom in clone2 after attach"); 473 if (rte_mbuf_refcnt_read(m) != 3) 474 GOTO_FAIL("invalid refcnt in m\n"); 475 476 /* detach the clones */ 477 rte_pktmbuf_detach(clone); 478 if (c_data != rte_pktmbuf_mtod(clone, char *)) 479 GOTO_FAIL("clone was not detached properly\n"); 480 if (rte_mbuf_refcnt_read(m) != 2) 481 GOTO_FAIL("invalid refcnt in m\n"); 482 483 rte_pktmbuf_detach(clone2); 484 if (c_data2 != rte_pktmbuf_mtod(clone2, char *)) 485 GOTO_FAIL("clone2 was not detached properly\n"); 486 if (rte_mbuf_refcnt_read(m) != 1) 487 GOTO_FAIL("invalid refcnt in m\n"); 488 489 /* free the clones and the initial mbuf */ 490 rte_pktmbuf_free(clone2); 491 rte_pktmbuf_free(clone); 492 rte_pktmbuf_free(m); 493 printf("%s ok\n", __func__); 494 return 0; 495 496 fail: 497 if (m) 498 rte_pktmbuf_free(m); 499 if (clone) 500 rte_pktmbuf_free(clone); 501 if (clone2) 502 rte_pktmbuf_free(clone2); 503 return -1; 504 } 505 #undef GOTO_FAIL 506 507 /* 508 * test allocation and free of mbufs 509 */ 510 static int 511 test_pktmbuf_pool(struct rte_mempool *pktmbuf_pool) 512 { 513 unsigned i; 514 struct rte_mbuf *m[NB_MBUF]; 515 int ret = 0; 516 517 for (i=0; i<NB_MBUF; i++) 518 m[i] = NULL; 519 520 /* alloc NB_MBUF mbufs */ 521 for (i=0; i<NB_MBUF; i++) { 522 m[i] = rte_pktmbuf_alloc(pktmbuf_pool); 523 if (m[i] == NULL) { 524 printf("rte_pktmbuf_alloc() failed (%u)\n", i); 525 ret = -1; 526 } 527 } 528 struct rte_mbuf *extra = NULL; 529 extra = rte_pktmbuf_alloc(pktmbuf_pool); 530 if(extra != NULL) { 531 printf("Error pool not empty"); 532 ret = -1; 533 } 534 extra = rte_pktmbuf_clone(m[0], pktmbuf_pool); 535 if(extra != NULL) { 536 printf("Error pool not empty"); 537 ret = -1; 538 } 539 /* free them */ 540 for (i=0; i<NB_MBUF; i++) { 541 if (m[i] != NULL) 542 rte_pktmbuf_free(m[i]); 543 } 544 545 return ret; 546 } 547 548 /* 549 * test that the pointer to the data on a packet mbuf is set properly 550 */ 551 static int 552 test_pktmbuf_pool_ptr(struct rte_mempool *pktmbuf_pool) 553 { 554 unsigned i; 555 struct rte_mbuf *m[NB_MBUF]; 556 int ret = 0; 557 558 for (i=0; i<NB_MBUF; i++) 559 m[i] = NULL; 560 561 /* alloc NB_MBUF mbufs */ 562 for (i=0; i<NB_MBUF; i++) { 563 m[i] = rte_pktmbuf_alloc(pktmbuf_pool); 564 if (m[i] == NULL) { 565 printf("rte_pktmbuf_alloc() failed (%u)\n", i); 566 ret = -1; 567 break; 568 } 569 m[i]->data_off += 64; 570 } 571 572 /* free them */ 573 for (i=0; i<NB_MBUF; i++) { 574 if (m[i] != NULL) 575 rte_pktmbuf_free(m[i]); 576 } 577 578 for (i=0; i<NB_MBUF; i++) 579 m[i] = NULL; 580 581 /* alloc NB_MBUF mbufs */ 582 for (i=0; i<NB_MBUF; i++) { 583 m[i] = rte_pktmbuf_alloc(pktmbuf_pool); 584 if (m[i] == NULL) { 585 printf("rte_pktmbuf_alloc() failed (%u)\n", i); 586 ret = -1; 587 break; 588 } 589 if (m[i]->data_off != RTE_PKTMBUF_HEADROOM) { 590 printf("invalid data_off\n"); 591 ret = -1; 592 } 593 } 594 595 /* free them */ 596 for (i=0; i<NB_MBUF; i++) { 597 if (m[i] != NULL) 598 rte_pktmbuf_free(m[i]); 599 } 600 601 return ret; 602 } 603 604 static int 605 test_pktmbuf_free_segment(struct rte_mempool *pktmbuf_pool) 606 { 607 unsigned i; 608 struct rte_mbuf *m[NB_MBUF]; 609 int ret = 0; 610 611 for (i=0; i<NB_MBUF; i++) 612 m[i] = NULL; 613 614 /* alloc NB_MBUF mbufs */ 615 for (i=0; i<NB_MBUF; i++) { 616 m[i] = rte_pktmbuf_alloc(pktmbuf_pool); 617 if (m[i] == NULL) { 618 printf("rte_pktmbuf_alloc() failed (%u)\n", i); 619 ret = -1; 620 } 621 } 622 623 /* free them */ 624 for (i=0; i<NB_MBUF; i++) { 625 if (m[i] != NULL) { 626 struct rte_mbuf *mb, *mt; 627 628 mb = m[i]; 629 while(mb != NULL) { 630 mt = mb; 631 mb = mb->next; 632 rte_pktmbuf_free_seg(mt); 633 } 634 } 635 } 636 637 return ret; 638 } 639 640 /* 641 * Stress test for rte_mbuf atomic refcnt. 642 * Implies that RTE_MBUF_REFCNT_ATOMIC is defined. 643 * For more efficiency, recommended to run with RTE_LIBRTE_MBUF_DEBUG defined. 644 */ 645 646 #ifdef RTE_MBUF_REFCNT_ATOMIC 647 648 static int 649 test_refcnt_slave(void *arg) 650 { 651 unsigned lcore, free; 652 void *mp = 0; 653 struct rte_ring *refcnt_mbuf_ring = arg; 654 655 lcore = rte_lcore_id(); 656 printf("%s started at lcore %u\n", __func__, lcore); 657 658 free = 0; 659 while (refcnt_stop_slaves == 0) { 660 if (rte_ring_dequeue(refcnt_mbuf_ring, &mp) == 0) { 661 free++; 662 rte_pktmbuf_free(mp); 663 } 664 } 665 666 refcnt_lcore[lcore] += free; 667 printf("%s finished at lcore %u, " 668 "number of freed mbufs: %u\n", 669 __func__, lcore, free); 670 return 0; 671 } 672 673 static void 674 test_refcnt_iter(unsigned int lcore, unsigned int iter, 675 struct rte_mempool *refcnt_pool, 676 struct rte_ring *refcnt_mbuf_ring) 677 { 678 uint16_t ref; 679 unsigned i, n, tref, wn; 680 struct rte_mbuf *m; 681 682 tref = 0; 683 684 /* For each mbuf in the pool: 685 * - allocate mbuf, 686 * - increment it's reference up to N+1, 687 * - enqueue it N times into the ring for slave cores to free. 688 */ 689 for (i = 0, n = rte_mempool_avail_count(refcnt_pool); 690 i != n && (m = rte_pktmbuf_alloc(refcnt_pool)) != NULL; 691 i++) { 692 ref = RTE_MAX(rte_rand() % REFCNT_MAX_REF, 1UL); 693 tref += ref; 694 if ((ref & 1) != 0) { 695 rte_pktmbuf_refcnt_update(m, ref); 696 while (ref-- != 0) 697 rte_ring_enqueue(refcnt_mbuf_ring, m); 698 } else { 699 while (ref-- != 0) { 700 rte_pktmbuf_refcnt_update(m, 1); 701 rte_ring_enqueue(refcnt_mbuf_ring, m); 702 } 703 } 704 rte_pktmbuf_free(m); 705 } 706 707 if (i != n) 708 rte_panic("(lcore=%u, iter=%u): was able to allocate only " 709 "%u from %u mbufs\n", lcore, iter, i, n); 710 711 /* wait till slave lcores will consume all mbufs */ 712 while (!rte_ring_empty(refcnt_mbuf_ring)) 713 ; 714 715 /* check that all mbufs are back into mempool by now */ 716 for (wn = 0; wn != REFCNT_MAX_TIMEOUT; wn++) { 717 if ((i = rte_mempool_avail_count(refcnt_pool)) == n) { 718 refcnt_lcore[lcore] += tref; 719 printf("%s(lcore=%u, iter=%u) completed, " 720 "%u references processed\n", 721 __func__, lcore, iter, tref); 722 return; 723 } 724 rte_delay_ms(100); 725 } 726 727 rte_panic("(lcore=%u, iter=%u): after %us only " 728 "%u of %u mbufs left free\n", lcore, iter, wn, i, n); 729 } 730 731 static int 732 test_refcnt_master(struct rte_mempool *refcnt_pool, 733 struct rte_ring *refcnt_mbuf_ring) 734 { 735 unsigned i, lcore; 736 737 lcore = rte_lcore_id(); 738 printf("%s started at lcore %u\n", __func__, lcore); 739 740 for (i = 0; i != REFCNT_MAX_ITER; i++) 741 test_refcnt_iter(lcore, i, refcnt_pool, refcnt_mbuf_ring); 742 743 refcnt_stop_slaves = 1; 744 rte_wmb(); 745 746 printf("%s finished at lcore %u\n", __func__, lcore); 747 return 0; 748 } 749 750 #endif 751 752 static int 753 test_refcnt_mbuf(void) 754 { 755 #ifdef RTE_MBUF_REFCNT_ATOMIC 756 unsigned int master, slave, tref; 757 int ret = -1; 758 struct rte_mempool *refcnt_pool = NULL; 759 struct rte_ring *refcnt_mbuf_ring = NULL; 760 761 if (rte_lcore_count() < 2) { 762 printf("Not enough cores for test_refcnt_mbuf, expecting at least 2\n"); 763 return TEST_SKIPPED; 764 } 765 766 printf("starting %s, at %u lcores\n", __func__, rte_lcore_count()); 767 768 /* create refcnt pool & ring if they don't exist */ 769 770 refcnt_pool = rte_pktmbuf_pool_create(MAKE_STRING(refcnt_pool), 771 REFCNT_MBUF_NUM, 0, 0, 0, 772 SOCKET_ID_ANY); 773 if (refcnt_pool == NULL) { 774 printf("%s: cannot allocate " MAKE_STRING(refcnt_pool) "\n", 775 __func__); 776 return -1; 777 } 778 779 refcnt_mbuf_ring = rte_ring_create("refcnt_mbuf_ring", 780 rte_align32pow2(REFCNT_RING_SIZE), SOCKET_ID_ANY, 781 RING_F_SP_ENQ); 782 if (refcnt_mbuf_ring == NULL) { 783 printf("%s: cannot allocate " MAKE_STRING(refcnt_mbuf_ring) 784 "\n", __func__); 785 goto err; 786 } 787 788 refcnt_stop_slaves = 0; 789 memset(refcnt_lcore, 0, sizeof (refcnt_lcore)); 790 791 rte_eal_mp_remote_launch(test_refcnt_slave, refcnt_mbuf_ring, 792 SKIP_MASTER); 793 794 test_refcnt_master(refcnt_pool, refcnt_mbuf_ring); 795 796 rte_eal_mp_wait_lcore(); 797 798 /* check that we porcessed all references */ 799 tref = 0; 800 master = rte_get_master_lcore(); 801 802 RTE_LCORE_FOREACH_SLAVE(slave) 803 tref += refcnt_lcore[slave]; 804 805 if (tref != refcnt_lcore[master]) 806 rte_panic("refernced mbufs: %u, freed mbufs: %u\n", 807 tref, refcnt_lcore[master]); 808 809 rte_mempool_dump(stdout, refcnt_pool); 810 rte_ring_dump(stdout, refcnt_mbuf_ring); 811 812 ret = 0; 813 814 err: 815 rte_mempool_free(refcnt_pool); 816 rte_ring_free(refcnt_mbuf_ring); 817 return ret; 818 #else 819 return 0; 820 #endif 821 } 822 823 #include <unistd.h> 824 #include <sys/wait.h> 825 826 /* use fork() to test mbuf errors panic */ 827 static int 828 verify_mbuf_check_panics(struct rte_mbuf *buf) 829 { 830 int pid; 831 int status; 832 833 pid = fork(); 834 835 if (pid == 0) { 836 rte_mbuf_sanity_check(buf, 1); /* should panic */ 837 exit(0); /* return normally if it doesn't panic */ 838 } else if (pid < 0){ 839 printf("Fork Failed\n"); 840 return -1; 841 } 842 wait(&status); 843 if(status == 0) 844 return -1; 845 846 return 0; 847 } 848 849 static int 850 test_failing_mbuf_sanity_check(struct rte_mempool *pktmbuf_pool) 851 { 852 struct rte_mbuf *buf; 853 struct rte_mbuf badbuf; 854 855 printf("Checking rte_mbuf_sanity_check for failure conditions\n"); 856 857 /* get a good mbuf to use to make copies */ 858 buf = rte_pktmbuf_alloc(pktmbuf_pool); 859 if (buf == NULL) 860 return -1; 861 printf("Checking good mbuf initially\n"); 862 if (verify_mbuf_check_panics(buf) != -1) 863 return -1; 864 865 printf("Now checking for error conditions\n"); 866 867 if (verify_mbuf_check_panics(NULL)) { 868 printf("Error with NULL mbuf test\n"); 869 return -1; 870 } 871 872 badbuf = *buf; 873 badbuf.pool = NULL; 874 if (verify_mbuf_check_panics(&badbuf)) { 875 printf("Error with bad-pool mbuf test\n"); 876 return -1; 877 } 878 879 badbuf = *buf; 880 badbuf.buf_iova = 0; 881 if (verify_mbuf_check_panics(&badbuf)) { 882 printf("Error with bad-physaddr mbuf test\n"); 883 return -1; 884 } 885 886 badbuf = *buf; 887 badbuf.buf_addr = NULL; 888 if (verify_mbuf_check_panics(&badbuf)) { 889 printf("Error with bad-addr mbuf test\n"); 890 return -1; 891 } 892 893 badbuf = *buf; 894 badbuf.refcnt = 0; 895 if (verify_mbuf_check_panics(&badbuf)) { 896 printf("Error with bad-refcnt(0) mbuf test\n"); 897 return -1; 898 } 899 900 badbuf = *buf; 901 badbuf.refcnt = UINT16_MAX; 902 if (verify_mbuf_check_panics(&badbuf)) { 903 printf("Error with bad-refcnt(MAX) mbuf test\n"); 904 return -1; 905 } 906 907 return 0; 908 } 909 910 static int 911 test_mbuf_linearize(struct rte_mempool *pktmbuf_pool, int pkt_len, 912 int nb_segs) 913 { 914 915 struct rte_mbuf *m = NULL, *mbuf = NULL; 916 uint8_t *data; 917 int data_len = 0; 918 int remain; 919 int seg, seg_len; 920 int i; 921 922 if (pkt_len < 1) { 923 printf("Packet size must be 1 or more (is %d)\n", pkt_len); 924 return -1; 925 } 926 927 if (nb_segs < 1) { 928 printf("Number of segments must be 1 or more (is %d)\n", 929 nb_segs); 930 return -1; 931 } 932 933 seg_len = pkt_len / nb_segs; 934 if (seg_len == 0) 935 seg_len = 1; 936 937 remain = pkt_len; 938 939 /* Create chained mbuf_src and fill it generated data */ 940 for (seg = 0; remain > 0; seg++) { 941 942 m = rte_pktmbuf_alloc(pktmbuf_pool); 943 if (m == NULL) { 944 printf("Cannot create segment for source mbuf"); 945 goto fail; 946 } 947 948 /* Make sure if tailroom is zeroed */ 949 memset(rte_pktmbuf_mtod(m, uint8_t *), 0, 950 rte_pktmbuf_tailroom(m)); 951 952 data_len = remain; 953 if (data_len > seg_len) 954 data_len = seg_len; 955 956 data = (uint8_t *)rte_pktmbuf_append(m, data_len); 957 if (data == NULL) { 958 printf("Cannot append %d bytes to the mbuf\n", 959 data_len); 960 goto fail; 961 } 962 963 for (i = 0; i < data_len; i++) 964 data[i] = (seg * seg_len + i) % 0x0ff; 965 966 if (seg == 0) 967 mbuf = m; 968 else 969 rte_pktmbuf_chain(mbuf, m); 970 971 remain -= data_len; 972 } 973 974 /* Create destination buffer to store coalesced data */ 975 if (rte_pktmbuf_linearize(mbuf)) { 976 printf("Mbuf linearization failed\n"); 977 goto fail; 978 } 979 980 if (!rte_pktmbuf_is_contiguous(mbuf)) { 981 printf("Source buffer should be contiguous after " 982 "linearization\n"); 983 goto fail; 984 } 985 986 data = rte_pktmbuf_mtod(mbuf, uint8_t *); 987 988 for (i = 0; i < pkt_len; i++) 989 if (data[i] != (i % 0x0ff)) { 990 printf("Incorrect data in linearized mbuf\n"); 991 goto fail; 992 } 993 994 rte_pktmbuf_free(mbuf); 995 return 0; 996 997 fail: 998 if (mbuf) 999 rte_pktmbuf_free(mbuf); 1000 return -1; 1001 } 1002 1003 static int 1004 test_mbuf_linearize_check(struct rte_mempool *pktmbuf_pool) 1005 { 1006 struct test_mbuf_array { 1007 int size; 1008 int nb_segs; 1009 } mbuf_array[] = { 1010 { 128, 1 }, 1011 { 64, 64 }, 1012 { 512, 10 }, 1013 { 250, 11 }, 1014 { 123, 8 }, 1015 }; 1016 unsigned int i; 1017 1018 printf("Test mbuf linearize API\n"); 1019 1020 for (i = 0; i < RTE_DIM(mbuf_array); i++) 1021 if (test_mbuf_linearize(pktmbuf_pool, mbuf_array[i].size, 1022 mbuf_array[i].nb_segs)) { 1023 printf("Test failed for %d, %d\n", mbuf_array[i].size, 1024 mbuf_array[i].nb_segs); 1025 return -1; 1026 } 1027 1028 return 0; 1029 } 1030 1031 /* 1032 * Helper function for test_tx_ofload 1033 */ 1034 static inline void 1035 set_tx_offload(struct rte_mbuf *mb, uint64_t il2, uint64_t il3, uint64_t il4, 1036 uint64_t tso, uint64_t ol3, uint64_t ol2) 1037 { 1038 mb->l2_len = il2; 1039 mb->l3_len = il3; 1040 mb->l4_len = il4; 1041 mb->tso_segsz = tso; 1042 mb->outer_l3_len = ol3; 1043 mb->outer_l2_len = ol2; 1044 } 1045 1046 static int 1047 test_tx_offload(void) 1048 { 1049 struct rte_mbuf *mb; 1050 uint64_t tm, v1, v2; 1051 size_t sz; 1052 uint32_t i; 1053 1054 static volatile struct { 1055 uint16_t l2; 1056 uint16_t l3; 1057 uint16_t l4; 1058 uint16_t tso; 1059 } txof; 1060 1061 const uint32_t num = 0x10000; 1062 1063 txof.l2 = rte_rand() % (1 << RTE_MBUF_L2_LEN_BITS); 1064 txof.l3 = rte_rand() % (1 << RTE_MBUF_L3_LEN_BITS); 1065 txof.l4 = rte_rand() % (1 << RTE_MBUF_L4_LEN_BITS); 1066 txof.tso = rte_rand() % (1 << RTE_MBUF_TSO_SEGSZ_BITS); 1067 1068 printf("%s started, tx_offload = {\n" 1069 "\tl2_len=%#hx,\n" 1070 "\tl3_len=%#hx,\n" 1071 "\tl4_len=%#hx,\n" 1072 "\ttso_segsz=%#hx,\n" 1073 "\touter_l3_len=%#x,\n" 1074 "\touter_l2_len=%#x,\n" 1075 "};\n", 1076 __func__, 1077 txof.l2, txof.l3, txof.l4, txof.tso, txof.l3, txof.l2); 1078 1079 sz = sizeof(*mb) * num; 1080 mb = rte_zmalloc(NULL, sz, RTE_CACHE_LINE_SIZE); 1081 if (mb == NULL) { 1082 printf("%s failed, out of memory\n", __func__); 1083 return -ENOMEM; 1084 } 1085 1086 memset(mb, 0, sz); 1087 tm = rte_rdtsc_precise(); 1088 1089 for (i = 0; i != num; i++) 1090 set_tx_offload(mb + i, txof.l2, txof.l3, txof.l4, 1091 txof.tso, txof.l3, txof.l2); 1092 1093 tm = rte_rdtsc_precise() - tm; 1094 printf("%s set tx_offload by bit-fields: %u iterations, %" 1095 PRIu64 " cycles, %#Lf cycles/iter\n", 1096 __func__, num, tm, (long double)tm / num); 1097 1098 v1 = mb[rte_rand() % num].tx_offload; 1099 1100 memset(mb, 0, sz); 1101 tm = rte_rdtsc_precise(); 1102 1103 for (i = 0; i != num; i++) 1104 mb[i].tx_offload = rte_mbuf_tx_offload(txof.l2, txof.l3, 1105 txof.l4, txof.tso, txof.l3, txof.l2, 0); 1106 1107 tm = rte_rdtsc_precise() - tm; 1108 printf("%s set raw tx_offload: %u iterations, %" 1109 PRIu64 " cycles, %#Lf cycles/iter\n", 1110 __func__, num, tm, (long double)tm / num); 1111 1112 v2 = mb[rte_rand() % num].tx_offload; 1113 1114 rte_free(mb); 1115 1116 printf("%s finished\n" 1117 "expected tx_offload value: 0x%" PRIx64 ";\n" 1118 "rte_mbuf_tx_offload value: 0x%" PRIx64 ";\n", 1119 __func__, v1, v2); 1120 1121 return (v1 == v2) ? 0 : -EINVAL; 1122 } 1123 1124 static int 1125 test_mbuf(void) 1126 { 1127 int ret = -1; 1128 struct rte_mempool *pktmbuf_pool = NULL; 1129 struct rte_mempool *pktmbuf_pool2 = NULL; 1130 1131 1132 RTE_BUILD_BUG_ON(sizeof(struct rte_mbuf) != RTE_CACHE_LINE_MIN_SIZE * 2); 1133 1134 /* create pktmbuf pool if it does not exist */ 1135 pktmbuf_pool = rte_pktmbuf_pool_create("test_pktmbuf_pool", 1136 NB_MBUF, 32, 0, MBUF_DATA_SIZE, SOCKET_ID_ANY); 1137 1138 if (pktmbuf_pool == NULL) { 1139 printf("cannot allocate mbuf pool\n"); 1140 goto err; 1141 } 1142 1143 /* create a specific pktmbuf pool with a priv_size != 0 and no data 1144 * room size */ 1145 pktmbuf_pool2 = rte_pktmbuf_pool_create("test_pktmbuf_pool2", 1146 NB_MBUF, 32, MBUF2_PRIV_SIZE, 0, SOCKET_ID_ANY); 1147 1148 if (pktmbuf_pool2 == NULL) { 1149 printf("cannot allocate mbuf pool\n"); 1150 goto err; 1151 } 1152 1153 /* test multiple mbuf alloc */ 1154 if (test_pktmbuf_pool(pktmbuf_pool) < 0) { 1155 printf("test_mbuf_pool() failed\n"); 1156 goto err; 1157 } 1158 1159 /* do it another time to check that all mbufs were freed */ 1160 if (test_pktmbuf_pool(pktmbuf_pool) < 0) { 1161 printf("test_mbuf_pool() failed (2)\n"); 1162 goto err; 1163 } 1164 1165 /* test that the pointer to the data on a packet mbuf is set properly */ 1166 if (test_pktmbuf_pool_ptr(pktmbuf_pool) < 0) { 1167 printf("test_pktmbuf_pool_ptr() failed\n"); 1168 goto err; 1169 } 1170 1171 /* test data manipulation in mbuf */ 1172 if (test_one_pktmbuf(pktmbuf_pool) < 0) { 1173 printf("test_one_mbuf() failed\n"); 1174 goto err; 1175 } 1176 1177 1178 /* 1179 * do it another time, to check that allocation reinitialize 1180 * the mbuf correctly 1181 */ 1182 if (test_one_pktmbuf(pktmbuf_pool) < 0) { 1183 printf("test_one_mbuf() failed (2)\n"); 1184 goto err; 1185 } 1186 1187 if (test_pktmbuf_with_non_ascii_data(pktmbuf_pool) < 0) { 1188 printf("test_pktmbuf_with_non_ascii_data() failed\n"); 1189 goto err; 1190 } 1191 1192 /* test free pktmbuf segment one by one */ 1193 if (test_pktmbuf_free_segment(pktmbuf_pool) < 0) { 1194 printf("test_pktmbuf_free_segment() failed.\n"); 1195 goto err; 1196 } 1197 1198 if (testclone_testupdate_testdetach(pktmbuf_pool) < 0) { 1199 printf("testclone_and_testupdate() failed \n"); 1200 goto err; 1201 } 1202 1203 if (test_attach_from_different_pool(pktmbuf_pool, pktmbuf_pool2) < 0) { 1204 printf("test_attach_from_different_pool() failed\n"); 1205 goto err; 1206 } 1207 1208 if (test_refcnt_mbuf() < 0) { 1209 printf("test_refcnt_mbuf() failed \n"); 1210 goto err; 1211 } 1212 1213 if (test_failing_mbuf_sanity_check(pktmbuf_pool) < 0) { 1214 printf("test_failing_mbuf_sanity_check() failed\n"); 1215 goto err; 1216 } 1217 1218 if (test_mbuf_linearize_check(pktmbuf_pool) < 0) { 1219 printf("test_mbuf_linearize_check() failed\n"); 1220 goto err; 1221 } 1222 1223 if (test_tx_offload() < 0) { 1224 printf("test_tx_offload() failed\n"); 1225 goto err; 1226 } 1227 1228 ret = 0; 1229 err: 1230 rte_mempool_free(pktmbuf_pool); 1231 rte_mempool_free(pktmbuf_pool2); 1232 return ret; 1233 } 1234 1235 REGISTER_TEST_COMMAND(mbuf_autotest, test_mbuf); 1236