1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <string.h> 6 #include <stdarg.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <stdint.h> 10 #include <inttypes.h> 11 #include <errno.h> 12 #include <sys/queue.h> 13 14 #include <rte_common.h> 15 #include <rte_debug.h> 16 #include <rte_log.h> 17 #include <rte_memory.h> 18 #include <rte_memcpy.h> 19 #include <rte_launch.h> 20 #include <rte_eal.h> 21 #include <rte_per_lcore.h> 22 #include <rte_lcore.h> 23 #include <rte_atomic.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_ring.h> 26 #include <rte_mempool.h> 27 #include <rte_mbuf.h> 28 #include <rte_random.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc.h> 31 32 #include "test.h" 33 34 #define MBUF_DATA_SIZE 2048 35 #define NB_MBUF 128 36 #define MBUF_TEST_DATA_LEN 1464 37 #define MBUF_TEST_DATA_LEN2 50 38 #define MBUF_TEST_HDR1_LEN 20 39 #define MBUF_TEST_HDR2_LEN 30 40 #define MBUF_TEST_ALL_HDRS_LEN (MBUF_TEST_HDR1_LEN+MBUF_TEST_HDR2_LEN) 41 42 /* size of private data for mbuf in pktmbuf_pool2 */ 43 #define MBUF2_PRIV_SIZE 128 44 45 #define REFCNT_MAX_ITER 64 46 #define REFCNT_MAX_TIMEOUT 10 47 #define REFCNT_MAX_REF (RTE_MAX_LCORE) 48 #define REFCNT_MBUF_NUM 64 49 #define REFCNT_RING_SIZE (REFCNT_MBUF_NUM * REFCNT_MAX_REF) 50 51 #define MAGIC_DATA 0x42424242 52 53 #define MAKE_STRING(x) # x 54 55 #ifdef RTE_MBUF_REFCNT_ATOMIC 56 57 static volatile uint32_t refcnt_stop_slaves; 58 static unsigned refcnt_lcore[RTE_MAX_LCORE]; 59 60 #endif 61 62 /* 63 * MBUF 64 * ==== 65 * 66 * #. Allocate a mbuf pool. 67 * 68 * - The pool contains NB_MBUF elements, where each mbuf is MBUF_SIZE 69 * bytes long. 70 * 71 * #. Test multiple allocations of mbufs from this pool. 72 * 73 * - Allocate NB_MBUF and store pointers in a table. 74 * - If an allocation fails, return an error. 75 * - Free all these mbufs. 76 * - Repeat the same test to check that mbufs were freed correctly. 77 * 78 * #. Test data manipulation in pktmbuf. 79 * 80 * - Alloc an mbuf. 81 * - Append data using rte_pktmbuf_append(). 82 * - Test for error in rte_pktmbuf_append() when len is too large. 83 * - Trim data at the end of mbuf using rte_pktmbuf_trim(). 84 * - Test for error in rte_pktmbuf_trim() when len is too large. 85 * - Prepend a header using rte_pktmbuf_prepend(). 86 * - Test for error in rte_pktmbuf_prepend() when len is too large. 87 * - Remove data at the beginning of mbuf using rte_pktmbuf_adj(). 88 * - Test for error in rte_pktmbuf_adj() when len is too large. 89 * - Check that appended data is not corrupt. 90 * - Free the mbuf. 91 * - Between all these tests, check data_len and pkt_len, and 92 * that the mbuf is contiguous. 93 * - Repeat the test to check that allocation operations 94 * reinitialize the mbuf correctly. 95 * 96 * #. Test packet cloning 97 * - Clone a mbuf and verify the data 98 * - Clone the cloned mbuf and verify the data 99 * - Attach a mbuf to another that does not have the same priv_size. 100 */ 101 102 #define GOTO_FAIL(str, ...) do { \ 103 printf("mbuf test FAILED (l.%d): <" str ">\n", \ 104 __LINE__, ##__VA_ARGS__); \ 105 goto fail; \ 106 } while(0) 107 108 /* 109 * test data manipulation in mbuf with non-ascii data 110 */ 111 static int 112 test_pktmbuf_with_non_ascii_data(struct rte_mempool *pktmbuf_pool) 113 { 114 struct rte_mbuf *m = NULL; 115 char *data; 116 117 m = rte_pktmbuf_alloc(pktmbuf_pool); 118 if (m == NULL) 119 GOTO_FAIL("Cannot allocate mbuf"); 120 if (rte_pktmbuf_pkt_len(m) != 0) 121 GOTO_FAIL("Bad length"); 122 123 data = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN); 124 if (data == NULL) 125 GOTO_FAIL("Cannot append data"); 126 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) 127 GOTO_FAIL("Bad pkt length"); 128 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) 129 GOTO_FAIL("Bad data length"); 130 memset(data, 0xff, rte_pktmbuf_pkt_len(m)); 131 if (!rte_pktmbuf_is_contiguous(m)) 132 GOTO_FAIL("Buffer should be continuous"); 133 rte_pktmbuf_dump(stdout, m, MBUF_TEST_DATA_LEN); 134 135 rte_pktmbuf_free(m); 136 137 return 0; 138 139 fail: 140 if(m) { 141 rte_pktmbuf_free(m); 142 } 143 return -1; 144 } 145 146 /* 147 * test data manipulation in mbuf 148 */ 149 static int 150 test_one_pktmbuf(struct rte_mempool *pktmbuf_pool) 151 { 152 struct rte_mbuf *m = NULL; 153 char *data, *data2, *hdr; 154 unsigned i; 155 156 printf("Test pktmbuf API\n"); 157 158 /* alloc a mbuf */ 159 160 m = rte_pktmbuf_alloc(pktmbuf_pool); 161 if (m == NULL) 162 GOTO_FAIL("Cannot allocate mbuf"); 163 if (rte_pktmbuf_pkt_len(m) != 0) 164 GOTO_FAIL("Bad length"); 165 166 rte_pktmbuf_dump(stdout, m, 0); 167 168 /* append data */ 169 170 data = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN); 171 if (data == NULL) 172 GOTO_FAIL("Cannot append data"); 173 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) 174 GOTO_FAIL("Bad pkt length"); 175 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) 176 GOTO_FAIL("Bad data length"); 177 memset(data, 0x66, rte_pktmbuf_pkt_len(m)); 178 if (!rte_pktmbuf_is_contiguous(m)) 179 GOTO_FAIL("Buffer should be continuous"); 180 rte_pktmbuf_dump(stdout, m, MBUF_TEST_DATA_LEN); 181 rte_pktmbuf_dump(stdout, m, 2*MBUF_TEST_DATA_LEN); 182 183 /* this append should fail */ 184 185 data2 = rte_pktmbuf_append(m, (uint16_t)(rte_pktmbuf_tailroom(m) + 1)); 186 if (data2 != NULL) 187 GOTO_FAIL("Append should not succeed"); 188 189 /* append some more data */ 190 191 data2 = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN2); 192 if (data2 == NULL) 193 GOTO_FAIL("Cannot append data"); 194 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_DATA_LEN2) 195 GOTO_FAIL("Bad pkt length"); 196 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_DATA_LEN2) 197 GOTO_FAIL("Bad data length"); 198 if (!rte_pktmbuf_is_contiguous(m)) 199 GOTO_FAIL("Buffer should be continuous"); 200 201 /* trim data at the end of mbuf */ 202 203 if (rte_pktmbuf_trim(m, MBUF_TEST_DATA_LEN2) < 0) 204 GOTO_FAIL("Cannot trim data"); 205 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) 206 GOTO_FAIL("Bad pkt length"); 207 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) 208 GOTO_FAIL("Bad data length"); 209 if (!rte_pktmbuf_is_contiguous(m)) 210 GOTO_FAIL("Buffer should be continuous"); 211 212 /* this trim should fail */ 213 214 if (rte_pktmbuf_trim(m, (uint16_t)(rte_pktmbuf_data_len(m) + 1)) == 0) 215 GOTO_FAIL("trim should not succeed"); 216 217 /* prepend one header */ 218 219 hdr = rte_pktmbuf_prepend(m, MBUF_TEST_HDR1_LEN); 220 if (hdr == NULL) 221 GOTO_FAIL("Cannot prepend"); 222 if (data - hdr != MBUF_TEST_HDR1_LEN) 223 GOTO_FAIL("Prepend failed"); 224 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_HDR1_LEN) 225 GOTO_FAIL("Bad pkt length"); 226 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_HDR1_LEN) 227 GOTO_FAIL("Bad data length"); 228 if (!rte_pktmbuf_is_contiguous(m)) 229 GOTO_FAIL("Buffer should be continuous"); 230 memset(hdr, 0x55, MBUF_TEST_HDR1_LEN); 231 232 /* prepend another header */ 233 234 hdr = rte_pktmbuf_prepend(m, MBUF_TEST_HDR2_LEN); 235 if (hdr == NULL) 236 GOTO_FAIL("Cannot prepend"); 237 if (data - hdr != MBUF_TEST_ALL_HDRS_LEN) 238 GOTO_FAIL("Prepend failed"); 239 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_ALL_HDRS_LEN) 240 GOTO_FAIL("Bad pkt length"); 241 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_ALL_HDRS_LEN) 242 GOTO_FAIL("Bad data length"); 243 if (!rte_pktmbuf_is_contiguous(m)) 244 GOTO_FAIL("Buffer should be continuous"); 245 memset(hdr, 0x55, MBUF_TEST_HDR2_LEN); 246 247 rte_mbuf_sanity_check(m, 1); 248 rte_mbuf_sanity_check(m, 0); 249 rte_pktmbuf_dump(stdout, m, 0); 250 251 /* this prepend should fail */ 252 253 hdr = rte_pktmbuf_prepend(m, (uint16_t)(rte_pktmbuf_headroom(m) + 1)); 254 if (hdr != NULL) 255 GOTO_FAIL("prepend should not succeed"); 256 257 /* remove data at beginning of mbuf (adj) */ 258 259 if (data != rte_pktmbuf_adj(m, MBUF_TEST_ALL_HDRS_LEN)) 260 GOTO_FAIL("rte_pktmbuf_adj failed"); 261 if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) 262 GOTO_FAIL("Bad pkt length"); 263 if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) 264 GOTO_FAIL("Bad data length"); 265 if (!rte_pktmbuf_is_contiguous(m)) 266 GOTO_FAIL("Buffer should be continuous"); 267 268 /* this adj should fail */ 269 270 if (rte_pktmbuf_adj(m, (uint16_t)(rte_pktmbuf_data_len(m) + 1)) != NULL) 271 GOTO_FAIL("rte_pktmbuf_adj should not succeed"); 272 273 /* check data */ 274 275 if (!rte_pktmbuf_is_contiguous(m)) 276 GOTO_FAIL("Buffer should be continuous"); 277 278 for (i=0; i<MBUF_TEST_DATA_LEN; i++) { 279 if (data[i] != 0x66) 280 GOTO_FAIL("Data corrupted at offset %u", i); 281 } 282 283 /* free mbuf */ 284 285 rte_pktmbuf_free(m); 286 m = NULL; 287 return 0; 288 289 fail: 290 if (m) 291 rte_pktmbuf_free(m); 292 return -1; 293 } 294 295 static int 296 testclone_testupdate_testdetach(struct rte_mempool *pktmbuf_pool) 297 { 298 struct rte_mbuf *m = NULL; 299 struct rte_mbuf *clone = NULL; 300 struct rte_mbuf *clone2 = NULL; 301 unaligned_uint32_t *data; 302 303 /* alloc a mbuf */ 304 m = rte_pktmbuf_alloc(pktmbuf_pool); 305 if (m == NULL) 306 GOTO_FAIL("ooops not allocating mbuf"); 307 308 if (rte_pktmbuf_pkt_len(m) != 0) 309 GOTO_FAIL("Bad length"); 310 311 rte_pktmbuf_append(m, sizeof(uint32_t)); 312 data = rte_pktmbuf_mtod(m, unaligned_uint32_t *); 313 *data = MAGIC_DATA; 314 315 /* clone the allocated mbuf */ 316 clone = rte_pktmbuf_clone(m, pktmbuf_pool); 317 if (clone == NULL) 318 GOTO_FAIL("cannot clone data\n"); 319 320 data = rte_pktmbuf_mtod(clone, unaligned_uint32_t *); 321 if (*data != MAGIC_DATA) 322 GOTO_FAIL("invalid data in clone\n"); 323 324 if (rte_mbuf_refcnt_read(m) != 2) 325 GOTO_FAIL("invalid refcnt in m\n"); 326 327 /* free the clone */ 328 rte_pktmbuf_free(clone); 329 clone = NULL; 330 331 /* same test with a chained mbuf */ 332 m->next = rte_pktmbuf_alloc(pktmbuf_pool); 333 if (m->next == NULL) 334 GOTO_FAIL("Next Pkt Null\n"); 335 336 rte_pktmbuf_append(m->next, sizeof(uint32_t)); 337 data = rte_pktmbuf_mtod(m->next, unaligned_uint32_t *); 338 *data = MAGIC_DATA; 339 340 clone = rte_pktmbuf_clone(m, pktmbuf_pool); 341 if (clone == NULL) 342 GOTO_FAIL("cannot clone data\n"); 343 344 data = rte_pktmbuf_mtod(clone, unaligned_uint32_t *); 345 if (*data != MAGIC_DATA) 346 GOTO_FAIL("invalid data in clone\n"); 347 348 data = rte_pktmbuf_mtod(clone->next, unaligned_uint32_t *); 349 if (*data != MAGIC_DATA) 350 GOTO_FAIL("invalid data in clone->next\n"); 351 352 if (rte_mbuf_refcnt_read(m) != 2) 353 GOTO_FAIL("invalid refcnt in m\n"); 354 355 if (rte_mbuf_refcnt_read(m->next) != 2) 356 GOTO_FAIL("invalid refcnt in m->next\n"); 357 358 /* try to clone the clone */ 359 360 clone2 = rte_pktmbuf_clone(clone, pktmbuf_pool); 361 if (clone2 == NULL) 362 GOTO_FAIL("cannot clone the clone\n"); 363 364 data = rte_pktmbuf_mtod(clone2, unaligned_uint32_t *); 365 if (*data != MAGIC_DATA) 366 GOTO_FAIL("invalid data in clone2\n"); 367 368 data = rte_pktmbuf_mtod(clone2->next, unaligned_uint32_t *); 369 if (*data != MAGIC_DATA) 370 GOTO_FAIL("invalid data in clone2->next\n"); 371 372 if (rte_mbuf_refcnt_read(m) != 3) 373 GOTO_FAIL("invalid refcnt in m\n"); 374 375 if (rte_mbuf_refcnt_read(m->next) != 3) 376 GOTO_FAIL("invalid refcnt in m->next\n"); 377 378 /* free mbuf */ 379 rte_pktmbuf_free(m); 380 rte_pktmbuf_free(clone); 381 rte_pktmbuf_free(clone2); 382 383 m = NULL; 384 clone = NULL; 385 clone2 = NULL; 386 printf("%s ok\n", __func__); 387 return 0; 388 389 fail: 390 if (m) 391 rte_pktmbuf_free(m); 392 if (clone) 393 rte_pktmbuf_free(clone); 394 if (clone2) 395 rte_pktmbuf_free(clone2); 396 return -1; 397 } 398 399 static int 400 test_attach_from_different_pool(struct rte_mempool *pktmbuf_pool, 401 struct rte_mempool *pktmbuf_pool2) 402 { 403 struct rte_mbuf *m = NULL; 404 struct rte_mbuf *clone = NULL; 405 struct rte_mbuf *clone2 = NULL; 406 char *data, *c_data, *c_data2; 407 408 /* alloc a mbuf */ 409 m = rte_pktmbuf_alloc(pktmbuf_pool); 410 if (m == NULL) 411 GOTO_FAIL("cannot allocate mbuf"); 412 413 if (rte_pktmbuf_pkt_len(m) != 0) 414 GOTO_FAIL("Bad length"); 415 416 data = rte_pktmbuf_mtod(m, char *); 417 418 /* allocate a new mbuf from the second pool, and attach it to the first 419 * mbuf */ 420 clone = rte_pktmbuf_alloc(pktmbuf_pool2); 421 if (clone == NULL) 422 GOTO_FAIL("cannot allocate mbuf from second pool\n"); 423 424 /* check data room size and priv size, and erase priv */ 425 if (rte_pktmbuf_data_room_size(clone->pool) != 0) 426 GOTO_FAIL("data room size should be 0\n"); 427 if (rte_pktmbuf_priv_size(clone->pool) != MBUF2_PRIV_SIZE) 428 GOTO_FAIL("data room size should be %d\n", MBUF2_PRIV_SIZE); 429 memset(clone + 1, 0, MBUF2_PRIV_SIZE); 430 431 /* save data pointer to compare it after detach() */ 432 c_data = rte_pktmbuf_mtod(clone, char *); 433 if (c_data != (char *)clone + sizeof(*clone) + MBUF2_PRIV_SIZE) 434 GOTO_FAIL("bad data pointer in clone"); 435 if (rte_pktmbuf_headroom(clone) != 0) 436 GOTO_FAIL("bad headroom in clone"); 437 438 rte_pktmbuf_attach(clone, m); 439 440 if (rte_pktmbuf_mtod(clone, char *) != data) 441 GOTO_FAIL("clone was not attached properly\n"); 442 if (rte_pktmbuf_headroom(clone) != RTE_PKTMBUF_HEADROOM) 443 GOTO_FAIL("bad headroom in clone after attach"); 444 if (rte_mbuf_refcnt_read(m) != 2) 445 GOTO_FAIL("invalid refcnt in m\n"); 446 447 /* allocate a new mbuf from the second pool, and attach it to the first 448 * cloned mbuf */ 449 clone2 = rte_pktmbuf_alloc(pktmbuf_pool2); 450 if (clone2 == NULL) 451 GOTO_FAIL("cannot allocate clone2 from second pool\n"); 452 453 /* check data room size and priv size, and erase priv */ 454 if (rte_pktmbuf_data_room_size(clone2->pool) != 0) 455 GOTO_FAIL("data room size should be 0\n"); 456 if (rte_pktmbuf_priv_size(clone2->pool) != MBUF2_PRIV_SIZE) 457 GOTO_FAIL("data room size should be %d\n", MBUF2_PRIV_SIZE); 458 memset(clone2 + 1, 0, MBUF2_PRIV_SIZE); 459 460 /* save data pointer to compare it after detach() */ 461 c_data2 = rte_pktmbuf_mtod(clone2, char *); 462 if (c_data2 != (char *)clone2 + sizeof(*clone2) + MBUF2_PRIV_SIZE) 463 GOTO_FAIL("bad data pointer in clone2"); 464 if (rte_pktmbuf_headroom(clone2) != 0) 465 GOTO_FAIL("bad headroom in clone2"); 466 467 rte_pktmbuf_attach(clone2, clone); 468 469 if (rte_pktmbuf_mtod(clone2, char *) != data) 470 GOTO_FAIL("clone2 was not attached properly\n"); 471 if (rte_pktmbuf_headroom(clone2) != RTE_PKTMBUF_HEADROOM) 472 GOTO_FAIL("bad headroom in clone2 after attach"); 473 if (rte_mbuf_refcnt_read(m) != 3) 474 GOTO_FAIL("invalid refcnt in m\n"); 475 476 /* detach the clones */ 477 rte_pktmbuf_detach(clone); 478 if (c_data != rte_pktmbuf_mtod(clone, char *)) 479 GOTO_FAIL("clone was not detached properly\n"); 480 if (rte_mbuf_refcnt_read(m) != 2) 481 GOTO_FAIL("invalid refcnt in m\n"); 482 483 rte_pktmbuf_detach(clone2); 484 if (c_data2 != rte_pktmbuf_mtod(clone2, char *)) 485 GOTO_FAIL("clone2 was not detached properly\n"); 486 if (rte_mbuf_refcnt_read(m) != 1) 487 GOTO_FAIL("invalid refcnt in m\n"); 488 489 /* free the clones and the initial mbuf */ 490 rte_pktmbuf_free(clone2); 491 rte_pktmbuf_free(clone); 492 rte_pktmbuf_free(m); 493 printf("%s ok\n", __func__); 494 return 0; 495 496 fail: 497 if (m) 498 rte_pktmbuf_free(m); 499 if (clone) 500 rte_pktmbuf_free(clone); 501 if (clone2) 502 rte_pktmbuf_free(clone2); 503 return -1; 504 } 505 #undef GOTO_FAIL 506 507 /* 508 * test allocation and free of mbufs 509 */ 510 static int 511 test_pktmbuf_pool(struct rte_mempool *pktmbuf_pool) 512 { 513 unsigned i; 514 struct rte_mbuf *m[NB_MBUF]; 515 int ret = 0; 516 517 for (i=0; i<NB_MBUF; i++) 518 m[i] = NULL; 519 520 /* alloc NB_MBUF mbufs */ 521 for (i=0; i<NB_MBUF; i++) { 522 m[i] = rte_pktmbuf_alloc(pktmbuf_pool); 523 if (m[i] == NULL) { 524 printf("rte_pktmbuf_alloc() failed (%u)\n", i); 525 ret = -1; 526 } 527 } 528 struct rte_mbuf *extra = NULL; 529 extra = rte_pktmbuf_alloc(pktmbuf_pool); 530 if(extra != NULL) { 531 printf("Error pool not empty"); 532 ret = -1; 533 } 534 extra = rte_pktmbuf_clone(m[0], pktmbuf_pool); 535 if(extra != NULL) { 536 printf("Error pool not empty"); 537 ret = -1; 538 } 539 /* free them */ 540 for (i=0; i<NB_MBUF; i++) { 541 if (m[i] != NULL) 542 rte_pktmbuf_free(m[i]); 543 } 544 545 return ret; 546 } 547 548 /* 549 * test that the pointer to the data on a packet mbuf is set properly 550 */ 551 static int 552 test_pktmbuf_pool_ptr(struct rte_mempool *pktmbuf_pool) 553 { 554 unsigned i; 555 struct rte_mbuf *m[NB_MBUF]; 556 int ret = 0; 557 558 for (i=0; i<NB_MBUF; i++) 559 m[i] = NULL; 560 561 /* alloc NB_MBUF mbufs */ 562 for (i=0; i<NB_MBUF; i++) { 563 m[i] = rte_pktmbuf_alloc(pktmbuf_pool); 564 if (m[i] == NULL) { 565 printf("rte_pktmbuf_alloc() failed (%u)\n", i); 566 ret = -1; 567 break; 568 } 569 m[i]->data_off += 64; 570 } 571 572 /* free them */ 573 for (i=0; i<NB_MBUF; i++) { 574 if (m[i] != NULL) 575 rte_pktmbuf_free(m[i]); 576 } 577 578 for (i=0; i<NB_MBUF; i++) 579 m[i] = NULL; 580 581 /* alloc NB_MBUF mbufs */ 582 for (i=0; i<NB_MBUF; i++) { 583 m[i] = rte_pktmbuf_alloc(pktmbuf_pool); 584 if (m[i] == NULL) { 585 printf("rte_pktmbuf_alloc() failed (%u)\n", i); 586 ret = -1; 587 break; 588 } 589 if (m[i]->data_off != RTE_PKTMBUF_HEADROOM) { 590 printf("invalid data_off\n"); 591 ret = -1; 592 } 593 } 594 595 /* free them */ 596 for (i=0; i<NB_MBUF; i++) { 597 if (m[i] != NULL) 598 rte_pktmbuf_free(m[i]); 599 } 600 601 return ret; 602 } 603 604 static int 605 test_pktmbuf_free_segment(struct rte_mempool *pktmbuf_pool) 606 { 607 unsigned i; 608 struct rte_mbuf *m[NB_MBUF]; 609 int ret = 0; 610 611 for (i=0; i<NB_MBUF; i++) 612 m[i] = NULL; 613 614 /* alloc NB_MBUF mbufs */ 615 for (i=0; i<NB_MBUF; i++) { 616 m[i] = rte_pktmbuf_alloc(pktmbuf_pool); 617 if (m[i] == NULL) { 618 printf("rte_pktmbuf_alloc() failed (%u)\n", i); 619 ret = -1; 620 } 621 } 622 623 /* free them */ 624 for (i=0; i<NB_MBUF; i++) { 625 if (m[i] != NULL) { 626 struct rte_mbuf *mb, *mt; 627 628 mb = m[i]; 629 while(mb != NULL) { 630 mt = mb; 631 mb = mb->next; 632 rte_pktmbuf_free_seg(mt); 633 } 634 } 635 } 636 637 return ret; 638 } 639 640 /* 641 * Stress test for rte_mbuf atomic refcnt. 642 * Implies that RTE_MBUF_REFCNT_ATOMIC is defined. 643 * For more efficiency, recommended to run with RTE_LIBRTE_MBUF_DEBUG defined. 644 */ 645 646 #ifdef RTE_MBUF_REFCNT_ATOMIC 647 648 static int 649 test_refcnt_slave(void *arg) 650 { 651 unsigned lcore, free; 652 void *mp = 0; 653 struct rte_ring *refcnt_mbuf_ring = arg; 654 655 lcore = rte_lcore_id(); 656 printf("%s started at lcore %u\n", __func__, lcore); 657 658 free = 0; 659 while (refcnt_stop_slaves == 0) { 660 if (rte_ring_dequeue(refcnt_mbuf_ring, &mp) == 0) { 661 free++; 662 rte_pktmbuf_free(mp); 663 } 664 } 665 666 refcnt_lcore[lcore] += free; 667 printf("%s finished at lcore %u, " 668 "number of freed mbufs: %u\n", 669 __func__, lcore, free); 670 return 0; 671 } 672 673 static void 674 test_refcnt_iter(unsigned int lcore, unsigned int iter, 675 struct rte_mempool *refcnt_pool, 676 struct rte_ring *refcnt_mbuf_ring) 677 { 678 uint16_t ref; 679 unsigned i, n, tref, wn; 680 struct rte_mbuf *m; 681 682 tref = 0; 683 684 /* For each mbuf in the pool: 685 * - allocate mbuf, 686 * - increment it's reference up to N+1, 687 * - enqueue it N times into the ring for slave cores to free. 688 */ 689 for (i = 0, n = rte_mempool_avail_count(refcnt_pool); 690 i != n && (m = rte_pktmbuf_alloc(refcnt_pool)) != NULL; 691 i++) { 692 ref = RTE_MAX(rte_rand() % REFCNT_MAX_REF, 1UL); 693 tref += ref; 694 if ((ref & 1) != 0) { 695 rte_pktmbuf_refcnt_update(m, ref); 696 while (ref-- != 0) 697 rte_ring_enqueue(refcnt_mbuf_ring, m); 698 } else { 699 while (ref-- != 0) { 700 rte_pktmbuf_refcnt_update(m, 1); 701 rte_ring_enqueue(refcnt_mbuf_ring, m); 702 } 703 } 704 rte_pktmbuf_free(m); 705 } 706 707 if (i != n) 708 rte_panic("(lcore=%u, iter=%u): was able to allocate only " 709 "%u from %u mbufs\n", lcore, iter, i, n); 710 711 /* wait till slave lcores will consume all mbufs */ 712 while (!rte_ring_empty(refcnt_mbuf_ring)) 713 ; 714 715 /* check that all mbufs are back into mempool by now */ 716 for (wn = 0; wn != REFCNT_MAX_TIMEOUT; wn++) { 717 if ((i = rte_mempool_avail_count(refcnt_pool)) == n) { 718 refcnt_lcore[lcore] += tref; 719 printf("%s(lcore=%u, iter=%u) completed, " 720 "%u references processed\n", 721 __func__, lcore, iter, tref); 722 return; 723 } 724 rte_delay_ms(100); 725 } 726 727 rte_panic("(lcore=%u, iter=%u): after %us only " 728 "%u of %u mbufs left free\n", lcore, iter, wn, i, n); 729 } 730 731 static int 732 test_refcnt_master(struct rte_mempool *refcnt_pool, 733 struct rte_ring *refcnt_mbuf_ring) 734 { 735 unsigned i, lcore; 736 737 lcore = rte_lcore_id(); 738 printf("%s started at lcore %u\n", __func__, lcore); 739 740 for (i = 0; i != REFCNT_MAX_ITER; i++) 741 test_refcnt_iter(lcore, i, refcnt_pool, refcnt_mbuf_ring); 742 743 refcnt_stop_slaves = 1; 744 rte_wmb(); 745 746 printf("%s finished at lcore %u\n", __func__, lcore); 747 return 0; 748 } 749 750 #endif 751 752 static int 753 test_refcnt_mbuf(void) 754 { 755 #ifdef RTE_MBUF_REFCNT_ATOMIC 756 unsigned lnum, master, slave, tref; 757 int ret = -1; 758 struct rte_mempool *refcnt_pool = NULL; 759 struct rte_ring *refcnt_mbuf_ring = NULL; 760 761 if ((lnum = rte_lcore_count()) == 1) { 762 printf("skipping %s, number of lcores: %u is not enough\n", 763 __func__, lnum); 764 return 0; 765 } 766 767 printf("starting %s, at %u lcores\n", __func__, lnum); 768 769 /* create refcnt pool & ring if they don't exist */ 770 771 refcnt_pool = rte_pktmbuf_pool_create(MAKE_STRING(refcnt_pool), 772 REFCNT_MBUF_NUM, 0, 0, 0, 773 SOCKET_ID_ANY); 774 if (refcnt_pool == NULL) { 775 printf("%s: cannot allocate " MAKE_STRING(refcnt_pool) "\n", 776 __func__); 777 return -1; 778 } 779 780 refcnt_mbuf_ring = rte_ring_create("refcnt_mbuf_ring", 781 rte_align32pow2(REFCNT_RING_SIZE), SOCKET_ID_ANY, 782 RING_F_SP_ENQ); 783 if (refcnt_mbuf_ring == NULL) { 784 printf("%s: cannot allocate " MAKE_STRING(refcnt_mbuf_ring) 785 "\n", __func__); 786 goto err; 787 } 788 789 refcnt_stop_slaves = 0; 790 memset(refcnt_lcore, 0, sizeof (refcnt_lcore)); 791 792 rte_eal_mp_remote_launch(test_refcnt_slave, refcnt_mbuf_ring, 793 SKIP_MASTER); 794 795 test_refcnt_master(refcnt_pool, refcnt_mbuf_ring); 796 797 rte_eal_mp_wait_lcore(); 798 799 /* check that we porcessed all references */ 800 tref = 0; 801 master = rte_get_master_lcore(); 802 803 RTE_LCORE_FOREACH_SLAVE(slave) 804 tref += refcnt_lcore[slave]; 805 806 if (tref != refcnt_lcore[master]) 807 rte_panic("refernced mbufs: %u, freed mbufs: %u\n", 808 tref, refcnt_lcore[master]); 809 810 rte_mempool_dump(stdout, refcnt_pool); 811 rte_ring_dump(stdout, refcnt_mbuf_ring); 812 813 ret = 0; 814 815 err: 816 rte_mempool_free(refcnt_pool); 817 rte_ring_free(refcnt_mbuf_ring); 818 return ret; 819 #else 820 return 0; 821 #endif 822 } 823 824 #include <unistd.h> 825 #include <sys/wait.h> 826 827 /* use fork() to test mbuf errors panic */ 828 static int 829 verify_mbuf_check_panics(struct rte_mbuf *buf) 830 { 831 int pid; 832 int status; 833 834 pid = fork(); 835 836 if (pid == 0) { 837 rte_mbuf_sanity_check(buf, 1); /* should panic */ 838 exit(0); /* return normally if it doesn't panic */ 839 } else if (pid < 0){ 840 printf("Fork Failed\n"); 841 return -1; 842 } 843 wait(&status); 844 if(status == 0) 845 return -1; 846 847 return 0; 848 } 849 850 static int 851 test_failing_mbuf_sanity_check(struct rte_mempool *pktmbuf_pool) 852 { 853 struct rte_mbuf *buf; 854 struct rte_mbuf badbuf; 855 856 printf("Checking rte_mbuf_sanity_check for failure conditions\n"); 857 858 /* get a good mbuf to use to make copies */ 859 buf = rte_pktmbuf_alloc(pktmbuf_pool); 860 if (buf == NULL) 861 return -1; 862 printf("Checking good mbuf initially\n"); 863 if (verify_mbuf_check_panics(buf) != -1) 864 return -1; 865 866 printf("Now checking for error conditions\n"); 867 868 if (verify_mbuf_check_panics(NULL)) { 869 printf("Error with NULL mbuf test\n"); 870 return -1; 871 } 872 873 badbuf = *buf; 874 badbuf.pool = NULL; 875 if (verify_mbuf_check_panics(&badbuf)) { 876 printf("Error with bad-pool mbuf test\n"); 877 return -1; 878 } 879 880 badbuf = *buf; 881 badbuf.buf_iova = 0; 882 if (verify_mbuf_check_panics(&badbuf)) { 883 printf("Error with bad-physaddr mbuf test\n"); 884 return -1; 885 } 886 887 badbuf = *buf; 888 badbuf.buf_addr = NULL; 889 if (verify_mbuf_check_panics(&badbuf)) { 890 printf("Error with bad-addr mbuf test\n"); 891 return -1; 892 } 893 894 badbuf = *buf; 895 badbuf.refcnt = 0; 896 if (verify_mbuf_check_panics(&badbuf)) { 897 printf("Error with bad-refcnt(0) mbuf test\n"); 898 return -1; 899 } 900 901 badbuf = *buf; 902 badbuf.refcnt = UINT16_MAX; 903 if (verify_mbuf_check_panics(&badbuf)) { 904 printf("Error with bad-refcnt(MAX) mbuf test\n"); 905 return -1; 906 } 907 908 return 0; 909 } 910 911 static int 912 test_mbuf_linearize(struct rte_mempool *pktmbuf_pool, int pkt_len, 913 int nb_segs) 914 { 915 916 struct rte_mbuf *m = NULL, *mbuf = NULL; 917 uint8_t *data; 918 int data_len = 0; 919 int remain; 920 int seg, seg_len; 921 int i; 922 923 if (pkt_len < 1) { 924 printf("Packet size must be 1 or more (is %d)\n", pkt_len); 925 return -1; 926 } 927 928 if (nb_segs < 1) { 929 printf("Number of segments must be 1 or more (is %d)\n", 930 nb_segs); 931 return -1; 932 } 933 934 seg_len = pkt_len / nb_segs; 935 if (seg_len == 0) 936 seg_len = 1; 937 938 remain = pkt_len; 939 940 /* Create chained mbuf_src and fill it generated data */ 941 for (seg = 0; remain > 0; seg++) { 942 943 m = rte_pktmbuf_alloc(pktmbuf_pool); 944 if (m == NULL) { 945 printf("Cannot create segment for source mbuf"); 946 goto fail; 947 } 948 949 /* Make sure if tailroom is zeroed */ 950 memset(rte_pktmbuf_mtod(m, uint8_t *), 0, 951 rte_pktmbuf_tailroom(m)); 952 953 data_len = remain; 954 if (data_len > seg_len) 955 data_len = seg_len; 956 957 data = (uint8_t *)rte_pktmbuf_append(m, data_len); 958 if (data == NULL) { 959 printf("Cannot append %d bytes to the mbuf\n", 960 data_len); 961 goto fail; 962 } 963 964 for (i = 0; i < data_len; i++) 965 data[i] = (seg * seg_len + i) % 0x0ff; 966 967 if (seg == 0) 968 mbuf = m; 969 else 970 rte_pktmbuf_chain(mbuf, m); 971 972 remain -= data_len; 973 } 974 975 /* Create destination buffer to store coalesced data */ 976 if (rte_pktmbuf_linearize(mbuf)) { 977 printf("Mbuf linearization failed\n"); 978 goto fail; 979 } 980 981 if (!rte_pktmbuf_is_contiguous(mbuf)) { 982 printf("Source buffer should be contiguous after " 983 "linearization\n"); 984 goto fail; 985 } 986 987 data = rte_pktmbuf_mtod(mbuf, uint8_t *); 988 989 for (i = 0; i < pkt_len; i++) 990 if (data[i] != (i % 0x0ff)) { 991 printf("Incorrect data in linearized mbuf\n"); 992 goto fail; 993 } 994 995 rte_pktmbuf_free(mbuf); 996 return 0; 997 998 fail: 999 if (mbuf) 1000 rte_pktmbuf_free(mbuf); 1001 return -1; 1002 } 1003 1004 static int 1005 test_mbuf_linearize_check(struct rte_mempool *pktmbuf_pool) 1006 { 1007 struct test_mbuf_array { 1008 int size; 1009 int nb_segs; 1010 } mbuf_array[] = { 1011 { 128, 1 }, 1012 { 64, 64 }, 1013 { 512, 10 }, 1014 { 250, 11 }, 1015 { 123, 8 }, 1016 }; 1017 unsigned int i; 1018 1019 printf("Test mbuf linearize API\n"); 1020 1021 for (i = 0; i < RTE_DIM(mbuf_array); i++) 1022 if (test_mbuf_linearize(pktmbuf_pool, mbuf_array[i].size, 1023 mbuf_array[i].nb_segs)) { 1024 printf("Test failed for %d, %d\n", mbuf_array[i].size, 1025 mbuf_array[i].nb_segs); 1026 return -1; 1027 } 1028 1029 return 0; 1030 } 1031 1032 /* 1033 * Helper function for test_tx_ofload 1034 */ 1035 static inline void 1036 set_tx_offload(struct rte_mbuf *mb, uint64_t il2, uint64_t il3, uint64_t il4, 1037 uint64_t tso, uint64_t ol3, uint64_t ol2) 1038 { 1039 mb->l2_len = il2; 1040 mb->l3_len = il3; 1041 mb->l4_len = il4; 1042 mb->tso_segsz = tso; 1043 mb->outer_l3_len = ol3; 1044 mb->outer_l2_len = ol2; 1045 } 1046 1047 static int 1048 test_tx_offload(void) 1049 { 1050 struct rte_mbuf *mb; 1051 uint64_t tm, v1, v2; 1052 size_t sz; 1053 uint32_t i; 1054 1055 static volatile struct { 1056 uint16_t l2; 1057 uint16_t l3; 1058 uint16_t l4; 1059 uint16_t tso; 1060 } txof; 1061 1062 const uint32_t num = 0x10000; 1063 1064 txof.l2 = rte_rand() % (1 << RTE_MBUF_L2_LEN_BITS); 1065 txof.l3 = rte_rand() % (1 << RTE_MBUF_L3_LEN_BITS); 1066 txof.l4 = rte_rand() % (1 << RTE_MBUF_L4_LEN_BITS); 1067 txof.tso = rte_rand() % (1 << RTE_MBUF_TSO_SEGSZ_BITS); 1068 1069 printf("%s started, tx_offload = {\n" 1070 "\tl2_len=%#hx,\n" 1071 "\tl3_len=%#hx,\n" 1072 "\tl4_len=%#hx,\n" 1073 "\ttso_segsz=%#hx,\n" 1074 "\touter_l3_len=%#x,\n" 1075 "\touter_l2_len=%#x,\n" 1076 "};\n", 1077 __func__, 1078 txof.l2, txof.l3, txof.l4, txof.tso, txof.l3, txof.l2); 1079 1080 sz = sizeof(*mb) * num; 1081 mb = rte_zmalloc(NULL, sz, RTE_CACHE_LINE_SIZE); 1082 if (mb == NULL) { 1083 printf("%s failed, out of memory\n", __func__); 1084 return -ENOMEM; 1085 } 1086 1087 memset(mb, 0, sz); 1088 tm = rte_rdtsc_precise(); 1089 1090 for (i = 0; i != num; i++) 1091 set_tx_offload(mb + i, txof.l2, txof.l3, txof.l4, 1092 txof.tso, txof.l3, txof.l2); 1093 1094 tm = rte_rdtsc_precise() - tm; 1095 printf("%s set tx_offload by bit-fields: %u iterations, %" 1096 PRIu64 " cycles, %#Lf cycles/iter\n", 1097 __func__, num, tm, (long double)tm / num); 1098 1099 v1 = mb[rte_rand() % num].tx_offload; 1100 1101 memset(mb, 0, sz); 1102 tm = rte_rdtsc_precise(); 1103 1104 for (i = 0; i != num; i++) 1105 mb[i].tx_offload = rte_mbuf_tx_offload(txof.l2, txof.l3, 1106 txof.l4, txof.tso, txof.l3, txof.l2, 0); 1107 1108 tm = rte_rdtsc_precise() - tm; 1109 printf("%s set raw tx_offload: %u iterations, %" 1110 PRIu64 " cycles, %#Lf cycles/iter\n", 1111 __func__, num, tm, (long double)tm / num); 1112 1113 v2 = mb[rte_rand() % num].tx_offload; 1114 1115 rte_free(mb); 1116 1117 printf("%s finished\n" 1118 "expected tx_offload value: 0x%" PRIx64 ";\n" 1119 "rte_mbuf_tx_offload value: 0x%" PRIx64 ";\n", 1120 __func__, v1, v2); 1121 1122 return (v1 == v2) ? 0 : -EINVAL; 1123 } 1124 1125 static int 1126 test_mbuf(void) 1127 { 1128 int ret = -1; 1129 struct rte_mempool *pktmbuf_pool = NULL; 1130 struct rte_mempool *pktmbuf_pool2 = NULL; 1131 1132 1133 RTE_BUILD_BUG_ON(sizeof(struct rte_mbuf) != RTE_CACHE_LINE_MIN_SIZE * 2); 1134 1135 /* create pktmbuf pool if it does not exist */ 1136 pktmbuf_pool = rte_pktmbuf_pool_create("test_pktmbuf_pool", 1137 NB_MBUF, 32, 0, MBUF_DATA_SIZE, SOCKET_ID_ANY); 1138 1139 if (pktmbuf_pool == NULL) { 1140 printf("cannot allocate mbuf pool\n"); 1141 goto err; 1142 } 1143 1144 /* create a specific pktmbuf pool with a priv_size != 0 and no data 1145 * room size */ 1146 pktmbuf_pool2 = rte_pktmbuf_pool_create("test_pktmbuf_pool2", 1147 NB_MBUF, 32, MBUF2_PRIV_SIZE, 0, SOCKET_ID_ANY); 1148 1149 if (pktmbuf_pool2 == NULL) { 1150 printf("cannot allocate mbuf pool\n"); 1151 goto err; 1152 } 1153 1154 /* test multiple mbuf alloc */ 1155 if (test_pktmbuf_pool(pktmbuf_pool) < 0) { 1156 printf("test_mbuf_pool() failed\n"); 1157 goto err; 1158 } 1159 1160 /* do it another time to check that all mbufs were freed */ 1161 if (test_pktmbuf_pool(pktmbuf_pool) < 0) { 1162 printf("test_mbuf_pool() failed (2)\n"); 1163 goto err; 1164 } 1165 1166 /* test that the pointer to the data on a packet mbuf is set properly */ 1167 if (test_pktmbuf_pool_ptr(pktmbuf_pool) < 0) { 1168 printf("test_pktmbuf_pool_ptr() failed\n"); 1169 goto err; 1170 } 1171 1172 /* test data manipulation in mbuf */ 1173 if (test_one_pktmbuf(pktmbuf_pool) < 0) { 1174 printf("test_one_mbuf() failed\n"); 1175 goto err; 1176 } 1177 1178 1179 /* 1180 * do it another time, to check that allocation reinitialize 1181 * the mbuf correctly 1182 */ 1183 if (test_one_pktmbuf(pktmbuf_pool) < 0) { 1184 printf("test_one_mbuf() failed (2)\n"); 1185 goto err; 1186 } 1187 1188 if (test_pktmbuf_with_non_ascii_data(pktmbuf_pool) < 0) { 1189 printf("test_pktmbuf_with_non_ascii_data() failed\n"); 1190 goto err; 1191 } 1192 1193 /* test free pktmbuf segment one by one */ 1194 if (test_pktmbuf_free_segment(pktmbuf_pool) < 0) { 1195 printf("test_pktmbuf_free_segment() failed.\n"); 1196 goto err; 1197 } 1198 1199 if (testclone_testupdate_testdetach(pktmbuf_pool) < 0) { 1200 printf("testclone_and_testupdate() failed \n"); 1201 goto err; 1202 } 1203 1204 if (test_attach_from_different_pool(pktmbuf_pool, pktmbuf_pool2) < 0) { 1205 printf("test_attach_from_different_pool() failed\n"); 1206 goto err; 1207 } 1208 1209 if (test_refcnt_mbuf()<0){ 1210 printf("test_refcnt_mbuf() failed \n"); 1211 goto err; 1212 } 1213 1214 if (test_failing_mbuf_sanity_check(pktmbuf_pool) < 0) { 1215 printf("test_failing_mbuf_sanity_check() failed\n"); 1216 goto err; 1217 } 1218 1219 if (test_mbuf_linearize_check(pktmbuf_pool) < 0) { 1220 printf("test_mbuf_linearize_check() failed\n"); 1221 goto err; 1222 } 1223 1224 if (test_tx_offload() < 0) { 1225 printf("test_tx_offload() failed\n"); 1226 goto err; 1227 } 1228 1229 ret = 0; 1230 err: 1231 rte_mempool_free(pktmbuf_pool); 1232 rte_mempool_free(pktmbuf_pool2); 1233 return ret; 1234 } 1235 1236 REGISTER_TEST_COMMAND(mbuf_autotest, test_mbuf); 1237