1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <string.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <stdint.h> 9 #include <inttypes.h> 10 #include <stdarg.h> 11 #include <errno.h> 12 #include <sys/queue.h> 13 14 #include <rte_common.h> 15 #include <rte_log.h> 16 #include <rte_debug.h> 17 #include <rte_memory.h> 18 #include <rte_launch.h> 19 #include <rte_cycles.h> 20 #include <rte_eal.h> 21 #include <rte_per_lcore.h> 22 #include <rte_lcore.h> 23 #include <rte_atomic.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_mempool.h> 26 #include <rte_spinlock.h> 27 #include <rte_malloc.h> 28 #include <rte_mbuf_pool_ops.h> 29 #include <rte_mbuf.h> 30 31 #include "test.h" 32 33 /* 34 * Mempool 35 * ======= 36 * 37 * Basic tests: done on one core with and without cache: 38 * 39 * - Get one object, put one object 40 * - Get two objects, put two objects 41 * - Get all objects, test that their content is not modified and 42 * put them back in the pool. 43 */ 44 45 #define MEMPOOL_ELT_SIZE 2048 46 #define MAX_KEEP 16 47 #define MEMPOOL_SIZE ((rte_lcore_count()*(MAX_KEEP+RTE_MEMPOOL_CACHE_MAX_SIZE))-1) 48 49 #define LOG_ERR() printf("test failed at %s():%d\n", __func__, __LINE__) 50 #define RET_ERR() do { \ 51 LOG_ERR(); \ 52 return -1; \ 53 } while (0) 54 #define GOTO_ERR(var, label) do { \ 55 LOG_ERR(); \ 56 var = -1; \ 57 goto label; \ 58 } while (0) 59 60 static rte_atomic32_t synchro; 61 62 /* 63 * save the object number in the first 4 bytes of object data. All 64 * other bytes are set to 0. 65 */ 66 static void 67 my_obj_init(struct rte_mempool *mp, __rte_unused void *arg, 68 void *obj, unsigned i) 69 { 70 uint32_t *objnum = obj; 71 72 memset(obj, 0, mp->elt_size); 73 *objnum = i; 74 } 75 76 /* basic tests (done on one core) */ 77 static int 78 test_mempool_basic(struct rte_mempool *mp, int use_external_cache) 79 { 80 uint32_t *objnum; 81 void **objtable; 82 void *obj, *obj2; 83 char *obj_data; 84 int ret = 0; 85 unsigned i, j; 86 int offset; 87 struct rte_mempool_cache *cache; 88 89 if (use_external_cache) { 90 /* Create a user-owned mempool cache. */ 91 cache = rte_mempool_cache_create(RTE_MEMPOOL_CACHE_MAX_SIZE, 92 SOCKET_ID_ANY); 93 if (cache == NULL) 94 RET_ERR(); 95 } else { 96 /* May be NULL if cache is disabled. */ 97 cache = rte_mempool_default_cache(mp, rte_lcore_id()); 98 } 99 100 /* dump the mempool status */ 101 rte_mempool_dump(stdout, mp); 102 103 printf("get an object\n"); 104 if (rte_mempool_generic_get(mp, &obj, 1, cache) < 0) 105 GOTO_ERR(ret, out); 106 rte_mempool_dump(stdout, mp); 107 108 /* tests that improve coverage */ 109 printf("get object count\n"); 110 /* We have to count the extra caches, one in this case. */ 111 offset = use_external_cache ? 1 * cache->len : 0; 112 if (rte_mempool_avail_count(mp) + offset != MEMPOOL_SIZE - 1) 113 GOTO_ERR(ret, out); 114 115 printf("get private data\n"); 116 if (rte_mempool_get_priv(mp) != (char *)mp + 117 MEMPOOL_HEADER_SIZE(mp, mp->cache_size)) 118 GOTO_ERR(ret, out); 119 120 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */ 121 printf("get physical address of an object\n"); 122 if (rte_mempool_virt2iova(obj) != rte_mem_virt2iova(obj)) 123 GOTO_ERR(ret, out); 124 #endif 125 126 printf("put the object back\n"); 127 rte_mempool_generic_put(mp, &obj, 1, cache); 128 rte_mempool_dump(stdout, mp); 129 130 printf("get 2 objects\n"); 131 if (rte_mempool_generic_get(mp, &obj, 1, cache) < 0) 132 GOTO_ERR(ret, out); 133 if (rte_mempool_generic_get(mp, &obj2, 1, cache) < 0) { 134 rte_mempool_generic_put(mp, &obj, 1, cache); 135 GOTO_ERR(ret, out); 136 } 137 rte_mempool_dump(stdout, mp); 138 139 printf("put the objects back\n"); 140 rte_mempool_generic_put(mp, &obj, 1, cache); 141 rte_mempool_generic_put(mp, &obj2, 1, cache); 142 rte_mempool_dump(stdout, mp); 143 144 /* 145 * get many objects: we cannot get them all because the cache 146 * on other cores may not be empty. 147 */ 148 objtable = malloc(MEMPOOL_SIZE * sizeof(void *)); 149 if (objtable == NULL) 150 GOTO_ERR(ret, out); 151 152 for (i = 0; i < MEMPOOL_SIZE; i++) { 153 if (rte_mempool_generic_get(mp, &objtable[i], 1, cache) < 0) 154 break; 155 } 156 157 /* 158 * for each object, check that its content was not modified, 159 * and put objects back in pool 160 */ 161 while (i--) { 162 obj = objtable[i]; 163 obj_data = obj; 164 objnum = obj; 165 if (*objnum > MEMPOOL_SIZE) { 166 printf("bad object number(%d)\n", *objnum); 167 ret = -1; 168 break; 169 } 170 for (j = sizeof(*objnum); j < mp->elt_size; j++) { 171 if (obj_data[j] != 0) 172 ret = -1; 173 } 174 175 rte_mempool_generic_put(mp, &objtable[i], 1, cache); 176 } 177 178 free(objtable); 179 if (ret == -1) 180 printf("objects were modified!\n"); 181 182 out: 183 if (use_external_cache) { 184 rte_mempool_cache_flush(cache, mp); 185 rte_mempool_cache_free(cache); 186 } 187 188 return ret; 189 } 190 191 static int test_mempool_creation_with_exceeded_cache_size(void) 192 { 193 struct rte_mempool *mp_cov; 194 195 mp_cov = rte_mempool_create("test_mempool_cache_too_big", 196 MEMPOOL_SIZE, 197 MEMPOOL_ELT_SIZE, 198 RTE_MEMPOOL_CACHE_MAX_SIZE + 32, 0, 199 NULL, NULL, 200 my_obj_init, NULL, 201 SOCKET_ID_ANY, 0); 202 203 if (mp_cov != NULL) { 204 rte_mempool_free(mp_cov); 205 RET_ERR(); 206 } 207 208 return 0; 209 } 210 211 static struct rte_mempool *mp_spsc; 212 static rte_spinlock_t scsp_spinlock; 213 static void *scsp_obj_table[MAX_KEEP]; 214 215 /* 216 * single producer function 217 */ 218 static int test_mempool_single_producer(void) 219 { 220 unsigned int i; 221 void *obj = NULL; 222 uint64_t start_cycles, end_cycles; 223 uint64_t duration = rte_get_timer_hz() / 4; 224 225 start_cycles = rte_get_timer_cycles(); 226 while (1) { 227 end_cycles = rte_get_timer_cycles(); 228 /* duration uses up, stop producing */ 229 if (start_cycles + duration < end_cycles) 230 break; 231 rte_spinlock_lock(&scsp_spinlock); 232 for (i = 0; i < MAX_KEEP; i ++) { 233 if (NULL != scsp_obj_table[i]) { 234 obj = scsp_obj_table[i]; 235 break; 236 } 237 } 238 rte_spinlock_unlock(&scsp_spinlock); 239 if (i >= MAX_KEEP) { 240 continue; 241 } 242 if (rte_mempool_from_obj(obj) != mp_spsc) { 243 printf("obj not owned by this mempool\n"); 244 RET_ERR(); 245 } 246 rte_mempool_put(mp_spsc, obj); 247 rte_spinlock_lock(&scsp_spinlock); 248 scsp_obj_table[i] = NULL; 249 rte_spinlock_unlock(&scsp_spinlock); 250 } 251 252 return 0; 253 } 254 255 /* 256 * single consumer function 257 */ 258 static int test_mempool_single_consumer(void) 259 { 260 unsigned int i; 261 void * obj; 262 uint64_t start_cycles, end_cycles; 263 uint64_t duration = rte_get_timer_hz() / 8; 264 265 start_cycles = rte_get_timer_cycles(); 266 while (1) { 267 end_cycles = rte_get_timer_cycles(); 268 /* duration uses up, stop consuming */ 269 if (start_cycles + duration < end_cycles) 270 break; 271 rte_spinlock_lock(&scsp_spinlock); 272 for (i = 0; i < MAX_KEEP; i ++) { 273 if (NULL == scsp_obj_table[i]) 274 break; 275 } 276 rte_spinlock_unlock(&scsp_spinlock); 277 if (i >= MAX_KEEP) 278 continue; 279 if (rte_mempool_get(mp_spsc, &obj) < 0) 280 break; 281 rte_spinlock_lock(&scsp_spinlock); 282 scsp_obj_table[i] = obj; 283 rte_spinlock_unlock(&scsp_spinlock); 284 } 285 286 return 0; 287 } 288 289 /* 290 * test function for mempool test based on singple consumer and single producer, 291 * can run on one lcore only 292 */ 293 static int 294 test_mempool_launch_single_consumer(__rte_unused void *arg) 295 { 296 return test_mempool_single_consumer(); 297 } 298 299 static void 300 my_mp_init(struct rte_mempool *mp, __rte_unused void *arg) 301 { 302 printf("mempool name is %s\n", mp->name); 303 /* nothing to be implemented here*/ 304 return ; 305 } 306 307 /* 308 * it tests the mempool operations based on singple producer and single consumer 309 */ 310 static int 311 test_mempool_sp_sc(void) 312 { 313 int ret = 0; 314 unsigned lcore_id = rte_lcore_id(); 315 unsigned lcore_next; 316 317 /* create a mempool with single producer/consumer ring */ 318 if (mp_spsc == NULL) { 319 mp_spsc = rte_mempool_create("test_mempool_sp_sc", MEMPOOL_SIZE, 320 MEMPOOL_ELT_SIZE, 0, 0, 321 my_mp_init, NULL, 322 my_obj_init, NULL, 323 SOCKET_ID_ANY, 324 MEMPOOL_F_NO_CACHE_ALIGN | MEMPOOL_F_SP_PUT | 325 MEMPOOL_F_SC_GET); 326 if (mp_spsc == NULL) 327 RET_ERR(); 328 } 329 if (rte_mempool_lookup("test_mempool_sp_sc") != mp_spsc) { 330 printf("Cannot lookup mempool from its name\n"); 331 ret = -1; 332 goto err; 333 } 334 lcore_next = rte_get_next_lcore(lcore_id, 0, 1); 335 if (lcore_next >= RTE_MAX_LCORE) { 336 ret = -1; 337 goto err; 338 } 339 if (rte_eal_lcore_role(lcore_next) != ROLE_RTE) { 340 ret = -1; 341 goto err; 342 } 343 rte_spinlock_init(&scsp_spinlock); 344 memset(scsp_obj_table, 0, sizeof(scsp_obj_table)); 345 rte_eal_remote_launch(test_mempool_launch_single_consumer, NULL, 346 lcore_next); 347 if (test_mempool_single_producer() < 0) 348 ret = -1; 349 350 if (rte_eal_wait_lcore(lcore_next) < 0) 351 ret = -1; 352 353 err: 354 rte_mempool_free(mp_spsc); 355 mp_spsc = NULL; 356 357 return ret; 358 } 359 360 /* 361 * it tests some more basic of mempool 362 */ 363 static int 364 test_mempool_basic_ex(struct rte_mempool *mp) 365 { 366 unsigned i; 367 void **obj; 368 void *err_obj; 369 int ret = -1; 370 371 if (mp == NULL) 372 return ret; 373 374 obj = rte_calloc("test_mempool_basic_ex", MEMPOOL_SIZE, 375 sizeof(void *), 0); 376 if (obj == NULL) { 377 printf("test_mempool_basic_ex fail to rte_malloc\n"); 378 return ret; 379 } 380 printf("test_mempool_basic_ex now mempool (%s) has %u free entries\n", 381 mp->name, rte_mempool_in_use_count(mp)); 382 if (rte_mempool_full(mp) != 1) { 383 printf("test_mempool_basic_ex the mempool should be full\n"); 384 goto fail_mp_basic_ex; 385 } 386 387 for (i = 0; i < MEMPOOL_SIZE; i ++) { 388 if (rte_mempool_get(mp, &obj[i]) < 0) { 389 printf("test_mp_basic_ex fail to get object for [%u]\n", 390 i); 391 goto fail_mp_basic_ex; 392 } 393 } 394 if (rte_mempool_get(mp, &err_obj) == 0) { 395 printf("test_mempool_basic_ex get an impossible obj\n"); 396 goto fail_mp_basic_ex; 397 } 398 printf("number: %u\n", i); 399 if (rte_mempool_empty(mp) != 1) { 400 printf("test_mempool_basic_ex the mempool should be empty\n"); 401 goto fail_mp_basic_ex; 402 } 403 404 for (i = 0; i < MEMPOOL_SIZE; i++) 405 rte_mempool_put(mp, obj[i]); 406 407 if (rte_mempool_full(mp) != 1) { 408 printf("test_mempool_basic_ex the mempool should be full\n"); 409 goto fail_mp_basic_ex; 410 } 411 412 ret = 0; 413 414 fail_mp_basic_ex: 415 if (obj != NULL) 416 rte_free((void *)obj); 417 418 return ret; 419 } 420 421 static int 422 test_mempool_same_name_twice_creation(void) 423 { 424 struct rte_mempool *mp_tc, *mp_tc2; 425 426 mp_tc = rte_mempool_create("test_mempool_same_name", MEMPOOL_SIZE, 427 MEMPOOL_ELT_SIZE, 0, 0, 428 NULL, NULL, 429 NULL, NULL, 430 SOCKET_ID_ANY, 0); 431 432 if (mp_tc == NULL) 433 RET_ERR(); 434 435 mp_tc2 = rte_mempool_create("test_mempool_same_name", MEMPOOL_SIZE, 436 MEMPOOL_ELT_SIZE, 0, 0, 437 NULL, NULL, 438 NULL, NULL, 439 SOCKET_ID_ANY, 0); 440 441 if (mp_tc2 != NULL) { 442 rte_mempool_free(mp_tc); 443 rte_mempool_free(mp_tc2); 444 RET_ERR(); 445 } 446 447 rte_mempool_free(mp_tc); 448 return 0; 449 } 450 451 static void 452 walk_cb(struct rte_mempool *mp, void *userdata __rte_unused) 453 { 454 printf("\t%s\n", mp->name); 455 } 456 457 struct mp_data { 458 int16_t ret; 459 }; 460 461 static void 462 test_mp_mem_init(struct rte_mempool *mp, 463 __rte_unused void *opaque, 464 __rte_unused struct rte_mempool_memhdr *memhdr, 465 __rte_unused unsigned int mem_idx) 466 { 467 struct mp_data *data = opaque; 468 469 if (mp == NULL) { 470 data->ret = -1; 471 return; 472 } 473 /* nothing to be implemented here*/ 474 data->ret = 0; 475 } 476 477 static int 478 test_mempool(void) 479 { 480 int ret = -1; 481 uint32_t nb_objs = 0; 482 uint32_t nb_mem_chunks = 0; 483 struct rte_mempool *mp_cache = NULL; 484 struct rte_mempool *mp_nocache = NULL; 485 struct rte_mempool *mp_stack_anon = NULL; 486 struct rte_mempool *mp_stack_mempool_iter = NULL; 487 struct rte_mempool *mp_stack = NULL; 488 struct rte_mempool *default_pool = NULL; 489 struct mp_data cb_arg = { 490 .ret = -1 491 }; 492 const char *default_pool_ops = rte_mbuf_best_mempool_ops(); 493 494 rte_atomic32_init(&synchro); 495 496 /* create a mempool (without cache) */ 497 mp_nocache = rte_mempool_create("test_nocache", MEMPOOL_SIZE, 498 MEMPOOL_ELT_SIZE, 0, 0, 499 NULL, NULL, 500 my_obj_init, NULL, 501 SOCKET_ID_ANY, 0); 502 503 if (mp_nocache == NULL) { 504 printf("cannot allocate mp_nocache mempool\n"); 505 GOTO_ERR(ret, err); 506 } 507 508 /* create a mempool (with cache) */ 509 mp_cache = rte_mempool_create("test_cache", MEMPOOL_SIZE, 510 MEMPOOL_ELT_SIZE, 511 RTE_MEMPOOL_CACHE_MAX_SIZE, 0, 512 NULL, NULL, 513 my_obj_init, NULL, 514 SOCKET_ID_ANY, 0); 515 516 if (mp_cache == NULL) { 517 printf("cannot allocate mp_cache mempool\n"); 518 GOTO_ERR(ret, err); 519 } 520 521 /* create an empty mempool */ 522 mp_stack_anon = rte_mempool_create_empty("test_stack_anon", 523 MEMPOOL_SIZE, 524 MEMPOOL_ELT_SIZE, 525 RTE_MEMPOOL_CACHE_MAX_SIZE, 0, 526 SOCKET_ID_ANY, 0); 527 528 if (mp_stack_anon == NULL) 529 GOTO_ERR(ret, err); 530 531 /* populate an empty mempool */ 532 ret = rte_mempool_populate_anon(mp_stack_anon); 533 printf("%s ret = %d\n", __func__, ret); 534 if (ret < 0) 535 GOTO_ERR(ret, err); 536 537 /* Try to populate when already populated */ 538 ret = rte_mempool_populate_anon(mp_stack_anon); 539 if (ret != 0) 540 GOTO_ERR(ret, err); 541 542 /* create a mempool */ 543 mp_stack_mempool_iter = rte_mempool_create("test_iter_obj", 544 MEMPOOL_SIZE, 545 MEMPOOL_ELT_SIZE, 546 RTE_MEMPOOL_CACHE_MAX_SIZE, 0, 547 NULL, NULL, 548 my_obj_init, NULL, 549 SOCKET_ID_ANY, 0); 550 551 if (mp_stack_mempool_iter == NULL) 552 GOTO_ERR(ret, err); 553 554 /* test to initialize mempool objects and memory */ 555 nb_objs = rte_mempool_obj_iter(mp_stack_mempool_iter, rte_pktmbuf_init, 556 NULL); 557 if (nb_objs == 0) 558 GOTO_ERR(ret, err); 559 560 nb_mem_chunks = rte_mempool_mem_iter(mp_stack_mempool_iter, 561 test_mp_mem_init, &cb_arg); 562 if (nb_mem_chunks == 0 || cb_arg.ret < 0) 563 GOTO_ERR(ret, err); 564 565 /* create a mempool with an external handler */ 566 mp_stack = rte_mempool_create_empty("test_stack", 567 MEMPOOL_SIZE, 568 MEMPOOL_ELT_SIZE, 569 RTE_MEMPOOL_CACHE_MAX_SIZE, 0, 570 SOCKET_ID_ANY, 0); 571 572 if (mp_stack == NULL) { 573 printf("cannot allocate mp_stack mempool\n"); 574 GOTO_ERR(ret, err); 575 } 576 if (rte_mempool_set_ops_byname(mp_stack, "stack", NULL) < 0) { 577 printf("cannot set stack handler\n"); 578 GOTO_ERR(ret, err); 579 } 580 if (rte_mempool_populate_default(mp_stack) < 0) { 581 printf("cannot populate mp_stack mempool\n"); 582 GOTO_ERR(ret, err); 583 } 584 rte_mempool_obj_iter(mp_stack, my_obj_init, NULL); 585 586 /* Create a mempool based on Default handler */ 587 printf("Testing %s mempool handler\n", default_pool_ops); 588 default_pool = rte_mempool_create_empty("default_pool", 589 MEMPOOL_SIZE, 590 MEMPOOL_ELT_SIZE, 591 RTE_MEMPOOL_CACHE_MAX_SIZE, 0, 592 SOCKET_ID_ANY, 0); 593 594 if (default_pool == NULL) { 595 printf("cannot allocate default mempool\n"); 596 GOTO_ERR(ret, err); 597 } 598 if (rte_mempool_set_ops_byname(default_pool, 599 default_pool_ops, NULL) < 0) { 600 printf("cannot set %s handler\n", default_pool_ops); 601 GOTO_ERR(ret, err); 602 } 603 if (rte_mempool_populate_default(default_pool) < 0) { 604 printf("cannot populate %s mempool\n", default_pool_ops); 605 GOTO_ERR(ret, err); 606 } 607 rte_mempool_obj_iter(default_pool, my_obj_init, NULL); 608 609 /* retrieve the mempool from its name */ 610 if (rte_mempool_lookup("test_nocache") != mp_nocache) { 611 printf("Cannot lookup mempool from its name\n"); 612 GOTO_ERR(ret, err); 613 } 614 615 printf("Walk into mempools:\n"); 616 rte_mempool_walk(walk_cb, NULL); 617 618 rte_mempool_list_dump(stdout); 619 620 /* basic tests without cache */ 621 if (test_mempool_basic(mp_nocache, 0) < 0) 622 GOTO_ERR(ret, err); 623 624 /* basic tests with cache */ 625 if (test_mempool_basic(mp_cache, 0) < 0) 626 GOTO_ERR(ret, err); 627 628 /* basic tests with user-owned cache */ 629 if (test_mempool_basic(mp_nocache, 1) < 0) 630 GOTO_ERR(ret, err); 631 632 /* more basic tests without cache */ 633 if (test_mempool_basic_ex(mp_nocache) < 0) 634 GOTO_ERR(ret, err); 635 636 /* mempool operation test based on single producer and single comsumer */ 637 if (test_mempool_sp_sc() < 0) 638 GOTO_ERR(ret, err); 639 640 if (test_mempool_creation_with_exceeded_cache_size() < 0) 641 GOTO_ERR(ret, err); 642 643 if (test_mempool_same_name_twice_creation() < 0) 644 GOTO_ERR(ret, err); 645 646 /* test the stack handler */ 647 if (test_mempool_basic(mp_stack, 1) < 0) 648 GOTO_ERR(ret, err); 649 650 if (test_mempool_basic(default_pool, 1) < 0) 651 GOTO_ERR(ret, err); 652 653 rte_mempool_list_dump(stdout); 654 655 ret = 0; 656 657 err: 658 rte_mempool_free(mp_nocache); 659 rte_mempool_free(mp_cache); 660 rte_mempool_free(mp_stack_anon); 661 rte_mempool_free(mp_stack_mempool_iter); 662 rte_mempool_free(mp_stack); 663 rte_mempool_free(default_pool); 664 665 return ret; 666 } 667 668 REGISTER_TEST_COMMAND(mempool_autotest, test_mempool); 669