1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 * Copyright(c) 2022 SmartShare Systems 4 */ 5 6 #include <string.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <stdint.h> 10 #include <inttypes.h> 11 #include <stdarg.h> 12 #include <errno.h> 13 #include <sys/queue.h> 14 15 #include <rte_common.h> 16 #include <rte_log.h> 17 #include <rte_debug.h> 18 #include <rte_memory.h> 19 #include <rte_launch.h> 20 #include <rte_cycles.h> 21 #include <rte_eal.h> 22 #include <rte_per_lcore.h> 23 #include <rte_lcore.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_mempool.h> 26 #include <rte_spinlock.h> 27 #include <rte_malloc.h> 28 #include <rte_mbuf_pool_ops.h> 29 30 #include "test.h" 31 32 /* 33 * Mempool performance 34 * ======= 35 * 36 * Each core get *n_keep* objects per bulk of *n_get_bulk*. Then, 37 * objects are put back in the pool per bulk of *n_put_bulk*. 38 * 39 * This sequence is done during TIME_S seconds. 40 * 41 * This test is done on the following configurations: 42 * 43 * - Cores configuration (*cores*) 44 * 45 * - One core with cache 46 * - Two cores with cache 47 * - Max. cores with cache 48 * - One core without cache 49 * - Two cores without cache 50 * - Max. cores without cache 51 * - One core with user-owned cache 52 * - Two cores with user-owned cache 53 * - Max. cores with user-owned cache 54 * 55 * - Bulk size (*n_get_bulk*, *n_put_bulk*) 56 * 57 * - Bulk get from 1 to 32 58 * - Bulk put from 1 to 32 59 * - Bulk get and put from 1 to 32, compile time constant 60 * 61 * - Number of kept objects (*n_keep*) 62 * 63 * - 32 64 * - 128 65 * - 512 66 */ 67 68 #define N 65536 69 #define TIME_S 5 70 #define MEMPOOL_ELT_SIZE 2048 71 #define MAX_KEEP 512 72 #define MEMPOOL_SIZE ((rte_lcore_count()*(MAX_KEEP+RTE_MEMPOOL_CACHE_MAX_SIZE))-1) 73 74 /* Number of pointers fitting into one cache line. */ 75 #define CACHE_LINE_BURST (RTE_CACHE_LINE_SIZE / sizeof(uintptr_t)) 76 77 #define LOG_ERR() printf("test failed at %s():%d\n", __func__, __LINE__) 78 #define RET_ERR() do { \ 79 LOG_ERR(); \ 80 return -1; \ 81 } while (0) 82 #define GOTO_ERR(var, label) do { \ 83 LOG_ERR(); \ 84 var = -1; \ 85 goto label; \ 86 } while (0) 87 88 static int use_external_cache; 89 static unsigned external_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 90 91 static RTE_ATOMIC(uint32_t) synchro; 92 93 /* number of objects in one bulk operation (get or put) */ 94 static unsigned n_get_bulk; 95 static unsigned n_put_bulk; 96 97 /* number of objects retrieved from mempool before putting them back */ 98 static unsigned n_keep; 99 100 /* true if we want to test with constant n_get_bulk and n_put_bulk */ 101 static int use_constant_values; 102 103 /* number of enqueues / dequeues */ 104 struct __rte_cache_aligned mempool_test_stats { 105 uint64_t enq_count; 106 }; 107 108 static struct mempool_test_stats stats[RTE_MAX_LCORE]; 109 110 /* 111 * save the object number in the first 4 bytes of object data. All 112 * other bytes are set to 0. 113 */ 114 static void 115 my_obj_init(struct rte_mempool *mp, __rte_unused void *arg, 116 void *obj, unsigned i) 117 { 118 uint32_t *objnum = obj; 119 memset(obj, 0, mp->elt_size); 120 *objnum = i; 121 } 122 123 static __rte_always_inline int 124 test_loop(struct rte_mempool *mp, struct rte_mempool_cache *cache, 125 unsigned int x_keep, unsigned int x_get_bulk, unsigned int x_put_bulk) 126 { 127 alignas(RTE_CACHE_LINE_SIZE) void *obj_table[MAX_KEEP]; 128 unsigned int idx; 129 unsigned int i; 130 int ret; 131 132 for (i = 0; likely(i < (N / x_keep)); i++) { 133 /* get x_keep objects by bulk of x_get_bulk */ 134 for (idx = 0; idx < x_keep; idx += x_get_bulk) { 135 ret = rte_mempool_generic_get(mp, 136 &obj_table[idx], 137 x_get_bulk, 138 cache); 139 if (unlikely(ret < 0)) { 140 rte_mempool_dump(stdout, mp); 141 return ret; 142 } 143 } 144 145 /* put the objects back by bulk of x_put_bulk */ 146 for (idx = 0; idx < x_keep; idx += x_put_bulk) { 147 rte_mempool_generic_put(mp, 148 &obj_table[idx], 149 x_put_bulk, 150 cache); 151 } 152 } 153 154 return 0; 155 } 156 157 static int 158 per_lcore_mempool_test(void *arg) 159 { 160 struct rte_mempool *mp = arg; 161 unsigned lcore_id = rte_lcore_id(); 162 int ret = 0; 163 uint64_t start_cycles, end_cycles; 164 uint64_t time_diff = 0, hz = rte_get_timer_hz(); 165 struct rte_mempool_cache *cache; 166 167 if (use_external_cache) { 168 /* Create a user-owned mempool cache. */ 169 cache = rte_mempool_cache_create(external_cache_size, 170 SOCKET_ID_ANY); 171 if (cache == NULL) 172 RET_ERR(); 173 } else { 174 /* May be NULL if cache is disabled. */ 175 cache = rte_mempool_default_cache(mp, lcore_id); 176 } 177 178 /* n_get_bulk and n_put_bulk must be divisors of n_keep */ 179 if (((n_keep / n_get_bulk) * n_get_bulk) != n_keep) 180 GOTO_ERR(ret, out); 181 if (((n_keep / n_put_bulk) * n_put_bulk) != n_keep) 182 GOTO_ERR(ret, out); 183 /* for constant n, n_get_bulk and n_put_bulk must be the same */ 184 if (use_constant_values && n_put_bulk != n_get_bulk) 185 GOTO_ERR(ret, out); 186 187 stats[lcore_id].enq_count = 0; 188 189 /* wait synchro for workers */ 190 if (lcore_id != rte_get_main_lcore()) 191 rte_wait_until_equal_32((uint32_t *)(uintptr_t)&synchro, 1, 192 rte_memory_order_relaxed); 193 194 start_cycles = rte_get_timer_cycles(); 195 196 while (time_diff/hz < TIME_S) { 197 if (!use_constant_values) 198 ret = test_loop(mp, cache, n_keep, n_get_bulk, n_put_bulk); 199 else if (n_get_bulk == 1) 200 ret = test_loop(mp, cache, n_keep, 1, 1); 201 else if (n_get_bulk == 4) 202 ret = test_loop(mp, cache, n_keep, 4, 4); 203 else if (n_get_bulk == CACHE_LINE_BURST) 204 ret = test_loop(mp, cache, n_keep, 205 CACHE_LINE_BURST, CACHE_LINE_BURST); 206 else if (n_get_bulk == 32) 207 ret = test_loop(mp, cache, n_keep, 32, 32); 208 else 209 ret = -1; 210 211 if (ret < 0) 212 GOTO_ERR(ret, out); 213 214 end_cycles = rte_get_timer_cycles(); 215 time_diff = end_cycles - start_cycles; 216 stats[lcore_id].enq_count += N; 217 } 218 219 out: 220 if (use_external_cache) { 221 rte_mempool_cache_flush(cache, mp); 222 rte_mempool_cache_free(cache); 223 } 224 225 return ret; 226 } 227 228 /* launch all the per-lcore test, and display the result */ 229 static int 230 launch_cores(struct rte_mempool *mp, unsigned int cores) 231 { 232 unsigned lcore_id; 233 uint64_t rate; 234 int ret; 235 unsigned cores_save = cores; 236 237 rte_atomic_store_explicit(&synchro, 0, rte_memory_order_relaxed); 238 239 /* reset stats */ 240 memset(stats, 0, sizeof(stats)); 241 242 printf("mempool_autotest cache=%u cores=%u n_get_bulk=%u " 243 "n_put_bulk=%u n_keep=%u constant_n=%u ", 244 use_external_cache ? 245 external_cache_size : (unsigned) mp->cache_size, 246 cores, n_get_bulk, n_put_bulk, n_keep, use_constant_values); 247 248 if (rte_mempool_avail_count(mp) != MEMPOOL_SIZE) { 249 printf("mempool is not full\n"); 250 return -1; 251 } 252 253 RTE_LCORE_FOREACH_WORKER(lcore_id) { 254 if (cores == 1) 255 break; 256 cores--; 257 rte_eal_remote_launch(per_lcore_mempool_test, 258 mp, lcore_id); 259 } 260 261 /* start synchro and launch test on main */ 262 rte_atomic_store_explicit(&synchro, 1, rte_memory_order_relaxed); 263 264 ret = per_lcore_mempool_test(mp); 265 266 cores = cores_save; 267 RTE_LCORE_FOREACH_WORKER(lcore_id) { 268 if (cores == 1) 269 break; 270 cores--; 271 if (rte_eal_wait_lcore(lcore_id) < 0) 272 ret = -1; 273 } 274 275 if (ret < 0) { 276 printf("per-lcore test returned -1\n"); 277 return -1; 278 } 279 280 rate = 0; 281 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) 282 rate += (stats[lcore_id].enq_count / TIME_S); 283 284 printf("rate_persec=%" PRIu64 "\n", rate); 285 286 return 0; 287 } 288 289 /* for a given number of core, launch all test cases */ 290 static int 291 do_one_mempool_test(struct rte_mempool *mp, unsigned int cores) 292 { 293 unsigned int bulk_tab_get[] = { 1, 4, CACHE_LINE_BURST, 32, 0 }; 294 unsigned int bulk_tab_put[] = { 1, 4, CACHE_LINE_BURST, 32, 0 }; 295 unsigned int keep_tab[] = { 32, 128, 512, 0 }; 296 unsigned *get_bulk_ptr; 297 unsigned *put_bulk_ptr; 298 unsigned *keep_ptr; 299 int ret; 300 301 for (get_bulk_ptr = bulk_tab_get; *get_bulk_ptr; get_bulk_ptr++) { 302 for (put_bulk_ptr = bulk_tab_put; *put_bulk_ptr; put_bulk_ptr++) { 303 for (keep_ptr = keep_tab; *keep_ptr; keep_ptr++) { 304 305 use_constant_values = 0; 306 n_get_bulk = *get_bulk_ptr; 307 n_put_bulk = *put_bulk_ptr; 308 n_keep = *keep_ptr; 309 ret = launch_cores(mp, cores); 310 if (ret < 0) 311 return -1; 312 313 /* replay test with constant values */ 314 if (n_get_bulk == n_put_bulk) { 315 use_constant_values = 1; 316 ret = launch_cores(mp, cores); 317 if (ret < 0) 318 return -1; 319 } 320 } 321 } 322 } 323 return 0; 324 } 325 326 static int 327 test_mempool_perf(void) 328 { 329 struct rte_mempool *mp_cache = NULL; 330 struct rte_mempool *mp_nocache = NULL; 331 struct rte_mempool *default_pool = NULL; 332 const char *default_pool_ops; 333 int ret = -1; 334 335 /* create a mempool (without cache) */ 336 mp_nocache = rte_mempool_create("perf_test_nocache", MEMPOOL_SIZE, 337 MEMPOOL_ELT_SIZE, 0, 0, 338 NULL, NULL, 339 my_obj_init, NULL, 340 SOCKET_ID_ANY, 0); 341 if (mp_nocache == NULL) 342 goto err; 343 344 /* create a mempool (with cache) */ 345 mp_cache = rte_mempool_create("perf_test_cache", MEMPOOL_SIZE, 346 MEMPOOL_ELT_SIZE, 347 RTE_MEMPOOL_CACHE_MAX_SIZE, 0, 348 NULL, NULL, 349 my_obj_init, NULL, 350 SOCKET_ID_ANY, 0); 351 if (mp_cache == NULL) 352 goto err; 353 354 default_pool_ops = rte_mbuf_best_mempool_ops(); 355 /* Create a mempool based on Default handler */ 356 default_pool = rte_mempool_create_empty("default_pool", 357 MEMPOOL_SIZE, 358 MEMPOOL_ELT_SIZE, 359 0, 0, 360 SOCKET_ID_ANY, 0); 361 362 if (default_pool == NULL) { 363 printf("cannot allocate %s mempool\n", default_pool_ops); 364 goto err; 365 } 366 367 if (rte_mempool_set_ops_byname(default_pool, default_pool_ops, NULL) 368 < 0) { 369 printf("cannot set %s handler\n", default_pool_ops); 370 goto err; 371 } 372 373 if (rte_mempool_populate_default(default_pool) < 0) { 374 printf("cannot populate %s mempool\n", default_pool_ops); 375 goto err; 376 } 377 378 rte_mempool_obj_iter(default_pool, my_obj_init, NULL); 379 380 /* performance test with 1, 2 and max cores */ 381 printf("start performance test (without cache)\n"); 382 383 if (do_one_mempool_test(mp_nocache, 1) < 0) 384 goto err; 385 386 if (do_one_mempool_test(mp_nocache, 2) < 0) 387 goto err; 388 389 if (do_one_mempool_test(mp_nocache, rte_lcore_count()) < 0) 390 goto err; 391 392 /* performance test with 1, 2 and max cores */ 393 printf("start performance test for %s (without cache)\n", 394 default_pool_ops); 395 396 if (do_one_mempool_test(default_pool, 1) < 0) 397 goto err; 398 399 if (do_one_mempool_test(default_pool, 2) < 0) 400 goto err; 401 402 if (do_one_mempool_test(default_pool, rte_lcore_count()) < 0) 403 goto err; 404 405 /* performance test with 1, 2 and max cores */ 406 printf("start performance test (with cache)\n"); 407 408 if (do_one_mempool_test(mp_cache, 1) < 0) 409 goto err; 410 411 if (do_one_mempool_test(mp_cache, 2) < 0) 412 goto err; 413 414 if (do_one_mempool_test(mp_cache, rte_lcore_count()) < 0) 415 goto err; 416 417 /* performance test with 1, 2 and max cores */ 418 printf("start performance test (with user-owned cache)\n"); 419 use_external_cache = 1; 420 421 if (do_one_mempool_test(mp_nocache, 1) < 0) 422 goto err; 423 424 if (do_one_mempool_test(mp_nocache, 2) < 0) 425 goto err; 426 427 if (do_one_mempool_test(mp_nocache, rte_lcore_count()) < 0) 428 goto err; 429 430 rte_mempool_list_dump(stdout); 431 432 ret = 0; 433 434 err: 435 rte_mempool_free(mp_cache); 436 rte_mempool_free(mp_nocache); 437 rte_mempool_free(default_pool); 438 return ret; 439 } 440 441 REGISTER_PERF_TEST(mempool_perf_autotest, test_mempool_perf); 442