1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include <math.h> 6 7 #include "test_perf_common.h" 8 9 #define NB_CRYPTODEV_DESCRIPTORS 128 10 #define DATA_SIZE 512 11 struct modex_test_data { 12 enum rte_crypto_asym_xform_type xform_type; 13 struct { 14 uint8_t data[DATA_SIZE]; 15 uint16_t len; 16 } base; 17 struct { 18 uint8_t data[DATA_SIZE]; 19 uint16_t len; 20 } exponent; 21 struct { 22 uint8_t data[DATA_SIZE]; 23 uint16_t len; 24 } modulus; 25 struct { 26 uint8_t data[DATA_SIZE]; 27 uint16_t len; 28 } reminder; 29 uint16_t result_len; 30 }; 31 32 static struct 33 modex_test_data modex_test_case = { 34 .xform_type = RTE_CRYPTO_ASYM_XFORM_MODEX, 35 .base = { 36 .data = { 37 0xF8, 0xBA, 0x1A, 0x55, 0xD0, 0x2F, 0x85, 38 0xAE, 0x96, 0x7B, 0xB6, 0x2F, 0xB6, 0xCD, 39 0xA8, 0xEB, 0x7E, 0x78, 0xA0, 0x50 40 }, 41 .len = 20, 42 }, 43 .exponent = { 44 .data = { 45 0x01, 0x00, 0x01 46 }, 47 .len = 3, 48 }, 49 .reminder = { 50 .data = { 51 0x2C, 0x60, 0x75, 0x45, 0x98, 0x9D, 0xE0, 0x72, 52 0xA0, 0x9D, 0x3A, 0x9E, 0x03, 0x38, 0x73, 0x3C, 53 0x31, 0x83, 0x04, 0xFE, 0x75, 0x43, 0xE6, 0x17, 54 0x5C, 0x01, 0x29, 0x51, 0x69, 0x33, 0x62, 0x2D, 55 0x78, 0xBE, 0xAE, 0xC4, 0xBC, 0xDE, 0x7E, 0x2C, 56 0x77, 0x84, 0xF2, 0xC5, 0x14, 0xB5, 0x2F, 0xF7, 57 0xC5, 0x94, 0xEF, 0x86, 0x75, 0x75, 0xB5, 0x11, 58 0xE5, 0x0E, 0x0A, 0x29, 0x76, 0xE2, 0xEA, 0x32, 59 0x0E, 0x43, 0x77, 0x7E, 0x2C, 0x27, 0xAC, 0x3B, 60 0x86, 0xA5, 0xDB, 0xC9, 0x48, 0x40, 0xE8, 0x99, 61 0x9A, 0x0A, 0x3D, 0xD6, 0x74, 0xFA, 0x2E, 0x2E, 62 0x5B, 0xAF, 0x8C, 0x99, 0x44, 0x2A, 0x67, 0x38, 63 0x27, 0x41, 0x59, 0x9D, 0xB8, 0x51, 0xC9, 0xF7, 64 0x43, 0x61, 0x31, 0x6E, 0xF1, 0x25, 0x38, 0x7F, 65 0xAE, 0xC6, 0xD0, 0xBB, 0x29, 0x76, 0x3F, 0x46, 66 0x2E, 0x1B, 0xE4, 0x67, 0x71, 0xE3, 0x87, 0x5A 67 }, 68 .len = 128, 69 }, 70 .modulus = { 71 .data = { 72 0xb3, 0xa1, 0xaf, 0xb7, 0x13, 0x08, 0x00, 0x0a, 73 0x35, 0xdc, 0x2b, 0x20, 0x8d, 0xa1, 0xb5, 0xce, 74 0x47, 0x8a, 0xc3, 0x80, 0xf4, 0x7d, 0x4a, 0xa2, 75 0x62, 0xfd, 0x61, 0x7f, 0xb5, 0xa8, 0xde, 0x0a, 76 0x17, 0x97, 0xa0, 0xbf, 0xdf, 0x56, 0x5a, 0x3d, 77 0x51, 0x56, 0x4f, 0x70, 0x70, 0x3f, 0x63, 0x6a, 78 0x44, 0x5b, 0xad, 0x84, 0x0d, 0x3f, 0x27, 0x6e, 79 0x3b, 0x34, 0x91, 0x60, 0x14, 0xb9, 0xaa, 0x72, 80 0xfd, 0xa3, 0x64, 0xd2, 0x03, 0xa7, 0x53, 0x87, 81 0x9e, 0x88, 0x0b, 0xc1, 0x14, 0x93, 0x1a, 0x62, 82 0xff, 0xb1, 0x5d, 0x74, 0xcd, 0x59, 0x63, 0x18, 83 0x11, 0x3d, 0x4f, 0xba, 0x75, 0xd4, 0x33, 0x4e, 84 0x23, 0x6b, 0x7b, 0x57, 0x44, 0xe1, 0xd3, 0x03, 85 0x13, 0xa6, 0xf0, 0x8b, 0x60, 0xb0, 0x9e, 0xee, 86 0x75, 0x08, 0x9d, 0x71, 0x63, 0x13, 0xcb, 0xa6, 87 0x81, 0x92, 0x14, 0x03, 0x22, 0x2d, 0xde, 0x55 88 }, 89 .len = 128, 90 }, 91 .result_len = 128, 92 }; 93 94 int 95 perf_test_result(struct evt_test *test, struct evt_options *opt) 96 { 97 RTE_SET_USED(opt); 98 int i; 99 uint64_t total = 0; 100 struct test_perf *t = evt_test_priv(test); 101 102 printf("Packet distribution across worker cores :\n"); 103 for (i = 0; i < t->nb_workers; i++) 104 total += t->worker[i].processed_pkts; 105 for (i = 0; i < t->nb_workers; i++) 106 printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:" 107 CLGRN" %3.2f"CLNRM"\n", i, 108 t->worker[i].processed_pkts, 109 (((double)t->worker[i].processed_pkts)/total) 110 * 100); 111 112 return t->result; 113 } 114 115 static inline int 116 perf_producer(void *arg) 117 { 118 int i; 119 struct prod_data *p = arg; 120 struct test_perf *t = p->t; 121 struct evt_options *opt = t->opt; 122 const uint8_t dev_id = p->dev_id; 123 const uint8_t port = p->port_id; 124 struct rte_mempool *pool = t->pool; 125 const uint64_t nb_pkts = t->nb_pkts; 126 const uint32_t nb_flows = t->nb_flows; 127 uint32_t flow_counter = 0; 128 uint64_t count = 0; 129 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 130 struct rte_event ev; 131 132 if (opt->verbose_level > 1) 133 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__, 134 rte_lcore_id(), dev_id, port, p->queue_id); 135 136 ev.event = 0; 137 ev.op = RTE_EVENT_OP_NEW; 138 ev.queue_id = p->queue_id; 139 ev.sched_type = t->opt->sched_type_list[0]; 140 ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 141 ev.event_type = RTE_EVENT_TYPE_CPU; 142 ev.sub_event_type = 0; /* stage 0 */ 143 144 while (count < nb_pkts && t->done == false) { 145 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 146 continue; 147 for (i = 0; i < BURST_SIZE; i++) { 148 ev.flow_id = flow_counter++ % nb_flows; 149 ev.event_ptr = m[i]; 150 m[i]->timestamp = rte_get_timer_cycles(); 151 while (rte_event_enqueue_burst(dev_id, 152 port, &ev, 1) != 1) { 153 if (t->done) 154 break; 155 rte_pause(); 156 m[i]->timestamp = rte_get_timer_cycles(); 157 } 158 } 159 count += BURST_SIZE; 160 } 161 162 return 0; 163 } 164 165 static inline int 166 perf_producer_burst(void *arg) 167 { 168 uint32_t i; 169 uint64_t timestamp; 170 struct rte_event_dev_info dev_info; 171 struct prod_data *p = arg; 172 struct test_perf *t = p->t; 173 struct evt_options *opt = t->opt; 174 const uint8_t dev_id = p->dev_id; 175 const uint8_t port = p->port_id; 176 struct rte_mempool *pool = t->pool; 177 const uint64_t nb_pkts = t->nb_pkts; 178 const uint32_t nb_flows = t->nb_flows; 179 uint32_t flow_counter = 0; 180 uint16_t enq = 0; 181 uint64_t count = 0; 182 struct perf_elt *m[MAX_PROD_ENQ_BURST_SIZE + 1]; 183 struct rte_event ev[MAX_PROD_ENQ_BURST_SIZE + 1]; 184 uint32_t burst_size = opt->prod_enq_burst_sz; 185 186 memset(m, 0, sizeof(*m) * (MAX_PROD_ENQ_BURST_SIZE + 1)); 187 rte_event_dev_info_get(dev_id, &dev_info); 188 if (dev_info.max_event_port_enqueue_depth < burst_size) 189 burst_size = dev_info.max_event_port_enqueue_depth; 190 191 if (opt->verbose_level > 1) 192 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__, 193 rte_lcore_id(), dev_id, port, p->queue_id); 194 195 for (i = 0; i < burst_size; i++) { 196 ev[i].op = RTE_EVENT_OP_NEW; 197 ev[i].queue_id = p->queue_id; 198 ev[i].sched_type = t->opt->sched_type_list[0]; 199 ev[i].priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 200 ev[i].event_type = RTE_EVENT_TYPE_CPU; 201 ev[i].sub_event_type = 0; /* stage 0 */ 202 } 203 204 while (count < nb_pkts && t->done == false) { 205 if (rte_mempool_get_bulk(pool, (void **)m, burst_size) < 0) 206 continue; 207 timestamp = rte_get_timer_cycles(); 208 for (i = 0; i < burst_size; i++) { 209 ev[i].flow_id = flow_counter++ % nb_flows; 210 ev[i].event_ptr = m[i]; 211 m[i]->timestamp = timestamp; 212 } 213 enq = rte_event_enqueue_burst(dev_id, port, ev, burst_size); 214 while (enq < burst_size) { 215 enq += rte_event_enqueue_burst(dev_id, port, 216 ev + enq, 217 burst_size - enq); 218 if (t->done) 219 break; 220 rte_pause(); 221 timestamp = rte_get_timer_cycles(); 222 for (i = enq; i < burst_size; i++) 223 m[i]->timestamp = timestamp; 224 } 225 count += burst_size; 226 } 227 return 0; 228 } 229 230 static inline int 231 perf_event_timer_producer(void *arg) 232 { 233 int i; 234 struct prod_data *p = arg; 235 struct test_perf *t = p->t; 236 struct evt_options *opt = t->opt; 237 uint32_t flow_counter = 0; 238 uint64_t count = 0; 239 uint64_t arm_latency = 0; 240 const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs; 241 const uint32_t nb_flows = t->nb_flows; 242 const uint64_t nb_timers = opt->nb_timers; 243 struct rte_mempool *pool = t->pool; 244 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 245 struct rte_event_timer_adapter **adptr = t->timer_adptr; 246 struct rte_event_timer tim; 247 uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; 248 249 memset(&tim, 0, sizeof(struct rte_event_timer)); 250 timeout_ticks = 251 opt->optm_timer_tick_nsec 252 ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) / 253 opt->optm_timer_tick_nsec) 254 : timeout_ticks; 255 timeout_ticks += timeout_ticks ? 0 : 1; 256 tim.ev.event_type = RTE_EVENT_TYPE_TIMER; 257 tim.ev.op = RTE_EVENT_OP_NEW; 258 tim.ev.sched_type = t->opt->sched_type_list[0]; 259 tim.ev.queue_id = p->queue_id; 260 tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 261 tim.state = RTE_EVENT_TIMER_NOT_ARMED; 262 tim.timeout_ticks = timeout_ticks; 263 264 if (opt->verbose_level > 1) 265 printf("%s(): lcore %d\n", __func__, rte_lcore_id()); 266 267 while (count < nb_timers && t->done == false) { 268 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 269 continue; 270 for (i = 0; i < BURST_SIZE; i++) { 271 rte_prefetch0(m[i + 1]); 272 m[i]->tim = tim; 273 m[i]->tim.ev.flow_id = flow_counter++ % nb_flows; 274 m[i]->tim.ev.event_ptr = m[i]; 275 m[i]->timestamp = rte_get_timer_cycles(); 276 while (rte_event_timer_arm_burst( 277 adptr[flow_counter % nb_timer_adptrs], 278 (struct rte_event_timer **)&m[i], 1) != 1) { 279 if (t->done) 280 break; 281 m[i]->timestamp = rte_get_timer_cycles(); 282 } 283 arm_latency += rte_get_timer_cycles() - m[i]->timestamp; 284 } 285 count += BURST_SIZE; 286 } 287 fflush(stdout); 288 rte_delay_ms(1000); 289 printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", 290 __func__, rte_lcore_id(), 291 count ? (float)(arm_latency / count) / 292 (rte_get_timer_hz() / 1000000) : 0); 293 return 0; 294 } 295 296 static inline int 297 perf_event_timer_producer_burst(void *arg) 298 { 299 int i; 300 struct prod_data *p = arg; 301 struct test_perf *t = p->t; 302 struct evt_options *opt = t->opt; 303 uint32_t flow_counter = 0; 304 uint64_t count = 0; 305 uint64_t arm_latency = 0; 306 const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs; 307 const uint32_t nb_flows = t->nb_flows; 308 const uint64_t nb_timers = opt->nb_timers; 309 struct rte_mempool *pool = t->pool; 310 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 311 struct rte_event_timer_adapter **adptr = t->timer_adptr; 312 struct rte_event_timer tim; 313 uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; 314 315 memset(&tim, 0, sizeof(struct rte_event_timer)); 316 timeout_ticks = 317 opt->optm_timer_tick_nsec 318 ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) / 319 opt->optm_timer_tick_nsec) 320 : timeout_ticks; 321 timeout_ticks += timeout_ticks ? 0 : 1; 322 tim.ev.event_type = RTE_EVENT_TYPE_TIMER; 323 tim.ev.op = RTE_EVENT_OP_NEW; 324 tim.ev.sched_type = t->opt->sched_type_list[0]; 325 tim.ev.queue_id = p->queue_id; 326 tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 327 tim.state = RTE_EVENT_TIMER_NOT_ARMED; 328 tim.timeout_ticks = timeout_ticks; 329 330 if (opt->verbose_level > 1) 331 printf("%s(): lcore %d\n", __func__, rte_lcore_id()); 332 333 while (count < nb_timers && t->done == false) { 334 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 335 continue; 336 for (i = 0; i < BURST_SIZE; i++) { 337 rte_prefetch0(m[i + 1]); 338 m[i]->tim = tim; 339 m[i]->tim.ev.flow_id = flow_counter++ % nb_flows; 340 m[i]->tim.ev.event_ptr = m[i]; 341 m[i]->timestamp = rte_get_timer_cycles(); 342 } 343 rte_event_timer_arm_tmo_tick_burst( 344 adptr[flow_counter % nb_timer_adptrs], 345 (struct rte_event_timer **)m, 346 tim.timeout_ticks, 347 BURST_SIZE); 348 arm_latency += rte_get_timer_cycles() - m[i - 1]->timestamp; 349 count += BURST_SIZE; 350 } 351 fflush(stdout); 352 rte_delay_ms(1000); 353 printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", 354 __func__, rte_lcore_id(), 355 count ? (float)(arm_latency / count) / 356 (rte_get_timer_hz() / 1000000) : 0); 357 return 0; 358 } 359 360 static inline void 361 crypto_adapter_enq_op_new(struct prod_data *p) 362 { 363 struct test_perf *t = p->t; 364 const uint32_t nb_flows = t->nb_flows; 365 const uint64_t nb_pkts = t->nb_pkts; 366 struct rte_mempool *pool = t->pool; 367 struct evt_options *opt = t->opt; 368 uint16_t qp_id = p->ca.cdev_qp_id; 369 uint8_t cdev_id = p->ca.cdev_id; 370 uint32_t flow_counter = 0; 371 struct rte_crypto_op *op; 372 struct rte_mbuf *m; 373 uint64_t count = 0; 374 uint16_t len; 375 376 if (opt->verbose_level > 1) 377 printf("%s(): lcore %d queue %d cdev_id %u cdev_qp_id %u\n", 378 __func__, rte_lcore_id(), p->queue_id, p->ca.cdev_id, 379 p->ca.cdev_qp_id); 380 381 len = opt->mbuf_sz ? opt->mbuf_sz : RTE_ETHER_MIN_LEN; 382 383 while (count < nb_pkts && t->done == false) { 384 if (opt->crypto_op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 385 struct rte_crypto_sym_op *sym_op; 386 387 op = rte_crypto_op_alloc(t->ca_op_pool, 388 RTE_CRYPTO_OP_TYPE_SYMMETRIC); 389 m = rte_pktmbuf_alloc(pool); 390 if (m == NULL) 391 continue; 392 393 rte_pktmbuf_append(m, len); 394 sym_op = op->sym; 395 sym_op->m_src = m; 396 sym_op->cipher.data.offset = 0; 397 sym_op->cipher.data.length = len; 398 rte_crypto_op_attach_sym_session( 399 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 400 } else { 401 struct rte_crypto_asym_op *asym_op; 402 uint8_t *result = rte_zmalloc(NULL, 403 modex_test_case.result_len, 0); 404 405 op = rte_crypto_op_alloc(t->ca_op_pool, 406 RTE_CRYPTO_OP_TYPE_ASYMMETRIC); 407 asym_op = op->asym; 408 asym_op->modex.base.data = modex_test_case.base.data; 409 asym_op->modex.base.length = modex_test_case.base.len; 410 asym_op->modex.result.data = result; 411 asym_op->modex.result.length = modex_test_case.result_len; 412 rte_crypto_op_attach_asym_session( 413 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 414 } 415 while (rte_cryptodev_enqueue_burst(cdev_id, qp_id, &op, 1) != 1 && 416 t->done == false) 417 rte_pause(); 418 419 count++; 420 } 421 } 422 423 static inline void 424 crypto_adapter_enq_op_fwd(struct prod_data *p) 425 { 426 const uint8_t dev_id = p->dev_id; 427 const uint8_t port = p->port_id; 428 struct test_perf *t = p->t; 429 const uint32_t nb_flows = t->nb_flows; 430 const uint64_t nb_pkts = t->nb_pkts; 431 struct rte_mempool *pool = t->pool; 432 struct evt_options *opt = t->opt; 433 uint32_t flow_counter = 0; 434 struct rte_crypto_op *op; 435 struct rte_event ev; 436 struct rte_mbuf *m; 437 uint64_t count = 0; 438 uint16_t len; 439 440 if (opt->verbose_level > 1) 441 printf("%s(): lcore %d port %d queue %d cdev_id %u cdev_qp_id %u\n", 442 __func__, rte_lcore_id(), port, p->queue_id, 443 p->ca.cdev_id, p->ca.cdev_qp_id); 444 445 ev.event = 0; 446 ev.op = RTE_EVENT_OP_NEW; 447 ev.queue_id = p->queue_id; 448 ev.sched_type = RTE_SCHED_TYPE_ATOMIC; 449 ev.event_type = RTE_EVENT_TYPE_CPU; 450 len = opt->mbuf_sz ? opt->mbuf_sz : RTE_ETHER_MIN_LEN; 451 452 while (count < nb_pkts && t->done == false) { 453 if (opt->crypto_op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 454 struct rte_crypto_sym_op *sym_op; 455 456 op = rte_crypto_op_alloc(t->ca_op_pool, 457 RTE_CRYPTO_OP_TYPE_SYMMETRIC); 458 m = rte_pktmbuf_alloc(pool); 459 if (m == NULL) 460 continue; 461 462 rte_pktmbuf_append(m, len); 463 sym_op = op->sym; 464 sym_op->m_src = m; 465 sym_op->cipher.data.offset = 0; 466 sym_op->cipher.data.length = len; 467 rte_crypto_op_attach_sym_session( 468 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 469 } else { 470 struct rte_crypto_asym_op *asym_op; 471 uint8_t *result = rte_zmalloc(NULL, 472 modex_test_case.result_len, 0); 473 474 op = rte_crypto_op_alloc(t->ca_op_pool, 475 RTE_CRYPTO_OP_TYPE_ASYMMETRIC); 476 asym_op = op->asym; 477 asym_op->modex.base.data = modex_test_case.base.data; 478 asym_op->modex.base.length = modex_test_case.base.len; 479 asym_op->modex.result.data = result; 480 asym_op->modex.result.length = modex_test_case.result_len; 481 rte_crypto_op_attach_asym_session( 482 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 483 } 484 ev.event_ptr = op; 485 486 while (rte_event_crypto_adapter_enqueue(dev_id, port, &ev, 1) != 1 && 487 t->done == false) 488 rte_pause(); 489 490 count++; 491 } 492 } 493 494 static inline int 495 perf_event_crypto_producer(void *arg) 496 { 497 struct prod_data *p = arg; 498 struct evt_options *opt = p->t->opt; 499 500 if (opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) 501 crypto_adapter_enq_op_new(p); 502 else 503 crypto_adapter_enq_op_fwd(p); 504 505 return 0; 506 } 507 508 static int 509 perf_producer_wrapper(void *arg) 510 { 511 struct prod_data *p = arg; 512 struct test_perf *t = p->t; 513 bool burst = evt_has_burst_mode(p->dev_id); 514 515 /* In case of synthetic producer, launch perf_producer or 516 * perf_producer_burst depending on producer enqueue burst size 517 */ 518 if (t->opt->prod_type == EVT_PROD_TYPE_SYNT && 519 t->opt->prod_enq_burst_sz == 1) 520 return perf_producer(arg); 521 else if (t->opt->prod_type == EVT_PROD_TYPE_SYNT && 522 t->opt->prod_enq_burst_sz > 1) { 523 if (!burst) 524 evt_err("This event device does not support burst mode"); 525 else 526 return perf_producer_burst(arg); 527 } 528 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && 529 !t->opt->timdev_use_burst) 530 return perf_event_timer_producer(arg); 531 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && 532 t->opt->timdev_use_burst) 533 return perf_event_timer_producer_burst(arg); 534 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) 535 return perf_event_crypto_producer(arg); 536 return 0; 537 } 538 539 static inline uint64_t 540 processed_pkts(struct test_perf *t) 541 { 542 uint8_t i; 543 uint64_t total = 0; 544 545 for (i = 0; i < t->nb_workers; i++) 546 total += t->worker[i].processed_pkts; 547 548 return total; 549 } 550 551 static inline uint64_t 552 total_latency(struct test_perf *t) 553 { 554 uint8_t i; 555 uint64_t total = 0; 556 557 for (i = 0; i < t->nb_workers; i++) 558 total += t->worker[i].latency; 559 560 return total; 561 } 562 563 564 int 565 perf_launch_lcores(struct evt_test *test, struct evt_options *opt, 566 int (*worker)(void *)) 567 { 568 int ret, lcore_id; 569 struct test_perf *t = evt_test_priv(test); 570 571 int port_idx = 0; 572 /* launch workers */ 573 RTE_LCORE_FOREACH_WORKER(lcore_id) { 574 if (!(opt->wlcores[lcore_id])) 575 continue; 576 577 ret = rte_eal_remote_launch(worker, 578 &t->worker[port_idx], lcore_id); 579 if (ret) { 580 evt_err("failed to launch worker %d", lcore_id); 581 return ret; 582 } 583 port_idx++; 584 } 585 586 /* launch producers */ 587 RTE_LCORE_FOREACH_WORKER(lcore_id) { 588 if (!(opt->plcores[lcore_id])) 589 continue; 590 591 ret = rte_eal_remote_launch(perf_producer_wrapper, 592 &t->prod[port_idx], lcore_id); 593 if (ret) { 594 evt_err("failed to launch perf_producer %d", lcore_id); 595 return ret; 596 } 597 port_idx++; 598 } 599 600 const uint64_t total_pkts = t->outstand_pkts; 601 602 uint64_t dead_lock_cycles = rte_get_timer_cycles(); 603 int64_t dead_lock_remaining = total_pkts; 604 const uint64_t dead_lock_sample = rte_get_timer_hz() * 5; 605 606 uint64_t perf_cycles = rte_get_timer_cycles(); 607 int64_t perf_remaining = total_pkts; 608 const uint64_t perf_sample = rte_get_timer_hz(); 609 610 static float total_mpps; 611 static uint64_t samples; 612 613 const uint64_t freq_mhz = rte_get_timer_hz() / 1000000; 614 int64_t remaining = t->outstand_pkts - processed_pkts(t); 615 616 while (t->done == false) { 617 const uint64_t new_cycles = rte_get_timer_cycles(); 618 619 if ((new_cycles - perf_cycles) > perf_sample) { 620 const uint64_t latency = total_latency(t); 621 const uint64_t pkts = processed_pkts(t); 622 623 remaining = t->outstand_pkts - pkts; 624 float mpps = (float)(perf_remaining-remaining)/1000000; 625 626 perf_remaining = remaining; 627 perf_cycles = new_cycles; 628 total_mpps += mpps; 629 ++samples; 630 if (opt->fwd_latency && pkts > 0) { 631 printf(CLGRN"\r%.3f mpps avg %.3f mpps [avg fwd latency %.3f us] "CLNRM, 632 mpps, total_mpps/samples, 633 (float)(latency/pkts)/freq_mhz); 634 } else { 635 printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM, 636 mpps, total_mpps/samples); 637 } 638 fflush(stdout); 639 640 if (remaining <= 0) { 641 t->result = EVT_TEST_SUCCESS; 642 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 643 opt->prod_type == 644 EVT_PROD_TYPE_EVENT_TIMER_ADPTR || 645 opt->prod_type == 646 EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { 647 t->done = true; 648 break; 649 } 650 } 651 } 652 653 if (new_cycles - dead_lock_cycles > dead_lock_sample && 654 (opt->prod_type == EVT_PROD_TYPE_SYNT || 655 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR || 656 opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)) { 657 remaining = t->outstand_pkts - processed_pkts(t); 658 if (dead_lock_remaining == remaining) { 659 rte_event_dev_dump(opt->dev_id, stdout); 660 evt_err("No schedules for seconds, deadlock"); 661 t->done = true; 662 break; 663 } 664 dead_lock_remaining = remaining; 665 dead_lock_cycles = new_cycles; 666 } 667 } 668 printf("\n"); 669 return 0; 670 } 671 672 static int 673 perf_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, 674 struct rte_event_port_conf prod_conf) 675 { 676 int ret = 0; 677 uint16_t prod; 678 struct rte_event_eth_rx_adapter_queue_conf queue_conf; 679 680 memset(&queue_conf, 0, 681 sizeof(struct rte_event_eth_rx_adapter_queue_conf)); 682 queue_conf.ev.sched_type = opt->sched_type_list[0]; 683 RTE_ETH_FOREACH_DEV(prod) { 684 uint32_t cap; 685 686 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id, 687 prod, &cap); 688 if (ret) { 689 evt_err("failed to get event rx adapter[%d]" 690 " capabilities", 691 opt->dev_id); 692 return ret; 693 } 694 queue_conf.ev.queue_id = prod * stride; 695 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id, 696 &prod_conf); 697 if (ret) { 698 evt_err("failed to create rx adapter[%d]", prod); 699 return ret; 700 } 701 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1, 702 &queue_conf); 703 if (ret) { 704 evt_err("failed to add rx queues to adapter[%d]", prod); 705 return ret; 706 } 707 708 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) { 709 uint32_t service_id; 710 711 rte_event_eth_rx_adapter_service_id_get(prod, 712 &service_id); 713 ret = evt_service_setup(service_id); 714 if (ret) { 715 evt_err("Failed to setup service core" 716 " for Rx adapter\n"); 717 return ret; 718 } 719 } 720 } 721 722 return ret; 723 } 724 725 static int 726 perf_event_timer_adapter_setup(struct test_perf *t) 727 { 728 int i; 729 int ret; 730 struct rte_event_timer_adapter_info adapter_info; 731 struct rte_event_timer_adapter *wl; 732 uint8_t nb_producers = evt_nr_active_lcores(t->opt->plcores); 733 uint8_t flags = RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES; 734 735 if (nb_producers == 1) 736 flags |= RTE_EVENT_TIMER_ADAPTER_F_SP_PUT; 737 738 for (i = 0; i < t->opt->nb_timer_adptrs; i++) { 739 struct rte_event_timer_adapter_conf config = { 740 .event_dev_id = t->opt->dev_id, 741 .timer_adapter_id = i, 742 .timer_tick_ns = t->opt->timer_tick_nsec, 743 .max_tmo_ns = t->opt->max_tmo_nsec, 744 .nb_timers = t->opt->pool_sz, 745 .flags = flags, 746 }; 747 748 wl = rte_event_timer_adapter_create(&config); 749 if (wl == NULL) { 750 evt_err("failed to create event timer ring %d", i); 751 return rte_errno; 752 } 753 754 memset(&adapter_info, 0, 755 sizeof(struct rte_event_timer_adapter_info)); 756 rte_event_timer_adapter_get_info(wl, &adapter_info); 757 t->opt->optm_timer_tick_nsec = adapter_info.min_resolution_ns; 758 759 if (!(adapter_info.caps & 760 RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) { 761 uint32_t service_id = -1U; 762 763 rte_event_timer_adapter_service_id_get(wl, 764 &service_id); 765 ret = evt_service_setup(service_id); 766 if (ret) { 767 evt_err("Failed to setup service core" 768 " for timer adapter\n"); 769 return ret; 770 } 771 rte_service_runstate_set(service_id, 1); 772 } 773 t->timer_adptr[i] = wl; 774 } 775 return 0; 776 } 777 778 static int 779 perf_event_crypto_adapter_setup(struct test_perf *t, struct prod_data *p) 780 { 781 struct evt_options *opt = t->opt; 782 uint32_t cap; 783 int ret; 784 785 ret = rte_event_crypto_adapter_caps_get(p->dev_id, p->ca.cdev_id, &cap); 786 if (ret) { 787 evt_err("Failed to get crypto adapter capabilities"); 788 return ret; 789 } 790 791 if (((opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) && 792 !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW)) || 793 ((opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_FORWARD) && 794 !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD))) { 795 evt_err("crypto adapter %s mode unsupported\n", 796 opt->crypto_adptr_mode ? "OP_FORWARD" : "OP_NEW"); 797 return -ENOTSUP; 798 } else if (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA)) { 799 evt_err("Storing crypto session not supported"); 800 return -ENOTSUP; 801 } 802 803 if (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) { 804 struct rte_event response_info; 805 806 response_info.event = 0; 807 response_info.sched_type = RTE_SCHED_TYPE_ATOMIC; 808 response_info.queue_id = p->queue_id; 809 ret = rte_event_crypto_adapter_queue_pair_add( 810 TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, 811 &response_info); 812 } else { 813 ret = rte_event_crypto_adapter_queue_pair_add( 814 TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, NULL); 815 } 816 817 return ret; 818 } 819 820 static struct rte_cryptodev_sym_session * 821 cryptodev_sym_sess_create(struct prod_data *p, struct test_perf *t) 822 { 823 struct rte_crypto_sym_xform cipher_xform; 824 struct rte_cryptodev_sym_session *sess; 825 826 cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; 827 cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL; 828 cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 829 cipher_xform.next = NULL; 830 831 sess = rte_cryptodev_sym_session_create(t->ca_sess_pool); 832 if (sess == NULL) { 833 evt_err("Failed to create sym session"); 834 return NULL; 835 } 836 837 if (rte_cryptodev_sym_session_init(p->ca.cdev_id, sess, &cipher_xform, 838 t->ca_sess_priv_pool)) { 839 evt_err("Failed to init session"); 840 return NULL; 841 } 842 843 return sess; 844 } 845 846 static void * 847 cryptodev_asym_sess_create(struct prod_data *p, struct test_perf *t) 848 { 849 const struct rte_cryptodev_asymmetric_xform_capability *capability; 850 struct rte_cryptodev_asym_capability_idx cap_idx; 851 struct rte_crypto_asym_xform xform; 852 void *sess; 853 854 xform.next = NULL; 855 xform.xform_type = RTE_CRYPTO_ASYM_XFORM_MODEX; 856 cap_idx.type = xform.xform_type; 857 capability = rte_cryptodev_asym_capability_get(p->ca.cdev_id, &cap_idx); 858 if (capability == NULL) { 859 evt_err("Device doesn't support MODEX. Test Skipped\n"); 860 return NULL; 861 } 862 863 xform.modex.modulus.data = modex_test_case.modulus.data; 864 xform.modex.modulus.length = modex_test_case.modulus.len; 865 xform.modex.exponent.data = modex_test_case.exponent.data; 866 xform.modex.exponent.length = modex_test_case.exponent.len; 867 868 if (rte_cryptodev_asym_session_create(p->ca.cdev_id, &xform, 869 t->ca_asym_sess_pool, &sess)) { 870 evt_err("Failed to create asym session"); 871 return NULL; 872 } 873 874 return sess; 875 } 876 877 int 878 perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, 879 uint8_t stride, uint8_t nb_queues, 880 const struct rte_event_port_conf *port_conf) 881 { 882 struct test_perf *t = evt_test_priv(test); 883 uint16_t port, prod; 884 int ret = -1; 885 886 /* setup one port per worker, linking to all queues */ 887 for (port = 0; port < evt_nr_active_lcores(opt->wlcores); 888 port++) { 889 struct worker_data *w = &t->worker[port]; 890 891 w->dev_id = opt->dev_id; 892 w->port_id = port; 893 w->t = t; 894 w->processed_pkts = 0; 895 w->latency = 0; 896 897 struct rte_event_port_conf conf = *port_conf; 898 conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_WORKER; 899 900 ret = rte_event_port_setup(opt->dev_id, port, &conf); 901 if (ret) { 902 evt_err("failed to setup port %d", port); 903 return ret; 904 } 905 906 ret = rte_event_port_link(opt->dev_id, port, NULL, NULL, 0); 907 if (ret != nb_queues) { 908 evt_err("failed to link all queues to port %d", port); 909 return -EINVAL; 910 } 911 } 912 913 /* port for producers, no links */ 914 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 915 for ( ; port < perf_nb_event_ports(opt); port++) { 916 struct prod_data *p = &t->prod[port]; 917 p->t = t; 918 } 919 920 struct rte_event_port_conf conf = *port_conf; 921 conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_PRODUCER; 922 923 ret = perf_event_rx_adapter_setup(opt, stride, conf); 924 if (ret) 925 return ret; 926 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 927 prod = 0; 928 for ( ; port < perf_nb_event_ports(opt); port++) { 929 struct prod_data *p = &t->prod[port]; 930 p->queue_id = prod * stride; 931 p->t = t; 932 prod++; 933 } 934 935 ret = perf_event_timer_adapter_setup(t); 936 if (ret) 937 return ret; 938 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { 939 struct rte_event_port_conf conf = *port_conf; 940 uint8_t cdev_id = 0; 941 uint16_t qp_id = 0; 942 943 ret = rte_event_crypto_adapter_create(TEST_PERF_CA_ID, 944 opt->dev_id, &conf, 0); 945 if (ret) { 946 evt_err("Failed to create crypto adapter"); 947 return ret; 948 } 949 950 prod = 0; 951 for (; port < perf_nb_event_ports(opt); port++) { 952 union rte_event_crypto_metadata m_data; 953 struct prod_data *p = &t->prod[port]; 954 uint32_t flow_id; 955 956 if (qp_id == rte_cryptodev_queue_pair_count(cdev_id)) { 957 cdev_id++; 958 qp_id = 0; 959 } 960 961 p->dev_id = opt->dev_id; 962 p->port_id = port; 963 p->queue_id = prod * stride; 964 p->ca.cdev_id = cdev_id; 965 p->ca.cdev_qp_id = qp_id; 966 p->ca.crypto_sess = rte_zmalloc_socket( 967 NULL, sizeof(void *) * t->nb_flows, 968 RTE_CACHE_LINE_SIZE, opt->socket_id); 969 p->t = t; 970 971 m_data.request_info.cdev_id = p->ca.cdev_id; 972 m_data.request_info.queue_pair_id = p->ca.cdev_qp_id; 973 m_data.response_info.sched_type = RTE_SCHED_TYPE_ATOMIC; 974 m_data.response_info.queue_id = p->queue_id; 975 976 for (flow_id = 0; flow_id < t->nb_flows; flow_id++) { 977 m_data.response_info.flow_id = flow_id; 978 if (opt->crypto_op_type == 979 RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 980 struct rte_cryptodev_sym_session *sess; 981 982 sess = cryptodev_sym_sess_create(p, t); 983 if (sess == NULL) 984 return -ENOMEM; 985 986 rte_cryptodev_session_event_mdata_set( 987 cdev_id, 988 sess, 989 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 990 RTE_CRYPTO_OP_WITH_SESSION, 991 &m_data, sizeof(m_data)); 992 p->ca.crypto_sess[flow_id] = sess; 993 } else { 994 void *sess; 995 996 sess = cryptodev_asym_sess_create(p, t); 997 if (sess == NULL) 998 return -ENOMEM; 999 rte_cryptodev_session_event_mdata_set( 1000 cdev_id, 1001 sess, 1002 RTE_CRYPTO_OP_TYPE_ASYMMETRIC, 1003 RTE_CRYPTO_OP_WITH_SESSION, 1004 &m_data, sizeof(m_data)); 1005 p->ca.crypto_sess[flow_id] = sess; 1006 } 1007 } 1008 1009 conf.event_port_cfg |= 1010 RTE_EVENT_PORT_CFG_HINT_PRODUCER | 1011 RTE_EVENT_PORT_CFG_HINT_CONSUMER; 1012 1013 ret = rte_event_port_setup(opt->dev_id, port, &conf); 1014 if (ret) { 1015 evt_err("failed to setup port %d", port); 1016 return ret; 1017 } 1018 1019 ret = perf_event_crypto_adapter_setup(t, p); 1020 if (ret) 1021 return ret; 1022 1023 qp_id++; 1024 prod++; 1025 } 1026 } else { 1027 prod = 0; 1028 for ( ; port < perf_nb_event_ports(opt); port++) { 1029 struct prod_data *p = &t->prod[port]; 1030 1031 p->dev_id = opt->dev_id; 1032 p->port_id = port; 1033 p->queue_id = prod * stride; 1034 p->t = t; 1035 1036 struct rte_event_port_conf conf = *port_conf; 1037 conf.event_port_cfg |= 1038 RTE_EVENT_PORT_CFG_HINT_PRODUCER | 1039 RTE_EVENT_PORT_CFG_HINT_CONSUMER; 1040 1041 ret = rte_event_port_setup(opt->dev_id, port, &conf); 1042 if (ret) { 1043 evt_err("failed to setup port %d", port); 1044 return ret; 1045 } 1046 prod++; 1047 } 1048 } 1049 1050 return ret; 1051 } 1052 1053 int 1054 perf_opt_check(struct evt_options *opt, uint64_t nb_queues) 1055 { 1056 unsigned int lcores; 1057 1058 /* N producer + N worker + main when producer cores are used 1059 * Else N worker + main when Rx adapter is used 1060 */ 1061 lcores = opt->prod_type == EVT_PROD_TYPE_SYNT ? 3 : 2; 1062 1063 if (rte_lcore_count() < lcores) { 1064 evt_err("test need minimum %d lcores", lcores); 1065 return -1; 1066 } 1067 1068 /* Validate worker lcores */ 1069 if (evt_lcores_has_overlap(opt->wlcores, rte_get_main_lcore())) { 1070 evt_err("worker lcores overlaps with main lcore"); 1071 return -1; 1072 } 1073 if (evt_lcores_has_overlap_multi(opt->wlcores, opt->plcores)) { 1074 evt_err("worker lcores overlaps producer lcores"); 1075 return -1; 1076 } 1077 if (evt_has_disabled_lcore(opt->wlcores)) { 1078 evt_err("one or more workers lcores are not enabled"); 1079 return -1; 1080 } 1081 if (!evt_has_active_lcore(opt->wlcores)) { 1082 evt_err("minimum one worker is required"); 1083 return -1; 1084 } 1085 1086 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 1087 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR || 1088 opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { 1089 /* Validate producer lcores */ 1090 if (evt_lcores_has_overlap(opt->plcores, 1091 rte_get_main_lcore())) { 1092 evt_err("producer lcores overlaps with main lcore"); 1093 return -1; 1094 } 1095 if (evt_has_disabled_lcore(opt->plcores)) { 1096 evt_err("one or more producer lcores are not enabled"); 1097 return -1; 1098 } 1099 if (!evt_has_active_lcore(opt->plcores)) { 1100 evt_err("minimum one producer is required"); 1101 return -1; 1102 } 1103 } 1104 1105 if (evt_has_invalid_stage(opt)) 1106 return -1; 1107 1108 if (evt_has_invalid_sched_type(opt)) 1109 return -1; 1110 1111 if (nb_queues > EVT_MAX_QUEUES) { 1112 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES); 1113 return -1; 1114 } 1115 if (perf_nb_event_ports(opt) > EVT_MAX_PORTS) { 1116 evt_err("number of ports exceeds %d", EVT_MAX_PORTS); 1117 return -1; 1118 } 1119 1120 /* Fixups */ 1121 if ((opt->nb_stages == 1 && 1122 opt->prod_type != EVT_PROD_TYPE_EVENT_TIMER_ADPTR) && 1123 opt->fwd_latency) { 1124 evt_info("fwd_latency is valid when nb_stages > 1, disabling"); 1125 opt->fwd_latency = 0; 1126 } 1127 1128 if (opt->fwd_latency && !opt->q_priority) { 1129 evt_info("enabled queue priority for latency measurement"); 1130 opt->q_priority = 1; 1131 } 1132 if (opt->nb_pkts == 0) 1133 opt->nb_pkts = INT64_MAX/evt_nr_active_lcores(opt->plcores); 1134 1135 return 0; 1136 } 1137 1138 void 1139 perf_opt_dump(struct evt_options *opt, uint8_t nb_queues) 1140 { 1141 evt_dump("nb_prod_lcores", "%d", evt_nr_active_lcores(opt->plcores)); 1142 evt_dump_producer_lcores(opt); 1143 evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores)); 1144 evt_dump_worker_lcores(opt); 1145 evt_dump_nb_stages(opt); 1146 evt_dump("nb_evdev_ports", "%d", perf_nb_event_ports(opt)); 1147 evt_dump("nb_evdev_queues", "%d", nb_queues); 1148 evt_dump_queue_priority(opt); 1149 evt_dump_sched_type_list(opt); 1150 evt_dump_producer_type(opt); 1151 evt_dump("prod_enq_burst_sz", "%d", opt->prod_enq_burst_sz); 1152 } 1153 1154 static void 1155 perf_event_port_flush(uint8_t dev_id __rte_unused, struct rte_event ev, 1156 void *args) 1157 { 1158 rte_mempool_put(args, ev.event_ptr); 1159 } 1160 1161 void 1162 perf_worker_cleanup(struct rte_mempool *const pool, uint8_t dev_id, 1163 uint8_t port_id, struct rte_event events[], uint16_t nb_enq, 1164 uint16_t nb_deq) 1165 { 1166 int i; 1167 1168 if (nb_deq) { 1169 for (i = nb_enq; i < nb_deq; i++) 1170 rte_mempool_put(pool, events[i].event_ptr); 1171 1172 for (i = 0; i < nb_deq; i++) 1173 events[i].op = RTE_EVENT_OP_RELEASE; 1174 rte_event_enqueue_burst(dev_id, port_id, events, nb_deq); 1175 } 1176 rte_event_port_quiesce(dev_id, port_id, perf_event_port_flush, pool); 1177 } 1178 1179 void 1180 perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt) 1181 { 1182 int i; 1183 struct test_perf *t = evt_test_priv(test); 1184 1185 if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 1186 for (i = 0; i < opt->nb_timer_adptrs; i++) 1187 rte_event_timer_adapter_stop(t->timer_adptr[i]); 1188 } 1189 rte_event_dev_stop(opt->dev_id); 1190 rte_event_dev_close(opt->dev_id); 1191 } 1192 1193 static inline void 1194 perf_elt_init(struct rte_mempool *mp, void *arg __rte_unused, 1195 void *obj, unsigned i __rte_unused) 1196 { 1197 memset(obj, 0, mp->elt_size); 1198 } 1199 1200 #define NB_RX_DESC 128 1201 #define NB_TX_DESC 512 1202 int 1203 perf_ethdev_setup(struct evt_test *test, struct evt_options *opt) 1204 { 1205 uint16_t i; 1206 int ret; 1207 struct test_perf *t = evt_test_priv(test); 1208 struct rte_eth_conf port_conf = { 1209 .rxmode = { 1210 .mq_mode = RTE_ETH_MQ_RX_RSS, 1211 .split_hdr_size = 0, 1212 }, 1213 .rx_adv_conf = { 1214 .rss_conf = { 1215 .rss_key = NULL, 1216 .rss_hf = RTE_ETH_RSS_IP, 1217 }, 1218 }, 1219 }; 1220 1221 if (opt->prod_type != EVT_PROD_TYPE_ETH_RX_ADPTR) 1222 return 0; 1223 1224 if (!rte_eth_dev_count_avail()) { 1225 evt_err("No ethernet ports found."); 1226 return -ENODEV; 1227 } 1228 1229 RTE_ETH_FOREACH_DEV(i) { 1230 struct rte_eth_dev_info dev_info; 1231 struct rte_eth_conf local_port_conf = port_conf; 1232 1233 ret = rte_eth_dev_info_get(i, &dev_info); 1234 if (ret != 0) { 1235 evt_err("Error during getting device (port %u) info: %s\n", 1236 i, strerror(-ret)); 1237 return ret; 1238 } 1239 1240 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 1241 dev_info.flow_type_rss_offloads; 1242 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 1243 port_conf.rx_adv_conf.rss_conf.rss_hf) { 1244 evt_info("Port %u modified RSS hash function based on hardware support," 1245 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 1246 i, 1247 port_conf.rx_adv_conf.rss_conf.rss_hf, 1248 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 1249 } 1250 1251 if (rte_eth_dev_configure(i, 1, 1, &local_port_conf) < 0) { 1252 evt_err("Failed to configure eth port [%d]", i); 1253 return -EINVAL; 1254 } 1255 1256 if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC, 1257 rte_socket_id(), NULL, t->pool) < 0) { 1258 evt_err("Failed to setup eth port [%d] rx_queue: %d.", 1259 i, 0); 1260 return -EINVAL; 1261 } 1262 1263 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC, 1264 rte_socket_id(), NULL) < 0) { 1265 evt_err("Failed to setup eth port [%d] tx_queue: %d.", 1266 i, 0); 1267 return -EINVAL; 1268 } 1269 1270 ret = rte_eth_promiscuous_enable(i); 1271 if (ret != 0) { 1272 evt_err("Failed to enable promiscuous mode for eth port [%d]: %s", 1273 i, rte_strerror(-ret)); 1274 return ret; 1275 } 1276 } 1277 1278 return 0; 1279 } 1280 1281 void 1282 perf_ethdev_rx_stop(struct evt_test *test, struct evt_options *opt) 1283 { 1284 uint16_t i; 1285 RTE_SET_USED(test); 1286 1287 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 1288 RTE_ETH_FOREACH_DEV(i) { 1289 rte_event_eth_rx_adapter_stop(i); 1290 rte_event_eth_rx_adapter_queue_del(i, i, -1); 1291 rte_eth_dev_rx_queue_stop(i, 0); 1292 } 1293 } 1294 } 1295 1296 void 1297 perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt) 1298 { 1299 uint16_t i; 1300 RTE_SET_USED(test); 1301 1302 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 1303 RTE_ETH_FOREACH_DEV(i) { 1304 rte_event_eth_tx_adapter_stop(i); 1305 rte_event_eth_tx_adapter_queue_del(i, i, -1); 1306 rte_eth_dev_tx_queue_stop(i, 0); 1307 rte_eth_dev_stop(i); 1308 } 1309 } 1310 } 1311 1312 int 1313 perf_cryptodev_setup(struct evt_test *test, struct evt_options *opt) 1314 { 1315 uint8_t cdev_count, cdev_id, nb_plcores, nb_qps; 1316 struct test_perf *t = evt_test_priv(test); 1317 unsigned int max_session_size; 1318 uint32_t nb_sessions; 1319 int ret; 1320 1321 if (opt->prod_type != EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) 1322 return 0; 1323 1324 cdev_count = rte_cryptodev_count(); 1325 if (cdev_count == 0) { 1326 evt_err("No crypto devices available\n"); 1327 return -ENODEV; 1328 } 1329 1330 t->ca_op_pool = rte_crypto_op_pool_create( 1331 "crypto_op_pool", opt->crypto_op_type, opt->pool_sz, 1332 128, sizeof(union rte_event_crypto_metadata), 1333 rte_socket_id()); 1334 if (t->ca_op_pool == NULL) { 1335 evt_err("Failed to create crypto op pool"); 1336 return -ENOMEM; 1337 } 1338 1339 nb_sessions = evt_nr_active_lcores(opt->plcores) * t->nb_flows; 1340 t->ca_asym_sess_pool = rte_cryptodev_asym_session_pool_create( 1341 "ca_asym_sess_pool", nb_sessions, 0, 1342 sizeof(union rte_event_crypto_metadata), SOCKET_ID_ANY); 1343 if (t->ca_asym_sess_pool == NULL) { 1344 evt_err("Failed to create sym session pool"); 1345 ret = -ENOMEM; 1346 goto err; 1347 } 1348 1349 t->ca_sess_pool = rte_cryptodev_sym_session_pool_create( 1350 "ca_sess_pool", nb_sessions, 0, 0, 1351 sizeof(union rte_event_crypto_metadata), SOCKET_ID_ANY); 1352 if (t->ca_sess_pool == NULL) { 1353 evt_err("Failed to create sym session pool"); 1354 ret = -ENOMEM; 1355 goto err; 1356 } 1357 1358 max_session_size = 0; 1359 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 1360 unsigned int session_size; 1361 1362 session_size = 1363 rte_cryptodev_sym_get_private_session_size(cdev_id); 1364 if (session_size > max_session_size) 1365 max_session_size = session_size; 1366 } 1367 1368 max_session_size += sizeof(union rte_event_crypto_metadata); 1369 t->ca_sess_priv_pool = rte_mempool_create( 1370 "ca_sess_priv_pool", nb_sessions, max_session_size, 0, 0, NULL, 1371 NULL, NULL, NULL, SOCKET_ID_ANY, 0); 1372 if (t->ca_sess_priv_pool == NULL) { 1373 evt_err("failed to create sym session private pool"); 1374 ret = -ENOMEM; 1375 goto err; 1376 } 1377 1378 /* 1379 * Calculate number of needed queue pairs, based on the amount of 1380 * available number of logical cores and crypto devices. For instance, 1381 * if there are 4 cores and 2 crypto devices, 2 queue pairs will be set 1382 * up per device. 1383 */ 1384 nb_plcores = evt_nr_active_lcores(opt->plcores); 1385 nb_qps = (nb_plcores % cdev_count) ? (nb_plcores / cdev_count) + 1 : 1386 nb_plcores / cdev_count; 1387 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 1388 struct rte_cryptodev_qp_conf qp_conf; 1389 struct rte_cryptodev_config conf; 1390 struct rte_cryptodev_info info; 1391 int qp_id; 1392 1393 rte_cryptodev_info_get(cdev_id, &info); 1394 if (nb_qps > info.max_nb_queue_pairs) { 1395 evt_err("Not enough queue pairs per cryptodev (%u)", 1396 nb_qps); 1397 ret = -EINVAL; 1398 goto err; 1399 } 1400 1401 conf.nb_queue_pairs = nb_qps; 1402 conf.socket_id = SOCKET_ID_ANY; 1403 conf.ff_disable = RTE_CRYPTODEV_FF_SECURITY; 1404 1405 ret = rte_cryptodev_configure(cdev_id, &conf); 1406 if (ret) { 1407 evt_err("Failed to configure cryptodev (%u)", cdev_id); 1408 goto err; 1409 } 1410 1411 qp_conf.nb_descriptors = NB_CRYPTODEV_DESCRIPTORS; 1412 qp_conf.mp_session = t->ca_sess_pool; 1413 qp_conf.mp_session_private = t->ca_sess_priv_pool; 1414 1415 for (qp_id = 0; qp_id < conf.nb_queue_pairs; qp_id++) { 1416 ret = rte_cryptodev_queue_pair_setup( 1417 cdev_id, qp_id, &qp_conf, 1418 rte_cryptodev_socket_id(cdev_id)); 1419 if (ret) { 1420 evt_err("Failed to setup queue pairs on cryptodev %u\n", 1421 cdev_id); 1422 goto err; 1423 } 1424 } 1425 } 1426 1427 return 0; 1428 err: 1429 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) 1430 rte_cryptodev_close(cdev_id); 1431 1432 rte_mempool_free(t->ca_op_pool); 1433 rte_mempool_free(t->ca_sess_pool); 1434 rte_mempool_free(t->ca_sess_priv_pool); 1435 rte_mempool_free(t->ca_asym_sess_pool); 1436 1437 return ret; 1438 } 1439 1440 void 1441 perf_cryptodev_destroy(struct evt_test *test, struct evt_options *opt) 1442 { 1443 uint8_t cdev_id, cdev_count = rte_cryptodev_count(); 1444 struct test_perf *t = evt_test_priv(test); 1445 uint16_t port; 1446 1447 if (opt->prod_type != EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) 1448 return; 1449 1450 for (port = t->nb_workers; port < perf_nb_event_ports(opt); port++) { 1451 struct rte_cryptodev_sym_session *sess; 1452 struct prod_data *p = &t->prod[port]; 1453 uint32_t flow_id; 1454 uint8_t cdev_id; 1455 1456 for (flow_id = 0; flow_id < t->nb_flows; flow_id++) { 1457 sess = p->ca.crypto_sess[flow_id]; 1458 cdev_id = p->ca.cdev_id; 1459 rte_cryptodev_sym_session_clear(cdev_id, sess); 1460 rte_cryptodev_sym_session_free(sess); 1461 } 1462 1463 rte_event_crypto_adapter_queue_pair_del( 1464 TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id); 1465 } 1466 1467 rte_event_crypto_adapter_free(TEST_PERF_CA_ID); 1468 1469 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 1470 rte_cryptodev_stop(cdev_id); 1471 rte_cryptodev_close(cdev_id); 1472 } 1473 1474 rte_mempool_free(t->ca_op_pool); 1475 rte_mempool_free(t->ca_sess_pool); 1476 rte_mempool_free(t->ca_sess_priv_pool); 1477 rte_mempool_free(t->ca_asym_sess_pool); 1478 } 1479 1480 int 1481 perf_mempool_setup(struct evt_test *test, struct evt_options *opt) 1482 { 1483 struct test_perf *t = evt_test_priv(test); 1484 1485 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 1486 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 1487 t->pool = rte_mempool_create(test->name, /* mempool name */ 1488 opt->pool_sz, /* number of elements*/ 1489 sizeof(struct perf_elt), /* element size*/ 1490 512, /* cache size*/ 1491 0, NULL, NULL, 1492 perf_elt_init, /* obj constructor */ 1493 NULL, opt->socket_id, 0); /* flags */ 1494 } else { 1495 t->pool = rte_pktmbuf_pool_create(test->name, /* mempool name */ 1496 opt->pool_sz, /* number of elements*/ 1497 512, /* cache size*/ 1498 0, 1499 RTE_MBUF_DEFAULT_BUF_SIZE, 1500 opt->socket_id); /* flags */ 1501 1502 } 1503 1504 if (t->pool == NULL) { 1505 evt_err("failed to create mempool"); 1506 return -ENOMEM; 1507 } 1508 1509 return 0; 1510 } 1511 1512 void 1513 perf_mempool_destroy(struct evt_test *test, struct evt_options *opt) 1514 { 1515 RTE_SET_USED(opt); 1516 struct test_perf *t = evt_test_priv(test); 1517 1518 rte_mempool_free(t->pool); 1519 } 1520 1521 int 1522 perf_test_setup(struct evt_test *test, struct evt_options *opt) 1523 { 1524 void *test_perf; 1525 1526 test_perf = rte_zmalloc_socket(test->name, sizeof(struct test_perf), 1527 RTE_CACHE_LINE_SIZE, opt->socket_id); 1528 if (test_perf == NULL) { 1529 evt_err("failed to allocate test_perf memory"); 1530 goto nomem; 1531 } 1532 test->test_priv = test_perf; 1533 1534 struct test_perf *t = evt_test_priv(test); 1535 1536 if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 1537 t->outstand_pkts = opt->nb_timers * 1538 evt_nr_active_lcores(opt->plcores); 1539 t->nb_pkts = opt->nb_timers; 1540 } else { 1541 t->outstand_pkts = opt->nb_pkts * 1542 evt_nr_active_lcores(opt->plcores); 1543 t->nb_pkts = opt->nb_pkts; 1544 } 1545 1546 t->nb_workers = evt_nr_active_lcores(opt->wlcores); 1547 t->done = false; 1548 t->nb_flows = opt->nb_flows; 1549 t->result = EVT_TEST_FAILED; 1550 t->opt = opt; 1551 memcpy(t->sched_type_list, opt->sched_type_list, 1552 sizeof(opt->sched_type_list)); 1553 return 0; 1554 nomem: 1555 return -ENOMEM; 1556 } 1557 1558 void 1559 perf_test_destroy(struct evt_test *test, struct evt_options *opt) 1560 { 1561 RTE_SET_USED(opt); 1562 1563 rte_free(test->test_priv); 1564 } 1565