1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include <math.h> 6 7 #include "test_perf_common.h" 8 9 #define NB_CRYPTODEV_DESCRIPTORS 1024 10 #define DATA_SIZE 512 11 struct modex_test_data { 12 enum rte_crypto_asym_xform_type xform_type; 13 struct { 14 uint8_t data[DATA_SIZE]; 15 uint16_t len; 16 } base; 17 struct { 18 uint8_t data[DATA_SIZE]; 19 uint16_t len; 20 } exponent; 21 struct { 22 uint8_t data[DATA_SIZE]; 23 uint16_t len; 24 } modulus; 25 struct { 26 uint8_t data[DATA_SIZE]; 27 uint16_t len; 28 } reminder; 29 uint16_t result_len; 30 }; 31 32 static struct 33 modex_test_data modex_test_case = { 34 .xform_type = RTE_CRYPTO_ASYM_XFORM_MODEX, 35 .base = { 36 .data = { 37 0xF8, 0xBA, 0x1A, 0x55, 0xD0, 0x2F, 0x85, 38 0xAE, 0x96, 0x7B, 0xB6, 0x2F, 0xB6, 0xCD, 39 0xA8, 0xEB, 0x7E, 0x78, 0xA0, 0x50 40 }, 41 .len = 20, 42 }, 43 .exponent = { 44 .data = { 45 0x01, 0x00, 0x01 46 }, 47 .len = 3, 48 }, 49 .reminder = { 50 .data = { 51 0x2C, 0x60, 0x75, 0x45, 0x98, 0x9D, 0xE0, 0x72, 52 0xA0, 0x9D, 0x3A, 0x9E, 0x03, 0x38, 0x73, 0x3C, 53 0x31, 0x83, 0x04, 0xFE, 0x75, 0x43, 0xE6, 0x17, 54 0x5C, 0x01, 0x29, 0x51, 0x69, 0x33, 0x62, 0x2D, 55 0x78, 0xBE, 0xAE, 0xC4, 0xBC, 0xDE, 0x7E, 0x2C, 56 0x77, 0x84, 0xF2, 0xC5, 0x14, 0xB5, 0x2F, 0xF7, 57 0xC5, 0x94, 0xEF, 0x86, 0x75, 0x75, 0xB5, 0x11, 58 0xE5, 0x0E, 0x0A, 0x29, 0x76, 0xE2, 0xEA, 0x32, 59 0x0E, 0x43, 0x77, 0x7E, 0x2C, 0x27, 0xAC, 0x3B, 60 0x86, 0xA5, 0xDB, 0xC9, 0x48, 0x40, 0xE8, 0x99, 61 0x9A, 0x0A, 0x3D, 0xD6, 0x74, 0xFA, 0x2E, 0x2E, 62 0x5B, 0xAF, 0x8C, 0x99, 0x44, 0x2A, 0x67, 0x38, 63 0x27, 0x41, 0x59, 0x9D, 0xB8, 0x51, 0xC9, 0xF7, 64 0x43, 0x61, 0x31, 0x6E, 0xF1, 0x25, 0x38, 0x7F, 65 0xAE, 0xC6, 0xD0, 0xBB, 0x29, 0x76, 0x3F, 0x46, 66 0x2E, 0x1B, 0xE4, 0x67, 0x71, 0xE3, 0x87, 0x5A 67 }, 68 .len = 128, 69 }, 70 .modulus = { 71 .data = { 72 0xb3, 0xa1, 0xaf, 0xb7, 0x13, 0x08, 0x00, 0x0a, 73 0x35, 0xdc, 0x2b, 0x20, 0x8d, 0xa1, 0xb5, 0xce, 74 0x47, 0x8a, 0xc3, 0x80, 0xf4, 0x7d, 0x4a, 0xa2, 75 0x62, 0xfd, 0x61, 0x7f, 0xb5, 0xa8, 0xde, 0x0a, 76 0x17, 0x97, 0xa0, 0xbf, 0xdf, 0x56, 0x5a, 0x3d, 77 0x51, 0x56, 0x4f, 0x70, 0x70, 0x3f, 0x63, 0x6a, 78 0x44, 0x5b, 0xad, 0x84, 0x0d, 0x3f, 0x27, 0x6e, 79 0x3b, 0x34, 0x91, 0x60, 0x14, 0xb9, 0xaa, 0x72, 80 0xfd, 0xa3, 0x64, 0xd2, 0x03, 0xa7, 0x53, 0x87, 81 0x9e, 0x88, 0x0b, 0xc1, 0x14, 0x93, 0x1a, 0x62, 82 0xff, 0xb1, 0x5d, 0x74, 0xcd, 0x59, 0x63, 0x18, 83 0x11, 0x3d, 0x4f, 0xba, 0x75, 0xd4, 0x33, 0x4e, 84 0x23, 0x6b, 0x7b, 0x57, 0x44, 0xe1, 0xd3, 0x03, 85 0x13, 0xa6, 0xf0, 0x8b, 0x60, 0xb0, 0x9e, 0xee, 86 0x75, 0x08, 0x9d, 0x71, 0x63, 0x13, 0xcb, 0xa6, 87 0x81, 0x92, 0x14, 0x03, 0x22, 0x2d, 0xde, 0x55 88 }, 89 .len = 128, 90 }, 91 .result_len = 128, 92 }; 93 94 int 95 perf_test_result(struct evt_test *test, struct evt_options *opt) 96 { 97 RTE_SET_USED(opt); 98 int i; 99 uint64_t total = 0; 100 struct test_perf *t = evt_test_priv(test); 101 102 printf("Packet distribution across worker cores :\n"); 103 for (i = 0; i < t->nb_workers; i++) 104 total += t->worker[i].processed_pkts; 105 for (i = 0; i < t->nb_workers; i++) 106 printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:" 107 CLGRN" %3.2f"CLNRM"\n", i, 108 t->worker[i].processed_pkts, 109 (((double)t->worker[i].processed_pkts)/total) 110 * 100); 111 112 return t->result; 113 } 114 115 static inline int 116 perf_producer(void *arg) 117 { 118 int i; 119 struct prod_data *p = arg; 120 struct test_perf *t = p->t; 121 struct evt_options *opt = t->opt; 122 const uint8_t dev_id = p->dev_id; 123 const uint8_t port = p->port_id; 124 struct rte_mempool *pool = t->pool; 125 const uint64_t nb_pkts = t->nb_pkts; 126 const uint32_t nb_flows = t->nb_flows; 127 uint32_t flow_counter = 0; 128 uint64_t count = 0; 129 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 130 struct rte_event ev; 131 132 if (opt->verbose_level > 1) 133 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__, 134 rte_lcore_id(), dev_id, port, p->queue_id); 135 136 ev.event = 0; 137 ev.op = RTE_EVENT_OP_NEW; 138 ev.queue_id = p->queue_id; 139 ev.sched_type = t->opt->sched_type_list[0]; 140 ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 141 ev.event_type = RTE_EVENT_TYPE_CPU; 142 ev.sub_event_type = 0; /* stage 0 */ 143 144 while (count < nb_pkts && t->done == false) { 145 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 146 continue; 147 for (i = 0; i < BURST_SIZE; i++) { 148 ev.flow_id = flow_counter++ % nb_flows; 149 ev.event_ptr = m[i]; 150 m[i]->timestamp = rte_get_timer_cycles(); 151 while (rte_event_enqueue_burst(dev_id, 152 port, &ev, 1) != 1) { 153 if (t->done) 154 break; 155 rte_pause(); 156 m[i]->timestamp = rte_get_timer_cycles(); 157 } 158 } 159 count += BURST_SIZE; 160 } 161 162 return 0; 163 } 164 165 static inline int 166 perf_producer_burst(void *arg) 167 { 168 uint32_t i; 169 uint64_t timestamp; 170 struct rte_event_dev_info dev_info; 171 struct prod_data *p = arg; 172 struct test_perf *t = p->t; 173 struct evt_options *opt = t->opt; 174 const uint8_t dev_id = p->dev_id; 175 const uint8_t port = p->port_id; 176 struct rte_mempool *pool = t->pool; 177 const uint64_t nb_pkts = t->nb_pkts; 178 const uint32_t nb_flows = t->nb_flows; 179 uint32_t flow_counter = 0; 180 uint16_t enq = 0; 181 uint64_t count = 0; 182 struct perf_elt *m[MAX_PROD_ENQ_BURST_SIZE + 1]; 183 struct rte_event ev[MAX_PROD_ENQ_BURST_SIZE + 1]; 184 uint32_t burst_size = opt->prod_enq_burst_sz; 185 186 memset(m, 0, sizeof(*m) * (MAX_PROD_ENQ_BURST_SIZE + 1)); 187 rte_event_dev_info_get(dev_id, &dev_info); 188 if (dev_info.max_event_port_enqueue_depth < burst_size) 189 burst_size = dev_info.max_event_port_enqueue_depth; 190 191 if (opt->verbose_level > 1) 192 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__, 193 rte_lcore_id(), dev_id, port, p->queue_id); 194 195 for (i = 0; i < burst_size; i++) { 196 ev[i].op = RTE_EVENT_OP_NEW; 197 ev[i].queue_id = p->queue_id; 198 ev[i].sched_type = t->opt->sched_type_list[0]; 199 ev[i].priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 200 ev[i].event_type = RTE_EVENT_TYPE_CPU; 201 ev[i].sub_event_type = 0; /* stage 0 */ 202 } 203 204 while (count < nb_pkts && t->done == false) { 205 if (rte_mempool_get_bulk(pool, (void **)m, burst_size) < 0) 206 continue; 207 timestamp = rte_get_timer_cycles(); 208 for (i = 0; i < burst_size; i++) { 209 ev[i].flow_id = flow_counter++ % nb_flows; 210 ev[i].event_ptr = m[i]; 211 m[i]->timestamp = timestamp; 212 } 213 enq = rte_event_enqueue_burst(dev_id, port, ev, burst_size); 214 while (enq < burst_size) { 215 enq += rte_event_enqueue_burst(dev_id, port, 216 ev + enq, 217 burst_size - enq); 218 if (t->done) 219 break; 220 rte_pause(); 221 timestamp = rte_get_timer_cycles(); 222 for (i = enq; i < burst_size; i++) 223 m[i]->timestamp = timestamp; 224 } 225 count += burst_size; 226 } 227 return 0; 228 } 229 230 static inline int 231 perf_event_timer_producer(void *arg) 232 { 233 int i; 234 struct prod_data *p = arg; 235 struct test_perf *t = p->t; 236 struct evt_options *opt = t->opt; 237 uint32_t flow_counter = 0; 238 uint64_t count = 0; 239 uint64_t arm_latency = 0; 240 const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs; 241 const uint32_t nb_flows = t->nb_flows; 242 const uint64_t nb_timers = opt->nb_timers; 243 struct rte_mempool *pool = t->pool; 244 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 245 struct rte_event_timer_adapter **adptr = t->timer_adptr; 246 struct rte_event_timer tim; 247 uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; 248 249 memset(&tim, 0, sizeof(struct rte_event_timer)); 250 timeout_ticks = 251 opt->optm_timer_tick_nsec 252 ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) / 253 opt->optm_timer_tick_nsec) 254 : timeout_ticks; 255 timeout_ticks += timeout_ticks ? 0 : 1; 256 tim.ev.event_type = RTE_EVENT_TYPE_TIMER; 257 tim.ev.op = RTE_EVENT_OP_NEW; 258 tim.ev.sched_type = t->opt->sched_type_list[0]; 259 tim.ev.queue_id = p->queue_id; 260 tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 261 tim.state = RTE_EVENT_TIMER_NOT_ARMED; 262 tim.timeout_ticks = timeout_ticks; 263 264 if (opt->verbose_level > 1) 265 printf("%s(): lcore %d\n", __func__, rte_lcore_id()); 266 267 while (count < nb_timers && t->done == false) { 268 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 269 continue; 270 for (i = 0; i < BURST_SIZE; i++) { 271 rte_prefetch0(m[i + 1]); 272 m[i]->tim = tim; 273 m[i]->tim.ev.flow_id = flow_counter++ % nb_flows; 274 m[i]->tim.ev.event_ptr = m[i]; 275 m[i]->timestamp = rte_get_timer_cycles(); 276 while (rte_event_timer_arm_burst( 277 adptr[flow_counter % nb_timer_adptrs], 278 (struct rte_event_timer **)&m[i], 1) != 1) { 279 if (t->done) 280 break; 281 m[i]->timestamp = rte_get_timer_cycles(); 282 } 283 arm_latency += rte_get_timer_cycles() - m[i]->timestamp; 284 } 285 count += BURST_SIZE; 286 } 287 fflush(stdout); 288 rte_delay_ms(1000); 289 printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", 290 __func__, rte_lcore_id(), 291 count ? (float)(arm_latency / count) / 292 (rte_get_timer_hz() / 1000000) : 0); 293 return 0; 294 } 295 296 static inline int 297 perf_event_timer_producer_burst(void *arg) 298 { 299 int i; 300 struct prod_data *p = arg; 301 struct test_perf *t = p->t; 302 struct evt_options *opt = t->opt; 303 uint32_t flow_counter = 0; 304 uint64_t count = 0; 305 uint64_t arm_latency = 0; 306 const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs; 307 const uint32_t nb_flows = t->nb_flows; 308 const uint64_t nb_timers = opt->nb_timers; 309 struct rte_mempool *pool = t->pool; 310 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 311 struct rte_event_timer_adapter **adptr = t->timer_adptr; 312 struct rte_event_timer tim; 313 uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; 314 315 memset(&tim, 0, sizeof(struct rte_event_timer)); 316 timeout_ticks = 317 opt->optm_timer_tick_nsec 318 ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) / 319 opt->optm_timer_tick_nsec) 320 : timeout_ticks; 321 timeout_ticks += timeout_ticks ? 0 : 1; 322 tim.ev.event_type = RTE_EVENT_TYPE_TIMER; 323 tim.ev.op = RTE_EVENT_OP_NEW; 324 tim.ev.sched_type = t->opt->sched_type_list[0]; 325 tim.ev.queue_id = p->queue_id; 326 tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 327 tim.state = RTE_EVENT_TIMER_NOT_ARMED; 328 tim.timeout_ticks = timeout_ticks; 329 330 if (opt->verbose_level > 1) 331 printf("%s(): lcore %d\n", __func__, rte_lcore_id()); 332 333 while (count < nb_timers && t->done == false) { 334 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 335 continue; 336 for (i = 0; i < BURST_SIZE; i++) { 337 rte_prefetch0(m[i + 1]); 338 m[i]->tim = tim; 339 m[i]->tim.ev.flow_id = flow_counter++ % nb_flows; 340 m[i]->tim.ev.event_ptr = m[i]; 341 m[i]->timestamp = rte_get_timer_cycles(); 342 } 343 rte_event_timer_arm_tmo_tick_burst( 344 adptr[flow_counter % nb_timer_adptrs], 345 (struct rte_event_timer **)m, 346 tim.timeout_ticks, 347 BURST_SIZE); 348 arm_latency += rte_get_timer_cycles() - m[i - 1]->timestamp; 349 count += BURST_SIZE; 350 } 351 fflush(stdout); 352 rte_delay_ms(1000); 353 printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", 354 __func__, rte_lcore_id(), 355 count ? (float)(arm_latency / count) / 356 (rte_get_timer_hz() / 1000000) : 0); 357 return 0; 358 } 359 360 static inline void 361 crypto_adapter_enq_op_new(struct prod_data *p) 362 { 363 struct test_perf *t = p->t; 364 const uint32_t nb_flows = t->nb_flows; 365 const uint64_t nb_pkts = t->nb_pkts; 366 struct rte_mempool *pool = t->pool; 367 struct evt_options *opt = t->opt; 368 uint16_t qp_id = p->ca.cdev_qp_id; 369 uint8_t cdev_id = p->ca.cdev_id; 370 uint64_t alloc_failures = 0; 371 uint32_t flow_counter = 0; 372 struct rte_crypto_op *op; 373 uint16_t len, offset; 374 struct rte_mbuf *m; 375 uint64_t count = 0; 376 377 if (opt->verbose_level > 1) 378 printf("%s(): lcore %d queue %d cdev_id %u cdev_qp_id %u\n", 379 __func__, rte_lcore_id(), p->queue_id, p->ca.cdev_id, 380 p->ca.cdev_qp_id); 381 382 offset = sizeof(struct perf_elt); 383 len = RTE_MAX(RTE_ETHER_MIN_LEN + offset, opt->mbuf_sz); 384 385 while (count < nb_pkts && t->done == false) { 386 if (opt->crypto_op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 387 struct rte_crypto_sym_op *sym_op; 388 389 op = rte_crypto_op_alloc(t->ca_op_pool, 390 RTE_CRYPTO_OP_TYPE_SYMMETRIC); 391 if (unlikely(op == NULL)) { 392 alloc_failures++; 393 continue; 394 } 395 396 m = rte_pktmbuf_alloc(pool); 397 if (unlikely(m == NULL)) { 398 alloc_failures++; 399 rte_crypto_op_free(op); 400 continue; 401 } 402 403 rte_pktmbuf_append(m, len); 404 sym_op = op->sym; 405 sym_op->m_src = m; 406 sym_op->cipher.data.offset = offset; 407 sym_op->cipher.data.length = len - offset; 408 rte_crypto_op_attach_sym_session( 409 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 410 } else { 411 struct rte_crypto_asym_op *asym_op; 412 uint8_t *result; 413 414 if (rte_mempool_get(pool, (void **)&result)) { 415 alloc_failures++; 416 continue; 417 } 418 419 op = rte_crypto_op_alloc(t->ca_op_pool, 420 RTE_CRYPTO_OP_TYPE_ASYMMETRIC); 421 if (unlikely(op == NULL)) { 422 alloc_failures++; 423 rte_mempool_put(pool, result); 424 continue; 425 } 426 427 asym_op = op->asym; 428 asym_op->modex.base.data = modex_test_case.base.data; 429 asym_op->modex.base.length = modex_test_case.base.len; 430 asym_op->modex.result.data = result; 431 asym_op->modex.result.length = modex_test_case.result_len; 432 rte_crypto_op_attach_asym_session( 433 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 434 } 435 while (rte_cryptodev_enqueue_burst(cdev_id, qp_id, &op, 1) != 1 && 436 t->done == false) 437 rte_pause(); 438 439 count++; 440 } 441 442 if (opt->verbose_level > 1 && alloc_failures) 443 printf("%s(): lcore %d allocation failures: %"PRIu64"\n", 444 __func__, rte_lcore_id(), alloc_failures); 445 } 446 447 static inline void 448 crypto_adapter_enq_op_fwd(struct prod_data *p) 449 { 450 const uint8_t dev_id = p->dev_id; 451 const uint8_t port = p->port_id; 452 struct test_perf *t = p->t; 453 const uint32_t nb_flows = t->nb_flows; 454 const uint64_t nb_pkts = t->nb_pkts; 455 struct rte_mempool *pool = t->pool; 456 struct evt_options *opt = t->opt; 457 uint64_t alloc_failures = 0; 458 uint32_t flow_counter = 0; 459 struct rte_crypto_op *op; 460 uint16_t len, offset; 461 struct rte_event ev; 462 struct rte_mbuf *m; 463 uint64_t count = 0; 464 465 if (opt->verbose_level > 1) 466 printf("%s(): lcore %d port %d queue %d cdev_id %u cdev_qp_id %u\n", 467 __func__, rte_lcore_id(), port, p->queue_id, 468 p->ca.cdev_id, p->ca.cdev_qp_id); 469 470 ev.event = 0; 471 ev.op = RTE_EVENT_OP_NEW; 472 ev.queue_id = p->queue_id; 473 ev.sched_type = RTE_SCHED_TYPE_ATOMIC; 474 ev.event_type = RTE_EVENT_TYPE_CPU; 475 476 offset = sizeof(struct perf_elt); 477 len = RTE_MAX(RTE_ETHER_MIN_LEN + offset, opt->mbuf_sz); 478 479 while (count < nb_pkts && t->done == false) { 480 if (opt->crypto_op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 481 struct rte_crypto_sym_op *sym_op; 482 483 op = rte_crypto_op_alloc(t->ca_op_pool, 484 RTE_CRYPTO_OP_TYPE_SYMMETRIC); 485 if (unlikely(op == NULL)) { 486 alloc_failures++; 487 continue; 488 } 489 490 m = rte_pktmbuf_alloc(pool); 491 if (unlikely(m == NULL)) { 492 alloc_failures++; 493 rte_crypto_op_free(op); 494 continue; 495 } 496 497 rte_pktmbuf_append(m, len); 498 sym_op = op->sym; 499 sym_op->m_src = m; 500 sym_op->cipher.data.offset = offset; 501 sym_op->cipher.data.length = len - offset; 502 rte_crypto_op_attach_sym_session( 503 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 504 } else { 505 struct rte_crypto_asym_op *asym_op; 506 uint8_t *result; 507 508 if (rte_mempool_get(pool, (void **)&result)) { 509 alloc_failures++; 510 continue; 511 } 512 513 op = rte_crypto_op_alloc(t->ca_op_pool, 514 RTE_CRYPTO_OP_TYPE_ASYMMETRIC); 515 if (unlikely(op == NULL)) { 516 alloc_failures++; 517 rte_mempool_put(pool, result); 518 continue; 519 } 520 521 asym_op = op->asym; 522 asym_op->modex.base.data = modex_test_case.base.data; 523 asym_op->modex.base.length = modex_test_case.base.len; 524 asym_op->modex.result.data = result; 525 asym_op->modex.result.length = modex_test_case.result_len; 526 rte_crypto_op_attach_asym_session( 527 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 528 } 529 ev.event_ptr = op; 530 531 while (rte_event_crypto_adapter_enqueue(dev_id, port, &ev, 1) != 1 && 532 t->done == false) 533 rte_pause(); 534 535 count++; 536 } 537 538 if (opt->verbose_level > 1 && alloc_failures) 539 printf("%s(): lcore %d allocation failures: %"PRIu64"\n", 540 __func__, rte_lcore_id(), alloc_failures); 541 } 542 543 static inline int 544 perf_event_crypto_producer(void *arg) 545 { 546 struct prod_data *p = arg; 547 struct evt_options *opt = p->t->opt; 548 549 if (opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) 550 crypto_adapter_enq_op_new(p); 551 else 552 crypto_adapter_enq_op_fwd(p); 553 554 return 0; 555 } 556 557 static int 558 perf_producer_wrapper(void *arg) 559 { 560 struct prod_data *p = arg; 561 struct test_perf *t = p->t; 562 bool burst = evt_has_burst_mode(p->dev_id); 563 564 /* In case of synthetic producer, launch perf_producer or 565 * perf_producer_burst depending on producer enqueue burst size 566 */ 567 if (t->opt->prod_type == EVT_PROD_TYPE_SYNT && 568 t->opt->prod_enq_burst_sz == 1) 569 return perf_producer(arg); 570 else if (t->opt->prod_type == EVT_PROD_TYPE_SYNT && 571 t->opt->prod_enq_burst_sz > 1) { 572 if (!burst) 573 evt_err("This event device does not support burst mode"); 574 else 575 return perf_producer_burst(arg); 576 } 577 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && 578 !t->opt->timdev_use_burst) 579 return perf_event_timer_producer(arg); 580 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && 581 t->opt->timdev_use_burst) 582 return perf_event_timer_producer_burst(arg); 583 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) 584 return perf_event_crypto_producer(arg); 585 return 0; 586 } 587 588 static inline uint64_t 589 processed_pkts(struct test_perf *t) 590 { 591 uint8_t i; 592 uint64_t total = 0; 593 594 for (i = 0; i < t->nb_workers; i++) 595 total += t->worker[i].processed_pkts; 596 597 return total; 598 } 599 600 static inline uint64_t 601 total_latency(struct test_perf *t) 602 { 603 uint8_t i; 604 uint64_t total = 0; 605 606 for (i = 0; i < t->nb_workers; i++) 607 total += t->worker[i].latency; 608 609 return total; 610 } 611 612 613 int 614 perf_launch_lcores(struct evt_test *test, struct evt_options *opt, 615 int (*worker)(void *)) 616 { 617 int ret, lcore_id; 618 struct test_perf *t = evt_test_priv(test); 619 620 int port_idx = 0; 621 /* launch workers */ 622 RTE_LCORE_FOREACH_WORKER(lcore_id) { 623 if (!(opt->wlcores[lcore_id])) 624 continue; 625 626 ret = rte_eal_remote_launch(worker, 627 &t->worker[port_idx], lcore_id); 628 if (ret) { 629 evt_err("failed to launch worker %d", lcore_id); 630 return ret; 631 } 632 port_idx++; 633 } 634 635 /* launch producers */ 636 RTE_LCORE_FOREACH_WORKER(lcore_id) { 637 if (!(opt->plcores[lcore_id])) 638 continue; 639 640 ret = rte_eal_remote_launch(perf_producer_wrapper, 641 &t->prod[port_idx], lcore_id); 642 if (ret) { 643 evt_err("failed to launch perf_producer %d", lcore_id); 644 return ret; 645 } 646 port_idx++; 647 } 648 649 const uint64_t total_pkts = t->outstand_pkts; 650 651 uint64_t dead_lock_cycles = rte_get_timer_cycles(); 652 int64_t dead_lock_remaining = total_pkts; 653 const uint64_t dead_lock_sample = rte_get_timer_hz() * 5; 654 655 uint64_t perf_cycles = rte_get_timer_cycles(); 656 int64_t perf_remaining = total_pkts; 657 const uint64_t perf_sample = rte_get_timer_hz(); 658 659 static float total_mpps; 660 static uint64_t samples; 661 662 const uint64_t freq_mhz = rte_get_timer_hz() / 1000000; 663 int64_t remaining = t->outstand_pkts - processed_pkts(t); 664 665 while (t->done == false) { 666 const uint64_t new_cycles = rte_get_timer_cycles(); 667 668 if ((new_cycles - perf_cycles) > perf_sample) { 669 const uint64_t latency = total_latency(t); 670 const uint64_t pkts = processed_pkts(t); 671 672 remaining = t->outstand_pkts - pkts; 673 float mpps = (float)(perf_remaining-remaining)/1000000; 674 675 perf_remaining = remaining; 676 perf_cycles = new_cycles; 677 total_mpps += mpps; 678 ++samples; 679 if (opt->fwd_latency && pkts > 0) { 680 printf(CLGRN"\r%.3f mpps avg %.3f mpps [avg fwd latency %.3f us] "CLNRM, 681 mpps, total_mpps/samples, 682 (float)(latency/pkts)/freq_mhz); 683 } else { 684 printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM, 685 mpps, total_mpps/samples); 686 } 687 fflush(stdout); 688 689 if (remaining <= 0) { 690 t->result = EVT_TEST_SUCCESS; 691 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 692 opt->prod_type == 693 EVT_PROD_TYPE_EVENT_TIMER_ADPTR || 694 opt->prod_type == 695 EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { 696 t->done = true; 697 break; 698 } 699 } 700 } 701 702 if (new_cycles - dead_lock_cycles > dead_lock_sample && 703 (opt->prod_type == EVT_PROD_TYPE_SYNT || 704 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR || 705 opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)) { 706 remaining = t->outstand_pkts - processed_pkts(t); 707 if (dead_lock_remaining == remaining) { 708 rte_event_dev_dump(opt->dev_id, stdout); 709 evt_err("No schedules for seconds, deadlock"); 710 t->done = true; 711 break; 712 } 713 dead_lock_remaining = remaining; 714 dead_lock_cycles = new_cycles; 715 } 716 } 717 printf("\n"); 718 return 0; 719 } 720 721 static int 722 perf_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, 723 struct rte_event_port_conf prod_conf) 724 { 725 int ret = 0; 726 uint16_t prod; 727 struct rte_event_eth_rx_adapter_queue_conf queue_conf; 728 729 memset(&queue_conf, 0, 730 sizeof(struct rte_event_eth_rx_adapter_queue_conf)); 731 queue_conf.ev.sched_type = opt->sched_type_list[0]; 732 RTE_ETH_FOREACH_DEV(prod) { 733 uint32_t cap; 734 735 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id, 736 prod, &cap); 737 if (ret) { 738 evt_err("failed to get event rx adapter[%d]" 739 " capabilities", 740 opt->dev_id); 741 return ret; 742 } 743 queue_conf.ev.queue_id = prod * stride; 744 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id, 745 &prod_conf); 746 if (ret) { 747 evt_err("failed to create rx adapter[%d]", prod); 748 return ret; 749 } 750 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1, 751 &queue_conf); 752 if (ret) { 753 evt_err("failed to add rx queues to adapter[%d]", prod); 754 return ret; 755 } 756 757 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) { 758 uint32_t service_id; 759 760 rte_event_eth_rx_adapter_service_id_get(prod, 761 &service_id); 762 ret = evt_service_setup(service_id); 763 if (ret) { 764 evt_err("Failed to setup service core" 765 " for Rx adapter\n"); 766 return ret; 767 } 768 } 769 } 770 771 return ret; 772 } 773 774 static int 775 perf_event_timer_adapter_setup(struct test_perf *t) 776 { 777 int i; 778 int ret; 779 struct rte_event_timer_adapter_info adapter_info; 780 struct rte_event_timer_adapter *wl; 781 uint8_t nb_producers = evt_nr_active_lcores(t->opt->plcores); 782 uint8_t flags = RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES; 783 784 if (nb_producers == 1) 785 flags |= RTE_EVENT_TIMER_ADAPTER_F_SP_PUT; 786 787 for (i = 0; i < t->opt->nb_timer_adptrs; i++) { 788 struct rte_event_timer_adapter_conf config = { 789 .event_dev_id = t->opt->dev_id, 790 .timer_adapter_id = i, 791 .timer_tick_ns = t->opt->timer_tick_nsec, 792 .max_tmo_ns = t->opt->max_tmo_nsec, 793 .nb_timers = t->opt->pool_sz, 794 .flags = flags, 795 }; 796 797 wl = rte_event_timer_adapter_create(&config); 798 if (wl == NULL) { 799 evt_err("failed to create event timer ring %d", i); 800 return rte_errno; 801 } 802 803 memset(&adapter_info, 0, 804 sizeof(struct rte_event_timer_adapter_info)); 805 rte_event_timer_adapter_get_info(wl, &adapter_info); 806 t->opt->optm_timer_tick_nsec = adapter_info.min_resolution_ns; 807 808 if (!(adapter_info.caps & 809 RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) { 810 uint32_t service_id = -1U; 811 812 rte_event_timer_adapter_service_id_get(wl, 813 &service_id); 814 ret = evt_service_setup(service_id); 815 if (ret) { 816 evt_err("Failed to setup service core" 817 " for timer adapter\n"); 818 return ret; 819 } 820 rte_service_runstate_set(service_id, 1); 821 } 822 t->timer_adptr[i] = wl; 823 } 824 return 0; 825 } 826 827 static int 828 perf_event_crypto_adapter_setup(struct test_perf *t, struct prod_data *p) 829 { 830 struct evt_options *opt = t->opt; 831 uint32_t cap; 832 int ret; 833 834 ret = rte_event_crypto_adapter_caps_get(p->dev_id, p->ca.cdev_id, &cap); 835 if (ret) { 836 evt_err("Failed to get crypto adapter capabilities"); 837 return ret; 838 } 839 840 if (((opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) && 841 !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW)) || 842 ((opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_FORWARD) && 843 !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD))) { 844 evt_err("crypto adapter %s mode unsupported\n", 845 opt->crypto_adptr_mode ? "OP_FORWARD" : "OP_NEW"); 846 return -ENOTSUP; 847 } else if (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA)) { 848 evt_err("Storing crypto session not supported"); 849 return -ENOTSUP; 850 } 851 852 if (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) { 853 struct rte_event_crypto_adapter_queue_conf conf; 854 855 memset(&conf, 0, sizeof(conf)); 856 conf.ev.sched_type = RTE_SCHED_TYPE_ATOMIC; 857 conf.ev.queue_id = p->queue_id; 858 ret = rte_event_crypto_adapter_queue_pair_add( 859 TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, &conf); 860 } else { 861 ret = rte_event_crypto_adapter_queue_pair_add( 862 TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, NULL); 863 } 864 865 return ret; 866 } 867 868 static void * 869 cryptodev_sym_sess_create(struct prod_data *p, struct test_perf *t) 870 { 871 struct rte_crypto_sym_xform cipher_xform; 872 void *sess; 873 874 cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; 875 cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL; 876 cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 877 cipher_xform.next = NULL; 878 879 sess = rte_cryptodev_sym_session_create(p->ca.cdev_id, &cipher_xform, 880 t->ca_sess_pool); 881 if (sess == NULL) { 882 evt_err("Failed to create sym session"); 883 return NULL; 884 } 885 886 return sess; 887 } 888 889 static void * 890 cryptodev_asym_sess_create(struct prod_data *p, struct test_perf *t) 891 { 892 const struct rte_cryptodev_asymmetric_xform_capability *capability; 893 struct rte_cryptodev_asym_capability_idx cap_idx; 894 struct rte_crypto_asym_xform xform; 895 void *sess; 896 897 xform.next = NULL; 898 xform.xform_type = RTE_CRYPTO_ASYM_XFORM_MODEX; 899 cap_idx.type = xform.xform_type; 900 capability = rte_cryptodev_asym_capability_get(p->ca.cdev_id, &cap_idx); 901 if (capability == NULL) { 902 evt_err("Device doesn't support MODEX. Test Skipped\n"); 903 return NULL; 904 } 905 906 xform.modex.modulus.data = modex_test_case.modulus.data; 907 xform.modex.modulus.length = modex_test_case.modulus.len; 908 xform.modex.exponent.data = modex_test_case.exponent.data; 909 xform.modex.exponent.length = modex_test_case.exponent.len; 910 911 if (rte_cryptodev_asym_session_create(p->ca.cdev_id, &xform, 912 t->ca_asym_sess_pool, &sess)) { 913 evt_err("Failed to create asym session"); 914 return NULL; 915 } 916 917 return sess; 918 } 919 920 int 921 perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, 922 uint8_t stride, uint8_t nb_queues, 923 const struct rte_event_port_conf *port_conf) 924 { 925 struct test_perf *t = evt_test_priv(test); 926 uint16_t port, prod; 927 int ret = -1; 928 929 /* setup one port per worker, linking to all queues */ 930 for (port = 0; port < evt_nr_active_lcores(opt->wlcores); 931 port++) { 932 struct worker_data *w = &t->worker[port]; 933 934 w->dev_id = opt->dev_id; 935 w->port_id = port; 936 w->t = t; 937 w->processed_pkts = 0; 938 w->latency = 0; 939 940 struct rte_event_port_conf conf = *port_conf; 941 conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_WORKER; 942 943 ret = rte_event_port_setup(opt->dev_id, port, &conf); 944 if (ret) { 945 evt_err("failed to setup port %d", port); 946 return ret; 947 } 948 949 ret = rte_event_port_link(opt->dev_id, port, NULL, NULL, 0); 950 if (ret != nb_queues) { 951 evt_err("failed to link all queues to port %d", port); 952 return -EINVAL; 953 } 954 } 955 956 /* port for producers, no links */ 957 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 958 for ( ; port < perf_nb_event_ports(opt); port++) { 959 struct prod_data *p = &t->prod[port]; 960 p->t = t; 961 } 962 963 struct rte_event_port_conf conf = *port_conf; 964 conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_PRODUCER; 965 966 ret = perf_event_rx_adapter_setup(opt, stride, conf); 967 if (ret) 968 return ret; 969 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 970 prod = 0; 971 for ( ; port < perf_nb_event_ports(opt); port++) { 972 struct prod_data *p = &t->prod[port]; 973 p->queue_id = prod * stride; 974 p->t = t; 975 prod++; 976 } 977 978 ret = perf_event_timer_adapter_setup(t); 979 if (ret) 980 return ret; 981 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { 982 struct rte_event_port_conf conf = *port_conf; 983 uint8_t cdev_id = 0; 984 uint16_t qp_id = 0; 985 986 ret = rte_event_crypto_adapter_create(TEST_PERF_CA_ID, 987 opt->dev_id, &conf, 0); 988 if (ret) { 989 evt_err("Failed to create crypto adapter"); 990 return ret; 991 } 992 993 prod = 0; 994 for (; port < perf_nb_event_ports(opt); port++) { 995 union rte_event_crypto_metadata m_data; 996 struct prod_data *p = &t->prod[port]; 997 uint32_t flow_id; 998 999 if (qp_id == rte_cryptodev_queue_pair_count(cdev_id)) { 1000 cdev_id++; 1001 qp_id = 0; 1002 } 1003 1004 p->dev_id = opt->dev_id; 1005 p->port_id = port; 1006 p->queue_id = prod * stride; 1007 p->ca.cdev_id = cdev_id; 1008 p->ca.cdev_qp_id = qp_id; 1009 p->ca.crypto_sess = rte_zmalloc_socket( 1010 NULL, sizeof(void *) * t->nb_flows, 1011 RTE_CACHE_LINE_SIZE, opt->socket_id); 1012 p->t = t; 1013 1014 ret = perf_event_crypto_adapter_setup(t, p); 1015 if (ret) 1016 return ret; 1017 1018 m_data.request_info.cdev_id = p->ca.cdev_id; 1019 m_data.request_info.queue_pair_id = p->ca.cdev_qp_id; 1020 m_data.response_info.sched_type = RTE_SCHED_TYPE_ATOMIC; 1021 m_data.response_info.queue_id = p->queue_id; 1022 1023 for (flow_id = 0; flow_id < t->nb_flows; flow_id++) { 1024 m_data.response_info.flow_id = flow_id; 1025 if (opt->crypto_op_type == 1026 RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 1027 void *sess; 1028 1029 sess = cryptodev_sym_sess_create(p, t); 1030 if (sess == NULL) 1031 return -ENOMEM; 1032 1033 ret = rte_cryptodev_session_event_mdata_set( 1034 cdev_id, 1035 sess, 1036 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 1037 RTE_CRYPTO_OP_WITH_SESSION, 1038 &m_data, sizeof(m_data)); 1039 if (ret) 1040 return ret; 1041 p->ca.crypto_sess[flow_id] = sess; 1042 } else { 1043 void *sess; 1044 1045 sess = cryptodev_asym_sess_create(p, t); 1046 if (sess == NULL) 1047 return -ENOMEM; 1048 ret = rte_cryptodev_session_event_mdata_set( 1049 cdev_id, 1050 sess, 1051 RTE_CRYPTO_OP_TYPE_ASYMMETRIC, 1052 RTE_CRYPTO_OP_WITH_SESSION, 1053 &m_data, sizeof(m_data)); 1054 if (ret) 1055 return ret; 1056 p->ca.crypto_sess[flow_id] = sess; 1057 } 1058 } 1059 1060 conf.event_port_cfg |= 1061 RTE_EVENT_PORT_CFG_HINT_PRODUCER | 1062 RTE_EVENT_PORT_CFG_HINT_CONSUMER; 1063 1064 ret = rte_event_port_setup(opt->dev_id, port, &conf); 1065 if (ret) { 1066 evt_err("failed to setup port %d", port); 1067 return ret; 1068 } 1069 1070 qp_id++; 1071 prod++; 1072 } 1073 } else { 1074 prod = 0; 1075 for ( ; port < perf_nb_event_ports(opt); port++) { 1076 struct prod_data *p = &t->prod[port]; 1077 1078 p->dev_id = opt->dev_id; 1079 p->port_id = port; 1080 p->queue_id = prod * stride; 1081 p->t = t; 1082 1083 struct rte_event_port_conf conf = *port_conf; 1084 conf.event_port_cfg |= 1085 RTE_EVENT_PORT_CFG_HINT_PRODUCER | 1086 RTE_EVENT_PORT_CFG_HINT_CONSUMER; 1087 1088 ret = rte_event_port_setup(opt->dev_id, port, &conf); 1089 if (ret) { 1090 evt_err("failed to setup port %d", port); 1091 return ret; 1092 } 1093 prod++; 1094 } 1095 } 1096 1097 return ret; 1098 } 1099 1100 int 1101 perf_opt_check(struct evt_options *opt, uint64_t nb_queues) 1102 { 1103 unsigned int lcores; 1104 1105 /* N producer + N worker + main when producer cores are used 1106 * Else N worker + main when Rx adapter is used 1107 */ 1108 lcores = opt->prod_type == EVT_PROD_TYPE_SYNT ? 3 : 2; 1109 1110 if (rte_lcore_count() < lcores) { 1111 evt_err("test need minimum %d lcores", lcores); 1112 return -1; 1113 } 1114 1115 /* Validate worker lcores */ 1116 if (evt_lcores_has_overlap(opt->wlcores, rte_get_main_lcore())) { 1117 evt_err("worker lcores overlaps with main lcore"); 1118 return -1; 1119 } 1120 if (evt_lcores_has_overlap_multi(opt->wlcores, opt->plcores)) { 1121 evt_err("worker lcores overlaps producer lcores"); 1122 return -1; 1123 } 1124 if (evt_has_disabled_lcore(opt->wlcores)) { 1125 evt_err("one or more workers lcores are not enabled"); 1126 return -1; 1127 } 1128 if (!evt_has_active_lcore(opt->wlcores)) { 1129 evt_err("minimum one worker is required"); 1130 return -1; 1131 } 1132 1133 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 1134 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR || 1135 opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { 1136 /* Validate producer lcores */ 1137 if (evt_lcores_has_overlap(opt->plcores, 1138 rte_get_main_lcore())) { 1139 evt_err("producer lcores overlaps with main lcore"); 1140 return -1; 1141 } 1142 if (evt_has_disabled_lcore(opt->plcores)) { 1143 evt_err("one or more producer lcores are not enabled"); 1144 return -1; 1145 } 1146 if (!evt_has_active_lcore(opt->plcores)) { 1147 evt_err("minimum one producer is required"); 1148 return -1; 1149 } 1150 } 1151 1152 if (evt_has_invalid_stage(opt)) 1153 return -1; 1154 1155 if (evt_has_invalid_sched_type(opt)) 1156 return -1; 1157 1158 if (nb_queues > EVT_MAX_QUEUES) { 1159 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES); 1160 return -1; 1161 } 1162 if (perf_nb_event_ports(opt) > EVT_MAX_PORTS) { 1163 evt_err("number of ports exceeds %d", EVT_MAX_PORTS); 1164 return -1; 1165 } 1166 1167 /* Fixups */ 1168 if ((opt->nb_stages == 1 && 1169 opt->prod_type != EVT_PROD_TYPE_EVENT_TIMER_ADPTR) && 1170 opt->fwd_latency) { 1171 evt_info("fwd_latency is valid when nb_stages > 1, disabling"); 1172 opt->fwd_latency = 0; 1173 } 1174 1175 if (opt->fwd_latency && !opt->q_priority) { 1176 evt_info("enabled queue priority for latency measurement"); 1177 opt->q_priority = 1; 1178 } 1179 if (opt->nb_pkts == 0) 1180 opt->nb_pkts = INT64_MAX/evt_nr_active_lcores(opt->plcores); 1181 1182 return 0; 1183 } 1184 1185 void 1186 perf_opt_dump(struct evt_options *opt, uint8_t nb_queues) 1187 { 1188 evt_dump("nb_prod_lcores", "%d", evt_nr_active_lcores(opt->plcores)); 1189 evt_dump_producer_lcores(opt); 1190 evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores)); 1191 evt_dump_worker_lcores(opt); 1192 evt_dump_nb_stages(opt); 1193 evt_dump("nb_evdev_ports", "%d", perf_nb_event_ports(opt)); 1194 evt_dump("nb_evdev_queues", "%d", nb_queues); 1195 evt_dump_queue_priority(opt); 1196 evt_dump_sched_type_list(opt); 1197 evt_dump_producer_type(opt); 1198 evt_dump("prod_enq_burst_sz", "%d", opt->prod_enq_burst_sz); 1199 } 1200 1201 static void 1202 perf_event_port_flush(uint8_t dev_id __rte_unused, struct rte_event ev, 1203 void *args) 1204 { 1205 rte_mempool_put(args, ev.event_ptr); 1206 } 1207 1208 void 1209 perf_worker_cleanup(struct rte_mempool *const pool, uint8_t dev_id, 1210 uint8_t port_id, struct rte_event events[], uint16_t nb_enq, 1211 uint16_t nb_deq) 1212 { 1213 int i; 1214 1215 if (nb_deq) { 1216 for (i = nb_enq; i < nb_deq; i++) 1217 rte_mempool_put(pool, events[i].event_ptr); 1218 1219 for (i = 0; i < nb_deq; i++) 1220 events[i].op = RTE_EVENT_OP_RELEASE; 1221 rte_event_enqueue_burst(dev_id, port_id, events, nb_deq); 1222 } 1223 rte_event_port_quiesce(dev_id, port_id, perf_event_port_flush, pool); 1224 } 1225 1226 void 1227 perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt) 1228 { 1229 int i; 1230 struct test_perf *t = evt_test_priv(test); 1231 1232 if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 1233 for (i = 0; i < opt->nb_timer_adptrs; i++) 1234 rte_event_timer_adapter_stop(t->timer_adptr[i]); 1235 } 1236 rte_event_dev_stop(opt->dev_id); 1237 rte_event_dev_close(opt->dev_id); 1238 } 1239 1240 static inline void 1241 perf_elt_init(struct rte_mempool *mp, void *arg __rte_unused, 1242 void *obj, unsigned i __rte_unused) 1243 { 1244 memset(obj, 0, mp->elt_size); 1245 } 1246 1247 #define NB_RX_DESC 128 1248 #define NB_TX_DESC 512 1249 int 1250 perf_ethdev_setup(struct evt_test *test, struct evt_options *opt) 1251 { 1252 uint16_t i; 1253 int ret; 1254 struct test_perf *t = evt_test_priv(test); 1255 struct rte_eth_conf port_conf = { 1256 .rxmode = { 1257 .mq_mode = RTE_ETH_MQ_RX_RSS, 1258 }, 1259 .rx_adv_conf = { 1260 .rss_conf = { 1261 .rss_key = NULL, 1262 .rss_hf = RTE_ETH_RSS_IP, 1263 }, 1264 }, 1265 }; 1266 1267 if (opt->prod_type != EVT_PROD_TYPE_ETH_RX_ADPTR) 1268 return 0; 1269 1270 if (!rte_eth_dev_count_avail()) { 1271 evt_err("No ethernet ports found."); 1272 return -ENODEV; 1273 } 1274 1275 RTE_ETH_FOREACH_DEV(i) { 1276 struct rte_eth_dev_info dev_info; 1277 struct rte_eth_conf local_port_conf = port_conf; 1278 1279 ret = rte_eth_dev_info_get(i, &dev_info); 1280 if (ret != 0) { 1281 evt_err("Error during getting device (port %u) info: %s\n", 1282 i, strerror(-ret)); 1283 return ret; 1284 } 1285 1286 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 1287 dev_info.flow_type_rss_offloads; 1288 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 1289 port_conf.rx_adv_conf.rss_conf.rss_hf) { 1290 evt_info("Port %u modified RSS hash function based on hardware support," 1291 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 1292 i, 1293 port_conf.rx_adv_conf.rss_conf.rss_hf, 1294 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 1295 } 1296 1297 if (rte_eth_dev_configure(i, 1, 1, &local_port_conf) < 0) { 1298 evt_err("Failed to configure eth port [%d]", i); 1299 return -EINVAL; 1300 } 1301 1302 if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC, 1303 rte_socket_id(), NULL, t->pool) < 0) { 1304 evt_err("Failed to setup eth port [%d] rx_queue: %d.", 1305 i, 0); 1306 return -EINVAL; 1307 } 1308 1309 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC, 1310 rte_socket_id(), NULL) < 0) { 1311 evt_err("Failed to setup eth port [%d] tx_queue: %d.", 1312 i, 0); 1313 return -EINVAL; 1314 } 1315 1316 ret = rte_eth_promiscuous_enable(i); 1317 if (ret != 0) { 1318 evt_err("Failed to enable promiscuous mode for eth port [%d]: %s", 1319 i, rte_strerror(-ret)); 1320 return ret; 1321 } 1322 } 1323 1324 return 0; 1325 } 1326 1327 void 1328 perf_ethdev_rx_stop(struct evt_test *test, struct evt_options *opt) 1329 { 1330 uint16_t i; 1331 RTE_SET_USED(test); 1332 1333 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 1334 RTE_ETH_FOREACH_DEV(i) { 1335 rte_event_eth_rx_adapter_stop(i); 1336 rte_event_eth_rx_adapter_queue_del(i, i, -1); 1337 rte_eth_dev_rx_queue_stop(i, 0); 1338 } 1339 } 1340 } 1341 1342 void 1343 perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt) 1344 { 1345 uint16_t i; 1346 RTE_SET_USED(test); 1347 1348 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 1349 RTE_ETH_FOREACH_DEV(i) { 1350 rte_event_eth_tx_adapter_stop(i); 1351 rte_event_eth_tx_adapter_queue_del(i, i, -1); 1352 rte_eth_dev_tx_queue_stop(i, 0); 1353 rte_eth_dev_stop(i); 1354 } 1355 } 1356 } 1357 1358 int 1359 perf_cryptodev_setup(struct evt_test *test, struct evt_options *opt) 1360 { 1361 uint8_t cdev_count, cdev_id, nb_plcores, nb_qps; 1362 struct test_perf *t = evt_test_priv(test); 1363 unsigned int max_session_size; 1364 uint32_t nb_sessions; 1365 int ret; 1366 1367 if (opt->prod_type != EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) 1368 return 0; 1369 1370 cdev_count = rte_cryptodev_count(); 1371 if (cdev_count == 0) { 1372 evt_err("No crypto devices available\n"); 1373 return -ENODEV; 1374 } 1375 1376 t->ca_op_pool = rte_crypto_op_pool_create( 1377 "crypto_op_pool", opt->crypto_op_type, opt->pool_sz, 1378 128, sizeof(union rte_event_crypto_metadata), 1379 rte_socket_id()); 1380 if (t->ca_op_pool == NULL) { 1381 evt_err("Failed to create crypto op pool"); 1382 return -ENOMEM; 1383 } 1384 1385 nb_sessions = evt_nr_active_lcores(opt->plcores) * t->nb_flows; 1386 t->ca_asym_sess_pool = rte_cryptodev_asym_session_pool_create( 1387 "ca_asym_sess_pool", nb_sessions, 0, 1388 sizeof(union rte_event_crypto_metadata), SOCKET_ID_ANY); 1389 if (t->ca_asym_sess_pool == NULL) { 1390 evt_err("Failed to create sym session pool"); 1391 ret = -ENOMEM; 1392 goto err; 1393 } 1394 1395 max_session_size = 0; 1396 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 1397 unsigned int session_size; 1398 1399 session_size = 1400 rte_cryptodev_sym_get_private_session_size(cdev_id); 1401 if (session_size > max_session_size) 1402 max_session_size = session_size; 1403 } 1404 1405 t->ca_sess_pool = rte_cryptodev_sym_session_pool_create( 1406 "ca_sess_pool", nb_sessions, max_session_size, 0, 1407 sizeof(union rte_event_crypto_metadata), SOCKET_ID_ANY); 1408 if (t->ca_sess_pool == NULL) { 1409 evt_err("Failed to create sym session pool"); 1410 ret = -ENOMEM; 1411 goto err; 1412 } 1413 1414 /* 1415 * Calculate number of needed queue pairs, based on the amount of 1416 * available number of logical cores and crypto devices. For instance, 1417 * if there are 4 cores and 2 crypto devices, 2 queue pairs will be set 1418 * up per device. 1419 */ 1420 nb_plcores = evt_nr_active_lcores(opt->plcores); 1421 nb_qps = (nb_plcores % cdev_count) ? (nb_plcores / cdev_count) + 1 : 1422 nb_plcores / cdev_count; 1423 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 1424 struct rte_cryptodev_qp_conf qp_conf; 1425 struct rte_cryptodev_config conf; 1426 struct rte_cryptodev_info info; 1427 int qp_id; 1428 1429 rte_cryptodev_info_get(cdev_id, &info); 1430 if (nb_qps > info.max_nb_queue_pairs) { 1431 evt_err("Not enough queue pairs per cryptodev (%u)", 1432 nb_qps); 1433 ret = -EINVAL; 1434 goto err; 1435 } 1436 1437 conf.nb_queue_pairs = nb_qps; 1438 conf.socket_id = SOCKET_ID_ANY; 1439 conf.ff_disable = RTE_CRYPTODEV_FF_SECURITY; 1440 1441 ret = rte_cryptodev_configure(cdev_id, &conf); 1442 if (ret) { 1443 evt_err("Failed to configure cryptodev (%u)", cdev_id); 1444 goto err; 1445 } 1446 1447 qp_conf.nb_descriptors = NB_CRYPTODEV_DESCRIPTORS; 1448 qp_conf.mp_session = t->ca_sess_pool; 1449 1450 for (qp_id = 0; qp_id < conf.nb_queue_pairs; qp_id++) { 1451 ret = rte_cryptodev_queue_pair_setup( 1452 cdev_id, qp_id, &qp_conf, 1453 rte_cryptodev_socket_id(cdev_id)); 1454 if (ret) { 1455 evt_err("Failed to setup queue pairs on cryptodev %u\n", 1456 cdev_id); 1457 goto err; 1458 } 1459 } 1460 } 1461 1462 return 0; 1463 err: 1464 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) 1465 rte_cryptodev_close(cdev_id); 1466 1467 rte_mempool_free(t->ca_op_pool); 1468 rte_mempool_free(t->ca_sess_pool); 1469 rte_mempool_free(t->ca_asym_sess_pool); 1470 1471 return ret; 1472 } 1473 1474 void 1475 perf_cryptodev_destroy(struct evt_test *test, struct evt_options *opt) 1476 { 1477 uint8_t cdev_id, cdev_count = rte_cryptodev_count(); 1478 struct test_perf *t = evt_test_priv(test); 1479 uint16_t port; 1480 1481 if (opt->prod_type != EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) 1482 return; 1483 1484 for (port = t->nb_workers; port < perf_nb_event_ports(opt); port++) { 1485 void *sess; 1486 struct prod_data *p = &t->prod[port]; 1487 uint32_t flow_id; 1488 uint8_t cdev_id; 1489 1490 for (flow_id = 0; flow_id < t->nb_flows; flow_id++) { 1491 sess = p->ca.crypto_sess[flow_id]; 1492 cdev_id = p->ca.cdev_id; 1493 rte_cryptodev_sym_session_free(cdev_id, sess); 1494 } 1495 1496 rte_event_crypto_adapter_queue_pair_del( 1497 TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id); 1498 } 1499 1500 rte_event_crypto_adapter_free(TEST_PERF_CA_ID); 1501 1502 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 1503 rte_cryptodev_stop(cdev_id); 1504 rte_cryptodev_close(cdev_id); 1505 } 1506 1507 rte_mempool_free(t->ca_op_pool); 1508 rte_mempool_free(t->ca_sess_pool); 1509 rte_mempool_free(t->ca_asym_sess_pool); 1510 } 1511 1512 int 1513 perf_mempool_setup(struct evt_test *test, struct evt_options *opt) 1514 { 1515 struct test_perf *t = evt_test_priv(test); 1516 1517 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 1518 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 1519 t->pool = rte_mempool_create(test->name, /* mempool name */ 1520 opt->pool_sz, /* number of elements*/ 1521 sizeof(struct perf_elt), /* element size*/ 1522 512, /* cache size*/ 1523 0, NULL, NULL, 1524 perf_elt_init, /* obj constructor */ 1525 NULL, opt->socket_id, 0); /* flags */ 1526 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR && 1527 opt->crypto_op_type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) { 1528 t->pool = rte_mempool_create(test->name, /* mempool name */ 1529 opt->pool_sz, /* number of elements*/ 1530 sizeof(struct perf_elt) + modex_test_case.result_len, 1531 /* element size*/ 1532 512, /* cache size*/ 1533 0, NULL, NULL, 1534 NULL, /* obj constructor */ 1535 NULL, opt->socket_id, 0); /* flags */ 1536 } else { 1537 t->pool = rte_pktmbuf_pool_create(test->name, /* mempool name */ 1538 opt->pool_sz, /* number of elements*/ 1539 512, /* cache size*/ 1540 0, 1541 RTE_MBUF_DEFAULT_BUF_SIZE, 1542 opt->socket_id); /* flags */ 1543 1544 } 1545 1546 if (t->pool == NULL) { 1547 evt_err("failed to create mempool"); 1548 return -ENOMEM; 1549 } 1550 1551 return 0; 1552 } 1553 1554 void 1555 perf_mempool_destroy(struct evt_test *test, struct evt_options *opt) 1556 { 1557 RTE_SET_USED(opt); 1558 struct test_perf *t = evt_test_priv(test); 1559 1560 rte_mempool_free(t->pool); 1561 } 1562 1563 int 1564 perf_test_setup(struct evt_test *test, struct evt_options *opt) 1565 { 1566 void *test_perf; 1567 1568 test_perf = rte_zmalloc_socket(test->name, sizeof(struct test_perf), 1569 RTE_CACHE_LINE_SIZE, opt->socket_id); 1570 if (test_perf == NULL) { 1571 evt_err("failed to allocate test_perf memory"); 1572 goto nomem; 1573 } 1574 test->test_priv = test_perf; 1575 1576 struct test_perf *t = evt_test_priv(test); 1577 1578 if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 1579 t->outstand_pkts = opt->nb_timers * 1580 evt_nr_active_lcores(opt->plcores); 1581 t->nb_pkts = opt->nb_timers; 1582 } else { 1583 t->outstand_pkts = opt->nb_pkts * 1584 evt_nr_active_lcores(opt->plcores); 1585 t->nb_pkts = opt->nb_pkts; 1586 } 1587 1588 t->nb_workers = evt_nr_active_lcores(opt->wlcores); 1589 t->done = false; 1590 t->nb_flows = opt->nb_flows; 1591 t->result = EVT_TEST_FAILED; 1592 t->opt = opt; 1593 memcpy(t->sched_type_list, opt->sched_type_list, 1594 sizeof(opt->sched_type_list)); 1595 return 0; 1596 nomem: 1597 return -ENOMEM; 1598 } 1599 1600 void 1601 perf_test_destroy(struct evt_test *test, struct evt_options *opt) 1602 { 1603 RTE_SET_USED(opt); 1604 1605 rte_free(test->test_priv); 1606 } 1607