1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include <math.h> 6 7 #include "test_perf_common.h" 8 9 #define NB_CRYPTODEV_DESCRIPTORS 1024 10 #define DATA_SIZE 512 11 #define IV_OFFSET (sizeof(struct rte_crypto_op) + \ 12 sizeof(struct rte_crypto_sym_op) + \ 13 sizeof(union rte_event_crypto_metadata)) 14 15 struct modex_test_data { 16 enum rte_crypto_asym_xform_type xform_type; 17 struct { 18 uint8_t data[DATA_SIZE]; 19 uint16_t len; 20 } base; 21 struct { 22 uint8_t data[DATA_SIZE]; 23 uint16_t len; 24 } exponent; 25 struct { 26 uint8_t data[DATA_SIZE]; 27 uint16_t len; 28 } modulus; 29 struct { 30 uint8_t data[DATA_SIZE]; 31 uint16_t len; 32 } reminder; 33 uint16_t result_len; 34 }; 35 36 static struct 37 modex_test_data modex_test_case = { 38 .xform_type = RTE_CRYPTO_ASYM_XFORM_MODEX, 39 .base = { 40 .data = { 41 0xF8, 0xBA, 0x1A, 0x55, 0xD0, 0x2F, 0x85, 42 0xAE, 0x96, 0x7B, 0xB6, 0x2F, 0xB6, 0xCD, 43 0xA8, 0xEB, 0x7E, 0x78, 0xA0, 0x50 44 }, 45 .len = 20, 46 }, 47 .exponent = { 48 .data = { 49 0x01, 0x00, 0x01 50 }, 51 .len = 3, 52 }, 53 .reminder = { 54 .data = { 55 0x2C, 0x60, 0x75, 0x45, 0x98, 0x9D, 0xE0, 0x72, 56 0xA0, 0x9D, 0x3A, 0x9E, 0x03, 0x38, 0x73, 0x3C, 57 0x31, 0x83, 0x04, 0xFE, 0x75, 0x43, 0xE6, 0x17, 58 0x5C, 0x01, 0x29, 0x51, 0x69, 0x33, 0x62, 0x2D, 59 0x78, 0xBE, 0xAE, 0xC4, 0xBC, 0xDE, 0x7E, 0x2C, 60 0x77, 0x84, 0xF2, 0xC5, 0x14, 0xB5, 0x2F, 0xF7, 61 0xC5, 0x94, 0xEF, 0x86, 0x75, 0x75, 0xB5, 0x11, 62 0xE5, 0x0E, 0x0A, 0x29, 0x76, 0xE2, 0xEA, 0x32, 63 0x0E, 0x43, 0x77, 0x7E, 0x2C, 0x27, 0xAC, 0x3B, 64 0x86, 0xA5, 0xDB, 0xC9, 0x48, 0x40, 0xE8, 0x99, 65 0x9A, 0x0A, 0x3D, 0xD6, 0x74, 0xFA, 0x2E, 0x2E, 66 0x5B, 0xAF, 0x8C, 0x99, 0x44, 0x2A, 0x67, 0x38, 67 0x27, 0x41, 0x59, 0x9D, 0xB8, 0x51, 0xC9, 0xF7, 68 0x43, 0x61, 0x31, 0x6E, 0xF1, 0x25, 0x38, 0x7F, 69 0xAE, 0xC6, 0xD0, 0xBB, 0x29, 0x76, 0x3F, 0x46, 70 0x2E, 0x1B, 0xE4, 0x67, 0x71, 0xE3, 0x87, 0x5A 71 }, 72 .len = 128, 73 }, 74 .modulus = { 75 .data = { 76 0xb3, 0xa1, 0xaf, 0xb7, 0x13, 0x08, 0x00, 0x0a, 77 0x35, 0xdc, 0x2b, 0x20, 0x8d, 0xa1, 0xb5, 0xce, 78 0x47, 0x8a, 0xc3, 0x80, 0xf4, 0x7d, 0x4a, 0xa2, 79 0x62, 0xfd, 0x61, 0x7f, 0xb5, 0xa8, 0xde, 0x0a, 80 0x17, 0x97, 0xa0, 0xbf, 0xdf, 0x56, 0x5a, 0x3d, 81 0x51, 0x56, 0x4f, 0x70, 0x70, 0x3f, 0x63, 0x6a, 82 0x44, 0x5b, 0xad, 0x84, 0x0d, 0x3f, 0x27, 0x6e, 83 0x3b, 0x34, 0x91, 0x60, 0x14, 0xb9, 0xaa, 0x72, 84 0xfd, 0xa3, 0x64, 0xd2, 0x03, 0xa7, 0x53, 0x87, 85 0x9e, 0x88, 0x0b, 0xc1, 0x14, 0x93, 0x1a, 0x62, 86 0xff, 0xb1, 0x5d, 0x74, 0xcd, 0x59, 0x63, 0x18, 87 0x11, 0x3d, 0x4f, 0xba, 0x75, 0xd4, 0x33, 0x4e, 88 0x23, 0x6b, 0x7b, 0x57, 0x44, 0xe1, 0xd3, 0x03, 89 0x13, 0xa6, 0xf0, 0x8b, 0x60, 0xb0, 0x9e, 0xee, 90 0x75, 0x08, 0x9d, 0x71, 0x63, 0x13, 0xcb, 0xa6, 91 0x81, 0x92, 0x14, 0x03, 0x22, 0x2d, 0xde, 0x55 92 }, 93 .len = 128, 94 }, 95 .result_len = 128, 96 }; 97 98 int 99 perf_test_result(struct evt_test *test, struct evt_options *opt) 100 { 101 RTE_SET_USED(opt); 102 int i; 103 uint64_t total = 0; 104 struct test_perf *t = evt_test_priv(test); 105 106 printf("Packet distribution across worker cores :\n"); 107 for (i = 0; i < t->nb_workers; i++) 108 total += t->worker[i].processed_pkts; 109 for (i = 0; i < t->nb_workers; i++) 110 printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:" 111 CLGRN" %3.2f"CLNRM"\n", i, 112 t->worker[i].processed_pkts, 113 (((double)t->worker[i].processed_pkts)/total) 114 * 100); 115 116 return t->result; 117 } 118 119 static inline int 120 perf_producer(void *arg) 121 { 122 int i; 123 struct prod_data *p = arg; 124 struct test_perf *t = p->t; 125 struct evt_options *opt = t->opt; 126 const uint8_t dev_id = p->dev_id; 127 const uint8_t port = p->port_id; 128 struct rte_mempool *pool = t->pool; 129 const uint64_t nb_pkts = t->nb_pkts; 130 const uint32_t nb_flows = t->nb_flows; 131 uint32_t flow_counter = 0; 132 uint64_t count = 0; 133 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 134 uint8_t enable_fwd_latency; 135 struct rte_event ev; 136 137 enable_fwd_latency = opt->fwd_latency; 138 if (opt->verbose_level > 1) 139 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__, 140 rte_lcore_id(), dev_id, port, p->queue_id); 141 142 ev.event = 0; 143 ev.op = RTE_EVENT_OP_NEW; 144 ev.queue_id = p->queue_id; 145 ev.sched_type = t->opt->sched_type_list[0]; 146 ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 147 ev.event_type = RTE_EVENT_TYPE_CPU; 148 ev.sub_event_type = 0; /* stage 0 */ 149 150 while (count < nb_pkts && t->done == false) { 151 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 152 continue; 153 for (i = 0; i < BURST_SIZE; i++) { 154 ev.flow_id = flow_counter++ % nb_flows; 155 ev.event_ptr = m[i]; 156 if (enable_fwd_latency) 157 m[i]->timestamp = rte_get_timer_cycles(); 158 while (rte_event_enqueue_new_burst(dev_id, port, &ev, 159 1) != 1) { 160 if (t->done) 161 break; 162 rte_pause(); 163 if (enable_fwd_latency) 164 m[i]->timestamp = 165 rte_get_timer_cycles(); 166 } 167 } 168 count += BURST_SIZE; 169 } 170 171 return 0; 172 } 173 174 static inline int 175 perf_producer_burst(void *arg) 176 { 177 uint32_t i; 178 uint64_t timestamp; 179 struct prod_data *p = arg; 180 struct test_perf *t = p->t; 181 struct evt_options *opt = t->opt; 182 const uint8_t dev_id = p->dev_id; 183 const uint8_t port = p->port_id; 184 struct rte_mempool *pool = t->pool; 185 const uint64_t nb_pkts = t->nb_pkts; 186 const uint32_t nb_flows = t->nb_flows; 187 uint32_t flow_counter = 0; 188 uint16_t enq = 0; 189 uint64_t count = 0; 190 struct perf_elt *m[opt->prod_enq_burst_sz + 1]; 191 struct rte_event ev[opt->prod_enq_burst_sz + 1]; 192 uint32_t burst_size = opt->prod_enq_burst_sz; 193 uint8_t enable_fwd_latency; 194 195 enable_fwd_latency = opt->fwd_latency; 196 memset(m, 0, sizeof(*m) * (opt->prod_enq_burst_sz + 1)); 197 if (opt->verbose_level > 1) 198 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__, 199 rte_lcore_id(), dev_id, port, p->queue_id); 200 201 for (i = 0; i < burst_size; i++) { 202 ev[i].op = RTE_EVENT_OP_NEW; 203 ev[i].queue_id = p->queue_id; 204 ev[i].sched_type = t->opt->sched_type_list[0]; 205 ev[i].priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 206 ev[i].event_type = RTE_EVENT_TYPE_CPU; 207 ev[i].sub_event_type = 0; /* stage 0 */ 208 } 209 210 while (count < nb_pkts && t->done == false) { 211 if (rte_mempool_get_bulk(pool, (void **)m, burst_size) < 0) 212 continue; 213 timestamp = rte_get_timer_cycles(); 214 for (i = 0; i < burst_size; i++) { 215 ev[i].flow_id = flow_counter++ % nb_flows; 216 ev[i].event_ptr = m[i]; 217 if (enable_fwd_latency) 218 m[i]->timestamp = timestamp; 219 } 220 enq = rte_event_enqueue_new_burst(dev_id, port, ev, burst_size); 221 while (enq < burst_size) { 222 enq += rte_event_enqueue_new_burst( 223 dev_id, port, ev + enq, burst_size - enq); 224 if (t->done) 225 break; 226 rte_pause(); 227 if (enable_fwd_latency) { 228 timestamp = rte_get_timer_cycles(); 229 for (i = enq; i < burst_size; i++) 230 m[i]->timestamp = timestamp; 231 } 232 } 233 count += burst_size; 234 } 235 return 0; 236 } 237 238 static inline int 239 perf_event_timer_producer(void *arg) 240 { 241 int i; 242 struct prod_data *p = arg; 243 struct test_perf *t = p->t; 244 struct evt_options *opt = t->opt; 245 uint32_t flow_counter = 0; 246 uint64_t count = 0; 247 uint64_t arm_latency = 0; 248 const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs; 249 const uint32_t nb_flows = t->nb_flows; 250 const uint64_t nb_timers = opt->nb_timers; 251 struct rte_mempool *pool = t->pool; 252 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 253 struct rte_event_timer_adapter **adptr = t->timer_adptr; 254 struct rte_event_timer tim; 255 uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; 256 257 memset(&tim, 0, sizeof(struct rte_event_timer)); 258 timeout_ticks = 259 opt->optm_timer_tick_nsec 260 ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) / 261 opt->optm_timer_tick_nsec) 262 : timeout_ticks; 263 timeout_ticks += timeout_ticks ? 0 : 1; 264 tim.ev.event_type = RTE_EVENT_TYPE_TIMER; 265 tim.ev.op = RTE_EVENT_OP_NEW; 266 tim.ev.sched_type = t->opt->sched_type_list[0]; 267 tim.ev.queue_id = p->queue_id; 268 tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 269 tim.state = RTE_EVENT_TIMER_NOT_ARMED; 270 tim.timeout_ticks = timeout_ticks; 271 272 if (opt->verbose_level > 1) 273 printf("%s(): lcore %d\n", __func__, rte_lcore_id()); 274 275 while (count < nb_timers && t->done == false) { 276 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 277 continue; 278 for (i = 0; i < BURST_SIZE; i++) { 279 rte_prefetch0(m[i + 1]); 280 m[i]->tim = tim; 281 m[i]->tim.ev.flow_id = flow_counter++ % nb_flows; 282 m[i]->tim.ev.event_ptr = m[i]; 283 m[i]->timestamp = rte_get_timer_cycles(); 284 while (rte_event_timer_arm_burst( 285 adptr[flow_counter % nb_timer_adptrs], 286 (struct rte_event_timer **)&m[i], 1) != 1) { 287 if (t->done) 288 break; 289 m[i]->timestamp = rte_get_timer_cycles(); 290 } 291 arm_latency += rte_get_timer_cycles() - m[i]->timestamp; 292 } 293 count += BURST_SIZE; 294 } 295 fflush(stdout); 296 rte_delay_ms(1000); 297 printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", 298 __func__, rte_lcore_id(), 299 count ? (float)(arm_latency / count) / 300 (rte_get_timer_hz() / 1000000) : 0); 301 return 0; 302 } 303 304 static inline int 305 perf_event_timer_producer_burst(void *arg) 306 { 307 int i; 308 struct prod_data *p = arg; 309 struct test_perf *t = p->t; 310 struct evt_options *opt = t->opt; 311 uint32_t flow_counter = 0; 312 uint64_t count = 0; 313 uint64_t arm_latency = 0; 314 const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs; 315 const uint32_t nb_flows = t->nb_flows; 316 const uint64_t nb_timers = opt->nb_timers; 317 struct rte_mempool *pool = t->pool; 318 struct perf_elt *m[BURST_SIZE + 1] = {NULL}; 319 struct rte_event_timer_adapter **adptr = t->timer_adptr; 320 struct rte_event_timer tim; 321 uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; 322 323 memset(&tim, 0, sizeof(struct rte_event_timer)); 324 timeout_ticks = 325 opt->optm_timer_tick_nsec 326 ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) / 327 opt->optm_timer_tick_nsec) 328 : timeout_ticks; 329 timeout_ticks += timeout_ticks ? 0 : 1; 330 tim.ev.event_type = RTE_EVENT_TYPE_TIMER; 331 tim.ev.op = RTE_EVENT_OP_NEW; 332 tim.ev.sched_type = t->opt->sched_type_list[0]; 333 tim.ev.queue_id = p->queue_id; 334 tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 335 tim.state = RTE_EVENT_TIMER_NOT_ARMED; 336 tim.timeout_ticks = timeout_ticks; 337 338 if (opt->verbose_level > 1) 339 printf("%s(): lcore %d\n", __func__, rte_lcore_id()); 340 341 while (count < nb_timers && t->done == false) { 342 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) 343 continue; 344 for (i = 0; i < BURST_SIZE; i++) { 345 rte_prefetch0(m[i + 1]); 346 m[i]->tim = tim; 347 m[i]->tim.ev.flow_id = flow_counter++ % nb_flows; 348 m[i]->tim.ev.event_ptr = m[i]; 349 m[i]->timestamp = rte_get_timer_cycles(); 350 } 351 rte_event_timer_arm_tmo_tick_burst( 352 adptr[flow_counter % nb_timer_adptrs], 353 (struct rte_event_timer **)m, 354 tim.timeout_ticks, 355 BURST_SIZE); 356 arm_latency += rte_get_timer_cycles() - m[i - 1]->timestamp; 357 count += BURST_SIZE; 358 } 359 fflush(stdout); 360 rte_delay_ms(1000); 361 printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", 362 __func__, rte_lcore_id(), 363 count ? (float)(arm_latency / count) / 364 (rte_get_timer_hz() / 1000000) : 0); 365 return 0; 366 } 367 368 static inline void 369 crypto_adapter_enq_op_new(struct prod_data *p) 370 { 371 struct test_perf *t = p->t; 372 const uint32_t nb_flows = t->nb_flows; 373 const uint64_t nb_pkts = t->nb_pkts; 374 struct rte_mempool *pool = t->pool; 375 uint16_t data_length, data_offset; 376 struct evt_options *opt = t->opt; 377 uint16_t qp_id = p->ca.cdev_qp_id; 378 uint8_t cdev_id = p->ca.cdev_id; 379 uint64_t alloc_failures = 0; 380 uint32_t flow_counter = 0; 381 struct rte_crypto_op *op; 382 uint16_t len, offset; 383 struct rte_mbuf *m; 384 uint64_t count = 0; 385 386 if (opt->verbose_level > 1) 387 printf("%s(): lcore %d queue %d cdev_id %u cdev_qp_id %u\n", 388 __func__, rte_lcore_id(), p->queue_id, p->ca.cdev_id, 389 p->ca.cdev_qp_id); 390 391 offset = sizeof(struct perf_elt); 392 len = RTE_MAX(RTE_ETHER_MIN_LEN + offset, opt->mbuf_sz); 393 394 if (opt->crypto_cipher_bit_mode) { 395 data_offset = offset << 3; 396 data_length = (len - offset) << 3; 397 } else { 398 data_offset = offset; 399 data_length = len - offset; 400 } 401 402 while (count < nb_pkts && t->done == false) { 403 if (opt->crypto_op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 404 struct rte_crypto_sym_op *sym_op; 405 406 op = rte_crypto_op_alloc(t->ca_op_pool, 407 RTE_CRYPTO_OP_TYPE_SYMMETRIC); 408 if (unlikely(op == NULL)) { 409 alloc_failures++; 410 continue; 411 } 412 413 m = rte_pktmbuf_alloc(pool); 414 if (unlikely(m == NULL)) { 415 alloc_failures++; 416 rte_crypto_op_free(op); 417 continue; 418 } 419 420 rte_pktmbuf_append(m, len); 421 sym_op = op->sym; 422 sym_op->m_src = m; 423 424 sym_op->cipher.data.offset = data_offset; 425 sym_op->cipher.data.length = data_length; 426 427 rte_crypto_op_attach_sym_session( 428 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 429 } else { 430 struct rte_crypto_asym_op *asym_op; 431 uint8_t *result; 432 433 if (rte_mempool_get(pool, (void **)&result)) { 434 alloc_failures++; 435 continue; 436 } 437 438 op = rte_crypto_op_alloc(t->ca_op_pool, 439 RTE_CRYPTO_OP_TYPE_ASYMMETRIC); 440 if (unlikely(op == NULL)) { 441 alloc_failures++; 442 rte_mempool_put(pool, result); 443 continue; 444 } 445 446 asym_op = op->asym; 447 asym_op->modex.base.data = modex_test_case.base.data; 448 asym_op->modex.base.length = modex_test_case.base.len; 449 asym_op->modex.result.data = result; 450 asym_op->modex.result.length = modex_test_case.result_len; 451 rte_crypto_op_attach_asym_session( 452 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 453 } 454 while (rte_cryptodev_enqueue_burst(cdev_id, qp_id, &op, 1) != 1 && 455 t->done == false) 456 rte_pause(); 457 458 count++; 459 } 460 461 if (opt->verbose_level > 1 && alloc_failures) 462 printf("%s(): lcore %d allocation failures: %"PRIu64"\n", 463 __func__, rte_lcore_id(), alloc_failures); 464 } 465 466 static inline void 467 crypto_adapter_enq_op_fwd(struct prod_data *p) 468 { 469 const uint8_t dev_id = p->dev_id; 470 const uint8_t port = p->port_id; 471 struct test_perf *t = p->t; 472 const uint32_t nb_flows = t->nb_flows; 473 const uint64_t nb_pkts = t->nb_pkts; 474 struct rte_mempool *pool = t->pool; 475 struct evt_options *opt = t->opt; 476 uint64_t alloc_failures = 0; 477 uint32_t flow_counter = 0; 478 struct rte_crypto_op *op; 479 uint16_t len, offset; 480 struct rte_event ev; 481 struct rte_mbuf *m; 482 uint64_t count = 0; 483 484 if (opt->verbose_level > 1) 485 printf("%s(): lcore %d port %d queue %d cdev_id %u cdev_qp_id %u\n", 486 __func__, rte_lcore_id(), port, p->queue_id, 487 p->ca.cdev_id, p->ca.cdev_qp_id); 488 489 ev.event = 0; 490 ev.op = RTE_EVENT_OP_NEW; 491 ev.queue_id = p->queue_id; 492 ev.sched_type = RTE_SCHED_TYPE_ATOMIC; 493 ev.event_type = RTE_EVENT_TYPE_CPU; 494 495 offset = sizeof(struct perf_elt); 496 len = RTE_MAX(RTE_ETHER_MIN_LEN + offset, opt->mbuf_sz); 497 498 while (count < nb_pkts && t->done == false) { 499 if (opt->crypto_op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 500 struct rte_crypto_sym_op *sym_op; 501 502 op = rte_crypto_op_alloc(t->ca_op_pool, 503 RTE_CRYPTO_OP_TYPE_SYMMETRIC); 504 if (unlikely(op == NULL)) { 505 alloc_failures++; 506 continue; 507 } 508 509 m = rte_pktmbuf_alloc(pool); 510 if (unlikely(m == NULL)) { 511 alloc_failures++; 512 rte_crypto_op_free(op); 513 continue; 514 } 515 516 rte_pktmbuf_append(m, len); 517 sym_op = op->sym; 518 sym_op->m_src = m; 519 sym_op->cipher.data.offset = offset; 520 sym_op->cipher.data.length = len - offset; 521 rte_crypto_op_attach_sym_session( 522 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 523 } else { 524 struct rte_crypto_asym_op *asym_op; 525 uint8_t *result; 526 527 if (rte_mempool_get(pool, (void **)&result)) { 528 alloc_failures++; 529 continue; 530 } 531 532 op = rte_crypto_op_alloc(t->ca_op_pool, 533 RTE_CRYPTO_OP_TYPE_ASYMMETRIC); 534 if (unlikely(op == NULL)) { 535 alloc_failures++; 536 rte_mempool_put(pool, result); 537 continue; 538 } 539 540 asym_op = op->asym; 541 asym_op->modex.base.data = modex_test_case.base.data; 542 asym_op->modex.base.length = modex_test_case.base.len; 543 asym_op->modex.result.data = result; 544 asym_op->modex.result.length = modex_test_case.result_len; 545 rte_crypto_op_attach_asym_session( 546 op, p->ca.crypto_sess[flow_counter++ % nb_flows]); 547 } 548 ev.event_ptr = op; 549 550 while (rte_event_crypto_adapter_enqueue(dev_id, port, &ev, 1) != 1 && 551 t->done == false) 552 rte_pause(); 553 554 count++; 555 } 556 557 if (opt->verbose_level > 1 && alloc_failures) 558 printf("%s(): lcore %d allocation failures: %"PRIu64"\n", 559 __func__, rte_lcore_id(), alloc_failures); 560 } 561 562 static inline void 563 dma_adapter_enq_op_fwd(struct prod_data *p) 564 { 565 struct test_perf *t = p->t; 566 const uint32_t nb_flows = t->nb_flows; 567 const uint64_t nb_pkts = t->nb_pkts; 568 struct rte_event_dma_adapter_op *op; 569 const uint8_t dev_id = p->dev_id; 570 struct evt_options *opt = t->opt; 571 const uint8_t port = p->port_id; 572 uint32_t flow_counter = 0; 573 struct rte_event ev; 574 uint64_t count = 0; 575 576 if (opt->verbose_level > 1) 577 printf("%s(): lcore %d port %d queue %d dma_dev_id %u dma_dev_vchan_id %u\n", 578 __func__, rte_lcore_id(), port, p->queue_id, 579 p->da.dma_dev_id, p->da.vchan_id); 580 581 ev.event = 0; 582 ev.op = RTE_EVENT_OP_NEW; 583 ev.queue_id = p->queue_id; 584 ev.sched_type = RTE_SCHED_TYPE_ATOMIC; 585 ev.event_type = RTE_EVENT_TYPE_CPU; 586 587 while (count < nb_pkts && t->done == false) { 588 op = p->da.dma_op[flow_counter++ % nb_flows]; 589 ev.event_ptr = op; 590 591 while (rte_event_dma_adapter_enqueue(dev_id, port, &ev, 1) != 1 && 592 t->done == false) 593 rte_pause(); 594 595 count++; 596 } 597 } 598 599 static inline int 600 perf_event_dma_producer(void *arg) 601 { 602 struct prod_data *p = arg; 603 604 /* Only fwd mode is supported. */ 605 dma_adapter_enq_op_fwd(p); 606 607 return 0; 608 } 609 610 static inline int 611 perf_event_crypto_producer(void *arg) 612 { 613 struct prod_data *p = arg; 614 struct evt_options *opt = p->t->opt; 615 616 if (opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) 617 crypto_adapter_enq_op_new(p); 618 else 619 crypto_adapter_enq_op_fwd(p); 620 621 return 0; 622 } 623 624 static void 625 crypto_adapter_enq_op_new_burst(struct prod_data *p) 626 { 627 const struct test_perf *t = p->t; 628 const struct evt_options *opt = t->opt; 629 630 struct rte_mbuf *m, *pkts_burst[MAX_PROD_ENQ_BURST_SIZE]; 631 struct rte_crypto_op *ops_burst[MAX_PROD_ENQ_BURST_SIZE]; 632 const uint32_t burst_size = opt->prod_enq_burst_sz; 633 uint8_t *result[MAX_PROD_ENQ_BURST_SIZE]; 634 const uint32_t nb_flows = t->nb_flows; 635 const uint64_t nb_pkts = t->nb_pkts; 636 uint16_t len, enq, nb_alloc, offset; 637 struct rte_mempool *pool = t->pool; 638 uint16_t qp_id = p->ca.cdev_qp_id; 639 uint8_t cdev_id = p->ca.cdev_id; 640 uint64_t alloc_failures = 0; 641 uint32_t flow_counter = 0; 642 uint64_t count = 0; 643 uint32_t i; 644 645 if (opt->verbose_level > 1) 646 printf("%s(): lcore %d queue %d cdev_id %u cdev_qp_id %u\n", 647 __func__, rte_lcore_id(), p->queue_id, p->ca.cdev_id, 648 p->ca.cdev_qp_id); 649 650 offset = sizeof(struct perf_elt); 651 len = RTE_MAX(RTE_ETHER_MIN_LEN + offset, opt->mbuf_sz); 652 653 while (count < nb_pkts && t->done == false) { 654 if (opt->crypto_op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 655 struct rte_crypto_sym_op *sym_op; 656 int ret; 657 658 nb_alloc = rte_crypto_op_bulk_alloc(t->ca_op_pool, 659 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops_burst, burst_size); 660 if (unlikely(nb_alloc != burst_size)) { 661 alloc_failures++; 662 continue; 663 } 664 665 ret = rte_pktmbuf_alloc_bulk(pool, pkts_burst, burst_size); 666 if (unlikely(ret != 0)) { 667 alloc_failures++; 668 rte_mempool_put_bulk(t->ca_op_pool, (void **)ops_burst, burst_size); 669 continue; 670 } 671 672 for (i = 0; i < burst_size; i++) { 673 m = pkts_burst[i]; 674 rte_pktmbuf_append(m, len); 675 sym_op = ops_burst[i]->sym; 676 sym_op->m_src = m; 677 sym_op->cipher.data.offset = offset; 678 sym_op->cipher.data.length = len - offset; 679 rte_crypto_op_attach_sym_session(ops_burst[i], 680 p->ca.crypto_sess[flow_counter++ % nb_flows]); 681 } 682 } else { 683 struct rte_crypto_asym_op *asym_op; 684 685 nb_alloc = rte_crypto_op_bulk_alloc(t->ca_op_pool, 686 RTE_CRYPTO_OP_TYPE_ASYMMETRIC, ops_burst, burst_size); 687 if (unlikely(nb_alloc != burst_size)) { 688 alloc_failures++; 689 continue; 690 } 691 692 if (rte_mempool_get_bulk(pool, (void **)result, burst_size)) { 693 alloc_failures++; 694 rte_mempool_put_bulk(t->ca_op_pool, (void **)ops_burst, burst_size); 695 continue; 696 } 697 698 for (i = 0; i < burst_size; i++) { 699 asym_op = ops_burst[i]->asym; 700 asym_op->modex.base.data = modex_test_case.base.data; 701 asym_op->modex.base.length = modex_test_case.base.len; 702 asym_op->modex.result.data = result[i]; 703 asym_op->modex.result.length = modex_test_case.result_len; 704 rte_crypto_op_attach_asym_session(ops_burst[i], 705 p->ca.crypto_sess[flow_counter++ % nb_flows]); 706 } 707 } 708 709 enq = 0; 710 while (!t->done) { 711 enq += rte_cryptodev_enqueue_burst(cdev_id, qp_id, ops_burst + enq, 712 burst_size - enq); 713 if (enq == burst_size) 714 break; 715 } 716 717 count += burst_size; 718 } 719 720 if (opt->verbose_level > 1 && alloc_failures) 721 printf("%s(): lcore %d allocation failures: %"PRIu64"\n", 722 __func__, rte_lcore_id(), alloc_failures); 723 } 724 725 static void 726 crypto_adapter_enq_op_fwd_burst(struct prod_data *p) 727 { 728 const struct test_perf *t = p->t; 729 const struct evt_options *opt = t->opt; 730 731 struct rte_mbuf *m, *pkts_burst[MAX_PROD_ENQ_BURST_SIZE]; 732 struct rte_crypto_op *ops_burst[MAX_PROD_ENQ_BURST_SIZE]; 733 const uint32_t burst_size = opt->prod_enq_burst_sz; 734 struct rte_event ev[MAX_PROD_ENQ_BURST_SIZE]; 735 uint8_t *result[MAX_PROD_ENQ_BURST_SIZE]; 736 const uint32_t nb_flows = t->nb_flows; 737 const uint64_t nb_pkts = t->nb_pkts; 738 uint16_t len, enq, nb_alloc, offset; 739 struct rte_mempool *pool = t->pool; 740 const uint8_t dev_id = p->dev_id; 741 const uint8_t port = p->port_id; 742 uint64_t alloc_failures = 0; 743 uint32_t flow_counter = 0; 744 uint64_t count = 0; 745 uint32_t i; 746 747 if (opt->verbose_level > 1) 748 printf("%s(): lcore %d port %d queue %d cdev_id %u cdev_qp_id %u\n", 749 __func__, rte_lcore_id(), port, p->queue_id, 750 p->ca.cdev_id, p->ca.cdev_qp_id); 751 752 offset = sizeof(struct perf_elt); 753 len = RTE_MAX(RTE_ETHER_MIN_LEN + offset, opt->mbuf_sz); 754 755 for (i = 0; i < burst_size; i++) { 756 ev[i].event = 0; 757 ev[i].op = RTE_EVENT_OP_NEW; 758 ev[i].queue_id = p->queue_id; 759 ev[i].sched_type = RTE_SCHED_TYPE_ATOMIC; 760 ev[i].event_type = RTE_EVENT_TYPE_CPU; 761 } 762 763 while (count < nb_pkts && t->done == false) { 764 if (opt->crypto_op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 765 struct rte_crypto_sym_op *sym_op; 766 int ret; 767 768 nb_alloc = rte_crypto_op_bulk_alloc(t->ca_op_pool, 769 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops_burst, burst_size); 770 if (unlikely(nb_alloc != burst_size)) { 771 alloc_failures++; 772 continue; 773 } 774 775 ret = rte_pktmbuf_alloc_bulk(pool, pkts_burst, burst_size); 776 if (unlikely(ret != 0)) { 777 alloc_failures++; 778 rte_mempool_put_bulk(t->ca_op_pool, (void **)ops_burst, burst_size); 779 continue; 780 } 781 782 for (i = 0; i < burst_size; i++) { 783 m = pkts_burst[i]; 784 rte_pktmbuf_append(m, len); 785 sym_op = ops_burst[i]->sym; 786 sym_op->m_src = m; 787 sym_op->cipher.data.offset = offset; 788 sym_op->cipher.data.length = len - offset; 789 rte_crypto_op_attach_sym_session(ops_burst[i], 790 p->ca.crypto_sess[flow_counter++ % nb_flows]); 791 ev[i].event_ptr = ops_burst[i]; 792 } 793 } else { 794 struct rte_crypto_asym_op *asym_op; 795 796 nb_alloc = rte_crypto_op_bulk_alloc(t->ca_op_pool, 797 RTE_CRYPTO_OP_TYPE_ASYMMETRIC, ops_burst, burst_size); 798 if (unlikely(nb_alloc != burst_size)) { 799 alloc_failures++; 800 continue; 801 } 802 803 if (rte_mempool_get_bulk(pool, (void **)result, burst_size)) { 804 alloc_failures++; 805 rte_mempool_put_bulk(t->ca_op_pool, (void **)ops_burst, burst_size); 806 continue; 807 } 808 809 for (i = 0; i < burst_size; i++) { 810 asym_op = ops_burst[i]->asym; 811 asym_op->modex.base.data = modex_test_case.base.data; 812 asym_op->modex.base.length = modex_test_case.base.len; 813 asym_op->modex.result.data = result[i]; 814 asym_op->modex.result.length = modex_test_case.result_len; 815 rte_crypto_op_attach_asym_session(ops_burst[i], 816 p->ca.crypto_sess[flow_counter++ % nb_flows]); 817 ev[i].event_ptr = ops_burst[i]; 818 } 819 } 820 821 enq = 0; 822 while (!t->done) { 823 enq += rte_event_crypto_adapter_enqueue(dev_id, port, ev + enq, 824 burst_size - enq); 825 if (enq == burst_size) 826 break; 827 } 828 829 count += burst_size; 830 } 831 832 if (opt->verbose_level > 1 && alloc_failures) 833 printf("%s(): lcore %d allocation failures: %"PRIu64"\n", 834 __func__, rte_lcore_id(), alloc_failures); 835 } 836 837 static inline int 838 perf_event_crypto_producer_burst(void *arg) 839 { 840 struct prod_data *p = arg; 841 struct evt_options *opt = p->t->opt; 842 843 if (opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) 844 crypto_adapter_enq_op_new_burst(p); 845 else 846 crypto_adapter_enq_op_fwd_burst(p); 847 848 return 0; 849 } 850 851 static int 852 perf_producer_wrapper(void *arg) 853 { 854 struct rte_event_dev_info dev_info; 855 struct prod_data *p = arg; 856 struct test_perf *t = p->t; 857 858 rte_event_dev_info_get(p->dev_id, &dev_info); 859 if (!t->opt->prod_enq_burst_sz) { 860 t->opt->prod_enq_burst_sz = MAX_PROD_ENQ_BURST_SIZE; 861 if (dev_info.max_event_port_enqueue_depth > 0 && 862 (uint32_t)dev_info.max_event_port_enqueue_depth < 863 t->opt->prod_enq_burst_sz) 864 t->opt->prod_enq_burst_sz = 865 dev_info.max_event_port_enqueue_depth; 866 } 867 868 /* In case of synthetic producer, launch perf_producer or 869 * perf_producer_burst depending on producer enqueue burst size 870 */ 871 if (t->opt->prod_type == EVT_PROD_TYPE_SYNT && 872 t->opt->prod_enq_burst_sz == 1) 873 return perf_producer(arg); 874 else if (t->opt->prod_type == EVT_PROD_TYPE_SYNT && 875 t->opt->prod_enq_burst_sz > 1) { 876 if (dev_info.max_event_port_enqueue_depth == 1) 877 evt_err("This event device does not support burst mode"); 878 else 879 return perf_producer_burst(arg); 880 } 881 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && 882 !t->opt->timdev_use_burst) 883 return perf_event_timer_producer(arg); 884 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && 885 t->opt->timdev_use_burst) 886 return perf_event_timer_producer_burst(arg); 887 else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { 888 if (t->opt->prod_enq_burst_sz > 1) 889 return perf_event_crypto_producer_burst(arg); 890 else 891 return perf_event_crypto_producer(arg); 892 } else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_DMA_ADPTR) 893 return perf_event_dma_producer(arg); 894 895 return 0; 896 } 897 898 static inline uint64_t 899 processed_pkts(struct test_perf *t) 900 { 901 uint8_t i; 902 uint64_t total = 0; 903 904 for (i = 0; i < t->nb_workers; i++) 905 total += t->worker[i].processed_pkts; 906 907 return total; 908 } 909 910 static inline uint64_t 911 total_latency(struct test_perf *t) 912 { 913 uint8_t i; 914 uint64_t total = 0; 915 916 for (i = 0; i < t->nb_workers; i++) 917 total += t->worker[i].latency; 918 919 return total; 920 } 921 922 923 int 924 perf_launch_lcores(struct evt_test *test, struct evt_options *opt, 925 int (*worker)(void *)) 926 { 927 int ret, lcore_id; 928 struct test_perf *t = evt_test_priv(test); 929 930 int port_idx = 0; 931 /* launch workers */ 932 RTE_LCORE_FOREACH_WORKER(lcore_id) { 933 if (!(opt->wlcores[lcore_id])) 934 continue; 935 936 ret = rte_eal_remote_launch(worker, 937 &t->worker[port_idx], lcore_id); 938 if (ret) { 939 evt_err("failed to launch worker %d", lcore_id); 940 return ret; 941 } 942 port_idx++; 943 } 944 945 /* launch producers */ 946 RTE_LCORE_FOREACH_WORKER(lcore_id) { 947 if (!(opt->plcores[lcore_id])) 948 continue; 949 950 ret = rte_eal_remote_launch(perf_producer_wrapper, 951 &t->prod[port_idx], lcore_id); 952 if (ret) { 953 evt_err("failed to launch perf_producer %d", lcore_id); 954 return ret; 955 } 956 port_idx++; 957 } 958 959 const uint64_t total_pkts = t->outstand_pkts; 960 961 uint64_t dead_lock_cycles = rte_get_timer_cycles(); 962 int64_t dead_lock_remaining = total_pkts; 963 const uint64_t dead_lock_sample = rte_get_timer_hz() * 5; 964 965 uint64_t perf_cycles = rte_get_timer_cycles(); 966 int64_t perf_remaining = total_pkts; 967 const uint64_t perf_sample = rte_get_timer_hz(); 968 969 static float total_mpps; 970 static uint64_t samples; 971 972 const uint64_t freq_mhz = rte_get_timer_hz() / 1000000; 973 int64_t remaining = t->outstand_pkts - processed_pkts(t); 974 975 while (t->done == false) { 976 const uint64_t new_cycles = rte_get_timer_cycles(); 977 978 if ((new_cycles - perf_cycles) > perf_sample) { 979 const uint64_t latency = total_latency(t); 980 const uint64_t pkts = processed_pkts(t); 981 982 remaining = t->outstand_pkts - pkts; 983 float mpps = (float)(perf_remaining-remaining)/1000000; 984 985 perf_remaining = remaining; 986 perf_cycles = new_cycles; 987 total_mpps += mpps; 988 ++samples; 989 if (opt->fwd_latency && pkts > 0) { 990 printf(CLGRN"\r%.3f mpps avg %.3f mpps [avg fwd latency %.3f us] "CLNRM, 991 mpps, total_mpps/samples, 992 (float)(latency/pkts)/freq_mhz); 993 } else { 994 printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM, 995 mpps, total_mpps/samples); 996 } 997 fflush(stdout); 998 999 if (remaining <= 0) { 1000 t->result = EVT_TEST_SUCCESS; 1001 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 1002 opt->prod_type == 1003 EVT_PROD_TYPE_EVENT_TIMER_ADPTR || 1004 opt->prod_type == 1005 EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR || 1006 opt->prod_type == 1007 EVT_PROD_TYPE_EVENT_DMA_ADPTR) { 1008 t->done = true; 1009 break; 1010 } 1011 } 1012 } 1013 1014 if (new_cycles - dead_lock_cycles > dead_lock_sample && 1015 (opt->prod_type == EVT_PROD_TYPE_SYNT || 1016 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR || 1017 opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR || 1018 opt->prod_type == EVT_PROD_TYPE_EVENT_DMA_ADPTR)) { 1019 remaining = t->outstand_pkts - processed_pkts(t); 1020 if (dead_lock_remaining == remaining) { 1021 rte_event_dev_dump(opt->dev_id, stdout); 1022 evt_err("No schedules for seconds, deadlock"); 1023 t->done = true; 1024 break; 1025 } 1026 dead_lock_remaining = remaining; 1027 dead_lock_cycles = new_cycles; 1028 } 1029 } 1030 printf("\n"); 1031 return 0; 1032 } 1033 1034 static int 1035 perf_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, 1036 struct rte_event_port_conf prod_conf) 1037 { 1038 int ret = 0; 1039 uint16_t prod; 1040 struct rte_event_eth_rx_adapter_queue_conf queue_conf; 1041 1042 memset(&queue_conf, 0, 1043 sizeof(struct rte_event_eth_rx_adapter_queue_conf)); 1044 queue_conf.ev.sched_type = opt->sched_type_list[0]; 1045 RTE_ETH_FOREACH_DEV(prod) { 1046 uint32_t cap; 1047 1048 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id, 1049 prod, &cap); 1050 if (ret) { 1051 evt_err("failed to get event rx adapter[%d]" 1052 " capabilities", 1053 opt->dev_id); 1054 return ret; 1055 } 1056 queue_conf.ev.queue_id = prod * stride; 1057 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id, 1058 &prod_conf); 1059 if (ret) { 1060 evt_err("failed to create rx adapter[%d]", prod); 1061 return ret; 1062 } 1063 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1, 1064 &queue_conf); 1065 if (ret) { 1066 evt_err("failed to add rx queues to adapter[%d]", prod); 1067 return ret; 1068 } 1069 1070 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) { 1071 uint32_t service_id; 1072 1073 rte_event_eth_rx_adapter_service_id_get(prod, 1074 &service_id); 1075 ret = evt_service_setup(service_id); 1076 if (ret) { 1077 evt_err("Failed to setup service core" 1078 " for Rx adapter\n"); 1079 return ret; 1080 } 1081 } 1082 } 1083 1084 return ret; 1085 } 1086 1087 static int 1088 perf_event_timer_adapter_setup(struct test_perf *t) 1089 { 1090 int i; 1091 int ret; 1092 struct rte_event_timer_adapter_info adapter_info; 1093 struct rte_event_timer_adapter *wl; 1094 uint8_t nb_producers = evt_nr_active_lcores(t->opt->plcores); 1095 uint8_t flags = RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES; 1096 1097 if (nb_producers == 1) 1098 flags |= RTE_EVENT_TIMER_ADAPTER_F_SP_PUT; 1099 1100 for (i = 0; i < t->opt->nb_timer_adptrs; i++) { 1101 struct rte_event_timer_adapter_conf config = { 1102 .event_dev_id = t->opt->dev_id, 1103 .timer_adapter_id = i, 1104 .timer_tick_ns = t->opt->timer_tick_nsec, 1105 .max_tmo_ns = t->opt->max_tmo_nsec, 1106 .nb_timers = t->opt->pool_sz, 1107 .flags = flags, 1108 }; 1109 1110 wl = rte_event_timer_adapter_create(&config); 1111 if (wl == NULL) { 1112 evt_err("failed to create event timer ring %d", i); 1113 return rte_errno; 1114 } 1115 1116 memset(&adapter_info, 0, 1117 sizeof(struct rte_event_timer_adapter_info)); 1118 rte_event_timer_adapter_get_info(wl, &adapter_info); 1119 t->opt->optm_timer_tick_nsec = adapter_info.min_resolution_ns; 1120 1121 if (!(adapter_info.caps & 1122 RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) { 1123 uint32_t service_id = -1U; 1124 1125 rte_event_timer_adapter_service_id_get(wl, 1126 &service_id); 1127 ret = evt_service_setup(service_id); 1128 if (ret) { 1129 evt_err("Failed to setup service core" 1130 " for timer adapter\n"); 1131 return ret; 1132 } 1133 rte_service_runstate_set(service_id, 1); 1134 } 1135 t->timer_adptr[i] = wl; 1136 } 1137 return 0; 1138 } 1139 1140 static int 1141 perf_event_crypto_adapter_setup(struct test_perf *t, struct prod_data *p) 1142 { 1143 struct rte_event_crypto_adapter_queue_conf conf; 1144 struct evt_options *opt = t->opt; 1145 uint32_t cap; 1146 int ret; 1147 1148 memset(&conf, 0, sizeof(conf)); 1149 1150 ret = rte_event_crypto_adapter_caps_get(p->dev_id, p->ca.cdev_id, &cap); 1151 if (ret) { 1152 evt_err("Failed to get crypto adapter capabilities"); 1153 return ret; 1154 } 1155 1156 if (((opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) && 1157 !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW)) || 1158 ((opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_FORWARD) && 1159 !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD))) { 1160 evt_err("crypto adapter %s mode unsupported\n", 1161 opt->crypto_adptr_mode ? "OP_FORWARD" : "OP_NEW"); 1162 return -ENOTSUP; 1163 } else if (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA)) { 1164 evt_err("Storing crypto session not supported"); 1165 return -ENOTSUP; 1166 } 1167 1168 if (opt->ena_vector) { 1169 struct rte_event_crypto_adapter_vector_limits limits; 1170 1171 if (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_EVENT_VECTOR)) { 1172 evt_err("Crypto adapter doesn't support event vector"); 1173 return -EINVAL; 1174 } 1175 1176 ret = rte_event_crypto_adapter_vector_limits_get(p->dev_id, p->ca.cdev_id, &limits); 1177 if (ret) { 1178 evt_err("Failed to get crypto adapter's vector limits"); 1179 return ret; 1180 } 1181 1182 if (opt->vector_size < limits.min_sz || opt->vector_size > limits.max_sz) { 1183 evt_err("Vector size [%d] not within limits max[%d] min[%d]", 1184 opt->vector_size, limits.max_sz, limits.min_sz); 1185 return -EINVAL; 1186 } 1187 1188 if (limits.log2_sz && !rte_is_power_of_2(opt->vector_size)) { 1189 evt_err("Vector size [%d] not power of 2", opt->vector_size); 1190 return -EINVAL; 1191 } 1192 1193 if (opt->vector_tmo_nsec > limits.max_timeout_ns || 1194 opt->vector_tmo_nsec < limits.min_timeout_ns) { 1195 evt_err("Vector timeout [%" PRIu64 "] not within limits " 1196 "max[%" PRIu64 "] min[%" PRIu64 "]", 1197 opt->vector_tmo_nsec, limits.max_timeout_ns, limits.min_timeout_ns); 1198 return -EINVAL; 1199 } 1200 1201 conf.vector_mp = t->ca_vector_pool; 1202 conf.vector_sz = opt->vector_size; 1203 conf.vector_timeout_ns = opt->vector_tmo_nsec; 1204 conf.flags |= RTE_EVENT_CRYPTO_ADAPTER_EVENT_VECTOR; 1205 } 1206 1207 if (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) { 1208 conf.ev.sched_type = RTE_SCHED_TYPE_ATOMIC; 1209 conf.ev.queue_id = p->queue_id; 1210 } 1211 1212 ret = rte_event_crypto_adapter_queue_pair_add( 1213 TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, &conf); 1214 1215 return ret; 1216 } 1217 1218 static int 1219 perf_event_dma_adapter_setup(struct test_perf *t, struct prod_data *p) 1220 { 1221 struct evt_options *opt = t->opt; 1222 struct rte_event event; 1223 uint32_t cap; 1224 int ret; 1225 1226 ret = rte_event_dma_adapter_caps_get(p->dev_id, p->da.dma_dev_id, &cap); 1227 if (ret) { 1228 evt_err("Failed to get dma adapter capabilities"); 1229 return ret; 1230 } 1231 1232 if (((opt->dma_adptr_mode == RTE_EVENT_DMA_ADAPTER_OP_NEW) && 1233 !(cap & RTE_EVENT_DMA_ADAPTER_CAP_INTERNAL_PORT_OP_NEW)) || 1234 ((opt->dma_adptr_mode == RTE_EVENT_DMA_ADAPTER_OP_FORWARD) && 1235 !(cap & RTE_EVENT_DMA_ADAPTER_CAP_INTERNAL_PORT_OP_FWD))) { 1236 evt_err("dma adapter %s mode unsupported\n", 1237 opt->dma_adptr_mode ? "OP_FORWARD" : "OP_NEW"); 1238 return -ENOTSUP; 1239 } 1240 1241 if (cap & RTE_EVENT_DMA_ADAPTER_CAP_INTERNAL_PORT_VCHAN_EV_BIND) 1242 ret = rte_event_dma_adapter_vchan_add(TEST_PERF_DA_ID, p->da.dma_dev_id, 1243 p->da.vchan_id, &event); 1244 else 1245 ret = rte_event_dma_adapter_vchan_add(TEST_PERF_DA_ID, p->da.dma_dev_id, 1246 p->da.vchan_id, NULL); 1247 1248 return ret; 1249 } 1250 1251 static void * 1252 cryptodev_sym_sess_create(struct prod_data *p, struct test_perf *t) 1253 { 1254 const struct rte_cryptodev_symmetric_capability *cap; 1255 struct rte_cryptodev_sym_capability_idx cap_idx; 1256 enum rte_crypto_cipher_algorithm cipher_algo; 1257 struct rte_crypto_sym_xform cipher_xform; 1258 struct evt_options *opt = t->opt; 1259 uint16_t key_size; 1260 uint16_t iv_size; 1261 void *sess; 1262 1263 cipher_algo = opt->crypto_cipher_alg; 1264 key_size = opt->crypto_cipher_key_sz; 1265 iv_size = opt->crypto_cipher_iv_sz; 1266 1267 /* Check if device supports the algorithm */ 1268 cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; 1269 cap_idx.algo.cipher = cipher_algo; 1270 1271 cap = rte_cryptodev_sym_capability_get(p->ca.cdev_id, &cap_idx); 1272 if (cap == NULL) { 1273 evt_err("Device doesn't support cipher algorithm [%s]. Test Skipped\n", 1274 rte_cryptodev_get_cipher_algo_string(cipher_algo)); 1275 return NULL; 1276 } 1277 1278 /* Check if device supports key size and IV size */ 1279 if (rte_cryptodev_sym_capability_check_cipher(cap, key_size, 1280 iv_size) < 0) { 1281 evt_err("Device doesn't support cipher configuration:\n" 1282 "cipher algo [%s], key sz [%d], iv sz [%d]. Test Skipped\n", 1283 rte_cryptodev_get_cipher_algo_string(cipher_algo), key_size, iv_size); 1284 return NULL; 1285 } 1286 1287 cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; 1288 cipher_xform.cipher.algo = cipher_algo; 1289 cipher_xform.cipher.key.data = opt->crypto_cipher_key; 1290 cipher_xform.cipher.key.length = key_size; 1291 cipher_xform.cipher.iv.length = iv_size; 1292 cipher_xform.cipher.iv.offset = IV_OFFSET; 1293 cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 1294 cipher_xform.next = NULL; 1295 1296 sess = rte_cryptodev_sym_session_create(p->ca.cdev_id, &cipher_xform, 1297 t->ca_sess_pool); 1298 if (sess == NULL) { 1299 evt_err("Failed to create sym session"); 1300 return NULL; 1301 } 1302 1303 return sess; 1304 } 1305 1306 static void * 1307 cryptodev_asym_sess_create(struct prod_data *p, struct test_perf *t) 1308 { 1309 const struct rte_cryptodev_asymmetric_xform_capability *capability; 1310 struct rte_cryptodev_asym_capability_idx cap_idx; 1311 struct rte_crypto_asym_xform xform; 1312 void *sess; 1313 1314 xform.next = NULL; 1315 xform.xform_type = RTE_CRYPTO_ASYM_XFORM_MODEX; 1316 cap_idx.type = xform.xform_type; 1317 capability = rte_cryptodev_asym_capability_get(p->ca.cdev_id, &cap_idx); 1318 if (capability == NULL) { 1319 evt_err("Device doesn't support MODEX. Test Skipped\n"); 1320 return NULL; 1321 } 1322 1323 xform.modex.modulus.data = modex_test_case.modulus.data; 1324 xform.modex.modulus.length = modex_test_case.modulus.len; 1325 xform.modex.exponent.data = modex_test_case.exponent.data; 1326 xform.modex.exponent.length = modex_test_case.exponent.len; 1327 1328 if (rte_cryptodev_asym_session_create(p->ca.cdev_id, &xform, 1329 t->ca_asym_sess_pool, &sess)) { 1330 evt_err("Failed to create asym session"); 1331 return NULL; 1332 } 1333 1334 return sess; 1335 } 1336 1337 int 1338 perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, 1339 uint8_t stride, uint8_t nb_queues, 1340 const struct rte_event_port_conf *port_conf) 1341 { 1342 struct test_perf *t = evt_test_priv(test); 1343 uint16_t port, prod; 1344 int ret = -1; 1345 1346 /* setup one port per worker, linking to all queues */ 1347 for (port = 0; port < evt_nr_active_lcores(opt->wlcores); 1348 port++) { 1349 struct worker_data *w = &t->worker[port]; 1350 1351 w->dev_id = opt->dev_id; 1352 w->port_id = port; 1353 w->t = t; 1354 w->processed_pkts = 0; 1355 w->latency = 0; 1356 1357 struct rte_event_port_conf conf = *port_conf; 1358 conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_WORKER; 1359 1360 ret = rte_event_port_setup(opt->dev_id, port, &conf); 1361 if (ret) { 1362 evt_err("failed to setup port %d", port); 1363 return ret; 1364 } 1365 1366 ret = rte_event_port_link(opt->dev_id, port, NULL, NULL, 0); 1367 if (ret != nb_queues) { 1368 evt_err("failed to link all queues to port %d", port); 1369 return -EINVAL; 1370 } 1371 } 1372 1373 /* port for producers, no links */ 1374 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 1375 for ( ; port < perf_nb_event_ports(opt); port++) { 1376 struct prod_data *p = &t->prod[port]; 1377 p->t = t; 1378 } 1379 1380 struct rte_event_port_conf conf = *port_conf; 1381 conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_PRODUCER; 1382 1383 ret = perf_event_rx_adapter_setup(opt, stride, conf); 1384 if (ret) 1385 return ret; 1386 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 1387 prod = 0; 1388 for ( ; port < perf_nb_event_ports(opt); port++) { 1389 struct prod_data *p = &t->prod[port]; 1390 p->queue_id = prod * stride; 1391 p->t = t; 1392 prod++; 1393 } 1394 1395 ret = perf_event_timer_adapter_setup(t); 1396 if (ret) 1397 return ret; 1398 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { 1399 struct rte_event_port_conf conf = *port_conf; 1400 uint8_t cdev_id = 0; 1401 uint16_t qp_id = 0; 1402 1403 ret = rte_event_crypto_adapter_create(TEST_PERF_CA_ID, 1404 opt->dev_id, &conf, 0); 1405 if (ret) { 1406 evt_err("Failed to create crypto adapter"); 1407 return ret; 1408 } 1409 1410 prod = 0; 1411 for (; port < perf_nb_event_ports(opt); port++) { 1412 union rte_event_crypto_metadata m_data; 1413 struct prod_data *p = &t->prod[port]; 1414 uint32_t flow_id; 1415 1416 if (qp_id == rte_cryptodev_queue_pair_count(cdev_id)) { 1417 cdev_id++; 1418 qp_id = 0; 1419 } 1420 1421 p->dev_id = opt->dev_id; 1422 p->port_id = port; 1423 p->queue_id = prod * stride; 1424 p->ca.cdev_id = cdev_id; 1425 p->ca.cdev_qp_id = qp_id; 1426 p->ca.crypto_sess = rte_zmalloc_socket( 1427 NULL, sizeof(void *) * t->nb_flows, 1428 RTE_CACHE_LINE_SIZE, opt->socket_id); 1429 p->t = t; 1430 1431 ret = perf_event_crypto_adapter_setup(t, p); 1432 if (ret) 1433 return ret; 1434 1435 m_data.request_info.cdev_id = p->ca.cdev_id; 1436 m_data.request_info.queue_pair_id = p->ca.cdev_qp_id; 1437 m_data.response_info.sched_type = RTE_SCHED_TYPE_ATOMIC; 1438 m_data.response_info.queue_id = p->queue_id; 1439 1440 for (flow_id = 0; flow_id < t->nb_flows; flow_id++) { 1441 m_data.response_info.flow_id = flow_id; 1442 if (opt->crypto_op_type == 1443 RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 1444 void *sess; 1445 1446 sess = cryptodev_sym_sess_create(p, t); 1447 if (sess == NULL) 1448 return -ENOMEM; 1449 1450 ret = rte_cryptodev_session_event_mdata_set( 1451 cdev_id, 1452 sess, 1453 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 1454 RTE_CRYPTO_OP_WITH_SESSION, 1455 &m_data, sizeof(m_data)); 1456 if (ret) 1457 return ret; 1458 p->ca.crypto_sess[flow_id] = sess; 1459 } else { 1460 void *sess; 1461 1462 sess = cryptodev_asym_sess_create(p, t); 1463 if (sess == NULL) 1464 return -ENOMEM; 1465 ret = rte_cryptodev_session_event_mdata_set( 1466 cdev_id, 1467 sess, 1468 RTE_CRYPTO_OP_TYPE_ASYMMETRIC, 1469 RTE_CRYPTO_OP_WITH_SESSION, 1470 &m_data, sizeof(m_data)); 1471 if (ret) 1472 return ret; 1473 p->ca.crypto_sess[flow_id] = sess; 1474 } 1475 } 1476 1477 conf.event_port_cfg |= 1478 RTE_EVENT_PORT_CFG_HINT_PRODUCER | 1479 RTE_EVENT_PORT_CFG_HINT_CONSUMER; 1480 1481 ret = rte_event_port_setup(opt->dev_id, port, &conf); 1482 if (ret) { 1483 evt_err("failed to setup port %d", port); 1484 return ret; 1485 } 1486 1487 qp_id++; 1488 prod++; 1489 } 1490 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_DMA_ADPTR) { 1491 struct rte_event_port_conf conf = *port_conf; 1492 struct rte_event_dma_adapter_op *op; 1493 struct rte_mempool *pool = t->pool; 1494 uint8_t dma_dev_id = 0; 1495 uint16_t vchan_id = 0; 1496 1497 ret = rte_event_dma_adapter_create(TEST_PERF_DA_ID, opt->dev_id, &conf, 0); 1498 if (ret) { 1499 evt_err("Failed to create dma adapter"); 1500 return ret; 1501 } 1502 1503 prod = 0; 1504 for (; port < perf_nb_event_ports(opt); port++) { 1505 struct prod_data *p = &t->prod[port]; 1506 uint32_t flow_id; 1507 1508 p->dev_id = opt->dev_id; 1509 p->port_id = port; 1510 p->queue_id = prod * stride; 1511 p->da.dma_dev_id = dma_dev_id; 1512 p->da.vchan_id = vchan_id; 1513 p->da.dma_op = rte_zmalloc_socket(NULL, sizeof(void *) * t->nb_flows, 1514 RTE_CACHE_LINE_SIZE, opt->socket_id); 1515 1516 p->t = t; 1517 1518 ret = perf_event_dma_adapter_setup(t, p); 1519 if (ret) 1520 return ret; 1521 1522 for (flow_id = 0; flow_id < t->nb_flows; flow_id++) { 1523 rte_mempool_get(t->da_op_pool, (void **)&op); 1524 1525 op->src_dst_seg[0].addr = rte_pktmbuf_iova(rte_pktmbuf_alloc(pool)); 1526 op->src_dst_seg[1].addr = rte_pktmbuf_iova(rte_pktmbuf_alloc(pool)); 1527 op->src_dst_seg[0].length = 1024; 1528 op->src_dst_seg[1].length = 1024; 1529 op->nb_src = 1; 1530 op->nb_dst = 1; 1531 op->flags = RTE_DMA_OP_FLAG_SUBMIT; 1532 op->op_mp = t->da_op_pool; 1533 op->dma_dev_id = dma_dev_id; 1534 op->vchan = vchan_id; 1535 1536 p->da.dma_op[flow_id] = op; 1537 } 1538 1539 conf.event_port_cfg |= 1540 RTE_EVENT_PORT_CFG_HINT_PRODUCER | 1541 RTE_EVENT_PORT_CFG_HINT_CONSUMER; 1542 1543 ret = rte_event_port_setup(opt->dev_id, port, &conf); 1544 if (ret) { 1545 evt_err("failed to setup port %d", port); 1546 return ret; 1547 } 1548 1549 prod++; 1550 } 1551 } else { 1552 prod = 0; 1553 for ( ; port < perf_nb_event_ports(opt); port++) { 1554 struct prod_data *p = &t->prod[port]; 1555 1556 p->dev_id = opt->dev_id; 1557 p->port_id = port; 1558 p->queue_id = prod * stride; 1559 p->t = t; 1560 1561 struct rte_event_port_conf conf = *port_conf; 1562 conf.event_port_cfg |= 1563 RTE_EVENT_PORT_CFG_HINT_PRODUCER | 1564 RTE_EVENT_PORT_CFG_HINT_CONSUMER; 1565 1566 ret = rte_event_port_setup(opt->dev_id, port, &conf); 1567 if (ret) { 1568 evt_err("failed to setup port %d", port); 1569 return ret; 1570 } 1571 prod++; 1572 } 1573 } 1574 1575 return ret; 1576 } 1577 1578 int 1579 perf_opt_check(struct evt_options *opt, uint64_t nb_queues) 1580 { 1581 unsigned int lcores; 1582 1583 /* N producer + N worker + main when producer cores are used 1584 * Else N worker + main when Rx adapter is used 1585 */ 1586 lcores = opt->prod_type == EVT_PROD_TYPE_SYNT ? 3 : 2; 1587 1588 if (rte_lcore_count() < lcores) { 1589 evt_err("test need minimum %d lcores", lcores); 1590 return -1; 1591 } 1592 1593 /* Validate worker lcores */ 1594 if (evt_lcores_has_overlap(opt->wlcores, rte_get_main_lcore())) { 1595 evt_err("worker lcores overlaps with main lcore"); 1596 return -1; 1597 } 1598 if (evt_lcores_has_overlap_multi(opt->wlcores, opt->plcores)) { 1599 evt_err("worker lcores overlaps producer lcores"); 1600 return -1; 1601 } 1602 if (evt_has_disabled_lcore(opt->wlcores)) { 1603 evt_err("one or more workers lcores are not enabled"); 1604 return -1; 1605 } 1606 if (!evt_has_active_lcore(opt->wlcores)) { 1607 evt_err("minimum one worker is required"); 1608 return -1; 1609 } 1610 1611 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 1612 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR || 1613 opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR || 1614 opt->prod_type == EVT_PROD_TYPE_EVENT_DMA_ADPTR) { 1615 /* Validate producer lcores */ 1616 if (evt_lcores_has_overlap(opt->plcores, 1617 rte_get_main_lcore())) { 1618 evt_err("producer lcores overlaps with main lcore"); 1619 return -1; 1620 } 1621 if (evt_has_disabled_lcore(opt->plcores)) { 1622 evt_err("one or more producer lcores are not enabled"); 1623 return -1; 1624 } 1625 if (!evt_has_active_lcore(opt->plcores)) { 1626 evt_err("minimum one producer is required"); 1627 return -1; 1628 } 1629 } 1630 1631 if (evt_has_invalid_stage(opt)) 1632 return -1; 1633 1634 if (evt_has_invalid_sched_type(opt)) 1635 return -1; 1636 1637 if (nb_queues > EVT_MAX_QUEUES) { 1638 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES); 1639 return -1; 1640 } 1641 if (perf_nb_event_ports(opt) > EVT_MAX_PORTS) { 1642 evt_err("number of ports exceeds %d", EVT_MAX_PORTS); 1643 return -1; 1644 } 1645 1646 /* Fixups */ 1647 if ((opt->nb_stages == 1 && 1648 opt->prod_type != EVT_PROD_TYPE_EVENT_TIMER_ADPTR) && 1649 opt->fwd_latency) { 1650 evt_info("fwd_latency is valid when nb_stages > 1, disabling"); 1651 opt->fwd_latency = 0; 1652 } 1653 1654 if (opt->fwd_latency && !opt->q_priority) { 1655 evt_info("enabled queue priority for latency measurement"); 1656 opt->q_priority = 1; 1657 } 1658 if (opt->nb_pkts == 0) 1659 opt->nb_pkts = INT64_MAX/evt_nr_active_lcores(opt->plcores); 1660 1661 return 0; 1662 } 1663 1664 void 1665 perf_opt_dump(struct evt_options *opt, uint8_t nb_queues) 1666 { 1667 evt_dump("nb_prod_lcores", "%d", evt_nr_active_lcores(opt->plcores)); 1668 evt_dump_producer_lcores(opt); 1669 evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores)); 1670 evt_dump_worker_lcores(opt); 1671 evt_dump_nb_stages(opt); 1672 evt_dump("nb_evdev_ports", "%d", perf_nb_event_ports(opt)); 1673 evt_dump("nb_evdev_queues", "%d", nb_queues); 1674 evt_dump_queue_priority(opt); 1675 evt_dump_sched_type_list(opt); 1676 evt_dump_producer_type(opt); 1677 evt_dump("prod_enq_burst_sz", "%d", opt->prod_enq_burst_sz); 1678 } 1679 1680 static void 1681 perf_event_port_flush(uint8_t dev_id __rte_unused, struct rte_event ev, 1682 void *args) 1683 { 1684 rte_mempool_put(args, ev.event_ptr); 1685 } 1686 1687 void 1688 perf_worker_cleanup(struct rte_mempool *const pool, uint8_t dev_id, 1689 uint8_t port_id, struct rte_event events[], uint16_t nb_enq, 1690 uint16_t nb_deq) 1691 { 1692 int i; 1693 1694 if (nb_deq) { 1695 for (i = nb_enq; i < nb_deq; i++) 1696 rte_mempool_put(pool, events[i].event_ptr); 1697 1698 for (i = 0; i < nb_deq; i++) 1699 events[i].op = RTE_EVENT_OP_RELEASE; 1700 rte_event_enqueue_burst(dev_id, port_id, events, nb_deq); 1701 } 1702 rte_event_port_quiesce(dev_id, port_id, perf_event_port_flush, pool); 1703 } 1704 1705 void 1706 perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt) 1707 { 1708 int i; 1709 struct test_perf *t = evt_test_priv(test); 1710 1711 if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 1712 for (i = 0; i < opt->nb_timer_adptrs; i++) 1713 rte_event_timer_adapter_stop(t->timer_adptr[i]); 1714 } 1715 rte_event_dev_stop(opt->dev_id); 1716 rte_event_dev_close(opt->dev_id); 1717 } 1718 1719 static inline void 1720 perf_elt_init(struct rte_mempool *mp, void *arg __rte_unused, 1721 void *obj, unsigned i __rte_unused) 1722 { 1723 memset(obj, 0, mp->elt_size); 1724 } 1725 1726 #define NB_RX_DESC 128 1727 #define NB_TX_DESC 512 1728 int 1729 perf_ethdev_setup(struct evt_test *test, struct evt_options *opt) 1730 { 1731 uint16_t i; 1732 int ret; 1733 struct test_perf *t = evt_test_priv(test); 1734 struct rte_eth_conf port_conf = { 1735 .rxmode = { 1736 .mq_mode = RTE_ETH_MQ_RX_RSS, 1737 }, 1738 .rx_adv_conf = { 1739 .rss_conf = { 1740 .rss_key = NULL, 1741 .rss_hf = RTE_ETH_RSS_IP, 1742 }, 1743 }, 1744 }; 1745 1746 if (opt->prod_type != EVT_PROD_TYPE_ETH_RX_ADPTR) 1747 return 0; 1748 1749 if (!rte_eth_dev_count_avail()) { 1750 evt_err("No ethernet ports found."); 1751 return -ENODEV; 1752 } 1753 1754 RTE_ETH_FOREACH_DEV(i) { 1755 struct rte_eth_dev_info dev_info; 1756 struct rte_eth_conf local_port_conf = port_conf; 1757 1758 ret = rte_eth_dev_info_get(i, &dev_info); 1759 if (ret != 0) { 1760 evt_err("Error during getting device (port %u) info: %s\n", 1761 i, strerror(-ret)); 1762 return ret; 1763 } 1764 1765 local_port_conf.rx_adv_conf.rss_conf.rss_hf &= 1766 dev_info.flow_type_rss_offloads; 1767 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != 1768 port_conf.rx_adv_conf.rss_conf.rss_hf) { 1769 evt_info("Port %u modified RSS hash function based on hardware support," 1770 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 1771 i, 1772 port_conf.rx_adv_conf.rss_conf.rss_hf, 1773 local_port_conf.rx_adv_conf.rss_conf.rss_hf); 1774 } 1775 1776 if (rte_eth_dev_configure(i, 1, 1, &local_port_conf) < 0) { 1777 evt_err("Failed to configure eth port [%d]", i); 1778 return -EINVAL; 1779 } 1780 1781 if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC, 1782 rte_socket_id(), NULL, t->pool) < 0) { 1783 evt_err("Failed to setup eth port [%d] rx_queue: %d.", 1784 i, 0); 1785 return -EINVAL; 1786 } 1787 1788 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC, 1789 rte_socket_id(), NULL) < 0) { 1790 evt_err("Failed to setup eth port [%d] tx_queue: %d.", 1791 i, 0); 1792 return -EINVAL; 1793 } 1794 1795 ret = rte_eth_promiscuous_enable(i); 1796 if (ret != 0) { 1797 evt_err("Failed to enable promiscuous mode for eth port [%d]: %s", 1798 i, rte_strerror(-ret)); 1799 return ret; 1800 } 1801 } 1802 1803 return 0; 1804 } 1805 1806 void 1807 perf_ethdev_rx_stop(struct evt_test *test, struct evt_options *opt) 1808 { 1809 uint16_t i; 1810 RTE_SET_USED(test); 1811 1812 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 1813 RTE_ETH_FOREACH_DEV(i) { 1814 rte_event_eth_rx_adapter_stop(i); 1815 rte_event_eth_rx_adapter_queue_del(i, i, -1); 1816 rte_eth_dev_rx_queue_stop(i, 0); 1817 } 1818 } 1819 } 1820 1821 void 1822 perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt) 1823 { 1824 uint16_t i; 1825 RTE_SET_USED(test); 1826 1827 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { 1828 RTE_ETH_FOREACH_DEV(i) { 1829 rte_event_eth_tx_adapter_stop(i); 1830 rte_event_eth_tx_adapter_queue_del(i, i, -1); 1831 rte_eth_dev_tx_queue_stop(i, 0); 1832 rte_eth_dev_stop(i); 1833 } 1834 } 1835 } 1836 1837 int 1838 perf_cryptodev_setup(struct evt_test *test, struct evt_options *opt) 1839 { 1840 uint8_t cdev_count, cdev_id, nb_plcores, nb_qps; 1841 struct test_perf *t = evt_test_priv(test); 1842 unsigned int max_session_size; 1843 uint32_t nb_sessions; 1844 int ret; 1845 1846 if (opt->prod_type != EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) 1847 return 0; 1848 1849 cdev_count = rte_cryptodev_count(); 1850 if (cdev_count == 0) { 1851 evt_err("No crypto devices available\n"); 1852 return -ENODEV; 1853 } 1854 1855 t->ca_op_pool = rte_crypto_op_pool_create( 1856 "crypto_op_pool", opt->crypto_op_type, opt->pool_sz, 1857 128, sizeof(union rte_event_crypto_metadata) + EVT_CRYPTO_MAX_IV_SIZE, 1858 rte_socket_id()); 1859 if (t->ca_op_pool == NULL) { 1860 evt_err("Failed to create crypto op pool"); 1861 return -ENOMEM; 1862 } 1863 1864 nb_sessions = evt_nr_active_lcores(opt->plcores) * t->nb_flows; 1865 t->ca_asym_sess_pool = rte_cryptodev_asym_session_pool_create( 1866 "ca_asym_sess_pool", nb_sessions, 0, 1867 sizeof(union rte_event_crypto_metadata), SOCKET_ID_ANY); 1868 if (t->ca_asym_sess_pool == NULL) { 1869 evt_err("Failed to create sym session pool"); 1870 ret = -ENOMEM; 1871 goto err; 1872 } 1873 1874 max_session_size = 0; 1875 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 1876 unsigned int session_size; 1877 1878 session_size = 1879 rte_cryptodev_sym_get_private_session_size(cdev_id); 1880 if (session_size > max_session_size) 1881 max_session_size = session_size; 1882 } 1883 1884 t->ca_sess_pool = rte_cryptodev_sym_session_pool_create( 1885 "ca_sess_pool", nb_sessions, max_session_size, 0, 1886 sizeof(union rte_event_crypto_metadata), SOCKET_ID_ANY); 1887 if (t->ca_sess_pool == NULL) { 1888 evt_err("Failed to create sym session pool"); 1889 ret = -ENOMEM; 1890 goto err; 1891 } 1892 1893 if (opt->ena_vector) { 1894 unsigned int nb_elem = (opt->pool_sz / opt->vector_size) * 2; 1895 nb_elem = RTE_MAX(512U, nb_elem); 1896 nb_elem += evt_nr_active_lcores(opt->wlcores) * 32; 1897 t->ca_vector_pool = rte_event_vector_pool_create("vector_pool", nb_elem, 32, 1898 opt->vector_size, opt->socket_id); 1899 if (t->ca_vector_pool == NULL) { 1900 evt_err("Failed to create event vector pool"); 1901 ret = -ENOMEM; 1902 goto err; 1903 } 1904 } 1905 1906 /* 1907 * Calculate number of needed queue pairs, based on the amount of 1908 * available number of logical cores and crypto devices. For instance, 1909 * if there are 4 cores and 2 crypto devices, 2 queue pairs will be set 1910 * up per device. 1911 */ 1912 nb_plcores = evt_nr_active_lcores(opt->plcores); 1913 nb_qps = (nb_plcores % cdev_count) ? (nb_plcores / cdev_count) + 1 : 1914 nb_plcores / cdev_count; 1915 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 1916 struct rte_cryptodev_qp_conf qp_conf; 1917 struct rte_cryptodev_config conf; 1918 struct rte_cryptodev_info info; 1919 int qp_id; 1920 1921 rte_cryptodev_info_get(cdev_id, &info); 1922 if (nb_qps > info.max_nb_queue_pairs) { 1923 evt_err("Not enough queue pairs per cryptodev (%u)", 1924 nb_qps); 1925 ret = -EINVAL; 1926 goto err; 1927 } 1928 1929 conf.nb_queue_pairs = nb_qps; 1930 conf.socket_id = SOCKET_ID_ANY; 1931 conf.ff_disable = RTE_CRYPTODEV_FF_SECURITY; 1932 1933 ret = rte_cryptodev_configure(cdev_id, &conf); 1934 if (ret) { 1935 evt_err("Failed to configure cryptodev (%u)", cdev_id); 1936 goto err; 1937 } 1938 1939 qp_conf.nb_descriptors = NB_CRYPTODEV_DESCRIPTORS; 1940 qp_conf.mp_session = t->ca_sess_pool; 1941 1942 for (qp_id = 0; qp_id < conf.nb_queue_pairs; qp_id++) { 1943 ret = rte_cryptodev_queue_pair_setup( 1944 cdev_id, qp_id, &qp_conf, 1945 rte_cryptodev_socket_id(cdev_id)); 1946 if (ret) { 1947 evt_err("Failed to setup queue pairs on cryptodev %u\n", 1948 cdev_id); 1949 goto err; 1950 } 1951 } 1952 } 1953 1954 return 0; 1955 err: 1956 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) 1957 rte_cryptodev_close(cdev_id); 1958 1959 rte_mempool_free(t->ca_op_pool); 1960 rte_mempool_free(t->ca_sess_pool); 1961 rte_mempool_free(t->ca_asym_sess_pool); 1962 rte_mempool_free(t->ca_vector_pool); 1963 1964 return ret; 1965 } 1966 1967 void 1968 perf_cryptodev_destroy(struct evt_test *test, struct evt_options *opt) 1969 { 1970 uint8_t cdev_id, cdev_count = rte_cryptodev_count(); 1971 struct test_perf *t = evt_test_priv(test); 1972 uint16_t port; 1973 1974 if (opt->prod_type != EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) 1975 return; 1976 1977 for (port = t->nb_workers; port < perf_nb_event_ports(opt); port++) { 1978 void *sess; 1979 struct prod_data *p = &t->prod[port]; 1980 uint32_t flow_id; 1981 uint8_t cdev_id; 1982 1983 for (flow_id = 0; flow_id < t->nb_flows; flow_id++) { 1984 sess = p->ca.crypto_sess[flow_id]; 1985 cdev_id = p->ca.cdev_id; 1986 rte_cryptodev_sym_session_free(cdev_id, sess); 1987 } 1988 1989 rte_event_crypto_adapter_queue_pair_del( 1990 TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id); 1991 } 1992 1993 rte_event_crypto_adapter_free(TEST_PERF_CA_ID); 1994 1995 for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { 1996 rte_cryptodev_stop(cdev_id); 1997 rte_cryptodev_close(cdev_id); 1998 } 1999 2000 rte_mempool_free(t->ca_op_pool); 2001 rte_mempool_free(t->ca_sess_pool); 2002 rte_mempool_free(t->ca_asym_sess_pool); 2003 rte_mempool_free(t->ca_vector_pool); 2004 } 2005 2006 int 2007 perf_dmadev_setup(struct evt_test *test, struct evt_options *opt) 2008 { 2009 const struct rte_dma_conf conf = { .nb_vchans = 1}; 2010 const struct rte_dma_vchan_conf qconf = { 2011 .direction = RTE_DMA_DIR_MEM_TO_MEM, 2012 .nb_desc = 1024, 2013 }; 2014 struct test_perf *t = evt_test_priv(test); 2015 uint8_t dma_dev_count, dma_dev_id = 0; 2016 unsigned int elt_size; 2017 int vchan_id; 2018 int ret; 2019 2020 if (opt->prod_type != EVT_PROD_TYPE_EVENT_DMA_ADPTR) 2021 return 0; 2022 2023 dma_dev_count = rte_dma_count_avail(); 2024 if (dma_dev_count == 0) { 2025 evt_err("No dma devices available\n"); 2026 return -ENODEV; 2027 } 2028 2029 elt_size = sizeof(struct rte_event_dma_adapter_op) + (sizeof(struct rte_dma_sge) * 2); 2030 t->da_op_pool = rte_mempool_create("dma_op_pool", opt->pool_sz, elt_size, 256, 2031 0, NULL, NULL, NULL, NULL, rte_socket_id(), 0); 2032 if (t->da_op_pool == NULL) { 2033 evt_err("Failed to create dma op pool"); 2034 return -ENOMEM; 2035 } 2036 2037 ret = rte_dma_configure(dma_dev_id, &conf); 2038 if (ret) { 2039 evt_err("Failed to configure dma dev (%u)", dma_dev_id); 2040 goto err; 2041 } 2042 2043 for (vchan_id = 0; vchan_id < conf.nb_vchans; vchan_id++) { 2044 ret = rte_dma_vchan_setup(dma_dev_id, vchan_id, &qconf); 2045 if (ret) { 2046 evt_err("Failed to setup vchan on dma dev %u\n", 2047 dma_dev_id); 2048 goto err; 2049 } 2050 } 2051 2052 return 0; 2053 err: 2054 rte_dma_close(dma_dev_id); 2055 rte_mempool_free(t->da_op_pool); 2056 2057 return ret; 2058 } 2059 2060 void 2061 perf_dmadev_destroy(struct evt_test *test, struct evt_options *opt) 2062 { 2063 uint8_t dma_dev_id = 0; 2064 struct test_perf *t = evt_test_priv(test); 2065 uint16_t port; 2066 2067 if (opt->prod_type != EVT_PROD_TYPE_EVENT_DMA_ADPTR) 2068 return; 2069 2070 for (port = t->nb_workers; port < perf_nb_event_ports(opt); port++) { 2071 struct prod_data *p = &t->prod[port]; 2072 struct rte_event_dma_adapter_op *op; 2073 uint32_t flow_id; 2074 2075 for (flow_id = 0; flow_id < t->nb_flows; flow_id++) { 2076 op = p->da.dma_op[flow_id]; 2077 2078 rte_pktmbuf_free((struct rte_mbuf *)(uintptr_t)op->src_dst_seg[0].addr); 2079 rte_pktmbuf_free((struct rte_mbuf *)(uintptr_t)op->src_dst_seg[1].addr); 2080 rte_mempool_put(op->op_mp, op); 2081 } 2082 2083 rte_event_dma_adapter_vchan_del(TEST_PERF_DA_ID, p->da.dma_dev_id, p->da.vchan_id); 2084 } 2085 2086 rte_event_dma_adapter_free(TEST_PERF_DA_ID); 2087 2088 rte_dma_stop(dma_dev_id); 2089 rte_dma_close(dma_dev_id); 2090 2091 rte_mempool_free(t->da_op_pool); 2092 } 2093 2094 int 2095 perf_mempool_setup(struct evt_test *test, struct evt_options *opt) 2096 { 2097 struct test_perf *t = evt_test_priv(test); 2098 unsigned int cache_sz; 2099 2100 cache_sz = RTE_MIN(RTE_MEMPOOL_CACHE_MAX_SIZE, (opt->pool_sz / 1.5) / t->nb_workers); 2101 if (opt->prod_type == EVT_PROD_TYPE_SYNT || 2102 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 2103 t->pool = rte_mempool_create(test->name, /* mempool name */ 2104 opt->pool_sz, /* number of elements*/ 2105 sizeof(struct perf_elt), /* element size*/ 2106 cache_sz, /* cache size*/ 2107 0, NULL, NULL, 2108 perf_elt_init, /* obj constructor */ 2109 NULL, opt->socket_id, 0); /* flags */ 2110 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR && 2111 opt->crypto_op_type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) { 2112 t->pool = rte_mempool_create(test->name, /* mempool name */ 2113 opt->pool_sz, /* number of elements*/ 2114 sizeof(struct perf_elt) + modex_test_case.result_len, 2115 /* element size*/ 2116 cache_sz, /* cache size*/ 2117 0, NULL, NULL, 2118 NULL, /* obj constructor */ 2119 NULL, opt->socket_id, 0); /* flags */ 2120 } else { 2121 t->pool = rte_pktmbuf_pool_create(test->name, /* mempool name */ 2122 opt->pool_sz, /* number of elements*/ 2123 cache_sz, /* cache size*/ 2124 0, 2125 RTE_MBUF_DEFAULT_BUF_SIZE, 2126 opt->socket_id); /* flags */ 2127 } 2128 2129 if (t->pool == NULL) { 2130 evt_err("failed to create mempool"); 2131 return -ENOMEM; 2132 } 2133 2134 return 0; 2135 } 2136 2137 void 2138 perf_mempool_destroy(struct evt_test *test, struct evt_options *opt) 2139 { 2140 RTE_SET_USED(opt); 2141 struct test_perf *t = evt_test_priv(test); 2142 2143 rte_mempool_free(t->pool); 2144 } 2145 2146 int 2147 perf_test_setup(struct evt_test *test, struct evt_options *opt) 2148 { 2149 void *test_perf; 2150 2151 test_perf = rte_zmalloc_socket(test->name, sizeof(struct test_perf), 2152 RTE_CACHE_LINE_SIZE, opt->socket_id); 2153 if (test_perf == NULL) { 2154 evt_err("failed to allocate test_perf memory"); 2155 goto nomem; 2156 } 2157 test->test_priv = test_perf; 2158 2159 struct test_perf *t = evt_test_priv(test); 2160 2161 if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { 2162 t->outstand_pkts = opt->nb_timers * 2163 evt_nr_active_lcores(opt->plcores); 2164 t->nb_pkts = opt->nb_timers; 2165 } else { 2166 t->outstand_pkts = opt->nb_pkts * 2167 evt_nr_active_lcores(opt->plcores); 2168 t->nb_pkts = opt->nb_pkts; 2169 } 2170 2171 t->nb_workers = evt_nr_active_lcores(opt->wlcores); 2172 t->done = false; 2173 t->nb_flows = opt->nb_flows; 2174 t->result = EVT_TEST_FAILED; 2175 t->opt = opt; 2176 memcpy(t->sched_type_list, opt->sched_type_list, 2177 sizeof(opt->sched_type_list)); 2178 return 0; 2179 nomem: 2180 return -ENOMEM; 2181 } 2182 2183 void 2184 perf_test_destroy(struct evt_test *test, struct evt_options *opt) 2185 { 2186 RTE_SET_USED(opt); 2187 2188 rte_free(test->test_priv); 2189 } 2190