1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <rte_malloc.h> 34 #include <rte_cycles.h> 35 #include <rte_crypto.h> 36 #include <rte_cryptodev.h> 37 38 #include "cperf_test_verify.h" 39 #include "cperf_ops.h" 40 41 struct cperf_verify_ctx { 42 uint8_t dev_id; 43 uint16_t qp_id; 44 uint8_t lcore_id; 45 46 struct rte_mempool *pkt_mbuf_pool_in; 47 struct rte_mempool *pkt_mbuf_pool_out; 48 struct rte_mbuf **mbufs_in; 49 struct rte_mbuf **mbufs_out; 50 51 struct rte_mempool *crypto_op_pool; 52 53 struct rte_cryptodev_sym_session *sess; 54 55 cperf_populate_ops_t populate_ops; 56 57 const struct cperf_options *options; 58 const struct cperf_test_vector *test_vector; 59 }; 60 61 struct cperf_op_result { 62 enum rte_crypto_op_status status; 63 }; 64 65 static void 66 cperf_verify_test_free(struct cperf_verify_ctx *ctx, uint32_t mbuf_nb) 67 { 68 uint32_t i; 69 70 if (ctx) { 71 if (ctx->sess) 72 rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); 73 74 if (ctx->mbufs_in) { 75 for (i = 0; i < mbuf_nb; i++) 76 rte_pktmbuf_free(ctx->mbufs_in[i]); 77 78 rte_free(ctx->mbufs_in); 79 } 80 81 if (ctx->mbufs_out) { 82 for (i = 0; i < mbuf_nb; i++) { 83 if (ctx->mbufs_out[i] != NULL) 84 rte_pktmbuf_free(ctx->mbufs_out[i]); 85 } 86 87 rte_free(ctx->mbufs_out); 88 } 89 90 if (ctx->pkt_mbuf_pool_in) 91 rte_mempool_free(ctx->pkt_mbuf_pool_in); 92 93 if (ctx->pkt_mbuf_pool_out) 94 rte_mempool_free(ctx->pkt_mbuf_pool_out); 95 96 if (ctx->crypto_op_pool) 97 rte_mempool_free(ctx->crypto_op_pool); 98 99 rte_free(ctx); 100 } 101 } 102 103 static struct rte_mbuf * 104 cperf_mbuf_create(struct rte_mempool *mempool, 105 uint32_t segments_nb, 106 const struct cperf_options *options, 107 const struct cperf_test_vector *test_vector) 108 { 109 struct rte_mbuf *mbuf; 110 uint32_t segment_sz = options->buffer_sz / segments_nb; 111 uint32_t last_sz = options->buffer_sz % segments_nb; 112 uint8_t *mbuf_data; 113 uint8_t *test_data = 114 (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 115 test_vector->plaintext.data : 116 test_vector->ciphertext.data; 117 118 mbuf = rte_pktmbuf_alloc(mempool); 119 if (mbuf == NULL) 120 goto error; 121 122 mbuf_data = (uint8_t *)rte_pktmbuf_append(mbuf, segment_sz); 123 if (mbuf_data == NULL) 124 goto error; 125 126 memcpy(mbuf_data, test_data, segment_sz); 127 test_data += segment_sz; 128 segments_nb--; 129 130 while (segments_nb) { 131 struct rte_mbuf *m; 132 133 m = rte_pktmbuf_alloc(mempool); 134 if (m == NULL) 135 goto error; 136 137 rte_pktmbuf_chain(mbuf, m); 138 139 mbuf_data = (uint8_t *)rte_pktmbuf_append(mbuf, segment_sz); 140 if (mbuf_data == NULL) 141 goto error; 142 143 memcpy(mbuf_data, test_data, segment_sz); 144 test_data += segment_sz; 145 segments_nb--; 146 } 147 148 if (last_sz) { 149 mbuf_data = (uint8_t *)rte_pktmbuf_append(mbuf, last_sz); 150 if (mbuf_data == NULL) 151 goto error; 152 153 memcpy(mbuf_data, test_data, last_sz); 154 } 155 156 if (options->op_type != CPERF_CIPHER_ONLY) { 157 mbuf_data = (uint8_t *)rte_pktmbuf_append(mbuf, 158 options->auth_digest_sz); 159 if (mbuf_data == NULL) 160 goto error; 161 } 162 163 if (options->op_type == CPERF_AEAD) { 164 uint8_t *aead = (uint8_t *)rte_pktmbuf_prepend(mbuf, 165 RTE_ALIGN_CEIL(options->auth_aad_sz, 16)); 166 167 if (aead == NULL) 168 goto error; 169 170 memcpy(aead, test_vector->aad.data, test_vector->aad.length); 171 } 172 173 return mbuf; 174 error: 175 if (mbuf != NULL) 176 rte_pktmbuf_free(mbuf); 177 178 return NULL; 179 } 180 181 void * 182 cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id, 183 const struct cperf_options *options, 184 const struct cperf_test_vector *test_vector, 185 const struct cperf_op_fns *op_fns) 186 { 187 struct cperf_verify_ctx *ctx = NULL; 188 unsigned int mbuf_idx = 0; 189 char pool_name[32] = ""; 190 191 ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0); 192 if (ctx == NULL) 193 goto err; 194 195 ctx->dev_id = dev_id; 196 ctx->qp_id = qp_id; 197 198 ctx->populate_ops = op_fns->populate_ops; 199 ctx->options = options; 200 ctx->test_vector = test_vector; 201 202 ctx->sess = op_fns->sess_create(dev_id, options, test_vector); 203 if (ctx->sess == NULL) 204 goto err; 205 206 snprintf(pool_name, sizeof(pool_name), "cperf_pool_in_cdev_%d", 207 dev_id); 208 209 ctx->pkt_mbuf_pool_in = rte_pktmbuf_pool_create(pool_name, 210 options->pool_sz * options->segments_nb, 0, 0, 211 RTE_PKTMBUF_HEADROOM + 212 RTE_CACHE_LINE_ROUNDUP( 213 (options->buffer_sz / options->segments_nb) + 214 (options->buffer_sz % options->segments_nb) + 215 options->auth_digest_sz), 216 rte_socket_id()); 217 218 if (ctx->pkt_mbuf_pool_in == NULL) 219 goto err; 220 221 /* Generate mbufs_in with plaintext populated for test */ 222 ctx->mbufs_in = rte_malloc(NULL, 223 (sizeof(struct rte_mbuf *) * ctx->options->pool_sz), 0); 224 225 for (mbuf_idx = 0; mbuf_idx < options->pool_sz; mbuf_idx++) { 226 ctx->mbufs_in[mbuf_idx] = cperf_mbuf_create( 227 ctx->pkt_mbuf_pool_in, options->segments_nb, 228 options, test_vector); 229 if (ctx->mbufs_in[mbuf_idx] == NULL) 230 goto err; 231 } 232 233 if (options->out_of_place == 1) { 234 235 snprintf(pool_name, sizeof(pool_name), "cperf_pool_out_cdev_%d", 236 dev_id); 237 238 ctx->pkt_mbuf_pool_out = rte_pktmbuf_pool_create( 239 pool_name, options->pool_sz, 0, 0, 240 RTE_PKTMBUF_HEADROOM + 241 RTE_CACHE_LINE_ROUNDUP( 242 options->buffer_sz + 243 options->auth_digest_sz), 244 rte_socket_id()); 245 246 if (ctx->pkt_mbuf_pool_out == NULL) 247 goto err; 248 } 249 250 ctx->mbufs_out = rte_malloc(NULL, 251 (sizeof(struct rte_mbuf *) * 252 ctx->options->pool_sz), 0); 253 254 for (mbuf_idx = 0; mbuf_idx < options->pool_sz; mbuf_idx++) { 255 if (options->out_of_place == 1) { 256 ctx->mbufs_out[mbuf_idx] = cperf_mbuf_create( 257 ctx->pkt_mbuf_pool_out, 1, 258 options, test_vector); 259 if (ctx->mbufs_out[mbuf_idx] == NULL) 260 goto err; 261 } else { 262 ctx->mbufs_out[mbuf_idx] = NULL; 263 } 264 } 265 266 snprintf(pool_name, sizeof(pool_name), "cperf_op_pool_cdev_%d", 267 dev_id); 268 269 ctx->crypto_op_pool = rte_crypto_op_pool_create(pool_name, 270 RTE_CRYPTO_OP_TYPE_SYMMETRIC, options->pool_sz, 0, 0, 271 rte_socket_id()); 272 if (ctx->crypto_op_pool == NULL) 273 goto err; 274 275 return ctx; 276 err: 277 cperf_verify_test_free(ctx, mbuf_idx); 278 279 return NULL; 280 } 281 282 static int 283 cperf_verify_op(struct rte_crypto_op *op, 284 const struct cperf_options *options, 285 const struct cperf_test_vector *vector) 286 { 287 const struct rte_mbuf *m; 288 uint32_t len; 289 uint16_t nb_segs; 290 uint8_t *data; 291 uint32_t cipher_offset, auth_offset; 292 uint8_t cipher, auth; 293 int res = 0; 294 295 if (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS) 296 return 1; 297 298 if (op->sym->m_dst) 299 m = op->sym->m_dst; 300 else 301 m = op->sym->m_src; 302 nb_segs = m->nb_segs; 303 len = 0; 304 while (m && nb_segs != 0) { 305 len += m->data_len; 306 m = m->next; 307 nb_segs--; 308 } 309 310 data = rte_malloc(NULL, len, 0); 311 if (data == NULL) 312 return 1; 313 314 if (op->sym->m_dst) 315 m = op->sym->m_dst; 316 else 317 m = op->sym->m_src; 318 nb_segs = m->nb_segs; 319 len = 0; 320 while (m && nb_segs != 0) { 321 memcpy(data + len, rte_pktmbuf_mtod(m, uint8_t *), 322 m->data_len); 323 len += m->data_len; 324 m = m->next; 325 nb_segs--; 326 } 327 328 switch (options->op_type) { 329 case CPERF_CIPHER_ONLY: 330 cipher = 1; 331 cipher_offset = 0; 332 auth = 0; 333 auth_offset = 0; 334 break; 335 case CPERF_CIPHER_THEN_AUTH: 336 cipher = 1; 337 cipher_offset = 0; 338 auth = 1; 339 auth_offset = vector->plaintext.length; 340 break; 341 case CPERF_AUTH_ONLY: 342 cipher = 0; 343 cipher_offset = 0; 344 auth = 1; 345 auth_offset = vector->plaintext.length; 346 break; 347 case CPERF_AUTH_THEN_CIPHER: 348 cipher = 1; 349 cipher_offset = 0; 350 auth = 1; 351 auth_offset = vector->plaintext.length; 352 break; 353 case CPERF_AEAD: 354 cipher = 1; 355 cipher_offset = vector->aad.length; 356 auth = 1; 357 auth_offset = vector->aad.length + vector->plaintext.length; 358 break; 359 } 360 361 if (cipher == 1) { 362 if (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 363 res += memcmp(data + cipher_offset, 364 vector->ciphertext.data, 365 vector->ciphertext.length); 366 else 367 res += memcmp(data + cipher_offset, 368 vector->plaintext.data, 369 vector->plaintext.length); 370 } 371 372 if (auth == 1) { 373 if (options->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE) 374 res += memcmp(data + auth_offset, 375 vector->digest.data, 376 options->auth_digest_sz); 377 } 378 379 return !!res; 380 } 381 382 int 383 cperf_verify_test_runner(void *test_ctx) 384 { 385 struct cperf_verify_ctx *ctx = test_ctx; 386 387 uint64_t ops_enqd = 0, ops_enqd_total = 0, ops_enqd_failed = 0; 388 uint64_t ops_deqd = 0, ops_deqd_total = 0, ops_deqd_failed = 0; 389 uint64_t ops_failed = 0; 390 391 static int only_once; 392 393 uint64_t i, m_idx = 0; 394 uint16_t ops_unused = 0; 395 396 struct rte_crypto_op *ops[ctx->options->burst_sz]; 397 struct rte_crypto_op *ops_processed[ctx->options->burst_sz]; 398 399 uint32_t lcore = rte_lcore_id(); 400 401 #ifdef CPERF_LINEARIZATION_ENABLE 402 struct rte_cryptodev_info dev_info; 403 int linearize = 0; 404 405 /* Check if source mbufs require coalescing */ 406 if (ctx->options->segments_nb > 1) { 407 rte_cryptodev_info_get(ctx->dev_id, &dev_info); 408 if ((dev_info.feature_flags & 409 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER) == 0) 410 linearize = 1; 411 } 412 #endif /* CPERF_LINEARIZATION_ENABLE */ 413 414 ctx->lcore_id = lcore; 415 416 if (!ctx->options->csv) 417 printf("\n# Running verify test on device: %u, lcore: %u\n", 418 ctx->dev_id, lcore); 419 420 while (ops_enqd_total < ctx->options->total_ops) { 421 422 uint16_t burst_size = ((ops_enqd_total + ctx->options->burst_sz) 423 <= ctx->options->total_ops) ? 424 ctx->options->burst_sz : 425 ctx->options->total_ops - 426 ops_enqd_total; 427 428 uint16_t ops_needed = burst_size - ops_unused; 429 430 /* Allocate crypto ops from pool */ 431 if (ops_needed != rte_crypto_op_bulk_alloc( 432 ctx->crypto_op_pool, 433 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 434 ops, ops_needed)) 435 return -1; 436 437 /* Setup crypto op, attach mbuf etc */ 438 (ctx->populate_ops)(ops, &ctx->mbufs_in[m_idx], 439 &ctx->mbufs_out[m_idx], 440 ops_needed, ctx->sess, ctx->options, 441 ctx->test_vector); 442 443 #ifdef CPERF_LINEARIZATION_ENABLE 444 if (linearize) { 445 /* PMD doesn't support scatter-gather and source buffer 446 * is segmented. 447 * We need to linearize it before enqueuing. 448 */ 449 for (i = 0; i < burst_size; i++) 450 rte_pktmbuf_linearize(ops[i]->sym->m_src); 451 } 452 #endif /* CPERF_LINEARIZATION_ENABLE */ 453 454 /* Enqueue burst of ops on crypto device */ 455 ops_enqd = rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, 456 ops, burst_size); 457 if (ops_enqd < burst_size) 458 ops_enqd_failed++; 459 460 /** 461 * Calculate number of ops not enqueued (mainly for hw 462 * accelerators whose ingress queue can fill up). 463 */ 464 ops_unused = burst_size - ops_enqd; 465 ops_enqd_total += ops_enqd; 466 467 468 /* Dequeue processed burst of ops from crypto device */ 469 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, 470 ops_processed, ctx->options->burst_sz); 471 472 m_idx += ops_needed; 473 if (m_idx + ctx->options->burst_sz > ctx->options->pool_sz) 474 m_idx = 0; 475 476 if (ops_deqd == 0) { 477 /** 478 * Count dequeue polls which didn't return any 479 * processed operations. This statistic is mainly 480 * relevant to hw accelerators. 481 */ 482 ops_deqd_failed++; 483 continue; 484 } 485 486 for (i = 0; i < ops_deqd; i++) { 487 if (cperf_verify_op(ops_processed[i], ctx->options, 488 ctx->test_vector)) 489 ops_failed++; 490 /* free crypto ops so they can be reused. We don't free 491 * the mbufs here as we don't want to reuse them as 492 * the crypto operation will change the data and cause 493 * failures. 494 */ 495 rte_crypto_op_free(ops_processed[i]); 496 } 497 ops_deqd_total += ops_deqd; 498 } 499 500 /* Dequeue any operations still in the crypto device */ 501 502 while (ops_deqd_total < ctx->options->total_ops) { 503 /* Sending 0 length burst to flush sw crypto device */ 504 rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0); 505 506 /* dequeue burst */ 507 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, 508 ops_processed, ctx->options->burst_sz); 509 if (ops_deqd == 0) { 510 ops_deqd_failed++; 511 continue; 512 } 513 514 for (i = 0; i < ops_deqd; i++) { 515 if (cperf_verify_op(ops_processed[i], ctx->options, 516 ctx->test_vector)) 517 ops_failed++; 518 /* free crypto ops so they can be reused. We don't free 519 * the mbufs here as we don't want to reuse them as 520 * the crypto operation will change the data and cause 521 * failures. 522 */ 523 rte_crypto_op_free(ops_processed[i]); 524 } 525 ops_deqd_total += ops_deqd; 526 } 527 528 if (!ctx->options->csv) { 529 if (!only_once) 530 printf("%12s%12s%12s%12s%12s%12s%12s%12s\n\n", 531 "lcore id", "Buf Size", "Burst size", 532 "Enqueued", "Dequeued", "Failed Enq", 533 "Failed Deq", "Failed Ops"); 534 only_once = 1; 535 536 printf("%12u%12u%12u%12"PRIu64"%12"PRIu64"%12"PRIu64 537 "%12"PRIu64"%12"PRIu64"\n", 538 ctx->lcore_id, 539 ctx->options->buffer_sz, 540 ctx->options->burst_sz, 541 ops_enqd_total, 542 ops_deqd_total, 543 ops_enqd_failed, 544 ops_deqd_failed, 545 ops_failed); 546 } else { 547 if (!only_once) 548 printf("\n# lcore id, Buffer Size(B), " 549 "Burst Size,Enqueued,Dequeued,Failed Enq," 550 "Failed Deq,Failed Ops\n"); 551 only_once = 1; 552 553 printf("%10u;%10u;%u;%"PRIu64";%"PRIu64";%"PRIu64";%"PRIu64";" 554 "%"PRIu64"\n", 555 ctx->lcore_id, 556 ctx->options->buffer_sz, 557 ctx->options->burst_sz, 558 ops_enqd_total, 559 ops_deqd_total, 560 ops_enqd_failed, 561 ops_deqd_failed, 562 ops_failed); 563 } 564 565 return 0; 566 } 567 568 569 570 void 571 cperf_verify_test_destructor(void *arg) 572 { 573 struct cperf_verify_ctx *ctx = arg; 574 575 if (ctx == NULL) 576 return; 577 578 cperf_verify_test_free(ctx, ctx->options->pool_sz); 579 } 580