1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <stdlib.h> 6 7 #include <rte_malloc.h> 8 #include <rte_cycles.h> 9 #include <rte_crypto.h> 10 #include <rte_cryptodev.h> 11 12 #include "cperf_test_verify.h" 13 #include "cperf_ops.h" 14 #include "cperf_test_common.h" 15 16 struct cperf_verify_ctx { 17 uint8_t dev_id; 18 uint16_t qp_id; 19 uint8_t lcore_id; 20 21 struct rte_mempool *pool; 22 23 void *sess; 24 25 cperf_populate_ops_t populate_ops; 26 27 uint32_t src_buf_offset; 28 uint32_t dst_buf_offset; 29 30 const struct cperf_options *options; 31 const struct cperf_test_vector *test_vector; 32 }; 33 34 struct cperf_op_result { 35 enum rte_crypto_op_status status; 36 }; 37 38 static void 39 cperf_verify_test_free(struct cperf_verify_ctx *ctx) 40 { 41 if (ctx == NULL) 42 return; 43 44 if (ctx->sess != NULL) { 45 if (ctx->options->op_type == CPERF_ASYM_MODEX) 46 rte_cryptodev_asym_session_free(ctx->dev_id, ctx->sess); 47 #ifdef RTE_LIB_SECURITY 48 else if (ctx->options->op_type == CPERF_PDCP || 49 ctx->options->op_type == CPERF_DOCSIS || 50 ctx->options->op_type == CPERF_IPSEC) { 51 struct rte_security_ctx *sec_ctx = 52 rte_cryptodev_get_sec_ctx(ctx->dev_id); 53 rte_security_session_destroy(sec_ctx, ctx->sess); 54 } 55 #endif 56 else 57 rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); 58 } 59 60 rte_mempool_free(ctx->pool); 61 rte_free(ctx); 62 } 63 64 void * 65 cperf_verify_test_constructor(struct rte_mempool *sess_mp, 66 uint8_t dev_id, uint16_t qp_id, 67 const struct cperf_options *options, 68 const struct cperf_test_vector *test_vector, 69 const struct cperf_op_fns *op_fns) 70 { 71 struct cperf_verify_ctx *ctx = NULL; 72 73 ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0); 74 if (ctx == NULL) 75 goto err; 76 77 ctx->dev_id = dev_id; 78 ctx->qp_id = qp_id; 79 80 ctx->populate_ops = op_fns->populate_ops; 81 ctx->options = options; 82 ctx->test_vector = test_vector; 83 84 /* IV goes at the end of the crypto operation */ 85 uint16_t iv_offset = sizeof(struct rte_crypto_op) + 86 sizeof(struct rte_crypto_sym_op); 87 88 ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, 89 test_vector, iv_offset); 90 if (ctx->sess == NULL) 91 goto err; 92 93 if (cperf_alloc_common_memory(options, test_vector, dev_id, qp_id, 0, 94 &ctx->src_buf_offset, &ctx->dst_buf_offset, 95 &ctx->pool) < 0) 96 goto err; 97 98 return ctx; 99 err: 100 cperf_verify_test_free(ctx); 101 102 return NULL; 103 } 104 105 static int 106 cperf_verify_op(struct rte_crypto_op *op, 107 const struct cperf_options *options, 108 const struct cperf_test_vector *vector) 109 { 110 const struct rte_mbuf *m; 111 uint32_t len; 112 uint16_t nb_segs; 113 uint8_t *data; 114 uint32_t cipher_offset, auth_offset; 115 uint8_t cipher, auth; 116 int res = 0; 117 118 if (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS) 119 return 1; 120 121 if (op->sym->m_dst) 122 m = op->sym->m_dst; 123 else 124 m = op->sym->m_src; 125 nb_segs = m->nb_segs; 126 len = 0; 127 while (m && nb_segs != 0) { 128 len += m->data_len; 129 m = m->next; 130 nb_segs--; 131 } 132 133 data = rte_malloc(NULL, len, 0); 134 if (data == NULL) 135 return 1; 136 137 if (op->sym->m_dst) 138 m = op->sym->m_dst; 139 else 140 m = op->sym->m_src; 141 nb_segs = m->nb_segs; 142 len = 0; 143 while (m && nb_segs != 0) { 144 memcpy(data + len, rte_pktmbuf_mtod(m, uint8_t *), 145 m->data_len); 146 len += m->data_len; 147 m = m->next; 148 nb_segs--; 149 } 150 151 switch (options->op_type) { 152 case CPERF_CIPHER_ONLY: 153 cipher = 1; 154 cipher_offset = 0; 155 auth = 0; 156 auth_offset = 0; 157 break; 158 case CPERF_CIPHER_THEN_AUTH: 159 cipher = 1; 160 cipher_offset = 0; 161 auth = 1; 162 auth_offset = options->test_buffer_size; 163 break; 164 case CPERF_AUTH_ONLY: 165 cipher = 0; 166 cipher_offset = 0; 167 auth = 1; 168 auth_offset = options->test_buffer_size; 169 break; 170 case CPERF_AUTH_THEN_CIPHER: 171 cipher = 1; 172 cipher_offset = 0; 173 auth = 1; 174 auth_offset = options->test_buffer_size; 175 break; 176 case CPERF_AEAD: 177 cipher = 1; 178 cipher_offset = 0; 179 auth = 1; 180 auth_offset = options->test_buffer_size; 181 break; 182 default: 183 res = 1; 184 goto out; 185 } 186 187 if (cipher == 1) { 188 if (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 189 res += memcmp(data + cipher_offset, 190 vector->ciphertext.data, 191 options->test_buffer_size); 192 else 193 res += memcmp(data + cipher_offset, 194 vector->plaintext.data, 195 options->test_buffer_size); 196 } 197 198 if (auth == 1) { 199 if (options->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE) 200 res += memcmp(data + auth_offset, 201 vector->digest.data, 202 options->digest_sz); 203 } 204 205 out: 206 rte_free(data); 207 return !!res; 208 } 209 210 int 211 cperf_verify_test_runner(void *test_ctx) 212 { 213 struct cperf_verify_ctx *ctx = test_ctx; 214 215 uint64_t ops_enqd = 0, ops_enqd_total = 0, ops_enqd_failed = 0; 216 uint64_t ops_deqd = 0, ops_deqd_total = 0, ops_deqd_failed = 0; 217 uint64_t ops_failed = 0; 218 219 static uint16_t display_once; 220 221 uint64_t i; 222 uint16_t ops_unused = 0; 223 uint32_t imix_idx = 0; 224 225 struct rte_crypto_op *ops[ctx->options->max_burst_size]; 226 struct rte_crypto_op *ops_processed[ctx->options->max_burst_size]; 227 228 uint32_t lcore = rte_lcore_id(); 229 230 #ifdef CPERF_LINEARIZATION_ENABLE 231 struct rte_cryptodev_info dev_info; 232 int linearize = 0; 233 234 /* Check if source mbufs require coalescing */ 235 if (ctx->options->segment_sz < ctx->options->max_buffer_size) { 236 rte_cryptodev_info_get(ctx->dev_id, &dev_info); 237 if ((dev_info.feature_flags & 238 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER) == 0) 239 linearize = 1; 240 } 241 #endif /* CPERF_LINEARIZATION_ENABLE */ 242 243 ctx->lcore_id = lcore; 244 245 if (!ctx->options->csv) 246 printf("\n# Running verify test on device: %u, lcore: %u\n", 247 ctx->dev_id, lcore); 248 249 uint16_t iv_offset = sizeof(struct rte_crypto_op) + 250 sizeof(struct rte_crypto_sym_op); 251 252 while (ops_enqd_total < ctx->options->total_ops) { 253 254 uint16_t burst_size = ((ops_enqd_total + ctx->options->max_burst_size) 255 <= ctx->options->total_ops) ? 256 ctx->options->max_burst_size : 257 ctx->options->total_ops - 258 ops_enqd_total; 259 260 uint16_t ops_needed = burst_size - ops_unused; 261 262 /* Allocate objects containing crypto operations and mbufs */ 263 if (rte_mempool_get_bulk(ctx->pool, (void **)ops, 264 ops_needed) != 0) { 265 RTE_LOG(ERR, USER1, 266 "Failed to allocate more crypto operations " 267 "from the crypto operation pool.\n" 268 "Consider increasing the pool size " 269 "with --pool-sz\n"); 270 return -1; 271 } 272 273 /* Setup crypto op, attach mbuf etc */ 274 (ctx->populate_ops)(ops, ctx->src_buf_offset, 275 ctx->dst_buf_offset, 276 ops_needed, ctx->sess, ctx->options, 277 ctx->test_vector, iv_offset, &imix_idx, NULL); 278 279 280 /* Populate the mbuf with the test vector, for verification */ 281 for (i = 0; i < ops_needed; i++) 282 cperf_mbuf_set(ops[i]->sym->m_src, 283 ctx->options, 284 ctx->test_vector); 285 286 #ifdef CPERF_LINEARIZATION_ENABLE 287 if (linearize) { 288 /* PMD doesn't support scatter-gather and source buffer 289 * is segmented. 290 * We need to linearize it before enqueuing. 291 */ 292 for (i = 0; i < burst_size; i++) 293 rte_pktmbuf_linearize(ops[i]->sym->m_src); 294 } 295 #endif /* CPERF_LINEARIZATION_ENABLE */ 296 297 /* Enqueue burst of ops on crypto device */ 298 ops_enqd = rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, 299 ops, burst_size); 300 if (ops_enqd < burst_size) 301 ops_enqd_failed++; 302 303 /** 304 * Calculate number of ops not enqueued (mainly for hw 305 * accelerators whose ingress queue can fill up). 306 */ 307 ops_unused = burst_size - ops_enqd; 308 ops_enqd_total += ops_enqd; 309 310 311 /* Dequeue processed burst of ops from crypto device */ 312 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, 313 ops_processed, ctx->options->max_burst_size); 314 315 if (ops_deqd == 0) { 316 /** 317 * Count dequeue polls which didn't return any 318 * processed operations. This statistic is mainly 319 * relevant to hw accelerators. 320 */ 321 ops_deqd_failed++; 322 continue; 323 } 324 325 for (i = 0; i < ops_deqd; i++) { 326 if (cperf_verify_op(ops_processed[i], ctx->options, 327 ctx->test_vector)) 328 ops_failed++; 329 } 330 /* Free crypto ops so they can be reused. */ 331 rte_mempool_put_bulk(ctx->pool, 332 (void **)ops_processed, ops_deqd); 333 ops_deqd_total += ops_deqd; 334 } 335 336 /* Dequeue any operations still in the crypto device */ 337 338 while (ops_deqd_total < ctx->options->total_ops) { 339 /* Sending 0 length burst to flush sw crypto device */ 340 rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0); 341 342 /* dequeue burst */ 343 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, 344 ops_processed, ctx->options->max_burst_size); 345 if (ops_deqd == 0) { 346 ops_deqd_failed++; 347 continue; 348 } 349 350 for (i = 0; i < ops_deqd; i++) { 351 if (cperf_verify_op(ops_processed[i], ctx->options, 352 ctx->test_vector)) 353 ops_failed++; 354 } 355 /* Free crypto ops so they can be reused. */ 356 rte_mempool_put_bulk(ctx->pool, 357 (void **)ops_processed, ops_deqd); 358 ops_deqd_total += ops_deqd; 359 } 360 361 uint16_t exp = 0; 362 if (!ctx->options->csv) { 363 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0, 364 __ATOMIC_RELAXED, __ATOMIC_RELAXED)) 365 printf("%12s%12s%12s%12s%12s%12s%12s%12s\n\n", 366 "lcore id", "Buf Size", "Burst size", 367 "Enqueued", "Dequeued", "Failed Enq", 368 "Failed Deq", "Failed Ops"); 369 370 printf("%12u%12u%12u%12"PRIu64"%12"PRIu64"%12"PRIu64 371 "%12"PRIu64"%12"PRIu64"\n", 372 ctx->lcore_id, 373 ctx->options->max_buffer_size, 374 ctx->options->max_burst_size, 375 ops_enqd_total, 376 ops_deqd_total, 377 ops_enqd_failed, 378 ops_deqd_failed, 379 ops_failed); 380 } else { 381 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0, 382 __ATOMIC_RELAXED, __ATOMIC_RELAXED)) 383 printf("\n# lcore id, Buffer Size(B), " 384 "Burst Size,Enqueued,Dequeued,Failed Enq," 385 "Failed Deq,Failed Ops\n"); 386 387 printf("%10u,%10u,%u,%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64"," 388 "%"PRIu64"\n", 389 ctx->lcore_id, 390 ctx->options->max_buffer_size, 391 ctx->options->max_burst_size, 392 ops_enqd_total, 393 ops_deqd_total, 394 ops_enqd_failed, 395 ops_deqd_failed, 396 ops_failed); 397 } 398 399 return 0; 400 } 401 402 403 404 void 405 cperf_verify_test_destructor(void *arg) 406 { 407 struct cperf_verify_ctx *ctx = arg; 408 409 if (ctx == NULL) 410 return; 411 412 cperf_verify_test_free(ctx); 413 } 414