1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <stdlib.h> 6 7 #include <rte_malloc.h> 8 #include <rte_cycles.h> 9 #include <rte_crypto.h> 10 #include <rte_cryptodev.h> 11 12 #include "cperf_test_verify.h" 13 #include "cperf_ops.h" 14 #include "cperf_test_common.h" 15 16 struct cperf_verify_ctx { 17 uint8_t dev_id; 18 uint16_t qp_id; 19 uint8_t lcore_id; 20 21 struct rte_mempool *pool; 22 23 void *sess; 24 25 cperf_populate_ops_t populate_ops; 26 27 uint32_t src_buf_offset; 28 uint32_t dst_buf_offset; 29 30 const struct cperf_options *options; 31 const struct cperf_test_vector *test_vector; 32 }; 33 34 struct cperf_op_result { 35 enum rte_crypto_op_status status; 36 }; 37 38 static void 39 cperf_verify_test_free(struct cperf_verify_ctx *ctx) 40 { 41 if (ctx == NULL) 42 return; 43 44 if (ctx->sess != NULL) { 45 if (ctx->options->op_type == CPERF_ASYM_MODEX) 46 rte_cryptodev_asym_session_free(ctx->dev_id, ctx->sess); 47 #ifdef RTE_LIB_SECURITY 48 else if (ctx->options->op_type == CPERF_PDCP || 49 ctx->options->op_type == CPERF_DOCSIS || 50 ctx->options->op_type == CPERF_IPSEC) { 51 void *sec_ctx = rte_cryptodev_get_sec_ctx(ctx->dev_id); 52 53 rte_security_session_destroy(sec_ctx, ctx->sess); 54 } 55 #endif 56 else 57 rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); 58 } 59 60 rte_mempool_free(ctx->pool); 61 rte_free(ctx); 62 } 63 64 void * 65 cperf_verify_test_constructor(struct rte_mempool *sess_mp, 66 uint8_t dev_id, uint16_t qp_id, 67 const struct cperf_options *options, 68 const struct cperf_test_vector *test_vector, 69 const struct cperf_op_fns *op_fns) 70 { 71 struct cperf_verify_ctx *ctx = NULL; 72 73 ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0); 74 if (ctx == NULL) 75 goto err; 76 77 ctx->dev_id = dev_id; 78 ctx->qp_id = qp_id; 79 80 ctx->populate_ops = op_fns->populate_ops; 81 ctx->options = options; 82 ctx->test_vector = test_vector; 83 84 /* IV goes at the end of the crypto operation */ 85 uint16_t iv_offset = sizeof(struct rte_crypto_op) + 86 sizeof(struct rte_crypto_sym_op); 87 88 ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, 89 test_vector, iv_offset); 90 if (ctx->sess == NULL) 91 goto err; 92 93 if (cperf_alloc_common_memory(options, test_vector, dev_id, qp_id, 0, 94 &ctx->src_buf_offset, &ctx->dst_buf_offset, 95 &ctx->pool) < 0) 96 goto err; 97 98 return ctx; 99 err: 100 cperf_verify_test_free(ctx); 101 102 return NULL; 103 } 104 105 static int 106 cperf_verify_op(struct rte_crypto_op *op, 107 const struct cperf_options *options, 108 const struct cperf_test_vector *vector) 109 { 110 const struct rte_mbuf *m; 111 uint32_t len; 112 uint16_t nb_segs; 113 uint8_t *data; 114 uint32_t cipher_offset, auth_offset = 0; 115 bool cipher = false; 116 bool digest_verify = false; 117 bool is_encrypt = false; 118 int res = 0; 119 120 if (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS) 121 return 1; 122 123 if (op->sym->m_dst) 124 m = op->sym->m_dst; 125 else 126 m = op->sym->m_src; 127 nb_segs = m->nb_segs; 128 len = 0; 129 while (m && nb_segs != 0) { 130 len += m->data_len; 131 m = m->next; 132 nb_segs--; 133 } 134 135 data = rte_malloc(NULL, len, 0); 136 if (data == NULL) 137 return 1; 138 139 if (op->sym->m_dst) 140 m = op->sym->m_dst; 141 else 142 m = op->sym->m_src; 143 nb_segs = m->nb_segs; 144 len = 0; 145 while (m && nb_segs != 0) { 146 memcpy(data + len, rte_pktmbuf_mtod(m, uint8_t *), 147 m->data_len); 148 len += m->data_len; 149 m = m->next; 150 nb_segs--; 151 } 152 153 switch (options->op_type) { 154 case CPERF_CIPHER_ONLY: 155 cipher = true; 156 cipher_offset = 0; 157 is_encrypt = options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT; 158 break; 159 case CPERF_AUTH_ONLY: 160 cipher_offset = 0; 161 if (options->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE) { 162 auth_offset = options->test_buffer_size; 163 digest_verify = true; 164 } 165 break; 166 case CPERF_CIPHER_THEN_AUTH: 167 case CPERF_AUTH_THEN_CIPHER: 168 cipher = true; 169 cipher_offset = 0; 170 if (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 171 auth_offset = options->test_buffer_size; 172 digest_verify = true; 173 is_encrypt = true; 174 } 175 break; 176 case CPERF_AEAD: 177 cipher = true; 178 cipher_offset = 0; 179 if (options->aead_op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { 180 auth_offset = options->test_buffer_size; 181 digest_verify = true; 182 is_encrypt = true; 183 } 184 break; 185 default: 186 res = 1; 187 goto out; 188 } 189 190 if (cipher) { 191 if (is_encrypt) 192 res += !!memcmp(data + cipher_offset, 193 vector->ciphertext.data, 194 options->test_buffer_size); 195 else 196 res += !!memcmp(data + cipher_offset, 197 vector->plaintext.data, 198 options->test_buffer_size); 199 } 200 201 if (digest_verify) 202 res += !!memcmp(data + auth_offset, vector->digest.data, options->digest_sz); 203 204 out: 205 rte_free(data); 206 return !!res; 207 } 208 209 int 210 cperf_verify_test_runner(void *test_ctx) 211 { 212 struct cperf_verify_ctx *ctx = test_ctx; 213 214 uint64_t ops_enqd = 0, ops_enqd_total = 0, ops_enqd_failed = 0; 215 uint64_t ops_deqd = 0, ops_deqd_total = 0, ops_deqd_failed = 0; 216 uint64_t ops_failed = 0; 217 218 static uint16_t display_once; 219 220 uint64_t i; 221 uint16_t ops_unused = 0; 222 uint32_t imix_idx = 0; 223 224 struct rte_crypto_op *ops[ctx->options->max_burst_size]; 225 struct rte_crypto_op *ops_processed[ctx->options->max_burst_size]; 226 227 uint32_t lcore = rte_lcore_id(); 228 229 #ifdef CPERF_LINEARIZATION_ENABLE 230 struct rte_cryptodev_info dev_info; 231 int linearize = 0; 232 233 /* Check if source mbufs require coalescing */ 234 if (ctx->options->segment_sz < ctx->options->max_buffer_size) { 235 rte_cryptodev_info_get(ctx->dev_id, &dev_info); 236 if ((dev_info.feature_flags & 237 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER) == 0) 238 linearize = 1; 239 } 240 #endif /* CPERF_LINEARIZATION_ENABLE */ 241 242 ctx->lcore_id = lcore; 243 244 if (!ctx->options->csv) 245 printf("\n# Running verify test on device: %u, lcore: %u\n", 246 ctx->dev_id, lcore); 247 248 uint16_t iv_offset = sizeof(struct rte_crypto_op) + 249 sizeof(struct rte_crypto_sym_op); 250 251 while (ops_enqd_total < ctx->options->total_ops) { 252 253 uint16_t burst_size = ((ops_enqd_total + ctx->options->max_burst_size) 254 <= ctx->options->total_ops) ? 255 ctx->options->max_burst_size : 256 ctx->options->total_ops - 257 ops_enqd_total; 258 259 uint16_t ops_needed = burst_size - ops_unused; 260 261 /* Allocate objects containing crypto operations and mbufs */ 262 if (rte_mempool_get_bulk(ctx->pool, (void **)ops, 263 ops_needed) != 0) { 264 RTE_LOG(ERR, USER1, 265 "Failed to allocate more crypto operations " 266 "from the crypto operation pool.\n" 267 "Consider increasing the pool size " 268 "with --pool-sz\n"); 269 return -1; 270 } 271 272 /* Setup crypto op, attach mbuf etc */ 273 (ctx->populate_ops)(ops, ctx->src_buf_offset, 274 ctx->dst_buf_offset, 275 ops_needed, ctx->sess, ctx->options, 276 ctx->test_vector, iv_offset, &imix_idx, NULL); 277 278 279 /* Populate the mbuf with the test vector, for verification */ 280 for (i = 0; i < ops_needed; i++) 281 cperf_mbuf_set(ops[i]->sym->m_src, 282 ctx->options, 283 ctx->test_vector); 284 285 #ifdef CPERF_LINEARIZATION_ENABLE 286 if (linearize) { 287 /* PMD doesn't support scatter-gather and source buffer 288 * is segmented. 289 * We need to linearize it before enqueuing. 290 */ 291 for (i = 0; i < burst_size; i++) 292 rte_pktmbuf_linearize(ops[i]->sym->m_src); 293 } 294 #endif /* CPERF_LINEARIZATION_ENABLE */ 295 296 /* Enqueue burst of ops on crypto device */ 297 ops_enqd = rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, 298 ops, burst_size); 299 if (ops_enqd < burst_size) 300 ops_enqd_failed++; 301 302 /** 303 * Calculate number of ops not enqueued (mainly for hw 304 * accelerators whose ingress queue can fill up). 305 */ 306 ops_unused = burst_size - ops_enqd; 307 ops_enqd_total += ops_enqd; 308 309 310 /* Dequeue processed burst of ops from crypto device */ 311 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, 312 ops_processed, ctx->options->max_burst_size); 313 314 if (ops_deqd == 0) { 315 /** 316 * Count dequeue polls which didn't return any 317 * processed operations. This statistic is mainly 318 * relevant to hw accelerators. 319 */ 320 ops_deqd_failed++; 321 continue; 322 } 323 324 for (i = 0; i < ops_deqd; i++) { 325 if (cperf_verify_op(ops_processed[i], ctx->options, 326 ctx->test_vector)) 327 ops_failed++; 328 } 329 /* Free crypto ops so they can be reused. */ 330 rte_mempool_put_bulk(ctx->pool, 331 (void **)ops_processed, ops_deqd); 332 ops_deqd_total += ops_deqd; 333 } 334 335 /* Dequeue any operations still in the crypto device */ 336 337 while (ops_deqd_total < ctx->options->total_ops) { 338 /* Sending 0 length burst to flush sw crypto device */ 339 rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0); 340 341 /* dequeue burst */ 342 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, 343 ops_processed, ctx->options->max_burst_size); 344 if (ops_deqd == 0) { 345 ops_deqd_failed++; 346 continue; 347 } 348 349 for (i = 0; i < ops_deqd; i++) { 350 if (cperf_verify_op(ops_processed[i], ctx->options, 351 ctx->test_vector)) 352 ops_failed++; 353 } 354 /* Free crypto ops so they can be reused. */ 355 rte_mempool_put_bulk(ctx->pool, 356 (void **)ops_processed, ops_deqd); 357 ops_deqd_total += ops_deqd; 358 } 359 360 uint16_t exp = 0; 361 if (!ctx->options->csv) { 362 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0, 363 __ATOMIC_RELAXED, __ATOMIC_RELAXED)) 364 printf("%12s%12s%12s%12s%12s%12s%12s%12s\n\n", 365 "lcore id", "Buf Size", "Burst size", 366 "Enqueued", "Dequeued", "Failed Enq", 367 "Failed Deq", "Failed Ops"); 368 369 printf("%12u%12u%12u%12"PRIu64"%12"PRIu64"%12"PRIu64 370 "%12"PRIu64"%12"PRIu64"\n", 371 ctx->lcore_id, 372 ctx->options->max_buffer_size, 373 ctx->options->max_burst_size, 374 ops_enqd_total, 375 ops_deqd_total, 376 ops_enqd_failed, 377 ops_deqd_failed, 378 ops_failed); 379 } else { 380 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0, 381 __ATOMIC_RELAXED, __ATOMIC_RELAXED)) 382 printf("\n# lcore id, Buffer Size(B), " 383 "Burst Size,Enqueued,Dequeued,Failed Enq," 384 "Failed Deq,Failed Ops\n"); 385 386 printf("%10u,%10u,%u,%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64"," 387 "%"PRIu64"\n", 388 ctx->lcore_id, 389 ctx->options->max_buffer_size, 390 ctx->options->max_burst_size, 391 ops_enqd_total, 392 ops_deqd_total, 393 ops_enqd_failed, 394 ops_deqd_failed, 395 ops_failed); 396 } 397 398 return 0; 399 } 400 401 402 403 void 404 cperf_verify_test_destructor(void *arg) 405 { 406 struct cperf_verify_ctx *ctx = arg; 407 408 if (ctx == NULL) 409 return; 410 411 cperf_verify_test_free(ctx); 412 } 413