1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017-2019 NXP 3 */ 4 5 #include <fcntl.h> 6 #include <unistd.h> 7 #include <sched.h> 8 #include <net/if.h> 9 10 #include <rte_byteorder.h> 11 #include <rte_common.h> 12 #include <rte_cryptodev_pmd.h> 13 #include <rte_crypto.h> 14 #include <rte_cryptodev.h> 15 #include <rte_bus_vdev.h> 16 #include <rte_malloc.h> 17 #include <rte_security_driver.h> 18 #include <rte_hexdump.h> 19 20 #include <caam_jr_capabilities.h> 21 #include <caam_jr_config.h> 22 #include <caam_jr_hw_specific.h> 23 #include <caam_jr_pvt.h> 24 #include <caam_jr_desc.h> 25 #include <caam_jr_log.h> 26 27 /* RTA header files */ 28 #include <desc/common.h> 29 #include <desc/algo.h> 30 #include <dpaa_of.h> 31 #ifdef RTE_LIBRTE_PMD_CAAM_JR_DEBUG 32 #define CAAM_JR_DBG 1 33 #else 34 #define CAAM_JR_DBG 0 35 #endif 36 #define CRYPTODEV_NAME_CAAM_JR_PMD crypto_caam_jr 37 static uint8_t cryptodev_driver_id; 38 39 /* Lists the states possible for the SEC user space driver. */ 40 enum sec_driver_state_e { 41 SEC_DRIVER_STATE_IDLE, /* Driver not initialized */ 42 SEC_DRIVER_STATE_STARTED, /* Driver initialized and can be used*/ 43 SEC_DRIVER_STATE_RELEASE, /* Driver release is in progress */ 44 }; 45 46 /* Job rings used for communication with SEC HW */ 47 static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS]; 48 49 /* The current state of SEC user space driver */ 50 static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE; 51 52 /* The number of job rings used by SEC user space driver */ 53 static int g_job_rings_no; 54 static int g_job_rings_max; 55 56 struct sec_outring_entry { 57 phys_addr_t desc; /* Pointer to completed descriptor */ 58 uint32_t status; /* Status for completed descriptor */ 59 } __rte_packed; 60 61 /* virtual address conversin when mempool support is available for ctx */ 62 static inline phys_addr_t 63 caam_jr_vtop_ctx(struct caam_jr_op_ctx *ctx, void *vaddr) 64 { 65 return (size_t)vaddr - ctx->vtop_offset; 66 } 67 68 static inline void 69 caam_jr_op_ending(struct caam_jr_op_ctx *ctx) 70 { 71 /* report op status to sym->op and then free the ctx memory */ 72 rte_mempool_put(ctx->ctx_pool, (void *)ctx); 73 } 74 75 static inline struct caam_jr_op_ctx * 76 caam_jr_alloc_ctx(struct caam_jr_session *ses) 77 { 78 struct caam_jr_op_ctx *ctx; 79 int ret; 80 81 ret = rte_mempool_get(ses->ctx_pool, (void **)(&ctx)); 82 if (!ctx || ret) { 83 CAAM_JR_DP_WARN("Alloc sec descriptor failed!"); 84 return NULL; 85 } 86 /* 87 * Clear SG memory. There are 16 SG entries of 16 Bytes each. 88 * one call to dcbz_64() clear 64 bytes, hence calling it 4 times 89 * to clear all the SG entries. caam_jr_alloc_ctx() is called for 90 * each packet, memset is costlier than dcbz_64(). 91 */ 92 dcbz_64(&ctx->sg[SG_CACHELINE_0]); 93 dcbz_64(&ctx->sg[SG_CACHELINE_1]); 94 dcbz_64(&ctx->sg[SG_CACHELINE_2]); 95 dcbz_64(&ctx->sg[SG_CACHELINE_3]); 96 97 ctx->ctx_pool = ses->ctx_pool; 98 ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx); 99 100 return ctx; 101 } 102 103 static 104 void caam_jr_stats_get(struct rte_cryptodev *dev, 105 struct rte_cryptodev_stats *stats) 106 { 107 struct caam_jr_qp **qp = (struct caam_jr_qp **) 108 dev->data->queue_pairs; 109 int i; 110 111 PMD_INIT_FUNC_TRACE(); 112 if (stats == NULL) { 113 CAAM_JR_ERR("Invalid stats ptr NULL"); 114 return; 115 } 116 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 117 if (qp[i] == NULL) { 118 CAAM_JR_WARN("Uninitialised queue pair"); 119 continue; 120 } 121 122 stats->enqueued_count += qp[i]->tx_pkts; 123 stats->dequeued_count += qp[i]->rx_pkts; 124 stats->enqueue_err_count += qp[i]->tx_errs; 125 stats->dequeue_err_count += qp[i]->rx_errs; 126 CAAM_JR_INFO("extra stats:\n\tRX Poll ERR = %" PRIu64 127 "\n\tTX Ring Full = %" PRIu64, 128 qp[i]->rx_poll_err, 129 qp[i]->tx_ring_full); 130 } 131 } 132 133 static 134 void caam_jr_stats_reset(struct rte_cryptodev *dev) 135 { 136 int i; 137 struct caam_jr_qp **qp = (struct caam_jr_qp **) 138 (dev->data->queue_pairs); 139 140 PMD_INIT_FUNC_TRACE(); 141 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 142 if (qp[i] == NULL) { 143 CAAM_JR_WARN("Uninitialised queue pair"); 144 continue; 145 } 146 qp[i]->rx_pkts = 0; 147 qp[i]->rx_errs = 0; 148 qp[i]->rx_poll_err = 0; 149 qp[i]->tx_pkts = 0; 150 qp[i]->tx_errs = 0; 151 qp[i]->tx_ring_full = 0; 152 } 153 } 154 155 static inline int 156 is_cipher_only(struct caam_jr_session *ses) 157 { 158 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && 159 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL)); 160 } 161 162 static inline int 163 is_auth_only(struct caam_jr_session *ses) 164 { 165 return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) && 166 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); 167 } 168 169 static inline int 170 is_aead(struct caam_jr_session *ses) 171 { 172 return ((ses->cipher_alg == 0) && 173 (ses->auth_alg == 0) && 174 (ses->aead_alg != 0)); 175 } 176 177 static inline int 178 is_auth_cipher(struct caam_jr_session *ses) 179 { 180 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && 181 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) && 182 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC)); 183 } 184 185 static inline int 186 is_proto_ipsec(struct caam_jr_session *ses) 187 { 188 return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC); 189 } 190 191 static inline int 192 is_encode(struct caam_jr_session *ses) 193 { 194 return ses->dir == DIR_ENC; 195 } 196 197 static inline int 198 is_decode(struct caam_jr_session *ses) 199 { 200 return ses->dir == DIR_DEC; 201 } 202 203 static inline void 204 caam_auth_alg(struct caam_jr_session *ses, struct alginfo *alginfo_a) 205 { 206 switch (ses->auth_alg) { 207 case RTE_CRYPTO_AUTH_NULL: 208 ses->digest_length = 0; 209 break; 210 case RTE_CRYPTO_AUTH_MD5_HMAC: 211 alginfo_a->algtype = 212 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 213 OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5; 214 alginfo_a->algmode = OP_ALG_AAI_HMAC; 215 break; 216 case RTE_CRYPTO_AUTH_SHA1_HMAC: 217 alginfo_a->algtype = 218 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 219 OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1; 220 alginfo_a->algmode = OP_ALG_AAI_HMAC; 221 break; 222 case RTE_CRYPTO_AUTH_SHA224_HMAC: 223 alginfo_a->algtype = 224 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 225 OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224; 226 alginfo_a->algmode = OP_ALG_AAI_HMAC; 227 break; 228 case RTE_CRYPTO_AUTH_SHA256_HMAC: 229 alginfo_a->algtype = 230 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 231 OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256; 232 alginfo_a->algmode = OP_ALG_AAI_HMAC; 233 break; 234 case RTE_CRYPTO_AUTH_SHA384_HMAC: 235 alginfo_a->algtype = 236 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 237 OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384; 238 alginfo_a->algmode = OP_ALG_AAI_HMAC; 239 break; 240 case RTE_CRYPTO_AUTH_SHA512_HMAC: 241 alginfo_a->algtype = 242 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 243 OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512; 244 alginfo_a->algmode = OP_ALG_AAI_HMAC; 245 break; 246 default: 247 CAAM_JR_DEBUG("unsupported auth alg %u", ses->auth_alg); 248 } 249 } 250 251 static inline void 252 caam_cipher_alg(struct caam_jr_session *ses, struct alginfo *alginfo_c) 253 { 254 switch (ses->cipher_alg) { 255 case RTE_CRYPTO_CIPHER_NULL: 256 break; 257 case RTE_CRYPTO_CIPHER_AES_CBC: 258 alginfo_c->algtype = 259 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 260 OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES; 261 alginfo_c->algmode = OP_ALG_AAI_CBC; 262 break; 263 case RTE_CRYPTO_CIPHER_3DES_CBC: 264 alginfo_c->algtype = 265 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 266 OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES; 267 alginfo_c->algmode = OP_ALG_AAI_CBC; 268 break; 269 case RTE_CRYPTO_CIPHER_AES_CTR: 270 alginfo_c->algtype = 271 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 272 OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES; 273 alginfo_c->algmode = OP_ALG_AAI_CTR; 274 break; 275 default: 276 CAAM_JR_DEBUG("unsupported cipher alg %d", ses->cipher_alg); 277 } 278 } 279 280 static inline void 281 caam_aead_alg(struct caam_jr_session *ses, struct alginfo *alginfo) 282 { 283 switch (ses->aead_alg) { 284 case RTE_CRYPTO_AEAD_AES_GCM: 285 alginfo->algtype = OP_ALG_ALGSEL_AES; 286 alginfo->algmode = OP_ALG_AAI_GCM; 287 break; 288 default: 289 CAAM_JR_DEBUG("unsupported AEAD alg %d", ses->aead_alg); 290 } 291 } 292 293 /* prepare command block of the session */ 294 static int 295 caam_jr_prep_cdb(struct caam_jr_session *ses) 296 { 297 struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0}; 298 int32_t shared_desc_len = 0; 299 struct sec_cdb *cdb; 300 int err; 301 #if CAAM_BYTE_ORDER == CORE_BYTE_ORDER 302 int swap = false; 303 #else 304 int swap = true; 305 #endif 306 307 if (ses->cdb) 308 caam_jr_dma_free(ses->cdb); 309 310 cdb = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, sizeof(struct sec_cdb)); 311 if (!cdb) { 312 CAAM_JR_ERR("failed to allocate memory for cdb\n"); 313 return -1; 314 } 315 316 ses->cdb = cdb; 317 318 memset(cdb, 0, sizeof(struct sec_cdb)); 319 320 if (is_cipher_only(ses)) { 321 caam_cipher_alg(ses, &alginfo_c); 322 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) { 323 CAAM_JR_ERR("not supported cipher alg"); 324 rte_free(cdb); 325 return -ENOTSUP; 326 } 327 328 alginfo_c.key = (size_t)ses->cipher_key.data; 329 alginfo_c.keylen = ses->cipher_key.length; 330 alginfo_c.key_enc_flags = 0; 331 alginfo_c.key_type = RTA_DATA_IMM; 332 333 shared_desc_len = cnstr_shdsc_blkcipher( 334 cdb->sh_desc, true, 335 swap, SHR_NEVER, &alginfo_c, 336 ses->iv.length, 337 ses->dir); 338 } else if (is_auth_only(ses)) { 339 caam_auth_alg(ses, &alginfo_a); 340 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) { 341 CAAM_JR_ERR("not supported auth alg"); 342 rte_free(cdb); 343 return -ENOTSUP; 344 } 345 346 alginfo_a.key = (size_t)ses->auth_key.data; 347 alginfo_a.keylen = ses->auth_key.length; 348 alginfo_a.key_enc_flags = 0; 349 alginfo_a.key_type = RTA_DATA_IMM; 350 351 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true, 352 swap, SHR_NEVER, &alginfo_a, 353 !ses->dir, 354 ses->digest_length); 355 } else if (is_aead(ses)) { 356 caam_aead_alg(ses, &alginfo); 357 if (alginfo.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) { 358 CAAM_JR_ERR("not supported aead alg"); 359 rte_free(cdb); 360 return -ENOTSUP; 361 } 362 alginfo.key = (size_t)ses->aead_key.data; 363 alginfo.keylen = ses->aead_key.length; 364 alginfo.key_enc_flags = 0; 365 alginfo.key_type = RTA_DATA_IMM; 366 367 if (ses->dir == DIR_ENC) 368 shared_desc_len = cnstr_shdsc_gcm_encap( 369 cdb->sh_desc, true, swap, 370 SHR_NEVER, &alginfo, 371 ses->iv.length, 372 ses->digest_length); 373 else 374 shared_desc_len = cnstr_shdsc_gcm_decap( 375 cdb->sh_desc, true, swap, 376 SHR_NEVER, &alginfo, 377 ses->iv.length, 378 ses->digest_length); 379 } else { 380 caam_cipher_alg(ses, &alginfo_c); 381 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) { 382 CAAM_JR_ERR("not supported cipher alg"); 383 rte_free(cdb); 384 return -ENOTSUP; 385 } 386 387 alginfo_c.key = (size_t)ses->cipher_key.data; 388 alginfo_c.keylen = ses->cipher_key.length; 389 alginfo_c.key_enc_flags = 0; 390 alginfo_c.key_type = RTA_DATA_IMM; 391 392 caam_auth_alg(ses, &alginfo_a); 393 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) { 394 CAAM_JR_ERR("not supported auth alg"); 395 rte_free(cdb); 396 return -ENOTSUP; 397 } 398 399 alginfo_a.key = (size_t)ses->auth_key.data; 400 alginfo_a.keylen = ses->auth_key.length; 401 alginfo_a.key_enc_flags = 0; 402 alginfo_a.key_type = RTA_DATA_IMM; 403 404 cdb->sh_desc[0] = alginfo_c.keylen; 405 cdb->sh_desc[1] = alginfo_a.keylen; 406 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, 407 MIN_JOB_DESC_SIZE, 408 (unsigned int *)cdb->sh_desc, 409 &cdb->sh_desc[2], 2); 410 411 if (err < 0) { 412 CAAM_JR_ERR("Crypto: Incorrect key lengths"); 413 rte_free(cdb); 414 return err; 415 } 416 if (cdb->sh_desc[2] & 1) 417 alginfo_c.key_type = RTA_DATA_IMM; 418 else { 419 alginfo_c.key = (size_t)caam_jr_mem_vtop( 420 (void *)(size_t)alginfo_c.key); 421 alginfo_c.key_type = RTA_DATA_PTR; 422 } 423 if (cdb->sh_desc[2] & (1<<1)) 424 alginfo_a.key_type = RTA_DATA_IMM; 425 else { 426 alginfo_a.key = (size_t)caam_jr_mem_vtop( 427 (void *)(size_t)alginfo_a.key); 428 alginfo_a.key_type = RTA_DATA_PTR; 429 } 430 cdb->sh_desc[0] = 0; 431 cdb->sh_desc[1] = 0; 432 cdb->sh_desc[2] = 0; 433 if (is_proto_ipsec(ses)) { 434 if (ses->dir == DIR_ENC) { 435 shared_desc_len = cnstr_shdsc_ipsec_new_encap( 436 cdb->sh_desc, 437 true, swap, SHR_SERIAL, 438 &ses->encap_pdb, 439 (uint8_t *)&ses->ip4_hdr, 440 &alginfo_c, &alginfo_a); 441 } else if (ses->dir == DIR_DEC) { 442 shared_desc_len = cnstr_shdsc_ipsec_new_decap( 443 cdb->sh_desc, 444 true, swap, SHR_SERIAL, 445 &ses->decap_pdb, 446 &alginfo_c, &alginfo_a); 447 } 448 } else { 449 /* Auth_only_len is overwritten in fd for each job */ 450 shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, 451 true, swap, SHR_SERIAL, 452 &alginfo_c, &alginfo_a, 453 ses->iv.length, 454 ses->digest_length, ses->dir); 455 } 456 } 457 458 if (shared_desc_len < 0) { 459 CAAM_JR_ERR("error in preparing command block"); 460 return shared_desc_len; 461 } 462 463 #if CAAM_JR_DBG 464 SEC_DUMP_DESC(cdb->sh_desc); 465 #endif 466 467 cdb->sh_hdr.hi.field.idlen = shared_desc_len; 468 469 return 0; 470 } 471 472 /* @brief Poll the HW for already processed jobs in the JR 473 * and silently discard the available jobs or notify them to UA 474 * with indicated error code. 475 * 476 * @param [in,out] job_ring The job ring to poll. 477 * @param [in] do_notify Can be #TRUE or #FALSE. Indicates if 478 * descriptors are to be discarded 479 * or notified to UA with given error_code. 480 * @param [out] notified_descs Number of notified descriptors. Can be NULL 481 * if do_notify is #FALSE 482 */ 483 static void 484 hw_flush_job_ring(struct sec_job_ring_t *job_ring, 485 uint32_t do_notify, 486 uint32_t *notified_descs) 487 { 488 int32_t jobs_no_to_discard = 0; 489 int32_t discarded_descs_no = 0; 490 491 CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]", 492 job_ring, job_ring->pidx, job_ring->cidx, do_notify); 493 494 jobs_no_to_discard = hw_get_no_finished_jobs(job_ring); 495 496 /* Discard all jobs */ 497 CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs", 498 job_ring, job_ring->pidx, job_ring->cidx, 499 jobs_no_to_discard); 500 501 while (jobs_no_to_discard > discarded_descs_no) { 502 discarded_descs_no++; 503 /* Now increment the consumer index for the current job ring, 504 * AFTER saving job in temporary location! 505 * Increment the consumer index for the current job ring 506 */ 507 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx, 508 SEC_JOB_RING_SIZE); 509 510 hw_remove_entries(job_ring, 1); 511 } 512 513 if (do_notify == true) { 514 ASSERT(notified_descs != NULL); 515 *notified_descs = discarded_descs_no; 516 } 517 } 518 519 /* @brief Poll the HW for already processed jobs in the JR 520 * and notify the available jobs to UA. 521 * 522 * @param [in] job_ring The job ring to poll. 523 * @param [in] limit The maximum number of jobs to notify. 524 * If set to negative value, all available jobs are 525 * notified. 526 * 527 * @retval >=0 for No of jobs notified to UA. 528 * @retval -1 for error 529 */ 530 static int 531 hw_poll_job_ring(struct sec_job_ring_t *job_ring, 532 struct rte_crypto_op **ops, int32_t limit, 533 struct caam_jr_qp *jr_qp) 534 { 535 int32_t jobs_no_to_notify = 0; /* the number of done jobs to notify*/ 536 int32_t number_of_jobs_available = 0; 537 int32_t notified_descs_no = 0; 538 uint32_t sec_error_code = 0; 539 struct job_descriptor *current_desc; 540 phys_addr_t current_desc_addr; 541 phys_addr_t *temp_addr; 542 struct caam_jr_op_ctx *ctx; 543 544 /* TODO check for ops have memory*/ 545 /* check here if any JR error that cannot be written 546 * in the output status word has occurred 547 */ 548 if (JR_REG_JRINT_JRE_EXTRACT(GET_JR_REG(JRINT, job_ring))) { 549 CAAM_JR_INFO("err received"); 550 sec_error_code = JR_REG_JRINT_ERR_TYPE_EXTRACT( 551 GET_JR_REG(JRINT, job_ring)); 552 if (unlikely(sec_error_code)) { 553 hw_job_ring_error_print(job_ring, sec_error_code); 554 return -1; 555 } 556 } 557 /* compute the number of jobs available in the job ring based on the 558 * producer and consumer index values. 559 */ 560 number_of_jobs_available = hw_get_no_finished_jobs(job_ring); 561 /* Compute the number of notifications that need to be raised to UA 562 * If limit > total number of done jobs -> notify all done jobs 563 * If limit = 0 -> error 564 * If limit < total number of done jobs -> notify a number 565 * of done jobs equal with limit 566 */ 567 jobs_no_to_notify = (limit > number_of_jobs_available) ? 568 number_of_jobs_available : limit; 569 CAAM_JR_DP_DEBUG( 570 "Jr[%p] pi[%d] ci[%d].limit =%d Available=%d.Jobs to notify=%d", 571 job_ring, job_ring->pidx, job_ring->cidx, 572 limit, number_of_jobs_available, jobs_no_to_notify); 573 574 rte_smp_rmb(); 575 576 while (jobs_no_to_notify > notified_descs_no) { 577 static uint64_t false_alarm; 578 static uint64_t real_poll; 579 580 /* Get job status here */ 581 sec_error_code = job_ring->output_ring[job_ring->cidx].status; 582 /* Get completed descriptor */ 583 temp_addr = &(job_ring->output_ring[job_ring->cidx].desc); 584 current_desc_addr = (phys_addr_t)sec_read_addr(temp_addr); 585 586 real_poll++; 587 /* todo check if it is false alarm no desc present */ 588 if (!current_desc_addr) { 589 false_alarm++; 590 printf("false alarm %" PRIu64 "real %" PRIu64 591 " sec_err =0x%x cidx Index =0%d\n", 592 false_alarm, real_poll, 593 sec_error_code, job_ring->cidx); 594 rte_panic("CAAM JR descriptor NULL"); 595 return notified_descs_no; 596 } 597 current_desc = (struct job_descriptor *) 598 caam_jr_dma_ptov(current_desc_addr); 599 /* now increment the consumer index for the current job ring, 600 * AFTER saving job in temporary location! 601 */ 602 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx, 603 SEC_JOB_RING_SIZE); 604 /* Signal that the job has been processed and the slot is free*/ 605 hw_remove_entries(job_ring, 1); 606 /*TODO for multiple ops, packets*/ 607 ctx = container_of(current_desc, struct caam_jr_op_ctx, jobdes); 608 if (unlikely(sec_error_code)) { 609 CAAM_JR_ERR("desc at cidx %d generated error 0x%x\n", 610 job_ring->cidx, sec_error_code); 611 hw_handle_job_ring_error(job_ring, sec_error_code); 612 //todo improve with exact errors 613 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; 614 jr_qp->rx_errs++; 615 } else { 616 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 617 #if CAAM_JR_DBG 618 if (ctx->op->sym->m_dst) { 619 rte_hexdump(stdout, "PROCESSED", 620 rte_pktmbuf_mtod(ctx->op->sym->m_dst, void *), 621 rte_pktmbuf_data_len(ctx->op->sym->m_dst)); 622 } else { 623 rte_hexdump(stdout, "PROCESSED", 624 rte_pktmbuf_mtod(ctx->op->sym->m_src, void *), 625 rte_pktmbuf_data_len(ctx->op->sym->m_src)); 626 } 627 #endif 628 } 629 if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 630 struct ip *ip4_hdr; 631 632 if (ctx->op->sym->m_dst) { 633 /*TODO check for ip header or other*/ 634 ip4_hdr = (struct ip *) 635 rte_pktmbuf_mtod(ctx->op->sym->m_dst, char*); 636 ctx->op->sym->m_dst->pkt_len = 637 rte_be_to_cpu_16(ip4_hdr->ip_len); 638 ctx->op->sym->m_dst->data_len = 639 rte_be_to_cpu_16(ip4_hdr->ip_len); 640 } else { 641 ip4_hdr = (struct ip *) 642 rte_pktmbuf_mtod(ctx->op->sym->m_src, char*); 643 ctx->op->sym->m_src->pkt_len = 644 rte_be_to_cpu_16(ip4_hdr->ip_len); 645 ctx->op->sym->m_src->data_len = 646 rte_be_to_cpu_16(ip4_hdr->ip_len); 647 } 648 } 649 *ops = ctx->op; 650 caam_jr_op_ending(ctx); 651 ops++; 652 notified_descs_no++; 653 } 654 return notified_descs_no; 655 } 656 657 static uint16_t 658 caam_jr_dequeue_burst(void *qp, struct rte_crypto_op **ops, 659 uint16_t nb_ops) 660 { 661 struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp; 662 struct sec_job_ring_t *ring = jr_qp->ring; 663 int num_rx; 664 int ret; 665 666 CAAM_JR_DP_DEBUG("Jr[%p]Polling. limit[%d]", ring, nb_ops); 667 668 /* Poll job ring 669 * If nb_ops < 0 -> poll JR until no more notifications are available. 670 * If nb_ops > 0 -> poll JR until limit is reached. 671 */ 672 673 /* Run hw poll job ring */ 674 num_rx = hw_poll_job_ring(ring, ops, nb_ops, jr_qp); 675 if (num_rx < 0) { 676 CAAM_JR_ERR("Error polling SEC engine (%d)", num_rx); 677 return 0; 678 } 679 680 CAAM_JR_DP_DEBUG("Jr[%p].Jobs notified[%d]. ", ring, num_rx); 681 682 if (ring->jr_mode == SEC_NOTIFICATION_TYPE_NAPI) { 683 if (num_rx < nb_ops) { 684 ret = caam_jr_enable_irqs(ring->irq_fd); 685 SEC_ASSERT(ret == 0, ret, 686 "Failed to enable irqs for job ring %p", ring); 687 } 688 } else if (ring->jr_mode == SEC_NOTIFICATION_TYPE_IRQ) { 689 690 /* Always enable IRQ generation when in pure IRQ mode */ 691 ret = caam_jr_enable_irqs(ring->irq_fd); 692 SEC_ASSERT(ret == 0, ret, 693 "Failed to enable irqs for job ring %p", ring); 694 } 695 696 jr_qp->rx_pkts += num_rx; 697 698 return num_rx; 699 } 700 701 /** 702 * packet looks like: 703 * |<----data_len------->| 704 * |ip_header|ah_header|icv|payload| 705 * ^ 706 * | 707 * mbuf->pkt.data 708 */ 709 static inline struct caam_jr_op_ctx * 710 build_auth_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses) 711 { 712 struct rte_crypto_sym_op *sym = op->sym; 713 struct rte_mbuf *mbuf = sym->m_src; 714 struct caam_jr_op_ctx *ctx; 715 struct sec4_sg_entry *sg; 716 int length; 717 struct sec_cdb *cdb; 718 uint64_t sdesc_offset; 719 struct sec_job_descriptor_t *jobdescr; 720 uint8_t extra_segs; 721 722 if (is_decode(ses)) 723 extra_segs = 2; 724 else 725 extra_segs = 1; 726 727 if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) { 728 CAAM_JR_DP_ERR("Auth: Max sec segs supported is %d", 729 MAX_SG_ENTRIES); 730 return NULL; 731 } 732 733 ctx = caam_jr_alloc_ctx(ses); 734 if (!ctx) 735 return NULL; 736 737 ctx->op = op; 738 739 cdb = ses->cdb; 740 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); 741 742 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; 743 744 SEC_JD_INIT(jobdescr); 745 SEC_JD_SET_SD(jobdescr, 746 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, 747 cdb->sh_hdr.hi.field.idlen); 748 749 /* output */ 750 SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr, 751 0, ses->digest_length); 752 753 /*input */ 754 sg = &ctx->sg[0]; 755 length = sym->auth.data.length; 756 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + sym->auth.data.offset); 757 sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset); 758 759 /* Successive segs */ 760 mbuf = mbuf->next; 761 while (mbuf) { 762 sg++; 763 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)); 764 sg->len = cpu_to_caam32(mbuf->data_len); 765 mbuf = mbuf->next; 766 } 767 768 if (is_decode(ses)) { 769 /* digest verification case */ 770 sg++; 771 /* hash result or digest, save digest first */ 772 rte_memcpy(ctx->digest, sym->auth.digest.data, 773 ses->digest_length); 774 #if CAAM_JR_DBG 775 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length); 776 #endif 777 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest)); 778 sg->len = cpu_to_caam32(ses->digest_length); 779 length += ses->digest_length; 780 } else { 781 sg->len -= ses->digest_length; 782 } 783 784 /* last element*/ 785 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 786 787 SEC_JD_SET_IN_PTR(jobdescr, 788 (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, length); 789 /* enabling sg list */ 790 (jobdescr)->seq_in.command.word |= 0x01000000; 791 792 return ctx; 793 } 794 795 static inline struct caam_jr_op_ctx * 796 build_auth_only(struct rte_crypto_op *op, struct caam_jr_session *ses) 797 { 798 struct rte_crypto_sym_op *sym = op->sym; 799 struct caam_jr_op_ctx *ctx; 800 struct sec4_sg_entry *sg; 801 rte_iova_t start_addr; 802 struct sec_cdb *cdb; 803 uint64_t sdesc_offset; 804 struct sec_job_descriptor_t *jobdescr; 805 806 ctx = caam_jr_alloc_ctx(ses); 807 if (!ctx) 808 return NULL; 809 810 ctx->op = op; 811 812 cdb = ses->cdb; 813 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); 814 815 start_addr = rte_pktmbuf_iova(sym->m_src); 816 817 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; 818 819 SEC_JD_INIT(jobdescr); 820 SEC_JD_SET_SD(jobdescr, 821 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, 822 cdb->sh_hdr.hi.field.idlen); 823 824 /* output */ 825 SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr, 826 0, ses->digest_length); 827 828 /*input */ 829 if (is_decode(ses)) { 830 sg = &ctx->sg[0]; 831 SEC_JD_SET_IN_PTR(jobdescr, 832 (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0, 833 (sym->auth.data.length + ses->digest_length)); 834 /* enabling sg list */ 835 (jobdescr)->seq_in.command.word |= 0x01000000; 836 837 /* hash result or digest, save digest first */ 838 rte_memcpy(ctx->digest, sym->auth.digest.data, 839 ses->digest_length); 840 sg->ptr = cpu_to_caam64(start_addr + sym->auth.data.offset); 841 sg->len = cpu_to_caam32(sym->auth.data.length); 842 843 #if CAAM_JR_DBG 844 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length); 845 #endif 846 /* let's check digest by hw */ 847 sg++; 848 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest)); 849 sg->len = cpu_to_caam32(ses->digest_length); 850 /* last element*/ 851 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 852 } else { 853 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)start_addr, 854 sym->auth.data.offset, sym->auth.data.length); 855 } 856 return ctx; 857 } 858 859 static inline struct caam_jr_op_ctx * 860 build_cipher_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses) 861 { 862 struct rte_crypto_sym_op *sym = op->sym; 863 struct rte_mbuf *mbuf = sym->m_src; 864 struct caam_jr_op_ctx *ctx; 865 struct sec4_sg_entry *sg, *in_sg; 866 int length; 867 struct sec_cdb *cdb; 868 uint64_t sdesc_offset; 869 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 870 ses->iv.offset); 871 struct sec_job_descriptor_t *jobdescr; 872 uint8_t reg_segs; 873 874 if (sym->m_dst) { 875 mbuf = sym->m_dst; 876 reg_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2; 877 } else { 878 mbuf = sym->m_src; 879 reg_segs = mbuf->nb_segs * 2 + 2; 880 } 881 882 if (reg_segs > MAX_SG_ENTRIES) { 883 CAAM_JR_DP_ERR("Cipher: Max sec segs supported is %d", 884 MAX_SG_ENTRIES); 885 return NULL; 886 } 887 888 ctx = caam_jr_alloc_ctx(ses); 889 if (!ctx) 890 return NULL; 891 892 ctx->op = op; 893 cdb = ses->cdb; 894 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); 895 896 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; 897 898 SEC_JD_INIT(jobdescr); 899 SEC_JD_SET_SD(jobdescr, 900 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, 901 cdb->sh_hdr.hi.field.idlen); 902 903 #if CAAM_JR_DBG 904 CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d", 905 sym->m_src->data_off, sym->cipher.data.offset, 906 sym->cipher.data.length, ses->iv.length); 907 #endif 908 /* output */ 909 if (sym->m_dst) 910 mbuf = sym->m_dst; 911 else 912 mbuf = sym->m_src; 913 914 sg = &ctx->sg[0]; 915 length = sym->cipher.data.length; 916 917 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) 918 + sym->cipher.data.offset); 919 sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset); 920 921 /* Successive segs */ 922 mbuf = mbuf->next; 923 while (mbuf) { 924 sg++; 925 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)); 926 sg->len = cpu_to_caam32(mbuf->data_len); 927 mbuf = mbuf->next; 928 } 929 /* last element*/ 930 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 931 932 SEC_JD_SET_OUT_PTR(jobdescr, 933 (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, 934 length); 935 /*enabling sg bit */ 936 (jobdescr)->seq_out.command.word |= 0x01000000; 937 938 /*input */ 939 sg++; 940 mbuf = sym->m_src; 941 in_sg = sg; 942 943 length = sym->cipher.data.length + ses->iv.length; 944 945 /* IV */ 946 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr)); 947 sg->len = cpu_to_caam32(ses->iv.length); 948 949 /* 1st seg */ 950 sg++; 951 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) 952 + sym->cipher.data.offset); 953 sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset); 954 955 /* Successive segs */ 956 mbuf = mbuf->next; 957 while (mbuf) { 958 sg++; 959 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)); 960 sg->len = cpu_to_caam32(mbuf->data_len); 961 mbuf = mbuf->next; 962 } 963 /* last element*/ 964 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 965 966 967 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, in_sg), 0, 968 length); 969 /*enabling sg bit */ 970 (jobdescr)->seq_in.command.word |= 0x01000000; 971 972 return ctx; 973 } 974 975 static inline struct caam_jr_op_ctx * 976 build_cipher_only(struct rte_crypto_op *op, struct caam_jr_session *ses) 977 { 978 struct rte_crypto_sym_op *sym = op->sym; 979 struct caam_jr_op_ctx *ctx; 980 struct sec4_sg_entry *sg; 981 rte_iova_t src_start_addr, dst_start_addr; 982 struct sec_cdb *cdb; 983 uint64_t sdesc_offset; 984 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 985 ses->iv.offset); 986 struct sec_job_descriptor_t *jobdescr; 987 988 ctx = caam_jr_alloc_ctx(ses); 989 if (!ctx) 990 return NULL; 991 992 ctx->op = op; 993 cdb = ses->cdb; 994 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); 995 996 src_start_addr = rte_pktmbuf_iova(sym->m_src); 997 if (sym->m_dst) 998 dst_start_addr = rte_pktmbuf_iova(sym->m_dst); 999 else 1000 dst_start_addr = src_start_addr; 1001 1002 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; 1003 1004 SEC_JD_INIT(jobdescr); 1005 SEC_JD_SET_SD(jobdescr, 1006 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, 1007 cdb->sh_hdr.hi.field.idlen); 1008 1009 #if CAAM_JR_DBG 1010 CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d", 1011 sym->m_src->data_off, sym->cipher.data.offset, 1012 sym->cipher.data.length, ses->iv.length); 1013 #endif 1014 /* output */ 1015 SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 1016 sym->cipher.data.offset, 1017 sym->cipher.data.length + ses->iv.length); 1018 1019 /*input */ 1020 sg = &ctx->sg[0]; 1021 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0, 1022 sym->cipher.data.length + ses->iv.length); 1023 /*enabling sg bit */ 1024 (jobdescr)->seq_in.command.word |= 0x01000000; 1025 1026 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr)); 1027 sg->len = cpu_to_caam32(ses->iv.length); 1028 1029 sg = &ctx->sg[1]; 1030 sg->ptr = cpu_to_caam64(src_start_addr + sym->cipher.data.offset); 1031 sg->len = cpu_to_caam32(sym->cipher.data.length); 1032 /* last element*/ 1033 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 1034 1035 return ctx; 1036 } 1037 1038 /* For decapsulation: 1039 * Input: 1040 * +----+----------------+--------------------------------+-----+ 1041 * | IV | Auth-only data | Authenticated & Encrypted data | ICV | 1042 * +----+----------------+--------------------------------+-----+ 1043 * Output: 1044 * +----+--------------------------+ 1045 * | Decrypted & authenticated data | 1046 * +----+--------------------------+ 1047 */ 1048 1049 static inline struct caam_jr_op_ctx * 1050 build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses) 1051 { 1052 struct rte_crypto_sym_op *sym = op->sym; 1053 struct caam_jr_op_ctx *ctx; 1054 struct sec4_sg_entry *sg, *out_sg, *in_sg; 1055 struct rte_mbuf *mbuf; 1056 uint32_t length = 0; 1057 struct sec_cdb *cdb; 1058 uint64_t sdesc_offset; 1059 uint8_t req_segs; 1060 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1061 ses->iv.offset); 1062 struct sec_job_descriptor_t *jobdescr; 1063 uint16_t auth_hdr_len = sym->cipher.data.offset - 1064 sym->auth.data.offset; 1065 uint16_t auth_tail_len = sym->auth.data.length - 1066 sym->cipher.data.length - auth_hdr_len; 1067 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len; 1068 1069 if (sym->m_dst) { 1070 mbuf = sym->m_dst; 1071 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3; 1072 } else { 1073 mbuf = sym->m_src; 1074 req_segs = mbuf->nb_segs * 2 + 3; 1075 } 1076 1077 if (req_segs > MAX_SG_ENTRIES) { 1078 CAAM_JR_DP_ERR("Cipher-Auth: Max sec segs supported is %d", 1079 MAX_SG_ENTRIES); 1080 return NULL; 1081 } 1082 1083 ctx = caam_jr_alloc_ctx(ses); 1084 if (!ctx) 1085 return NULL; 1086 1087 ctx->op = op; 1088 cdb = ses->cdb; 1089 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); 1090 1091 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; 1092 1093 SEC_JD_INIT(jobdescr); 1094 SEC_JD_SET_SD(jobdescr, 1095 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, 1096 cdb->sh_hdr.hi.field.idlen); 1097 1098 /* output */ 1099 if (sym->m_dst) 1100 mbuf = sym->m_dst; 1101 else 1102 mbuf = sym->m_src; 1103 1104 out_sg = &ctx->sg[0]; 1105 if (is_encode(ses)) 1106 length = sym->auth.data.length + ses->digest_length; 1107 else 1108 length = sym->auth.data.length; 1109 1110 sg = &ctx->sg[0]; 1111 1112 /* 1st seg */ 1113 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) 1114 + sym->auth.data.offset); 1115 sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset); 1116 1117 /* Successive segs */ 1118 mbuf = mbuf->next; 1119 while (mbuf) { 1120 sg++; 1121 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)); 1122 sg->len = cpu_to_caam32(mbuf->data_len); 1123 mbuf = mbuf->next; 1124 } 1125 1126 if (is_encode(ses)) { 1127 /* set auth output */ 1128 sg++; 1129 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr); 1130 sg->len = cpu_to_caam32(ses->digest_length); 1131 } 1132 /* last element*/ 1133 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 1134 1135 SEC_JD_SET_OUT_PTR(jobdescr, 1136 (uint64_t)caam_jr_dma_vtop(out_sg), 0, length); 1137 /* set sg bit */ 1138 (jobdescr)->seq_out.command.word |= 0x01000000; 1139 1140 /* input */ 1141 sg++; 1142 mbuf = sym->m_src; 1143 in_sg = sg; 1144 if (is_encode(ses)) 1145 length = ses->iv.length + sym->auth.data.length; 1146 else 1147 length = ses->iv.length + sym->auth.data.length 1148 + ses->digest_length; 1149 1150 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr)); 1151 sg->len = cpu_to_caam32(ses->iv.length); 1152 1153 sg++; 1154 /* 1st seg */ 1155 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) 1156 + sym->auth.data.offset); 1157 sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset); 1158 1159 /* Successive segs */ 1160 mbuf = mbuf->next; 1161 while (mbuf) { 1162 sg++; 1163 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)); 1164 sg->len = cpu_to_caam32(mbuf->data_len); 1165 mbuf = mbuf->next; 1166 } 1167 1168 if (is_decode(ses)) { 1169 sg++; 1170 rte_memcpy(ctx->digest, sym->auth.digest.data, 1171 ses->digest_length); 1172 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest)); 1173 sg->len = cpu_to_caam32(ses->digest_length); 1174 } 1175 /* last element*/ 1176 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 1177 1178 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(in_sg), 0, 1179 length); 1180 /* set sg bit */ 1181 (jobdescr)->seq_in.command.word |= 0x01000000; 1182 /* Auth_only_len is set as 0 in descriptor and it is 1183 * overwritten here in the jd which will update 1184 * the DPOVRD reg. 1185 */ 1186 if (auth_only_len) 1187 /* set sg bit */ 1188 (jobdescr)->dpovrd = 0x80000000 | auth_only_len; 1189 1190 return ctx; 1191 } 1192 1193 static inline struct caam_jr_op_ctx * 1194 build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses) 1195 { 1196 struct rte_crypto_sym_op *sym = op->sym; 1197 struct caam_jr_op_ctx *ctx; 1198 struct sec4_sg_entry *sg; 1199 rte_iova_t src_start_addr, dst_start_addr; 1200 uint32_t length = 0; 1201 struct sec_cdb *cdb; 1202 uint64_t sdesc_offset; 1203 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1204 ses->iv.offset); 1205 struct sec_job_descriptor_t *jobdescr; 1206 uint16_t auth_hdr_len = sym->cipher.data.offset - 1207 sym->auth.data.offset; 1208 uint16_t auth_tail_len = sym->auth.data.length - 1209 sym->cipher.data.length - auth_hdr_len; 1210 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len; 1211 1212 src_start_addr = rte_pktmbuf_iova(sym->m_src); 1213 if (sym->m_dst) 1214 dst_start_addr = rte_pktmbuf_iova(sym->m_dst); 1215 else 1216 dst_start_addr = src_start_addr; 1217 1218 ctx = caam_jr_alloc_ctx(ses); 1219 if (!ctx) 1220 return NULL; 1221 1222 ctx->op = op; 1223 cdb = ses->cdb; 1224 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); 1225 1226 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; 1227 1228 SEC_JD_INIT(jobdescr); 1229 SEC_JD_SET_SD(jobdescr, 1230 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, 1231 cdb->sh_hdr.hi.field.idlen); 1232 1233 /* input */ 1234 sg = &ctx->sg[0]; 1235 if (is_encode(ses)) { 1236 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr)); 1237 sg->len = cpu_to_caam32(ses->iv.length); 1238 length += ses->iv.length; 1239 1240 sg++; 1241 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset); 1242 sg->len = cpu_to_caam32(sym->auth.data.length); 1243 length += sym->auth.data.length; 1244 /* last element*/ 1245 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 1246 } else { 1247 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr)); 1248 sg->len = cpu_to_caam32(ses->iv.length); 1249 length += ses->iv.length; 1250 1251 sg++; 1252 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset); 1253 sg->len = cpu_to_caam32(sym->auth.data.length); 1254 length += sym->auth.data.length; 1255 1256 rte_memcpy(ctx->digest, sym->auth.digest.data, 1257 ses->digest_length); 1258 sg++; 1259 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest)); 1260 sg->len = cpu_to_caam32(ses->digest_length); 1261 length += ses->digest_length; 1262 /* last element*/ 1263 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 1264 } 1265 1266 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(&ctx->sg[0]), 0, 1267 length); 1268 /* set sg bit */ 1269 (jobdescr)->seq_in.command.word |= 0x01000000; 1270 1271 /* output */ 1272 sg = &ctx->sg[6]; 1273 1274 sg->ptr = cpu_to_caam64(dst_start_addr + sym->cipher.data.offset); 1275 sg->len = cpu_to_caam32(sym->cipher.data.length); 1276 length = sym->cipher.data.length; 1277 1278 if (is_encode(ses)) { 1279 /* set auth output */ 1280 sg++; 1281 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr); 1282 sg->len = cpu_to_caam32(ses->digest_length); 1283 length += ses->digest_length; 1284 } 1285 /* last element*/ 1286 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); 1287 1288 SEC_JD_SET_OUT_PTR(jobdescr, 1289 (uint64_t)caam_jr_dma_vtop(&ctx->sg[6]), 0, length); 1290 /* set sg bit */ 1291 (jobdescr)->seq_out.command.word |= 0x01000000; 1292 1293 /* Auth_only_len is set as 0 in descriptor and it is 1294 * overwritten here in the jd which will update 1295 * the DPOVRD reg. 1296 */ 1297 if (auth_only_len) 1298 /* set sg bit */ 1299 (jobdescr)->dpovrd = 0x80000000 | auth_only_len; 1300 1301 return ctx; 1302 } 1303 1304 static inline struct caam_jr_op_ctx * 1305 build_proto(struct rte_crypto_op *op, struct caam_jr_session *ses) 1306 { 1307 struct rte_crypto_sym_op *sym = op->sym; 1308 struct caam_jr_op_ctx *ctx = NULL; 1309 phys_addr_t src_start_addr, dst_start_addr; 1310 struct sec_cdb *cdb; 1311 uint64_t sdesc_offset; 1312 struct sec_job_descriptor_t *jobdescr; 1313 1314 ctx = caam_jr_alloc_ctx(ses); 1315 if (!ctx) 1316 return NULL; 1317 ctx->op = op; 1318 1319 src_start_addr = rte_pktmbuf_iova(sym->m_src); 1320 if (sym->m_dst) 1321 dst_start_addr = rte_pktmbuf_iova(sym->m_dst); 1322 else 1323 dst_start_addr = src_start_addr; 1324 1325 cdb = ses->cdb; 1326 sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); 1327 1328 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; 1329 1330 SEC_JD_INIT(jobdescr); 1331 SEC_JD_SET_SD(jobdescr, 1332 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, 1333 cdb->sh_hdr.hi.field.idlen); 1334 1335 /* output */ 1336 SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 0, 1337 sym->m_src->buf_len - sym->m_src->data_off); 1338 /* input */ 1339 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)src_start_addr, 0, 1340 sym->m_src->pkt_len); 1341 sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK; 1342 1343 return ctx; 1344 } 1345 1346 static int 1347 caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp) 1348 { 1349 struct sec_job_ring_t *ring = qp->ring; 1350 struct caam_jr_session *ses; 1351 struct caam_jr_op_ctx *ctx = NULL; 1352 struct sec_job_descriptor_t *jobdescr __rte_unused; 1353 #if CAAM_JR_DBG 1354 int i; 1355 #endif 1356 1357 switch (op->sess_type) { 1358 case RTE_CRYPTO_OP_WITH_SESSION: 1359 ses = (struct caam_jr_session *) 1360 get_sym_session_private_data(op->sym->session, 1361 cryptodev_driver_id); 1362 break; 1363 case RTE_CRYPTO_OP_SECURITY_SESSION: 1364 ses = (struct caam_jr_session *) 1365 get_sec_session_private_data( 1366 op->sym->sec_session); 1367 break; 1368 default: 1369 CAAM_JR_DP_ERR("sessionless crypto op not supported"); 1370 qp->tx_errs++; 1371 return -1; 1372 } 1373 1374 if (unlikely(!ses->qp || ses->qp != qp)) { 1375 CAAM_JR_DP_DEBUG("Old:sess->qp=%p New qp = %p\n", ses->qp, qp); 1376 ses->qp = qp; 1377 caam_jr_prep_cdb(ses); 1378 } 1379 1380 if (rte_pktmbuf_is_contiguous(op->sym->m_src)) { 1381 if (is_auth_cipher(ses)) 1382 ctx = build_cipher_auth(op, ses); 1383 else if (is_aead(ses)) 1384 goto err1; 1385 else if (is_auth_only(ses)) 1386 ctx = build_auth_only(op, ses); 1387 else if (is_cipher_only(ses)) 1388 ctx = build_cipher_only(op, ses); 1389 else if (is_proto_ipsec(ses)) 1390 ctx = build_proto(op, ses); 1391 } else { 1392 if (is_auth_cipher(ses)) 1393 ctx = build_cipher_auth_sg(op, ses); 1394 else if (is_aead(ses)) 1395 goto err1; 1396 else if (is_auth_only(ses)) 1397 ctx = build_auth_only_sg(op, ses); 1398 else if (is_cipher_only(ses)) 1399 ctx = build_cipher_only_sg(op, ses); 1400 } 1401 err1: 1402 if (unlikely(!ctx)) { 1403 qp->tx_errs++; 1404 CAAM_JR_ERR("not supported sec op"); 1405 return -1; 1406 } 1407 #if CAAM_JR_DBG 1408 if (is_decode(ses)) 1409 rte_hexdump(stdout, "DECODE", 1410 rte_pktmbuf_mtod(op->sym->m_src, void *), 1411 rte_pktmbuf_data_len(op->sym->m_src)); 1412 else 1413 rte_hexdump(stdout, "ENCODE", 1414 rte_pktmbuf_mtod(op->sym->m_src, void *), 1415 rte_pktmbuf_data_len(op->sym->m_src)); 1416 1417 printf("\n JD before conversion\n"); 1418 for (i = 0; i < 12; i++) 1419 printf("\n 0x%08x", ctx->jobdes.desc[i]); 1420 #endif 1421 1422 CAAM_JR_DP_DEBUG("Jr[%p] pi[%d] ci[%d].Before sending desc", 1423 ring, ring->pidx, ring->cidx); 1424 1425 /* todo - do we want to retry */ 1426 if (SEC_JOB_RING_IS_FULL(ring->pidx, ring->cidx, 1427 SEC_JOB_RING_SIZE, SEC_JOB_RING_SIZE)) { 1428 CAAM_JR_DP_DEBUG("Ring FULL Jr[%p] pi[%d] ci[%d].Size = %d", 1429 ring, ring->pidx, ring->cidx, SEC_JOB_RING_SIZE); 1430 caam_jr_op_ending(ctx); 1431 qp->tx_ring_full++; 1432 return -EBUSY; 1433 } 1434 1435 #if CORE_BYTE_ORDER != CAAM_BYTE_ORDER 1436 jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; 1437 1438 jobdescr->deschdr.command.word = 1439 cpu_to_caam32(jobdescr->deschdr.command.word); 1440 jobdescr->sd_ptr = cpu_to_caam64(jobdescr->sd_ptr); 1441 jobdescr->seq_out.command.word = 1442 cpu_to_caam32(jobdescr->seq_out.command.word); 1443 jobdescr->seq_out_ptr = cpu_to_caam64(jobdescr->seq_out_ptr); 1444 jobdescr->out_ext_length = cpu_to_caam32(jobdescr->out_ext_length); 1445 jobdescr->seq_in.command.word = 1446 cpu_to_caam32(jobdescr->seq_in.command.word); 1447 jobdescr->seq_in_ptr = cpu_to_caam64(jobdescr->seq_in_ptr); 1448 jobdescr->in_ext_length = cpu_to_caam32(jobdescr->in_ext_length); 1449 jobdescr->load_dpovrd.command.word = 1450 cpu_to_caam32(jobdescr->load_dpovrd.command.word); 1451 jobdescr->dpovrd = cpu_to_caam32(jobdescr->dpovrd); 1452 #endif 1453 1454 /* Set ptr in input ring to current descriptor */ 1455 sec_write_addr(&ring->input_ring[ring->pidx], 1456 (phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc)); 1457 rte_smp_wmb(); 1458 1459 /* Notify HW that a new job is enqueued */ 1460 hw_enqueue_desc_on_job_ring(ring); 1461 1462 /* increment the producer index for the current job ring */ 1463 ring->pidx = SEC_CIRCULAR_COUNTER(ring->pidx, SEC_JOB_RING_SIZE); 1464 1465 return 0; 1466 } 1467 1468 static uint16_t 1469 caam_jr_enqueue_burst(void *qp, struct rte_crypto_op **ops, 1470 uint16_t nb_ops) 1471 { 1472 /* Function to transmit the frames to given device and queuepair */ 1473 uint32_t loop; 1474 int32_t ret; 1475 struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp; 1476 uint16_t num_tx = 0; 1477 /*Prepare each packet which is to be sent*/ 1478 for (loop = 0; loop < nb_ops; loop++) { 1479 ret = caam_jr_enqueue_op(ops[loop], jr_qp); 1480 if (!ret) 1481 num_tx++; 1482 } 1483 1484 jr_qp->tx_pkts += num_tx; 1485 1486 return num_tx; 1487 } 1488 1489 /* Release queue pair */ 1490 static int 1491 caam_jr_queue_pair_release(struct rte_cryptodev *dev, 1492 uint16_t qp_id) 1493 { 1494 struct sec_job_ring_t *internals; 1495 struct caam_jr_qp *qp = NULL; 1496 1497 PMD_INIT_FUNC_TRACE(); 1498 CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id); 1499 1500 internals = dev->data->dev_private; 1501 if (qp_id >= internals->max_nb_queue_pairs) { 1502 CAAM_JR_ERR("Max supported qpid %d", 1503 internals->max_nb_queue_pairs); 1504 return -EINVAL; 1505 } 1506 1507 qp = &internals->qps[qp_id]; 1508 qp->ring = NULL; 1509 dev->data->queue_pairs[qp_id] = NULL; 1510 1511 return 0; 1512 } 1513 1514 /* Setup a queue pair */ 1515 static int 1516 caam_jr_queue_pair_setup( 1517 struct rte_cryptodev *dev, uint16_t qp_id, 1518 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, 1519 __rte_unused int socket_id) 1520 { 1521 struct sec_job_ring_t *internals; 1522 struct caam_jr_qp *qp = NULL; 1523 1524 PMD_INIT_FUNC_TRACE(); 1525 CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf); 1526 1527 internals = dev->data->dev_private; 1528 if (qp_id >= internals->max_nb_queue_pairs) { 1529 CAAM_JR_ERR("Max supported qpid %d", 1530 internals->max_nb_queue_pairs); 1531 return -EINVAL; 1532 } 1533 1534 qp = &internals->qps[qp_id]; 1535 qp->ring = internals; 1536 dev->data->queue_pairs[qp_id] = qp; 1537 1538 return 0; 1539 } 1540 1541 /* Returns the size of the aesni gcm session structure */ 1542 static unsigned int 1543 caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 1544 { 1545 PMD_INIT_FUNC_TRACE(); 1546 1547 return sizeof(struct caam_jr_session); 1548 } 1549 1550 static int 1551 caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused, 1552 struct rte_crypto_sym_xform *xform, 1553 struct caam_jr_session *session) 1554 { 1555 session->cipher_alg = xform->cipher.algo; 1556 session->iv.length = xform->cipher.iv.length; 1557 session->iv.offset = xform->cipher.iv.offset; 1558 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, 1559 RTE_CACHE_LINE_SIZE); 1560 if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) { 1561 CAAM_JR_ERR("No Memory for cipher key\n"); 1562 return -ENOMEM; 1563 } 1564 session->cipher_key.length = xform->cipher.key.length; 1565 1566 memcpy(session->cipher_key.data, xform->cipher.key.data, 1567 xform->cipher.key.length); 1568 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 1569 DIR_ENC : DIR_DEC; 1570 1571 return 0; 1572 } 1573 1574 static int 1575 caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused, 1576 struct rte_crypto_sym_xform *xform, 1577 struct caam_jr_session *session) 1578 { 1579 session->auth_alg = xform->auth.algo; 1580 session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, 1581 RTE_CACHE_LINE_SIZE); 1582 if (session->auth_key.data == NULL && xform->auth.key.length > 0) { 1583 CAAM_JR_ERR("No Memory for auth key\n"); 1584 return -ENOMEM; 1585 } 1586 session->auth_key.length = xform->auth.key.length; 1587 session->digest_length = xform->auth.digest_length; 1588 1589 memcpy(session->auth_key.data, xform->auth.key.data, 1590 xform->auth.key.length); 1591 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? 1592 DIR_ENC : DIR_DEC; 1593 1594 return 0; 1595 } 1596 1597 static int 1598 caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused, 1599 struct rte_crypto_sym_xform *xform, 1600 struct caam_jr_session *session) 1601 { 1602 session->aead_alg = xform->aead.algo; 1603 session->iv.length = xform->aead.iv.length; 1604 session->iv.offset = xform->aead.iv.offset; 1605 session->auth_only_len = xform->aead.aad_length; 1606 session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length, 1607 RTE_CACHE_LINE_SIZE); 1608 if (session->aead_key.data == NULL && xform->aead.key.length > 0) { 1609 CAAM_JR_ERR("No Memory for aead key\n"); 1610 return -ENOMEM; 1611 } 1612 session->aead_key.length = xform->aead.key.length; 1613 session->digest_length = xform->aead.digest_length; 1614 1615 memcpy(session->aead_key.data, xform->aead.key.data, 1616 xform->aead.key.length); 1617 session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 1618 DIR_ENC : DIR_DEC; 1619 1620 return 0; 1621 } 1622 1623 static int 1624 caam_jr_set_session_parameters(struct rte_cryptodev *dev, 1625 struct rte_crypto_sym_xform *xform, void *sess) 1626 { 1627 struct sec_job_ring_t *internals = dev->data->dev_private; 1628 struct caam_jr_session *session = sess; 1629 1630 PMD_INIT_FUNC_TRACE(); 1631 1632 if (unlikely(sess == NULL)) { 1633 CAAM_JR_ERR("invalid session struct"); 1634 return -EINVAL; 1635 } 1636 1637 /* Default IV length = 0 */ 1638 session->iv.length = 0; 1639 1640 /* Cipher Only */ 1641 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { 1642 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 1643 caam_jr_cipher_init(dev, xform, session); 1644 1645 /* Authentication Only */ 1646 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1647 xform->next == NULL) { 1648 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; 1649 caam_jr_auth_init(dev, xform, session); 1650 1651 /* Cipher then Authenticate */ 1652 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1653 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1654 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1655 caam_jr_cipher_init(dev, xform, session); 1656 caam_jr_auth_init(dev, xform->next, session); 1657 } else { 1658 CAAM_JR_ERR("Not supported: Auth then Cipher"); 1659 goto err1; 1660 } 1661 1662 /* Authenticate then Cipher */ 1663 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1664 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1665 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 1666 caam_jr_auth_init(dev, xform, session); 1667 caam_jr_cipher_init(dev, xform->next, session); 1668 } else { 1669 CAAM_JR_ERR("Not supported: Auth then Cipher"); 1670 goto err1; 1671 } 1672 1673 /* AEAD operation for AES-GCM kind of Algorithms */ 1674 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && 1675 xform->next == NULL) { 1676 caam_jr_aead_init(dev, xform, session); 1677 1678 } else { 1679 CAAM_JR_ERR("Invalid crypto type"); 1680 return -EINVAL; 1681 } 1682 session->ctx_pool = internals->ctx_pool; 1683 1684 return 0; 1685 1686 err1: 1687 rte_free(session->cipher_key.data); 1688 rte_free(session->auth_key.data); 1689 memset(session, 0, sizeof(struct caam_jr_session)); 1690 1691 return -EINVAL; 1692 } 1693 1694 static int 1695 caam_jr_sym_session_configure(struct rte_cryptodev *dev, 1696 struct rte_crypto_sym_xform *xform, 1697 struct rte_cryptodev_sym_session *sess, 1698 struct rte_mempool *mempool) 1699 { 1700 void *sess_private_data; 1701 int ret; 1702 1703 PMD_INIT_FUNC_TRACE(); 1704 1705 if (rte_mempool_get(mempool, &sess_private_data)) { 1706 CAAM_JR_ERR("Couldn't get object from session mempool"); 1707 return -ENOMEM; 1708 } 1709 1710 memset(sess_private_data, 0, sizeof(struct caam_jr_session)); 1711 ret = caam_jr_set_session_parameters(dev, xform, sess_private_data); 1712 if (ret != 0) { 1713 CAAM_JR_ERR("failed to configure session parameters"); 1714 /* Return session to mempool */ 1715 rte_mempool_put(mempool, sess_private_data); 1716 return ret; 1717 } 1718 1719 set_sym_session_private_data(sess, dev->driver_id, sess_private_data); 1720 1721 return 0; 1722 } 1723 1724 /* Clear the memory of session so it doesn't leave key material behind */ 1725 static void 1726 caam_jr_sym_session_clear(struct rte_cryptodev *dev, 1727 struct rte_cryptodev_sym_session *sess) 1728 { 1729 uint8_t index = dev->driver_id; 1730 void *sess_priv = get_sym_session_private_data(sess, index); 1731 struct caam_jr_session *s = (struct caam_jr_session *)sess_priv; 1732 1733 PMD_INIT_FUNC_TRACE(); 1734 1735 if (sess_priv) { 1736 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 1737 1738 rte_free(s->cipher_key.data); 1739 rte_free(s->auth_key.data); 1740 memset(s, 0, sizeof(struct caam_jr_session)); 1741 set_sym_session_private_data(sess, index, NULL); 1742 rte_mempool_put(sess_mp, sess_priv); 1743 } 1744 } 1745 1746 static int 1747 caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, 1748 struct rte_security_session_conf *conf, 1749 void *sess) 1750 { 1751 struct sec_job_ring_t *internals = dev->data->dev_private; 1752 struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; 1753 struct rte_crypto_auth_xform *auth_xform; 1754 struct rte_crypto_cipher_xform *cipher_xform; 1755 struct caam_jr_session *session = (struct caam_jr_session *)sess; 1756 1757 PMD_INIT_FUNC_TRACE(); 1758 1759 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 1760 cipher_xform = &conf->crypto_xform->cipher; 1761 auth_xform = &conf->crypto_xform->next->auth; 1762 } else { 1763 auth_xform = &conf->crypto_xform->auth; 1764 cipher_xform = &conf->crypto_xform->next->cipher; 1765 } 1766 session->proto_alg = conf->protocol; 1767 session->cipher_key.data = rte_zmalloc(NULL, 1768 cipher_xform->key.length, 1769 RTE_CACHE_LINE_SIZE); 1770 if (session->cipher_key.data == NULL && 1771 cipher_xform->key.length > 0) { 1772 CAAM_JR_ERR("No Memory for cipher key\n"); 1773 return -ENOMEM; 1774 } 1775 1776 session->cipher_key.length = cipher_xform->key.length; 1777 session->auth_key.data = rte_zmalloc(NULL, 1778 auth_xform->key.length, 1779 RTE_CACHE_LINE_SIZE); 1780 if (session->auth_key.data == NULL && 1781 auth_xform->key.length > 0) { 1782 CAAM_JR_ERR("No Memory for auth key\n"); 1783 rte_free(session->cipher_key.data); 1784 return -ENOMEM; 1785 } 1786 session->auth_key.length = auth_xform->key.length; 1787 memcpy(session->cipher_key.data, cipher_xform->key.data, 1788 cipher_xform->key.length); 1789 memcpy(session->auth_key.data, auth_xform->key.data, 1790 auth_xform->key.length); 1791 1792 switch (auth_xform->algo) { 1793 case RTE_CRYPTO_AUTH_SHA1_HMAC: 1794 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; 1795 break; 1796 case RTE_CRYPTO_AUTH_MD5_HMAC: 1797 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; 1798 break; 1799 case RTE_CRYPTO_AUTH_SHA256_HMAC: 1800 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; 1801 break; 1802 case RTE_CRYPTO_AUTH_SHA384_HMAC: 1803 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; 1804 break; 1805 case RTE_CRYPTO_AUTH_SHA512_HMAC: 1806 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; 1807 break; 1808 case RTE_CRYPTO_AUTH_AES_CMAC: 1809 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC; 1810 break; 1811 case RTE_CRYPTO_AUTH_NULL: 1812 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 1813 break; 1814 case RTE_CRYPTO_AUTH_SHA224_HMAC: 1815 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 1816 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 1817 case RTE_CRYPTO_AUTH_SHA1: 1818 case RTE_CRYPTO_AUTH_SHA256: 1819 case RTE_CRYPTO_AUTH_SHA512: 1820 case RTE_CRYPTO_AUTH_SHA224: 1821 case RTE_CRYPTO_AUTH_SHA384: 1822 case RTE_CRYPTO_AUTH_MD5: 1823 case RTE_CRYPTO_AUTH_AES_GMAC: 1824 case RTE_CRYPTO_AUTH_KASUMI_F9: 1825 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 1826 case RTE_CRYPTO_AUTH_ZUC_EIA3: 1827 CAAM_JR_ERR("Crypto: Unsupported auth alg %u\n", 1828 auth_xform->algo); 1829 goto out; 1830 default: 1831 CAAM_JR_ERR("Crypto: Undefined Auth specified %u\n", 1832 auth_xform->algo); 1833 goto out; 1834 } 1835 1836 switch (cipher_xform->algo) { 1837 case RTE_CRYPTO_CIPHER_AES_CBC: 1838 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; 1839 break; 1840 case RTE_CRYPTO_CIPHER_3DES_CBC: 1841 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; 1842 break; 1843 case RTE_CRYPTO_CIPHER_AES_CTR: 1844 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; 1845 break; 1846 case RTE_CRYPTO_CIPHER_NULL: 1847 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 1848 case RTE_CRYPTO_CIPHER_3DES_ECB: 1849 case RTE_CRYPTO_CIPHER_AES_ECB: 1850 case RTE_CRYPTO_CIPHER_KASUMI_F8: 1851 CAAM_JR_ERR("Crypto: Unsupported Cipher alg %u\n", 1852 cipher_xform->algo); 1853 goto out; 1854 default: 1855 CAAM_JR_ERR("Crypto: Undefined Cipher specified %u\n", 1856 cipher_xform->algo); 1857 goto out; 1858 } 1859 1860 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 1861 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) + 1862 sizeof(session->ip4_hdr)); 1863 session->ip4_hdr.ip_v = IPVERSION; 1864 session->ip4_hdr.ip_hl = 5; 1865 session->ip4_hdr.ip_len = rte_cpu_to_be_16( 1866 sizeof(session->ip4_hdr)); 1867 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; 1868 session->ip4_hdr.ip_id = 0; 1869 session->ip4_hdr.ip_off = 0; 1870 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; 1871 session->ip4_hdr.ip_p = (ipsec_xform->proto == 1872 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP 1873 : IPPROTO_AH; 1874 session->ip4_hdr.ip_sum = 0; 1875 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip; 1876 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip; 1877 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *) 1878 (void *)&session->ip4_hdr, 1879 sizeof(struct ip)); 1880 1881 session->encap_pdb.options = 1882 (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | 1883 PDBOPTS_ESP_OIHI_PDB_INL | 1884 PDBOPTS_ESP_IVSRC | 1885 PDBHMO_ESP_ENCAP_DTTL; 1886 if (ipsec_xform->options.esn) 1887 session->encap_pdb.options |= PDBOPTS_ESP_ESN; 1888 session->encap_pdb.spi = ipsec_xform->spi; 1889 session->encap_pdb.ip_hdr_len = sizeof(struct ip); 1890 1891 session->dir = DIR_ENC; 1892 } else if (ipsec_xform->direction == 1893 RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { 1894 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); 1895 session->decap_pdb.options = sizeof(struct ip) << 16; 1896 if (ipsec_xform->options.esn) 1897 session->decap_pdb.options |= PDBOPTS_ESP_ESN; 1898 session->dir = DIR_DEC; 1899 } else 1900 goto out; 1901 session->ctx_pool = internals->ctx_pool; 1902 1903 return 0; 1904 out: 1905 rte_free(session->auth_key.data); 1906 rte_free(session->cipher_key.data); 1907 memset(session, 0, sizeof(struct caam_jr_session)); 1908 return -1; 1909 } 1910 1911 static int 1912 caam_jr_security_session_create(void *dev, 1913 struct rte_security_session_conf *conf, 1914 struct rte_security_session *sess, 1915 struct rte_mempool *mempool) 1916 { 1917 void *sess_private_data; 1918 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; 1919 int ret; 1920 1921 if (rte_mempool_get(mempool, &sess_private_data)) { 1922 CAAM_JR_ERR("Couldn't get object from session mempool"); 1923 return -ENOMEM; 1924 } 1925 1926 switch (conf->protocol) { 1927 case RTE_SECURITY_PROTOCOL_IPSEC: 1928 ret = caam_jr_set_ipsec_session(cdev, conf, 1929 sess_private_data); 1930 break; 1931 case RTE_SECURITY_PROTOCOL_MACSEC: 1932 return -ENOTSUP; 1933 default: 1934 return -EINVAL; 1935 } 1936 if (ret != 0) { 1937 CAAM_JR_ERR("failed to configure session parameters"); 1938 /* Return session to mempool */ 1939 rte_mempool_put(mempool, sess_private_data); 1940 return ret; 1941 } 1942 1943 set_sec_session_private_data(sess, sess_private_data); 1944 1945 return ret; 1946 } 1947 1948 /* Clear the memory of session so it doesn't leave key material behind */ 1949 static int 1950 caam_jr_security_session_destroy(void *dev __rte_unused, 1951 struct rte_security_session *sess) 1952 { 1953 PMD_INIT_FUNC_TRACE(); 1954 void *sess_priv = get_sec_session_private_data(sess); 1955 1956 struct caam_jr_session *s = (struct caam_jr_session *)sess_priv; 1957 1958 if (sess_priv) { 1959 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 1960 1961 rte_free(s->cipher_key.data); 1962 rte_free(s->auth_key.data); 1963 memset(sess, 0, sizeof(struct caam_jr_session)); 1964 set_sec_session_private_data(sess, NULL); 1965 rte_mempool_put(sess_mp, sess_priv); 1966 } 1967 return 0; 1968 } 1969 1970 1971 static int 1972 caam_jr_dev_configure(struct rte_cryptodev *dev, 1973 struct rte_cryptodev_config *config __rte_unused) 1974 { 1975 char str[20]; 1976 struct sec_job_ring_t *internals; 1977 1978 PMD_INIT_FUNC_TRACE(); 1979 1980 internals = dev->data->dev_private; 1981 snprintf(str, sizeof(str), "ctx_pool_%d", dev->data->dev_id); 1982 if (!internals->ctx_pool) { 1983 internals->ctx_pool = rte_mempool_create((const char *)str, 1984 CTX_POOL_NUM_BUFS, 1985 sizeof(struct caam_jr_op_ctx), 1986 CTX_POOL_CACHE_SIZE, 0, 1987 NULL, NULL, NULL, NULL, 1988 SOCKET_ID_ANY, 0); 1989 if (!internals->ctx_pool) { 1990 CAAM_JR_ERR("%s create failed\n", str); 1991 return -ENOMEM; 1992 } 1993 } else 1994 CAAM_JR_INFO("mempool already created for dev_id : %d", 1995 dev->data->dev_id); 1996 1997 return 0; 1998 } 1999 2000 static int 2001 caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused) 2002 { 2003 PMD_INIT_FUNC_TRACE(); 2004 return 0; 2005 } 2006 2007 static void 2008 caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused) 2009 { 2010 PMD_INIT_FUNC_TRACE(); 2011 } 2012 2013 static int 2014 caam_jr_dev_close(struct rte_cryptodev *dev) 2015 { 2016 struct sec_job_ring_t *internals; 2017 2018 PMD_INIT_FUNC_TRACE(); 2019 2020 if (dev == NULL) 2021 return -ENOMEM; 2022 2023 internals = dev->data->dev_private; 2024 rte_mempool_free(internals->ctx_pool); 2025 internals->ctx_pool = NULL; 2026 2027 return 0; 2028 } 2029 2030 static void 2031 caam_jr_dev_infos_get(struct rte_cryptodev *dev, 2032 struct rte_cryptodev_info *info) 2033 { 2034 struct sec_job_ring_t *internals = dev->data->dev_private; 2035 2036 PMD_INIT_FUNC_TRACE(); 2037 if (info != NULL) { 2038 info->max_nb_queue_pairs = internals->max_nb_queue_pairs; 2039 info->feature_flags = dev->feature_flags; 2040 info->capabilities = caam_jr_get_cryptodev_capabilities(); 2041 info->sym.max_nb_sessions = internals->max_nb_sessions; 2042 info->driver_id = cryptodev_driver_id; 2043 } 2044 } 2045 2046 static struct rte_cryptodev_ops caam_jr_ops = { 2047 .dev_configure = caam_jr_dev_configure, 2048 .dev_start = caam_jr_dev_start, 2049 .dev_stop = caam_jr_dev_stop, 2050 .dev_close = caam_jr_dev_close, 2051 .dev_infos_get = caam_jr_dev_infos_get, 2052 .stats_get = caam_jr_stats_get, 2053 .stats_reset = caam_jr_stats_reset, 2054 .queue_pair_setup = caam_jr_queue_pair_setup, 2055 .queue_pair_release = caam_jr_queue_pair_release, 2056 .sym_session_get_size = caam_jr_sym_session_get_size, 2057 .sym_session_configure = caam_jr_sym_session_configure, 2058 .sym_session_clear = caam_jr_sym_session_clear 2059 }; 2060 2061 static struct rte_security_ops caam_jr_security_ops = { 2062 .session_create = caam_jr_security_session_create, 2063 .session_update = NULL, 2064 .session_stats_get = NULL, 2065 .session_destroy = caam_jr_security_session_destroy, 2066 .set_pkt_metadata = NULL, 2067 .capabilities_get = caam_jr_get_security_capabilities 2068 }; 2069 2070 /* @brief Flush job rings of any processed descs. 2071 * The processed descs are silently dropped, 2072 * WITHOUT being notified to UA. 2073 */ 2074 static void 2075 close_job_ring(struct sec_job_ring_t *job_ring) 2076 { 2077 if (job_ring->irq_fd != -1) { 2078 /* Producer index is frozen. If consumer index is not equal 2079 * with producer index, then we have descs to flush. 2080 */ 2081 while (job_ring->pidx != job_ring->cidx) 2082 hw_flush_job_ring(job_ring, false, NULL); 2083 2084 /* free the uio job ring */ 2085 free_job_ring(job_ring->irq_fd); 2086 job_ring->irq_fd = -1; 2087 caam_jr_dma_free(job_ring->input_ring); 2088 caam_jr_dma_free(job_ring->output_ring); 2089 g_job_rings_no--; 2090 } 2091 } 2092 2093 /** @brief Release the software and hardware resources tied to a job ring. 2094 * @param [in] job_ring The job ring 2095 * 2096 * @retval 0 for success 2097 * @retval -1 for error 2098 */ 2099 static int 2100 shutdown_job_ring(struct sec_job_ring_t *job_ring) 2101 { 2102 int ret = 0; 2103 2104 PMD_INIT_FUNC_TRACE(); 2105 ASSERT(job_ring != NULL); 2106 ret = hw_shutdown_job_ring(job_ring); 2107 SEC_ASSERT(ret == 0, ret, 2108 "Failed to shutdown hardware job ring %p", 2109 job_ring); 2110 2111 if (job_ring->coalescing_en) 2112 hw_job_ring_disable_coalescing(job_ring); 2113 2114 if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) { 2115 ret = caam_jr_disable_irqs(job_ring->irq_fd); 2116 SEC_ASSERT(ret == 0, ret, 2117 "Failed to disable irqs for job ring %p", 2118 job_ring); 2119 } 2120 2121 return ret; 2122 } 2123 2124 /* 2125 * @brief Release the resources used by the SEC user space driver. 2126 * 2127 * Reset and release SEC's job rings indicated by the User Application at 2128 * init_job_ring() and free any memory allocated internally. 2129 * Call once during application tear down. 2130 * 2131 * @note In case there are any descriptors in-flight (descriptors received by 2132 * SEC driver for processing and for which no response was yet provided to UA), 2133 * the descriptors are discarded without any notifications to User Application. 2134 * 2135 * @retval ::0 is returned for a successful execution 2136 * @retval ::-1 is returned if SEC driver release is in progress 2137 */ 2138 static int 2139 caam_jr_dev_uninit(struct rte_cryptodev *dev) 2140 { 2141 struct sec_job_ring_t *internals; 2142 2143 PMD_INIT_FUNC_TRACE(); 2144 if (dev == NULL) 2145 return -ENODEV; 2146 2147 internals = dev->data->dev_private; 2148 rte_free(dev->security_ctx); 2149 2150 /* If any descriptors in flight , poll and wait 2151 * until all descriptors are received and silently discarded. 2152 */ 2153 if (internals) { 2154 shutdown_job_ring(internals); 2155 close_job_ring(internals); 2156 rte_mempool_free(internals->ctx_pool); 2157 } 2158 2159 CAAM_JR_INFO("Closing crypto device %s", dev->data->name); 2160 2161 /* last caam jr instance) */ 2162 if (g_job_rings_no == 0) 2163 g_driver_state = SEC_DRIVER_STATE_IDLE; 2164 2165 return SEC_SUCCESS; 2166 } 2167 2168 /* @brief Initialize the software and hardware resources tied to a job ring. 2169 * @param [in] jr_mode; Model to be used by SEC Driver to receive 2170 * notifications from SEC. Can be either 2171 * of the three: #SEC_NOTIFICATION_TYPE_NAPI 2172 * #SEC_NOTIFICATION_TYPE_IRQ or 2173 * #SEC_NOTIFICATION_TYPE_POLL 2174 * @param [in] NAPI_mode The NAPI work mode to configure a job ring at 2175 * startup. Used only when #SEC_NOTIFICATION_TYPE 2176 * is set to #SEC_NOTIFICATION_TYPE_NAPI. 2177 * @param [in] irq_coalescing_timer This value determines the maximum 2178 * amount of time after processing a 2179 * descriptor before raising an interrupt. 2180 * @param [in] irq_coalescing_count This value determines how many 2181 * descriptors are completed before 2182 * raising an interrupt. 2183 * @param [in] reg_base_addr, The job ring base address register 2184 * @param [in] irq_id The job ring interrupt identification number. 2185 * @retval job_ring_handle for successful job ring configuration 2186 * @retval NULL on error 2187 * 2188 */ 2189 static void * 2190 init_job_ring(void *reg_base_addr, int irq_id) 2191 { 2192 struct sec_job_ring_t *job_ring = NULL; 2193 int i, ret = 0; 2194 int jr_mode = SEC_NOTIFICATION_TYPE_POLL; 2195 int napi_mode = 0; 2196 int irq_coalescing_timer = 0; 2197 int irq_coalescing_count = 0; 2198 2199 for (i = 0; i < MAX_SEC_JOB_RINGS; i++) { 2200 if (g_job_rings[i].irq_fd == -1) { 2201 job_ring = &g_job_rings[i]; 2202 g_job_rings_no++; 2203 break; 2204 } 2205 } 2206 if (job_ring == NULL) { 2207 CAAM_JR_ERR("No free job ring\n"); 2208 return NULL; 2209 } 2210 2211 job_ring->register_base_addr = reg_base_addr; 2212 job_ring->jr_mode = jr_mode; 2213 job_ring->napi_mode = 0; 2214 job_ring->irq_fd = irq_id; 2215 2216 /* Allocate mem for input and output ring */ 2217 2218 /* Allocate memory for input ring */ 2219 job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, 2220 SEC_DMA_MEM_INPUT_RING_SIZE); 2221 memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE); 2222 2223 /* Allocate memory for output ring */ 2224 job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, 2225 SEC_DMA_MEM_OUTPUT_RING_SIZE); 2226 memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE); 2227 2228 /* Reset job ring in SEC hw and configure job ring registers */ 2229 ret = hw_reset_job_ring(job_ring); 2230 if (ret != 0) { 2231 CAAM_JR_ERR("Failed to reset hardware job ring"); 2232 goto cleanup; 2233 } 2234 2235 if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) { 2236 /* When SEC US driver works in NAPI mode, the UA can select 2237 * if the driver starts with IRQs on or off. 2238 */ 2239 if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) { 2240 CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p", 2241 job_ring); 2242 ret = caam_jr_enable_irqs(job_ring->irq_fd); 2243 if (ret != 0) { 2244 CAAM_JR_ERR("Failed to enable irqs for job ring"); 2245 goto cleanup; 2246 } 2247 } 2248 } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) { 2249 /* When SEC US driver works in pure interrupt mode, 2250 * IRQ's are always enabled. 2251 */ 2252 CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p", 2253 job_ring); 2254 ret = caam_jr_enable_irqs(job_ring->irq_fd); 2255 if (ret != 0) { 2256 CAAM_JR_ERR("Failed to enable irqs for job ring"); 2257 goto cleanup; 2258 } 2259 } 2260 if (irq_coalescing_timer || irq_coalescing_count) { 2261 hw_job_ring_set_coalescing_param(job_ring, 2262 irq_coalescing_timer, 2263 irq_coalescing_count); 2264 2265 hw_job_ring_enable_coalescing(job_ring); 2266 job_ring->coalescing_en = 1; 2267 } 2268 2269 job_ring->jr_state = SEC_JOB_RING_STATE_STARTED; 2270 job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS; 2271 job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS; 2272 2273 return job_ring; 2274 cleanup: 2275 caam_jr_dma_free(job_ring->output_ring); 2276 caam_jr_dma_free(job_ring->input_ring); 2277 return NULL; 2278 } 2279 2280 2281 static int 2282 caam_jr_dev_init(const char *name, 2283 struct rte_vdev_device *vdev, 2284 struct rte_cryptodev_pmd_init_params *init_params) 2285 { 2286 struct rte_cryptodev *dev; 2287 struct rte_security_ctx *security_instance; 2288 struct uio_job_ring *job_ring; 2289 char str[RTE_CRYPTODEV_NAME_MAX_LEN]; 2290 2291 PMD_INIT_FUNC_TRACE(); 2292 2293 /* Validate driver state */ 2294 if (g_driver_state == SEC_DRIVER_STATE_IDLE) { 2295 g_job_rings_max = sec_configure(); 2296 if (!g_job_rings_max) { 2297 CAAM_JR_ERR("No job ring detected on UIO !!!!"); 2298 return -1; 2299 } 2300 /* Update driver state */ 2301 g_driver_state = SEC_DRIVER_STATE_STARTED; 2302 } 2303 2304 if (g_job_rings_no >= g_job_rings_max) { 2305 CAAM_JR_ERR("No more job rings available max=%d!!!!", 2306 g_job_rings_max); 2307 return -1; 2308 } 2309 2310 job_ring = config_job_ring(); 2311 if (job_ring == NULL) { 2312 CAAM_JR_ERR("failed to create job ring"); 2313 goto init_error; 2314 } 2315 2316 snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id); 2317 2318 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 2319 if (dev == NULL) { 2320 CAAM_JR_ERR("failed to create cryptodev vdev"); 2321 goto cleanup; 2322 } 2323 /*TODO free it during teardown*/ 2324 dev->data->dev_private = init_job_ring(job_ring->register_base_addr, 2325 job_ring->uio_fd); 2326 2327 if (!dev->data->dev_private) { 2328 CAAM_JR_ERR("Ring memory allocation failed\n"); 2329 goto cleanup2; 2330 } 2331 2332 dev->driver_id = cryptodev_driver_id; 2333 dev->dev_ops = &caam_jr_ops; 2334 2335 /* register rx/tx burst functions for data path */ 2336 dev->dequeue_burst = caam_jr_dequeue_burst; 2337 dev->enqueue_burst = caam_jr_enqueue_burst; 2338 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2339 RTE_CRYPTODEV_FF_HW_ACCELERATED | 2340 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 2341 RTE_CRYPTODEV_FF_SECURITY | 2342 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 2343 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | 2344 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 2345 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | 2346 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; 2347 2348 /* For secondary processes, we don't initialise any further as primary 2349 * has already done this work. Only check we don't need a different 2350 * RX function 2351 */ 2352 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 2353 CAAM_JR_WARN("Device already init by primary process"); 2354 return 0; 2355 } 2356 2357 /*TODO free it during teardown*/ 2358 security_instance = rte_malloc("caam_jr", 2359 sizeof(struct rte_security_ctx), 0); 2360 if (security_instance == NULL) { 2361 CAAM_JR_ERR("memory allocation failed\n"); 2362 //todo error handling. 2363 goto cleanup2; 2364 } 2365 2366 security_instance->device = (void *)dev; 2367 security_instance->ops = &caam_jr_security_ops; 2368 security_instance->sess_cnt = 0; 2369 dev->security_ctx = security_instance; 2370 2371 RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name); 2372 2373 return 0; 2374 2375 cleanup2: 2376 caam_jr_dev_uninit(dev); 2377 rte_cryptodev_pmd_release_device(dev); 2378 cleanup: 2379 free_job_ring(job_ring->uio_fd); 2380 init_error: 2381 CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed", 2382 init_params->name); 2383 2384 return -ENXIO; 2385 } 2386 2387 /** Initialise CAAM JR crypto device */ 2388 static int 2389 cryptodev_caam_jr_probe(struct rte_vdev_device *vdev) 2390 { 2391 int ret; 2392 2393 struct rte_cryptodev_pmd_init_params init_params = { 2394 "", 2395 sizeof(struct sec_job_ring_t), 2396 rte_socket_id(), 2397 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 2398 }; 2399 const char *name; 2400 const char *input_args; 2401 2402 name = rte_vdev_device_name(vdev); 2403 if (name == NULL) 2404 return -EINVAL; 2405 2406 input_args = rte_vdev_device_args(vdev); 2407 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 2408 2409 ret = of_init(); 2410 if (ret) { 2411 RTE_LOG(ERR, PMD, 2412 "of_init failed\n"); 2413 return -EINVAL; 2414 } 2415 /* if sec device version is not configured */ 2416 if (!rta_get_sec_era()) { 2417 const struct device_node *caam_node; 2418 2419 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { 2420 const uint32_t *prop = of_get_property(caam_node, 2421 "fsl,sec-era", 2422 NULL); 2423 if (prop) { 2424 rta_set_sec_era( 2425 INTL_SEC_ERA(rte_be_to_cpu_32(*prop))); 2426 break; 2427 } 2428 } 2429 } 2430 #ifdef RTE_LIBRTE_PMD_CAAM_JR_BE 2431 if (rta_get_sec_era() > RTA_SEC_ERA_8) { 2432 RTE_LOG(ERR, PMD, 2433 "CAAM is compiled in BE mode for device with sec era > 8???\n"); 2434 return -EINVAL; 2435 } 2436 #endif 2437 2438 return caam_jr_dev_init(name, vdev, &init_params); 2439 } 2440 2441 /** Uninitialise CAAM JR crypto device */ 2442 static int 2443 cryptodev_caam_jr_remove(struct rte_vdev_device *vdev) 2444 { 2445 struct rte_cryptodev *cryptodev; 2446 const char *name; 2447 2448 name = rte_vdev_device_name(vdev); 2449 if (name == NULL) 2450 return -EINVAL; 2451 2452 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 2453 if (cryptodev == NULL) 2454 return -ENODEV; 2455 2456 caam_jr_dev_uninit(cryptodev); 2457 2458 return rte_cryptodev_pmd_destroy(cryptodev); 2459 } 2460 2461 static void 2462 sec_job_rings_init(void) 2463 { 2464 int i; 2465 2466 for (i = 0; i < MAX_SEC_JOB_RINGS; i++) 2467 g_job_rings[i].irq_fd = -1; 2468 } 2469 2470 static struct rte_vdev_driver cryptodev_caam_jr_drv = { 2471 .probe = cryptodev_caam_jr_probe, 2472 .remove = cryptodev_caam_jr_remove 2473 }; 2474 2475 static struct cryptodev_driver caam_jr_crypto_drv; 2476 2477 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv); 2478 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD, 2479 "max_nb_queue_pairs=<int>" 2480 "socket_id=<int>"); 2481 RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver, 2482 cryptodev_driver_id); 2483 2484 RTE_INIT(caam_jr_init) 2485 { 2486 sec_uio_job_rings_init(); 2487 sec_job_rings_init(); 2488 } 2489 2490 RTE_LOG_REGISTER(caam_jr_logtype, pmd.crypto.caam, NOTICE); 2491