1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 #include <string.h> 5 6 #include <rte_common.h> 7 #include <rte_malloc.h> 8 9 #include "rte_port_sym_crypto.h" 10 11 #include "port_log.h" 12 13 /* 14 * Port Crypto Reader 15 */ 16 #ifdef RTE_PORT_STATS_COLLECT 17 18 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) \ 19 (port)->stats.n_pkts_in += (val) 20 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) \ 21 (port)->stats.n_pkts_drop += (val) 22 23 #else 24 25 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) 26 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) 27 28 #endif 29 30 struct rte_port_sym_crypto_reader { 31 struct rte_port_in_stats stats; 32 33 uint8_t cryptodev_id; 34 uint16_t queue_id; 35 struct rte_crypto_op *ops[RTE_PORT_IN_BURST_SIZE_MAX]; 36 rte_port_sym_crypto_reader_callback_fn f_callback; 37 void *arg_callback; 38 }; 39 40 static void * 41 rte_port_sym_crypto_reader_create(void *params, int socket_id) 42 { 43 struct rte_port_sym_crypto_reader_params *conf = 44 params; 45 struct rte_port_sym_crypto_reader *port; 46 47 /* Check input parameters */ 48 if (conf == NULL) { 49 PORT_LOG(ERR, "%s: params is NULL", __func__); 50 return NULL; 51 } 52 53 /* Memory allocation */ 54 port = rte_zmalloc_socket("PORT", sizeof(*port), 55 RTE_CACHE_LINE_SIZE, socket_id); 56 if (port == NULL) { 57 PORT_LOG(ERR, "%s: Failed to allocate port", __func__); 58 return NULL; 59 } 60 61 /* Initialization */ 62 port->cryptodev_id = conf->cryptodev_id; 63 port->queue_id = conf->queue_id; 64 port->f_callback = conf->f_callback; 65 port->arg_callback = conf->arg_callback; 66 67 return port; 68 } 69 70 static int 71 rte_port_sym_crypto_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) 72 { 73 struct rte_port_sym_crypto_reader *p = 74 port; 75 uint16_t rx_ops_cnt, i, n = 0; 76 77 rx_ops_cnt = rte_cryptodev_dequeue_burst(p->cryptodev_id, p->queue_id, 78 p->ops, n_pkts); 79 80 for (i = 0; i < rx_ops_cnt; i++) { 81 struct rte_crypto_op *op = p->ops[i]; 82 83 /** Drop failed pkts */ 84 if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) { 85 rte_pktmbuf_free(op->sym->m_src); 86 continue; 87 } 88 89 pkts[n++] = op->sym->m_src; 90 } 91 92 if (p->f_callback) 93 (*p->f_callback)(pkts, n, p->arg_callback); 94 95 RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(p, n); 96 RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(p, rx_ops_cnt - n); 97 98 return n; 99 } 100 101 static int 102 rte_port_sym_crypto_reader_free(void *port) 103 { 104 if (port == NULL) { 105 PORT_LOG(ERR, "%s: port is NULL", __func__); 106 return -EINVAL; 107 } 108 109 rte_free(port); 110 111 return 0; 112 } 113 114 static int rte_port_sym_crypto_reader_stats_read(void *port, 115 struct rte_port_in_stats *stats, int clear) 116 { 117 struct rte_port_sym_crypto_reader *p = 118 port; 119 120 if (stats != NULL) 121 memcpy(stats, &p->stats, sizeof(p->stats)); 122 123 if (clear) 124 memset(&p->stats, 0, sizeof(p->stats)); 125 126 return 0; 127 } 128 129 /* 130 * Port crypto Writer 131 */ 132 #ifdef RTE_PORT_STATS_COLLECT 133 134 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) \ 135 (port)->stats.n_pkts_in += (val) 136 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) \ 137 (port)->stats.n_pkts_drop += (val) 138 139 #else 140 141 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) 142 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) 143 144 #endif 145 146 struct rte_port_sym_crypto_writer { 147 struct rte_port_out_stats stats; 148 149 struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; 150 151 uint32_t tx_burst_sz; 152 uint32_t tx_buf_count; 153 uint64_t bsz_mask; 154 155 uint8_t cryptodev_id; 156 uint16_t queue_id; 157 uint16_t crypto_op_offset; 158 }; 159 160 static void * 161 rte_port_sym_crypto_writer_create(void *params, int socket_id) 162 { 163 struct rte_port_sym_crypto_writer_params *conf = 164 params; 165 struct rte_port_sym_crypto_writer *port; 166 167 /* Check input parameters */ 168 if ((conf == NULL) || 169 (conf->tx_burst_sz == 0) || 170 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || 171 (!rte_is_power_of_2(conf->tx_burst_sz))) { 172 PORT_LOG(ERR, "%s: Invalid input parameters", __func__); 173 return NULL; 174 } 175 176 /* Memory allocation */ 177 port = rte_zmalloc_socket("PORT", sizeof(*port), 178 RTE_CACHE_LINE_SIZE, socket_id); 179 if (port == NULL) { 180 PORT_LOG(ERR, "%s: Failed to allocate port", __func__); 181 return NULL; 182 } 183 184 /* Initialization */ 185 port->tx_burst_sz = conf->tx_burst_sz; 186 port->tx_buf_count = 0; 187 port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); 188 189 port->cryptodev_id = conf->cryptodev_id; 190 port->queue_id = conf->queue_id; 191 port->crypto_op_offset = conf->crypto_op_offset; 192 193 return port; 194 } 195 196 static inline void 197 send_burst(struct rte_port_sym_crypto_writer *p) 198 { 199 uint32_t nb_tx; 200 201 nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, 202 p->tx_buf, p->tx_buf_count); 203 204 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - 205 nb_tx); 206 for (; nb_tx < p->tx_buf_count; nb_tx++) 207 rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); 208 209 p->tx_buf_count = 0; 210 } 211 212 static int 213 rte_port_sym_crypto_writer_tx(void *port, struct rte_mbuf *pkt) 214 { 215 struct rte_port_sym_crypto_writer *p = 216 port; 217 218 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) 219 RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset); 220 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); 221 if (p->tx_buf_count >= p->tx_burst_sz) 222 send_burst(p); 223 224 return 0; 225 } 226 227 static int 228 rte_port_sym_crypto_writer_tx_bulk(void *port, 229 struct rte_mbuf **pkts, 230 uint64_t pkts_mask) 231 { 232 struct rte_port_sym_crypto_writer *p = 233 port; 234 uint64_t bsz_mask = p->bsz_mask; 235 uint32_t tx_buf_count = p->tx_buf_count; 236 uint64_t expr = (pkts_mask & (pkts_mask + 1)) | 237 ((pkts_mask & bsz_mask) ^ bsz_mask); 238 239 if (expr == 0) { 240 uint64_t n_pkts = rte_popcount64(pkts_mask); 241 uint32_t i; 242 243 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, n_pkts); 244 245 for (i = 0; i < n_pkts; i++) 246 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) 247 RTE_MBUF_METADATA_UINT8_PTR(pkts[i], 248 p->crypto_op_offset); 249 250 if (p->tx_buf_count >= p->tx_burst_sz) 251 send_burst(p); 252 } else { 253 for (; pkts_mask;) { 254 uint32_t pkt_index = rte_ctz64(pkts_mask); 255 uint64_t pkt_mask = 1LLU << pkt_index; 256 struct rte_mbuf *pkt = pkts[pkt_index]; 257 258 p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) 259 RTE_MBUF_METADATA_UINT8_PTR(pkt, 260 p->crypto_op_offset); 261 262 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); 263 pkts_mask &= ~pkt_mask; 264 } 265 266 p->tx_buf_count = tx_buf_count; 267 if (tx_buf_count >= p->tx_burst_sz) 268 send_burst(p); 269 } 270 271 return 0; 272 } 273 274 static int 275 rte_port_sym_crypto_writer_flush(void *port) 276 { 277 struct rte_port_sym_crypto_writer *p = 278 port; 279 280 if (p->tx_buf_count > 0) 281 send_burst(p); 282 283 return 0; 284 } 285 286 static int 287 rte_port_sym_crypto_writer_free(void *port) 288 { 289 if (port == NULL) { 290 PORT_LOG(ERR, "%s: Port is NULL", __func__); 291 return -EINVAL; 292 } 293 294 rte_port_sym_crypto_writer_flush(port); 295 rte_free(port); 296 297 return 0; 298 } 299 300 static int rte_port_sym_crypto_writer_stats_read(void *port, 301 struct rte_port_out_stats *stats, int clear) 302 { 303 struct rte_port_sym_crypto_writer *p = 304 port; 305 306 if (stats != NULL) 307 memcpy(stats, &p->stats, sizeof(p->stats)); 308 309 if (clear) 310 memset(&p->stats, 0, sizeof(p->stats)); 311 312 return 0; 313 } 314 315 /* 316 * Port crypto Writer Nodrop 317 */ 318 #ifdef RTE_PORT_STATS_COLLECT 319 320 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \ 321 (port)->stats.n_pkts_in += (val) 322 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \ 323 (port)->stats.n_pkts_drop += (val) 324 325 #else 326 327 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) 328 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) 329 330 #endif 331 332 struct rte_port_sym_crypto_writer_nodrop { 333 struct rte_port_out_stats stats; 334 335 struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; 336 uint32_t tx_burst_sz; 337 uint32_t tx_buf_count; 338 uint64_t bsz_mask; 339 uint64_t n_retries; 340 341 uint8_t cryptodev_id; 342 uint16_t queue_id; 343 uint16_t crypto_op_offset; 344 }; 345 346 static void * 347 rte_port_sym_crypto_writer_nodrop_create(void *params, int socket_id) 348 { 349 struct rte_port_sym_crypto_writer_nodrop_params *conf = 350 params; 351 struct rte_port_sym_crypto_writer_nodrop *port; 352 353 /* Check input parameters */ 354 if ((conf == NULL) || 355 (conf->tx_burst_sz == 0) || 356 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || 357 (!rte_is_power_of_2(conf->tx_burst_sz))) { 358 PORT_LOG(ERR, "%s: Invalid input parameters", __func__); 359 return NULL; 360 } 361 362 /* Memory allocation */ 363 port = rte_zmalloc_socket("PORT", sizeof(*port), 364 RTE_CACHE_LINE_SIZE, socket_id); 365 if (port == NULL) { 366 PORT_LOG(ERR, "%s: Failed to allocate port", __func__); 367 return NULL; 368 } 369 370 /* Initialization */ 371 port->cryptodev_id = conf->cryptodev_id; 372 port->queue_id = conf->queue_id; 373 port->crypto_op_offset = conf->crypto_op_offset; 374 port->tx_burst_sz = conf->tx_burst_sz; 375 port->tx_buf_count = 0; 376 port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); 377 378 /* 379 * When n_retries is 0 it means that we should wait for every packet to 380 * send no matter how many retries should it take. To limit number of 381 * branches in fast path, we use UINT64_MAX instead of branching. 382 */ 383 port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries; 384 385 return port; 386 } 387 388 static inline void 389 send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop *p) 390 { 391 uint32_t nb_tx = 0, i; 392 393 nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, 394 p->tx_buf, p->tx_buf_count); 395 396 /* We sent all the packets in a first try */ 397 if (nb_tx >= p->tx_buf_count) { 398 p->tx_buf_count = 0; 399 return; 400 } 401 402 for (i = 0; i < p->n_retries; i++) { 403 nb_tx += rte_cryptodev_enqueue_burst(p->cryptodev_id, 404 p->queue_id, p->tx_buf + nb_tx, 405 p->tx_buf_count - nb_tx); 406 407 /* We sent all the packets in more than one try */ 408 if (nb_tx >= p->tx_buf_count) { 409 p->tx_buf_count = 0; 410 return; 411 } 412 } 413 414 /* We didn't send the packets in maximum allowed attempts */ 415 RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(p, 416 p->tx_buf_count - nb_tx); 417 for ( ; nb_tx < p->tx_buf_count; nb_tx++) 418 rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); 419 420 p->tx_buf_count = 0; 421 } 422 423 static int 424 rte_port_sym_crypto_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) 425 { 426 struct rte_port_sym_crypto_writer_nodrop *p = 427 port; 428 429 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) 430 RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset); 431 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); 432 if (p->tx_buf_count >= p->tx_burst_sz) 433 send_burst_nodrop(p); 434 435 return 0; 436 } 437 438 static int 439 rte_port_sym_crypto_writer_nodrop_tx_bulk(void *port, 440 struct rte_mbuf **pkts, 441 uint64_t pkts_mask) 442 { 443 struct rte_port_sym_crypto_writer_nodrop *p = 444 port; 445 446 uint64_t bsz_mask = p->bsz_mask; 447 uint32_t tx_buf_count = p->tx_buf_count; 448 uint64_t expr = (pkts_mask & (pkts_mask + 1)) | 449 ((pkts_mask & bsz_mask) ^ bsz_mask); 450 451 if (expr == 0) { 452 uint64_t n_pkts = rte_popcount64(pkts_mask); 453 uint32_t i; 454 455 RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); 456 457 for (i = 0; i < n_pkts; i++) 458 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) 459 RTE_MBUF_METADATA_UINT8_PTR(pkts[i], 460 p->crypto_op_offset); 461 462 if (p->tx_buf_count >= p->tx_burst_sz) 463 send_burst_nodrop(p); 464 } else { 465 for ( ; pkts_mask; ) { 466 uint32_t pkt_index = rte_ctz64(pkts_mask); 467 uint64_t pkt_mask = 1LLU << pkt_index; 468 struct rte_mbuf *pkt = pkts[pkt_index]; 469 470 p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) 471 RTE_MBUF_METADATA_UINT8_PTR(pkt, 472 p->crypto_op_offset); 473 RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 474 1); 475 pkts_mask &= ~pkt_mask; 476 } 477 478 p->tx_buf_count = tx_buf_count; 479 if (tx_buf_count >= p->tx_burst_sz) 480 send_burst_nodrop(p); 481 } 482 483 return 0; 484 } 485 486 static int 487 rte_port_sym_crypto_writer_nodrop_flush(void *port) 488 { 489 struct rte_port_sym_crypto_writer_nodrop *p = 490 port; 491 492 if (p->tx_buf_count > 0) 493 send_burst_nodrop(p); 494 495 return 0; 496 } 497 498 static int 499 rte_port_sym_crypto_writer_nodrop_free(void *port) 500 { 501 if (port == NULL) { 502 PORT_LOG(ERR, "%s: Port is NULL", __func__); 503 return -EINVAL; 504 } 505 506 rte_port_sym_crypto_writer_nodrop_flush(port); 507 rte_free(port); 508 509 return 0; 510 } 511 512 static int rte_port_sym_crypto_writer_nodrop_stats_read(void *port, 513 struct rte_port_out_stats *stats, int clear) 514 { 515 struct rte_port_sym_crypto_writer_nodrop *p = 516 port; 517 518 if (stats != NULL) 519 memcpy(stats, &p->stats, sizeof(p->stats)); 520 521 if (clear) 522 memset(&p->stats, 0, sizeof(p->stats)); 523 524 return 0; 525 } 526 527 528 /* 529 * Summary of port operations 530 */ 531 struct rte_port_in_ops rte_port_sym_crypto_reader_ops = { 532 .f_create = rte_port_sym_crypto_reader_create, 533 .f_free = rte_port_sym_crypto_reader_free, 534 .f_rx = rte_port_sym_crypto_reader_rx, 535 .f_stats = rte_port_sym_crypto_reader_stats_read, 536 }; 537 538 struct rte_port_out_ops rte_port_sym_crypto_writer_ops = { 539 .f_create = rte_port_sym_crypto_writer_create, 540 .f_free = rte_port_sym_crypto_writer_free, 541 .f_tx = rte_port_sym_crypto_writer_tx, 542 .f_tx_bulk = rte_port_sym_crypto_writer_tx_bulk, 543 .f_flush = rte_port_sym_crypto_writer_flush, 544 .f_stats = rte_port_sym_crypto_writer_stats_read, 545 }; 546 547 struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops = { 548 .f_create = rte_port_sym_crypto_writer_nodrop_create, 549 .f_free = rte_port_sym_crypto_writer_nodrop_free, 550 .f_tx = rte_port_sym_crypto_writer_nodrop_tx, 551 .f_tx_bulk = rte_port_sym_crypto_writer_nodrop_tx_bulk, 552 .f_flush = rte_port_sym_crypto_writer_nodrop_flush, 553 .f_stats = rte_port_sym_crypto_writer_nodrop_stats_read, 554 }; 555