1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <string.h> 6 7 #include <rte_common.h> 8 #include <rte_bus_vdev.h> 9 #include <rte_malloc.h> 10 #include <rte_ring.h> 11 #include <rte_kvargs.h> 12 13 #include <rte_bbdev.h> 14 #include <rte_bbdev_pmd.h> 15 16 #include <phy_turbo.h> 17 #include <phy_crc.h> 18 #include <phy_rate_match.h> 19 #include <divide.h> 20 21 #define DRIVER_NAME turbo_sw 22 23 /* Turbo SW PMD logging ID */ 24 static int bbdev_turbo_sw_logtype; 25 26 /* Helper macro for logging */ 27 #define rte_bbdev_log(level, fmt, ...) \ 28 rte_log(RTE_LOG_ ## level, bbdev_turbo_sw_logtype, fmt "\n", \ 29 ##__VA_ARGS__) 30 31 #define rte_bbdev_log_debug(fmt, ...) \ 32 rte_bbdev_log(DEBUG, RTE_STR(__LINE__) ":%s() " fmt, __func__, \ 33 ##__VA_ARGS__) 34 35 #define DEINT_INPUT_BUF_SIZE (((RTE_BBDEV_MAX_CB_SIZE >> 3) + 1) * 48) 36 #define DEINT_OUTPUT_BUF_SIZE (DEINT_INPUT_BUF_SIZE * 6) 37 #define ADAPTER_OUTPUT_BUF_SIZE ((RTE_BBDEV_MAX_CB_SIZE + 4) * 48) 38 39 /* private data structure */ 40 struct bbdev_private { 41 unsigned int max_nb_queues; /**< Max number of queues */ 42 }; 43 44 /* Initialisation params structure that can be used by Turbo SW driver */ 45 struct turbo_sw_params { 46 int socket_id; /*< Turbo SW device socket */ 47 uint16_t queues_num; /*< Turbo SW device queues number */ 48 }; 49 50 /* Accecptable params for Turbo SW devices */ 51 #define TURBO_SW_MAX_NB_QUEUES_ARG "max_nb_queues" 52 #define TURBO_SW_SOCKET_ID_ARG "socket_id" 53 54 static const char * const turbo_sw_valid_params[] = { 55 TURBO_SW_MAX_NB_QUEUES_ARG, 56 TURBO_SW_SOCKET_ID_ARG 57 }; 58 59 /* queue */ 60 struct turbo_sw_queue { 61 /* Ring for processed (encoded/decoded) operations which are ready to 62 * be dequeued. 63 */ 64 struct rte_ring *processed_pkts; 65 /* Stores input for turbo encoder (used when CRC attachment is 66 * performed 67 */ 68 uint8_t *enc_in; 69 /* Stores output from turbo encoder */ 70 uint8_t *enc_out; 71 /* Alpha gamma buf for bblib_turbo_decoder() function */ 72 int8_t *ag; 73 /* Temp buf for bblib_turbo_decoder() function */ 74 uint16_t *code_block; 75 /* Input buf for bblib_rate_dematching_lte() function */ 76 uint8_t *deint_input; 77 /* Output buf for bblib_rate_dematching_lte() function */ 78 uint8_t *deint_output; 79 /* Output buf for bblib_turbodec_adapter_lte() function */ 80 uint8_t *adapter_output; 81 /* Operation type of this queue */ 82 enum rte_bbdev_op_type type; 83 } __rte_cache_aligned; 84 85 /* Calculate index based on Table 5.1.3-3 from TS34.212 */ 86 static inline int32_t 87 compute_idx(uint16_t k) 88 { 89 int32_t result = 0; 90 91 if (k < RTE_BBDEV_MIN_CB_SIZE || k > RTE_BBDEV_MAX_CB_SIZE) 92 return -1; 93 94 if (k > 2048) { 95 if ((k - 2048) % 64 != 0) 96 result = -1; 97 98 result = 124 + (k - 2048) / 64; 99 } else if (k <= 512) { 100 if ((k - 40) % 8 != 0) 101 result = -1; 102 103 result = (k - 40) / 8 + 1; 104 } else if (k <= 1024) { 105 if ((k - 512) % 16 != 0) 106 result = -1; 107 108 result = 60 + (k - 512) / 16; 109 } else { /* 1024 < k <= 2048 */ 110 if ((k - 1024) % 32 != 0) 111 result = -1; 112 113 result = 92 + (k - 1024) / 32; 114 } 115 116 return result; 117 } 118 119 /* Read flag value 0/1 from bitmap */ 120 static inline bool 121 check_bit(uint32_t bitmap, uint32_t bitmask) 122 { 123 return bitmap & bitmask; 124 } 125 126 /* Get device info */ 127 static void 128 info_get(struct rte_bbdev *dev, struct rte_bbdev_driver_info *dev_info) 129 { 130 struct bbdev_private *internals = dev->data->dev_private; 131 132 static const struct rte_bbdev_op_cap bbdev_capabilities[] = { 133 { 134 .type = RTE_BBDEV_OP_TURBO_DEC, 135 .cap.turbo_dec = { 136 .capability_flags = 137 RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE | 138 RTE_BBDEV_TURBO_POS_LLR_1_BIT_IN | 139 RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN | 140 RTE_BBDEV_TURBO_CRC_TYPE_24B | 141 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP | 142 RTE_BBDEV_TURBO_EARLY_TERMINATION, 143 .max_llr_modulus = 16, 144 .num_buffers_src = RTE_BBDEV_MAX_CODE_BLOCKS, 145 .num_buffers_hard_out = 146 RTE_BBDEV_MAX_CODE_BLOCKS, 147 .num_buffers_soft_out = 0, 148 } 149 }, 150 { 151 .type = RTE_BBDEV_OP_TURBO_ENC, 152 .cap.turbo_enc = { 153 .capability_flags = 154 RTE_BBDEV_TURBO_CRC_24B_ATTACH | 155 RTE_BBDEV_TURBO_CRC_24A_ATTACH | 156 RTE_BBDEV_TURBO_RATE_MATCH | 157 RTE_BBDEV_TURBO_RV_INDEX_BYPASS, 158 .num_buffers_src = RTE_BBDEV_MAX_CODE_BLOCKS, 159 .num_buffers_dst = RTE_BBDEV_MAX_CODE_BLOCKS, 160 } 161 }, 162 RTE_BBDEV_END_OF_CAPABILITIES_LIST() 163 }; 164 165 static struct rte_bbdev_queue_conf default_queue_conf = { 166 .queue_size = RTE_BBDEV_QUEUE_SIZE_LIMIT, 167 }; 168 169 static const enum rte_cpu_flag_t cpu_flag = RTE_CPUFLAG_SSE4_2; 170 171 default_queue_conf.socket = dev->data->socket_id; 172 173 dev_info->driver_name = RTE_STR(DRIVER_NAME); 174 dev_info->max_num_queues = internals->max_nb_queues; 175 dev_info->queue_size_lim = RTE_BBDEV_QUEUE_SIZE_LIMIT; 176 dev_info->hardware_accelerated = false; 177 dev_info->max_queue_priority = 0; 178 dev_info->default_queue_conf = default_queue_conf; 179 dev_info->capabilities = bbdev_capabilities; 180 dev_info->cpu_flag_reqs = &cpu_flag; 181 dev_info->min_alignment = 64; 182 183 rte_bbdev_log_debug("got device info from %u\n", dev->data->dev_id); 184 } 185 186 /* Release queue */ 187 static int 188 q_release(struct rte_bbdev *dev, uint16_t q_id) 189 { 190 struct turbo_sw_queue *q = dev->data->queues[q_id].queue_private; 191 192 if (q != NULL) { 193 rte_ring_free(q->processed_pkts); 194 rte_free(q->enc_out); 195 rte_free(q->enc_in); 196 rte_free(q->ag); 197 rte_free(q->code_block); 198 rte_free(q->deint_input); 199 rte_free(q->deint_output); 200 rte_free(q->adapter_output); 201 rte_free(q); 202 dev->data->queues[q_id].queue_private = NULL; 203 } 204 205 rte_bbdev_log_debug("released device queue %u:%u", 206 dev->data->dev_id, q_id); 207 return 0; 208 } 209 210 /* Setup a queue */ 211 static int 212 q_setup(struct rte_bbdev *dev, uint16_t q_id, 213 const struct rte_bbdev_queue_conf *queue_conf) 214 { 215 int ret; 216 struct turbo_sw_queue *q; 217 char name[RTE_RING_NAMESIZE]; 218 219 /* Allocate the queue data structure. */ 220 q = rte_zmalloc_socket(RTE_STR(DRIVER_NAME), sizeof(*q), 221 RTE_CACHE_LINE_SIZE, queue_conf->socket); 222 if (q == NULL) { 223 rte_bbdev_log(ERR, "Failed to allocate queue memory"); 224 return -ENOMEM; 225 } 226 227 /* Allocate memory for encoder output. */ 228 ret = snprintf(name, RTE_RING_NAMESIZE, RTE_STR(DRIVER_NAME)"_enc_out%u:%u", 229 dev->data->dev_id, q_id); 230 if ((ret < 0) || (ret >= (int)RTE_RING_NAMESIZE)) { 231 rte_bbdev_log(ERR, 232 "Creating queue name for device %u queue %u failed", 233 dev->data->dev_id, q_id); 234 return -ENAMETOOLONG; 235 } 236 q->enc_out = rte_zmalloc_socket(name, 237 ((RTE_BBDEV_MAX_TB_SIZE >> 3) + 3) * 238 sizeof(*q->enc_out) * 3, 239 RTE_CACHE_LINE_SIZE, queue_conf->socket); 240 if (q->enc_out == NULL) { 241 rte_bbdev_log(ERR, 242 "Failed to allocate queue memory for %s", name); 243 goto free_q; 244 } 245 246 /* Allocate memory for rate matching output. */ 247 ret = snprintf(name, RTE_RING_NAMESIZE, 248 RTE_STR(DRIVER_NAME)"_enc_in%u:%u", dev->data->dev_id, 249 q_id); 250 if ((ret < 0) || (ret >= (int)RTE_RING_NAMESIZE)) { 251 rte_bbdev_log(ERR, 252 "Creating queue name for device %u queue %u failed", 253 dev->data->dev_id, q_id); 254 return -ENAMETOOLONG; 255 } 256 q->enc_in = rte_zmalloc_socket(name, 257 (RTE_BBDEV_MAX_CB_SIZE >> 3) * sizeof(*q->enc_in), 258 RTE_CACHE_LINE_SIZE, queue_conf->socket); 259 if (q->enc_in == NULL) { 260 rte_bbdev_log(ERR, 261 "Failed to allocate queue memory for %s", name); 262 goto free_q; 263 } 264 265 /* Allocate memory for Aplha Gamma temp buffer. */ 266 ret = snprintf(name, RTE_RING_NAMESIZE, RTE_STR(DRIVER_NAME)"_ag%u:%u", 267 dev->data->dev_id, q_id); 268 if ((ret < 0) || (ret >= (int)RTE_RING_NAMESIZE)) { 269 rte_bbdev_log(ERR, 270 "Creating queue name for device %u queue %u failed", 271 dev->data->dev_id, q_id); 272 return -ENAMETOOLONG; 273 } 274 q->ag = rte_zmalloc_socket(name, 275 RTE_BBDEV_MAX_CB_SIZE * 10 * sizeof(*q->ag), 276 RTE_CACHE_LINE_SIZE, queue_conf->socket); 277 if (q->ag == NULL) { 278 rte_bbdev_log(ERR, 279 "Failed to allocate queue memory for %s", name); 280 goto free_q; 281 } 282 283 /* Allocate memory for code block temp buffer. */ 284 ret = snprintf(name, RTE_RING_NAMESIZE, RTE_STR(DRIVER_NAME)"_cb%u:%u", 285 dev->data->dev_id, q_id); 286 if ((ret < 0) || (ret >= (int)RTE_RING_NAMESIZE)) { 287 rte_bbdev_log(ERR, 288 "Creating queue name for device %u queue %u failed", 289 dev->data->dev_id, q_id); 290 return -ENAMETOOLONG; 291 } 292 q->code_block = rte_zmalloc_socket(name, 293 RTE_BBDEV_MAX_CB_SIZE * sizeof(*q->code_block), 294 RTE_CACHE_LINE_SIZE, queue_conf->socket); 295 if (q->code_block == NULL) { 296 rte_bbdev_log(ERR, 297 "Failed to allocate queue memory for %s", name); 298 goto free_q; 299 } 300 301 /* Allocate memory for Deinterleaver input. */ 302 ret = snprintf(name, RTE_RING_NAMESIZE, 303 RTE_STR(DRIVER_NAME)"_deint_input%u:%u", 304 dev->data->dev_id, q_id); 305 if ((ret < 0) || (ret >= (int)RTE_RING_NAMESIZE)) { 306 rte_bbdev_log(ERR, 307 "Creating queue name for device %u queue %u failed", 308 dev->data->dev_id, q_id); 309 return -ENAMETOOLONG; 310 } 311 q->deint_input = rte_zmalloc_socket(name, 312 DEINT_INPUT_BUF_SIZE * sizeof(*q->deint_input), 313 RTE_CACHE_LINE_SIZE, queue_conf->socket); 314 if (q->deint_input == NULL) { 315 rte_bbdev_log(ERR, 316 "Failed to allocate queue memory for %s", name); 317 goto free_q; 318 } 319 320 /* Allocate memory for Deinterleaver output. */ 321 ret = snprintf(name, RTE_RING_NAMESIZE, 322 RTE_STR(DRIVER_NAME)"_deint_output%u:%u", 323 dev->data->dev_id, q_id); 324 if ((ret < 0) || (ret >= (int)RTE_RING_NAMESIZE)) { 325 rte_bbdev_log(ERR, 326 "Creating queue name for device %u queue %u failed", 327 dev->data->dev_id, q_id); 328 return -ENAMETOOLONG; 329 } 330 q->deint_output = rte_zmalloc_socket(NULL, 331 DEINT_OUTPUT_BUF_SIZE * sizeof(*q->deint_output), 332 RTE_CACHE_LINE_SIZE, queue_conf->socket); 333 if (q->deint_output == NULL) { 334 rte_bbdev_log(ERR, 335 "Failed to allocate queue memory for %s", name); 336 goto free_q; 337 } 338 339 /* Allocate memory for Adapter output. */ 340 ret = snprintf(name, RTE_RING_NAMESIZE, 341 RTE_STR(DRIVER_NAME)"_adapter_output%u:%u", 342 dev->data->dev_id, q_id); 343 if ((ret < 0) || (ret >= (int)RTE_RING_NAMESIZE)) { 344 rte_bbdev_log(ERR, 345 "Creating queue name for device %u queue %u failed", 346 dev->data->dev_id, q_id); 347 return -ENAMETOOLONG; 348 } 349 q->adapter_output = rte_zmalloc_socket(NULL, 350 ADAPTER_OUTPUT_BUF_SIZE * sizeof(*q->adapter_output), 351 RTE_CACHE_LINE_SIZE, queue_conf->socket); 352 if (q->adapter_output == NULL) { 353 rte_bbdev_log(ERR, 354 "Failed to allocate queue memory for %s", name); 355 goto free_q; 356 } 357 358 /* Create ring for packets awaiting to be dequeued. */ 359 ret = snprintf(name, RTE_RING_NAMESIZE, RTE_STR(DRIVER_NAME)"%u:%u", 360 dev->data->dev_id, q_id); 361 if ((ret < 0) || (ret >= (int)RTE_RING_NAMESIZE)) { 362 rte_bbdev_log(ERR, 363 "Creating queue name for device %u queue %u failed", 364 dev->data->dev_id, q_id); 365 return -ENAMETOOLONG; 366 } 367 q->processed_pkts = rte_ring_create(name, queue_conf->queue_size, 368 queue_conf->socket, RING_F_SP_ENQ | RING_F_SC_DEQ); 369 if (q->processed_pkts == NULL) { 370 rte_bbdev_log(ERR, "Failed to create ring for %s", name); 371 goto free_q; 372 } 373 374 q->type = queue_conf->op_type; 375 376 dev->data->queues[q_id].queue_private = q; 377 rte_bbdev_log_debug("setup device queue %s", name); 378 return 0; 379 380 free_q: 381 rte_ring_free(q->processed_pkts); 382 rte_free(q->enc_out); 383 rte_free(q->enc_in); 384 rte_free(q->ag); 385 rte_free(q->code_block); 386 rte_free(q->deint_input); 387 rte_free(q->deint_output); 388 rte_free(q->adapter_output); 389 rte_free(q); 390 return -EFAULT; 391 } 392 393 static const struct rte_bbdev_ops pmd_ops = { 394 .info_get = info_get, 395 .queue_setup = q_setup, 396 .queue_release = q_release 397 }; 398 399 /* Checks if the encoder input buffer is correct. 400 * Returns 0 if it's valid, -1 otherwise. 401 */ 402 static inline int 403 is_enc_input_valid(const uint16_t k, const int32_t k_idx, 404 const uint16_t in_length) 405 { 406 if (k_idx < 0) { 407 rte_bbdev_log(ERR, "K Index is invalid"); 408 return -1; 409 } 410 411 if (in_length - (k >> 3) < 0) { 412 rte_bbdev_log(ERR, 413 "Mismatch between input length (%u bytes) and K (%u bits)", 414 in_length, k); 415 return -1; 416 } 417 418 if (k > RTE_BBDEV_MAX_CB_SIZE) { 419 rte_bbdev_log(ERR, "CB size (%u) is too big, max: %d", 420 k, RTE_BBDEV_MAX_CB_SIZE); 421 return -1; 422 } 423 424 return 0; 425 } 426 427 /* Checks if the decoder input buffer is correct. 428 * Returns 0 if it's valid, -1 otherwise. 429 */ 430 static inline int 431 is_dec_input_valid(int32_t k_idx, int16_t kw, int16_t in_length) 432 { 433 if (k_idx < 0) { 434 rte_bbdev_log(ERR, "K index is invalid"); 435 return -1; 436 } 437 438 if (in_length - kw < 0) { 439 rte_bbdev_log(ERR, 440 "Mismatch between input length (%u) and kw (%u)", 441 in_length, kw); 442 return -1; 443 } 444 445 if (kw > RTE_BBDEV_MAX_KW) { 446 rte_bbdev_log(ERR, "Input length (%u) is too big, max: %d", 447 kw, RTE_BBDEV_MAX_KW); 448 return -1; 449 } 450 451 return 0; 452 } 453 454 static inline void 455 process_enc_cb(struct turbo_sw_queue *q, struct rte_bbdev_enc_op *op, 456 uint8_t r, uint8_t c, uint16_t k, uint16_t ncb, 457 uint32_t e, struct rte_mbuf *m_in, struct rte_mbuf *m_out, 458 uint16_t in_offset, uint16_t out_offset, uint16_t total_left) 459 { 460 int ret; 461 int16_t k_idx; 462 uint16_t m; 463 uint8_t *in, *out0, *out1, *out2, *tmp_out, *rm_out; 464 uint64_t first_3_bytes = 0; 465 struct rte_bbdev_op_turbo_enc *enc = &op->turbo_enc; 466 struct bblib_crc_request crc_req; 467 struct bblib_crc_response crc_resp; 468 struct bblib_turbo_encoder_request turbo_req; 469 struct bblib_turbo_encoder_response turbo_resp; 470 struct bblib_rate_match_dl_request rm_req; 471 struct bblib_rate_match_dl_response rm_resp; 472 473 k_idx = compute_idx(k); 474 in = rte_pktmbuf_mtod_offset(m_in, uint8_t *, in_offset); 475 476 /* CRC24A (for TB) */ 477 if ((enc->op_flags & RTE_BBDEV_TURBO_CRC_24A_ATTACH) && 478 (enc->code_block_mode == 1)) { 479 ret = is_enc_input_valid(k - 24, k_idx, total_left); 480 if (ret != 0) { 481 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 482 return; 483 } 484 crc_req.data = in; 485 crc_req.len = (k - 24) >> 3; 486 /* Check if there is a room for CRC bits. If not use 487 * the temporary buffer. 488 */ 489 if (rte_pktmbuf_append(m_in, 3) == NULL) { 490 rte_memcpy(q->enc_in, in, (k - 24) >> 3); 491 in = q->enc_in; 492 } else { 493 /* Store 3 first bytes of next CB as they will be 494 * overwritten by CRC bytes. If it is the last CB then 495 * there is no point to store 3 next bytes and this 496 * if..else branch will be omitted. 497 */ 498 first_3_bytes = *((uint64_t *)&in[(k - 32) >> 3]); 499 } 500 501 crc_resp.data = in; 502 bblib_lte_crc24a_gen(&crc_req, &crc_resp); 503 } else if (enc->op_flags & RTE_BBDEV_TURBO_CRC_24B_ATTACH) { 504 /* CRC24B */ 505 ret = is_enc_input_valid(k - 24, k_idx, total_left); 506 if (ret != 0) { 507 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 508 return; 509 } 510 crc_req.data = in; 511 crc_req.len = (k - 24) >> 3; 512 /* Check if there is a room for CRC bits. If this is the last 513 * CB in TB. If not use temporary buffer. 514 */ 515 if ((c - r == 1) && (rte_pktmbuf_append(m_in, 3) == NULL)) { 516 rte_memcpy(q->enc_in, in, (k - 24) >> 3); 517 in = q->enc_in; 518 } else if (c - r > 1) { 519 /* Store 3 first bytes of next CB as they will be 520 * overwritten by CRC bytes. If it is the last CB then 521 * there is no point to store 3 next bytes and this 522 * if..else branch will be omitted. 523 */ 524 first_3_bytes = *((uint64_t *)&in[(k - 32) >> 3]); 525 } 526 527 crc_resp.data = in; 528 bblib_lte_crc24b_gen(&crc_req, &crc_resp); 529 } else { 530 ret = is_enc_input_valid(k, k_idx, total_left); 531 if (ret != 0) { 532 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 533 return; 534 } 535 } 536 537 /* Turbo encoder */ 538 539 /* Each bit layer output from turbo encoder is (k+4) bits long, i.e. 540 * input length + 4 tail bits. That's (k/8) + 1 bytes after rounding up. 541 * So dst_data's length should be 3*(k/8) + 3 bytes. 542 * In Rate-matching bypass case outputs pointers passed to encoder 543 * (out0, out1 and out2) can directly point to addresses of output from 544 * turbo_enc entity. 545 */ 546 if (enc->op_flags & RTE_BBDEV_TURBO_RATE_MATCH) { 547 out0 = q->enc_out; 548 out1 = RTE_PTR_ADD(out0, (k >> 3) + 1); 549 out2 = RTE_PTR_ADD(out1, (k >> 3) + 1); 550 } else { 551 out0 = (uint8_t *)rte_pktmbuf_append(m_out, (k >> 3) * 3 + 2); 552 if (out0 == NULL) { 553 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 554 rte_bbdev_log(ERR, 555 "Too little space in output mbuf"); 556 return; 557 } 558 enc->output.length += (k >> 3) * 3 + 2; 559 /* rte_bbdev_op_data.offset can be different than the 560 * offset of the appended bytes 561 */ 562 out0 = rte_pktmbuf_mtod_offset(m_out, uint8_t *, out_offset); 563 out1 = rte_pktmbuf_mtod_offset(m_out, uint8_t *, 564 out_offset + (k >> 3) + 1); 565 out2 = rte_pktmbuf_mtod_offset(m_out, uint8_t *, 566 out_offset + 2 * ((k >> 3) + 1)); 567 } 568 569 turbo_req.case_id = k_idx; 570 turbo_req.input_win = in; 571 turbo_req.length = k >> 3; 572 turbo_resp.output_win_0 = out0; 573 turbo_resp.output_win_1 = out1; 574 turbo_resp.output_win_2 = out2; 575 if (bblib_turbo_encoder(&turbo_req, &turbo_resp) != 0) { 576 op->status |= 1 << RTE_BBDEV_DRV_ERROR; 577 rte_bbdev_log(ERR, "Turbo Encoder failed"); 578 return; 579 } 580 581 /* Restore 3 first bytes of next CB if they were overwritten by CRC*/ 582 if (first_3_bytes != 0) 583 *((uint64_t *)&in[(k - 32) >> 3]) = first_3_bytes; 584 585 /* Rate-matching */ 586 if (enc->op_flags & RTE_BBDEV_TURBO_RATE_MATCH) { 587 uint8_t mask_id; 588 /* Integer round up division by 8 */ 589 uint16_t out_len = (e + 7) >> 3; 590 /* The mask array is indexed using E%8. E is an even number so 591 * there are only 4 possible values. 592 */ 593 const uint8_t mask_out[] = {0xFF, 0xC0, 0xF0, 0xFC}; 594 595 /* get output data starting address */ 596 rm_out = (uint8_t *)rte_pktmbuf_append(m_out, out_len); 597 if (rm_out == NULL) { 598 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 599 rte_bbdev_log(ERR, 600 "Too little space in output mbuf"); 601 return; 602 } 603 /* rte_bbdev_op_data.offset can be different than the offset 604 * of the appended bytes 605 */ 606 rm_out = rte_pktmbuf_mtod_offset(m_out, uint8_t *, out_offset); 607 608 /* index of current code block */ 609 rm_req.r = r; 610 /* total number of code block */ 611 rm_req.C = c; 612 /* For DL - 1, UL - 0 */ 613 rm_req.direction = 1; 614 /* According to 3ggp 36.212 Spec 5.1.4.1.2 section Nsoft, KMIMO 615 * and MDL_HARQ are used for Ncb calculation. As Ncb is already 616 * known we can adjust those parameters 617 */ 618 rm_req.Nsoft = ncb * rm_req.C; 619 rm_req.KMIMO = 1; 620 rm_req.MDL_HARQ = 1; 621 /* According to 3ggp 36.212 Spec 5.1.4.1.2 section Nl, Qm and G 622 * are used for E calculation. As E is already known we can 623 * adjust those parameters 624 */ 625 rm_req.NL = e; 626 rm_req.Qm = 1; 627 rm_req.G = rm_req.NL * rm_req.Qm * rm_req.C; 628 629 rm_req.rvidx = enc->rv_index; 630 rm_req.Kidx = k_idx - 1; 631 rm_req.nLen = k + 4; 632 rm_req.tin0 = out0; 633 rm_req.tin1 = out1; 634 rm_req.tin2 = out2; 635 rm_resp.output = rm_out; 636 rm_resp.OutputLen = out_len; 637 if (enc->op_flags & RTE_BBDEV_TURBO_RV_INDEX_BYPASS) 638 rm_req.bypass_rvidx = 1; 639 else 640 rm_req.bypass_rvidx = 0; 641 642 if (bblib_rate_match_dl(&rm_req, &rm_resp) != 0) { 643 op->status |= 1 << RTE_BBDEV_DRV_ERROR; 644 rte_bbdev_log(ERR, "Rate matching failed"); 645 return; 646 } 647 648 /* SW fills an entire last byte even if E%8 != 0. Clear the 649 * superfluous data bits for consistency with HW device. 650 */ 651 mask_id = (e & 7) >> 1; 652 rm_out[out_len - 1] &= mask_out[mask_id]; 653 654 enc->output.length += rm_resp.OutputLen; 655 } else { 656 /* Rate matching is bypassed */ 657 658 /* Completing last byte of out0 (where 4 tail bits are stored) 659 * by moving first 4 bits from out1 660 */ 661 tmp_out = (uint8_t *) --out1; 662 *tmp_out = *tmp_out | ((*(tmp_out + 1) & 0xF0) >> 4); 663 tmp_out++; 664 /* Shifting out1 data by 4 bits to the left */ 665 for (m = 0; m < k >> 3; ++m) { 666 uint8_t *first = tmp_out; 667 uint8_t second = *(tmp_out + 1); 668 *first = (*first << 4) | ((second & 0xF0) >> 4); 669 tmp_out++; 670 } 671 /* Shifting out2 data by 8 bits to the left */ 672 for (m = 0; m < (k >> 3) + 1; ++m) { 673 *tmp_out = *(tmp_out + 1); 674 tmp_out++; 675 } 676 *tmp_out = 0; 677 } 678 } 679 680 static inline void 681 enqueue_enc_one_op(struct turbo_sw_queue *q, struct rte_bbdev_enc_op *op) 682 { 683 uint8_t c, r, crc24_bits = 0; 684 uint16_t k, ncb; 685 uint32_t e; 686 struct rte_bbdev_op_turbo_enc *enc = &op->turbo_enc; 687 uint16_t in_offset = enc->input.offset; 688 uint16_t out_offset = enc->output.offset; 689 struct rte_mbuf *m_in = enc->input.data; 690 struct rte_mbuf *m_out = enc->output.data; 691 uint16_t total_left = enc->input.length; 692 693 /* Clear op status */ 694 op->status = 0; 695 696 if (total_left > RTE_BBDEV_MAX_TB_SIZE >> 3) { 697 rte_bbdev_log(ERR, "TB size (%u) is too big, max: %d", 698 total_left, RTE_BBDEV_MAX_TB_SIZE); 699 op->status = 1 << RTE_BBDEV_DATA_ERROR; 700 return; 701 } 702 703 if (m_in == NULL || m_out == NULL) { 704 rte_bbdev_log(ERR, "Invalid mbuf pointer"); 705 op->status = 1 << RTE_BBDEV_DATA_ERROR; 706 return; 707 } 708 709 if ((enc->op_flags & RTE_BBDEV_TURBO_CRC_24B_ATTACH) || 710 (enc->op_flags & RTE_BBDEV_TURBO_CRC_24A_ATTACH)) 711 crc24_bits = 24; 712 713 if (enc->code_block_mode == 0) { /* For Transport Block mode */ 714 c = enc->tb_params.c; 715 r = enc->tb_params.r; 716 } else {/* For Code Block mode */ 717 c = 1; 718 r = 0; 719 } 720 721 while (total_left > 0 && r < c) { 722 if (enc->code_block_mode == 0) { 723 k = (r < enc->tb_params.c_neg) ? 724 enc->tb_params.k_neg : enc->tb_params.k_pos; 725 ncb = (r < enc->tb_params.c_neg) ? 726 enc->tb_params.ncb_neg : enc->tb_params.ncb_pos; 727 e = (r < enc->tb_params.cab) ? 728 enc->tb_params.ea : enc->tb_params.eb; 729 } else { 730 k = enc->cb_params.k; 731 ncb = enc->cb_params.ncb; 732 e = enc->cb_params.e; 733 } 734 735 process_enc_cb(q, op, r, c, k, ncb, e, m_in, 736 m_out, in_offset, out_offset, total_left); 737 /* Update total_left */ 738 total_left -= (k - crc24_bits) >> 3; 739 /* Update offsets for next CBs (if exist) */ 740 in_offset += (k - crc24_bits) >> 3; 741 if (enc->op_flags & RTE_BBDEV_TURBO_RATE_MATCH) 742 out_offset += e >> 3; 743 else 744 out_offset += (k >> 3) * 3 + 2; 745 r++; 746 } 747 748 /* check if all input data was processed */ 749 if (total_left != 0) { 750 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 751 rte_bbdev_log(ERR, 752 "Mismatch between mbuf length and included CBs sizes"); 753 } 754 } 755 756 static inline uint16_t 757 enqueue_enc_all_ops(struct turbo_sw_queue *q, struct rte_bbdev_enc_op **ops, 758 uint16_t nb_ops) 759 { 760 uint16_t i; 761 762 for (i = 0; i < nb_ops; ++i) 763 enqueue_enc_one_op(q, ops[i]); 764 765 return rte_ring_enqueue_burst(q->processed_pkts, (void **)ops, nb_ops, 766 NULL); 767 } 768 769 /* Remove the padding bytes from a cyclic buffer. 770 * The input buffer is a data stream wk as described in 3GPP TS 36.212 section 771 * 5.1.4.1.2 starting from w0 and with length Ncb bytes. 772 * The output buffer is a data stream wk with pruned padding bytes. It's length 773 * is 3*D bytes and the order of non-padding bytes is preserved. 774 */ 775 static inline void 776 remove_nulls_from_circular_buf(const uint8_t *in, uint8_t *out, uint16_t k, 777 uint16_t ncb) 778 { 779 uint32_t in_idx, out_idx, c_idx; 780 const uint32_t d = k + 4; 781 const uint32_t kw = (ncb / 3); 782 const uint32_t nd = kw - d; 783 const uint32_t r_subblock = kw / RTE_BBDEV_C_SUBBLOCK; 784 /* Inter-column permutation pattern */ 785 const uint32_t P[RTE_BBDEV_C_SUBBLOCK] = {0, 16, 8, 24, 4, 20, 12, 28, 786 2, 18, 10, 26, 6, 22, 14, 30, 1, 17, 9, 25, 5, 21, 13, 787 29, 3, 19, 11, 27, 7, 23, 15, 31}; 788 in_idx = 0; 789 out_idx = 0; 790 791 /* The padding bytes are at the first Nd positions in the first row. */ 792 for (c_idx = 0; in_idx < kw; in_idx += r_subblock, ++c_idx) { 793 if (P[c_idx] < nd) { 794 rte_memcpy(&out[out_idx], &in[in_idx + 1], 795 r_subblock - 1); 796 out_idx += r_subblock - 1; 797 } else { 798 rte_memcpy(&out[out_idx], &in[in_idx], r_subblock); 799 out_idx += r_subblock; 800 } 801 } 802 803 /* First and second parity bits sub-blocks are interlaced. */ 804 for (c_idx = 0; in_idx < ncb - 2 * r_subblock; 805 in_idx += 2 * r_subblock, ++c_idx) { 806 uint32_t second_block_c_idx = P[c_idx]; 807 uint32_t third_block_c_idx = P[c_idx] + 1; 808 809 if (second_block_c_idx < nd && third_block_c_idx < nd) { 810 rte_memcpy(&out[out_idx], &in[in_idx + 2], 811 2 * r_subblock - 2); 812 out_idx += 2 * r_subblock - 2; 813 } else if (second_block_c_idx >= nd && 814 third_block_c_idx >= nd) { 815 rte_memcpy(&out[out_idx], &in[in_idx], 2 * r_subblock); 816 out_idx += 2 * r_subblock; 817 } else if (second_block_c_idx < nd) { 818 out[out_idx++] = in[in_idx]; 819 rte_memcpy(&out[out_idx], &in[in_idx + 2], 820 2 * r_subblock - 2); 821 out_idx += 2 * r_subblock - 2; 822 } else { 823 rte_memcpy(&out[out_idx], &in[in_idx + 1], 824 2 * r_subblock - 1); 825 out_idx += 2 * r_subblock - 1; 826 } 827 } 828 829 /* Last interlaced row is different - its last byte is the only padding 830 * byte. We can have from 4 up to 28 padding bytes (Nd) per sub-block. 831 * After interlacing the 1st and 2nd parity sub-blocks we can have 0, 1 832 * or 2 padding bytes each time we make a step of 2 * R_SUBBLOCK bytes 833 * (moving to another column). 2nd parity sub-block uses the same 834 * inter-column permutation pattern as the systematic and 1st parity 835 * sub-blocks but it adds '1' to the resulting index and calculates the 836 * modulus of the result and Kw. Last column is mapped to itself (id 31) 837 * so the first byte taken from the 2nd parity sub-block will be the 838 * 32nd (31+1) byte, then 64th etc. (step is C_SUBBLOCK == 32) and the 839 * last byte will be the first byte from the sub-block: 840 * (32 + 32 * (R_SUBBLOCK-1)) % Kw == Kw % Kw == 0. Nd can't be smaller 841 * than 4 so we know that bytes with ids 0, 1, 2 and 3 must be the 842 * padding bytes. The bytes from the 1st parity sub-block are the bytes 843 * from the 31st column - Nd can't be greater than 28 so we are sure 844 * that there are no padding bytes in 31st column. 845 */ 846 rte_memcpy(&out[out_idx], &in[in_idx], 2 * r_subblock - 1); 847 } 848 849 static inline void 850 move_padding_bytes(const uint8_t *in, uint8_t *out, uint16_t k, 851 uint16_t ncb) 852 { 853 uint16_t d = k + 4; 854 uint16_t kpi = ncb / 3; 855 uint16_t nd = kpi - d; 856 857 rte_memcpy(&out[nd], in, d); 858 rte_memcpy(&out[nd + kpi + 64], &in[kpi], d); 859 rte_memcpy(&out[(nd - 1) + 2 * (kpi + 64)], &in[2 * kpi], d); 860 } 861 862 static inline void 863 process_dec_cb(struct turbo_sw_queue *q, struct rte_bbdev_dec_op *op, 864 uint8_t c, uint16_t k, uint16_t kw, struct rte_mbuf *m_in, 865 struct rte_mbuf *m_out, uint16_t in_offset, uint16_t out_offset, 866 bool check_crc_24b, uint16_t crc24_overlap, uint16_t total_left) 867 { 868 int ret; 869 int32_t k_idx; 870 int32_t iter_cnt; 871 uint8_t *in, *out, *adapter_input; 872 int32_t ncb, ncb_without_null; 873 struct bblib_turbo_adapter_ul_response adapter_resp; 874 struct bblib_turbo_adapter_ul_request adapter_req; 875 struct bblib_turbo_decoder_request turbo_req; 876 struct bblib_turbo_decoder_response turbo_resp; 877 struct rte_bbdev_op_turbo_dec *dec = &op->turbo_dec; 878 879 k_idx = compute_idx(k); 880 881 ret = is_dec_input_valid(k_idx, kw, total_left); 882 if (ret != 0) { 883 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 884 return; 885 } 886 887 in = rte_pktmbuf_mtod_offset(m_in, uint8_t *, in_offset); 888 ncb = kw; 889 ncb_without_null = (k + 4) * 3; 890 891 if (check_bit(dec->op_flags, RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE)) { 892 struct bblib_deinterleave_ul_request deint_req; 893 struct bblib_deinterleave_ul_response deint_resp; 894 895 /* SW decoder accepts only a circular buffer without NULL bytes 896 * so the input needs to be converted. 897 */ 898 remove_nulls_from_circular_buf(in, q->deint_input, k, ncb); 899 900 deint_req.pharqbuffer = q->deint_input; 901 deint_req.ncb = ncb_without_null; 902 deint_resp.pinteleavebuffer = q->deint_output; 903 bblib_deinterleave_ul(&deint_req, &deint_resp); 904 } else 905 move_padding_bytes(in, q->deint_output, k, ncb); 906 907 adapter_input = q->deint_output; 908 909 if (dec->op_flags & RTE_BBDEV_TURBO_POS_LLR_1_BIT_IN) 910 adapter_req.isinverted = 1; 911 else if (dec->op_flags & RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN) 912 adapter_req.isinverted = 0; 913 else { 914 op->status |= 1 << RTE_BBDEV_DRV_ERROR; 915 rte_bbdev_log(ERR, "LLR format wasn't specified"); 916 return; 917 } 918 919 adapter_req.ncb = ncb_without_null; 920 adapter_req.pinteleavebuffer = adapter_input; 921 adapter_resp.pharqout = q->adapter_output; 922 bblib_turbo_adapter_ul(&adapter_req, &adapter_resp); 923 924 out = (uint8_t *)rte_pktmbuf_append(m_out, ((k - crc24_overlap) >> 3)); 925 if (out == NULL) { 926 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 927 rte_bbdev_log(ERR, "Too little space in output mbuf"); 928 return; 929 } 930 /* rte_bbdev_op_data.offset can be different than the offset of the 931 * appended bytes 932 */ 933 out = rte_pktmbuf_mtod_offset(m_out, uint8_t *, out_offset); 934 if (check_crc_24b) 935 turbo_req.c = c + 1; 936 else 937 turbo_req.c = c; 938 turbo_req.input = (int8_t *)q->adapter_output; 939 turbo_req.k = k; 940 turbo_req.k_idx = k_idx; 941 turbo_req.max_iter_num = dec->iter_max; 942 turbo_resp.ag_buf = q->ag; 943 turbo_resp.cb_buf = q->code_block; 944 turbo_resp.output = out; 945 iter_cnt = bblib_turbo_decoder(&turbo_req, &turbo_resp); 946 dec->hard_output.length += (k >> 3); 947 948 if (iter_cnt > 0) { 949 /* Temporary solution for returned iter_count from SDK */ 950 iter_cnt = (iter_cnt - 1) / 2; 951 dec->iter_count = RTE_MAX(iter_cnt, dec->iter_count); 952 } else { 953 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 954 rte_bbdev_log(ERR, "Turbo Decoder failed"); 955 return; 956 } 957 } 958 959 static inline void 960 enqueue_dec_one_op(struct turbo_sw_queue *q, struct rte_bbdev_dec_op *op) 961 { 962 uint8_t c, r = 0; 963 uint16_t kw, k = 0; 964 uint16_t crc24_overlap = 0; 965 struct rte_bbdev_op_turbo_dec *dec = &op->turbo_dec; 966 struct rte_mbuf *m_in = dec->input.data; 967 struct rte_mbuf *m_out = dec->hard_output.data; 968 uint16_t in_offset = dec->input.offset; 969 uint16_t total_left = dec->input.length; 970 uint16_t out_offset = dec->hard_output.offset; 971 972 /* Clear op status */ 973 op->status = 0; 974 975 if (m_in == NULL || m_out == NULL) { 976 rte_bbdev_log(ERR, "Invalid mbuf pointer"); 977 op->status = 1 << RTE_BBDEV_DATA_ERROR; 978 return; 979 } 980 981 if (dec->code_block_mode == 0) { /* For Transport Block mode */ 982 c = dec->tb_params.c; 983 } else { /* For Code Block mode */ 984 k = dec->cb_params.k; 985 c = 1; 986 } 987 988 if ((c > 1) && !check_bit(dec->op_flags, 989 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP)) 990 crc24_overlap = 24; 991 992 while (total_left > 0) { 993 if (dec->code_block_mode == 0) 994 k = (r < dec->tb_params.c_neg) ? 995 dec->tb_params.k_neg : dec->tb_params.k_pos; 996 997 /* Calculates circular buffer size (Kw). 998 * According to 3gpp 36.212 section 5.1.4.2 999 * Kw = 3 * Kpi, 1000 * where: 1001 * Kpi = nCol * nRow 1002 * where nCol is 32 and nRow can be calculated from: 1003 * D =< nCol * nRow 1004 * where D is the size of each output from turbo encoder block 1005 * (k + 4). 1006 */ 1007 kw = RTE_ALIGN_CEIL(k + 4, RTE_BBDEV_C_SUBBLOCK) * 3; 1008 1009 process_dec_cb(q, op, c, k, kw, m_in, m_out, in_offset, 1010 out_offset, check_bit(dec->op_flags, 1011 RTE_BBDEV_TURBO_CRC_TYPE_24B), crc24_overlap, 1012 total_left); 1013 /* To keep CRC24 attached to end of Code block, use 1014 * RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP flag as it 1015 * removed by default once verified. 1016 */ 1017 1018 /* Update total_left */ 1019 total_left -= kw; 1020 /* Update offsets for next CBs (if exist) */ 1021 in_offset += kw; 1022 out_offset += ((k - crc24_overlap) >> 3); 1023 r++; 1024 } 1025 if (total_left != 0) { 1026 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 1027 rte_bbdev_log(ERR, 1028 "Mismatch between mbuf length and included Circular buffer sizes"); 1029 } 1030 } 1031 1032 static inline uint16_t 1033 enqueue_dec_all_ops(struct turbo_sw_queue *q, struct rte_bbdev_dec_op **ops, 1034 uint16_t nb_ops) 1035 { 1036 uint16_t i; 1037 1038 for (i = 0; i < nb_ops; ++i) 1039 enqueue_dec_one_op(q, ops[i]); 1040 1041 return rte_ring_enqueue_burst(q->processed_pkts, (void **)ops, nb_ops, 1042 NULL); 1043 } 1044 1045 /* Enqueue burst */ 1046 static uint16_t 1047 enqueue_enc_ops(struct rte_bbdev_queue_data *q_data, 1048 struct rte_bbdev_enc_op **ops, uint16_t nb_ops) 1049 { 1050 void *queue = q_data->queue_private; 1051 struct turbo_sw_queue *q = queue; 1052 uint16_t nb_enqueued = 0; 1053 1054 nb_enqueued = enqueue_enc_all_ops(q, ops, nb_ops); 1055 1056 q_data->queue_stats.enqueue_err_count += nb_ops - nb_enqueued; 1057 q_data->queue_stats.enqueued_count += nb_enqueued; 1058 1059 return nb_enqueued; 1060 } 1061 1062 /* Enqueue burst */ 1063 static uint16_t 1064 enqueue_dec_ops(struct rte_bbdev_queue_data *q_data, 1065 struct rte_bbdev_dec_op **ops, uint16_t nb_ops) 1066 { 1067 void *queue = q_data->queue_private; 1068 struct turbo_sw_queue *q = queue; 1069 uint16_t nb_enqueued = 0; 1070 1071 nb_enqueued = enqueue_dec_all_ops(q, ops, nb_ops); 1072 1073 q_data->queue_stats.enqueue_err_count += nb_ops - nb_enqueued; 1074 q_data->queue_stats.enqueued_count += nb_enqueued; 1075 1076 return nb_enqueued; 1077 } 1078 1079 /* Dequeue decode burst */ 1080 static uint16_t 1081 dequeue_dec_ops(struct rte_bbdev_queue_data *q_data, 1082 struct rte_bbdev_dec_op **ops, uint16_t nb_ops) 1083 { 1084 struct turbo_sw_queue *q = q_data->queue_private; 1085 uint16_t nb_dequeued = rte_ring_dequeue_burst(q->processed_pkts, 1086 (void **)ops, nb_ops, NULL); 1087 q_data->queue_stats.dequeued_count += nb_dequeued; 1088 1089 return nb_dequeued; 1090 } 1091 1092 /* Dequeue encode burst */ 1093 static uint16_t 1094 dequeue_enc_ops(struct rte_bbdev_queue_data *q_data, 1095 struct rte_bbdev_enc_op **ops, uint16_t nb_ops) 1096 { 1097 struct turbo_sw_queue *q = q_data->queue_private; 1098 uint16_t nb_dequeued = rte_ring_dequeue_burst(q->processed_pkts, 1099 (void **)ops, nb_ops, NULL); 1100 q_data->queue_stats.dequeued_count += nb_dequeued; 1101 1102 return nb_dequeued; 1103 } 1104 1105 /* Parse 16bit integer from string argument */ 1106 static inline int 1107 parse_u16_arg(const char *key, const char *value, void *extra_args) 1108 { 1109 uint16_t *u16 = extra_args; 1110 unsigned int long result; 1111 1112 if ((value == NULL) || (extra_args == NULL)) 1113 return -EINVAL; 1114 errno = 0; 1115 result = strtoul(value, NULL, 0); 1116 if ((result >= (1 << 16)) || (errno != 0)) { 1117 rte_bbdev_log(ERR, "Invalid value %lu for %s", result, key); 1118 return -ERANGE; 1119 } 1120 *u16 = (uint16_t)result; 1121 return 0; 1122 } 1123 1124 /* Parse parameters used to create device */ 1125 static int 1126 parse_turbo_sw_params(struct turbo_sw_params *params, const char *input_args) 1127 { 1128 struct rte_kvargs *kvlist = NULL; 1129 int ret = 0; 1130 1131 if (params == NULL) 1132 return -EINVAL; 1133 if (input_args) { 1134 kvlist = rte_kvargs_parse(input_args, turbo_sw_valid_params); 1135 if (kvlist == NULL) 1136 return -EFAULT; 1137 1138 ret = rte_kvargs_process(kvlist, turbo_sw_valid_params[0], 1139 &parse_u16_arg, ¶ms->queues_num); 1140 if (ret < 0) 1141 goto exit; 1142 1143 ret = rte_kvargs_process(kvlist, turbo_sw_valid_params[1], 1144 &parse_u16_arg, ¶ms->socket_id); 1145 if (ret < 0) 1146 goto exit; 1147 1148 if (params->socket_id >= RTE_MAX_NUMA_NODES) { 1149 rte_bbdev_log(ERR, "Invalid socket, must be < %u", 1150 RTE_MAX_NUMA_NODES); 1151 goto exit; 1152 } 1153 } 1154 1155 exit: 1156 if (kvlist) 1157 rte_kvargs_free(kvlist); 1158 return ret; 1159 } 1160 1161 /* Create device */ 1162 static int 1163 turbo_sw_bbdev_create(struct rte_vdev_device *vdev, 1164 struct turbo_sw_params *init_params) 1165 { 1166 struct rte_bbdev *bbdev; 1167 const char *name = rte_vdev_device_name(vdev); 1168 1169 bbdev = rte_bbdev_allocate(name); 1170 if (bbdev == NULL) 1171 return -ENODEV; 1172 1173 bbdev->data->dev_private = rte_zmalloc_socket(name, 1174 sizeof(struct bbdev_private), RTE_CACHE_LINE_SIZE, 1175 init_params->socket_id); 1176 if (bbdev->data->dev_private == NULL) { 1177 rte_bbdev_release(bbdev); 1178 return -ENOMEM; 1179 } 1180 1181 bbdev->dev_ops = &pmd_ops; 1182 bbdev->device = &vdev->device; 1183 bbdev->data->socket_id = init_params->socket_id; 1184 bbdev->intr_handle = NULL; 1185 1186 /* register rx/tx burst functions for data path */ 1187 bbdev->dequeue_enc_ops = dequeue_enc_ops; 1188 bbdev->dequeue_dec_ops = dequeue_dec_ops; 1189 bbdev->enqueue_enc_ops = enqueue_enc_ops; 1190 bbdev->enqueue_dec_ops = enqueue_dec_ops; 1191 ((struct bbdev_private *) bbdev->data->dev_private)->max_nb_queues = 1192 init_params->queues_num; 1193 1194 return 0; 1195 } 1196 1197 /* Initialise device */ 1198 static int 1199 turbo_sw_bbdev_probe(struct rte_vdev_device *vdev) 1200 { 1201 struct turbo_sw_params init_params = { 1202 rte_socket_id(), 1203 RTE_BBDEV_DEFAULT_MAX_NB_QUEUES 1204 }; 1205 const char *name; 1206 const char *input_args; 1207 1208 if (vdev == NULL) 1209 return -EINVAL; 1210 1211 name = rte_vdev_device_name(vdev); 1212 if (name == NULL) 1213 return -EINVAL; 1214 input_args = rte_vdev_device_args(vdev); 1215 parse_turbo_sw_params(&init_params, input_args); 1216 1217 rte_bbdev_log_debug( 1218 "Initialising %s on NUMA node %d with max queues: %d\n", 1219 name, init_params.socket_id, init_params.queues_num); 1220 1221 return turbo_sw_bbdev_create(vdev, &init_params); 1222 } 1223 1224 /* Uninitialise device */ 1225 static int 1226 turbo_sw_bbdev_remove(struct rte_vdev_device *vdev) 1227 { 1228 struct rte_bbdev *bbdev; 1229 const char *name; 1230 1231 if (vdev == NULL) 1232 return -EINVAL; 1233 1234 name = rte_vdev_device_name(vdev); 1235 if (name == NULL) 1236 return -EINVAL; 1237 1238 bbdev = rte_bbdev_get_named_dev(name); 1239 if (bbdev == NULL) 1240 return -EINVAL; 1241 1242 rte_free(bbdev->data->dev_private); 1243 1244 return rte_bbdev_release(bbdev); 1245 } 1246 1247 static struct rte_vdev_driver bbdev_turbo_sw_pmd_drv = { 1248 .probe = turbo_sw_bbdev_probe, 1249 .remove = turbo_sw_bbdev_remove 1250 }; 1251 1252 RTE_PMD_REGISTER_VDEV(DRIVER_NAME, bbdev_turbo_sw_pmd_drv); 1253 RTE_PMD_REGISTER_PARAM_STRING(DRIVER_NAME, 1254 TURBO_SW_MAX_NB_QUEUES_ARG"=<int> " 1255 TURBO_SW_SOCKET_ID_ARG"=<int>"); 1256 1257 RTE_INIT(null_bbdev_init_log); 1258 static void 1259 null_bbdev_init_log(void) 1260 { 1261 bbdev_turbo_sw_logtype = rte_log_register("pmd.bb.turbo_sw"); 1262 if (bbdev_turbo_sw_logtype >= 0) 1263 rte_log_set_level(bbdev_turbo_sw_logtype, RTE_LOG_NOTICE); 1264 } 1265